You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by ji...@apache.org on 2017/04/26 21:03:16 UTC
[1/6] mesos git commit: Added a TODO about an issue in POSIX disk
isolator.
Repository: mesos
Updated Branches:
refs/heads/master 54e47b443 -> 17139c5e5
Added a TODO about an issue in POSIX disk isolator.
The 'exclusive' list might not be accurate if a new persistent volumes
is later added to the resources list. That might result in incorrect
sandbox usage being reported.
Review: https://reviews.apache.org/r/58643
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/dff76aa4
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/dff76aa4
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/dff76aa4
Branch: refs/heads/master
Commit: dff76aa4f639a19052e723d522f7057229e43fad
Parents: c0cfacf
Author: Jie Yu <yu...@gmail.com>
Authored: Fri Apr 21 17:26:39 2017 -0700
Committer: Jie Yu <yu...@gmail.com>
Committed: Wed Apr 26 13:41:53 2017 -0700
----------------------------------------------------------------------
src/slave/containerizer/mesos/isolators/posix/disk.cpp | 5 +++++
1 file changed, 5 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/dff76aa4/src/slave/containerizer/mesos/isolators/posix/disk.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/isolators/posix/disk.cpp b/src/slave/containerizer/mesos/isolators/posix/disk.cpp
index 126344e..335e9f7 100644
--- a/src/slave/containerizer/mesos/isolators/posix/disk.cpp
+++ b/src/slave/containerizer/mesos/isolators/posix/disk.cpp
@@ -280,6 +280,11 @@ Future<Bytes> PosixDiskIsolatorProcess::collect(
const Owned<Info>& info = infos[containerId];
// Volume paths to exclude from sandbox disk usage calculation.
+ //
+ // TODO(jieyu): The 'excludes' list might change when a new
+ // persistent volume is added to the list. That might result in the
+ // 'du' process to incorrectly include the disk usage of the newly
+ // added persistent volume to the usage of the sandbox.
vector<string> excludes;
if (path == info->directory) {
foreachkey (const string& exclude, info->paths) {
[6/6] mesos git commit: Fixed a style issue in v1 mesos.proto.
Posted by ji...@apache.org.
Fixed a style issue in v1 mesos.proto.
Review: https://reviews.apache.org/r/58646
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/17139c5e
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/17139c5e
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/17139c5e
Branch: refs/heads/master
Commit: 17139c5e57a1a8b153c7f2584eb7f28922859825
Parents: df5038f
Author: Jie Yu <yu...@gmail.com>
Authored: Fri Apr 21 19:46:24 2017 -0700
Committer: Jie Yu <yu...@gmail.com>
Committed: Wed Apr 26 13:52:23 2017 -0700
----------------------------------------------------------------------
include/mesos/v1/mesos.proto | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/17139c5e/include/mesos/v1/mesos.proto
----------------------------------------------------------------------
diff --git a/include/mesos/v1/mesos.proto b/include/mesos/v1/mesos.proto
index 7423da9..f7c05a8 100644
--- a/include/mesos/v1/mesos.proto
+++ b/include/mesos/v1/mesos.proto
@@ -1046,6 +1046,7 @@ message Resource {
optional SharedInfo shared = 10;
}
+
/**
* When the network bandwidth caps are enabled and the container
* is over its limit, outbound packets may be either delayed or
[3/6] mesos git commit: Cancelled the disk usage collection if no
longer needed.
Posted by ji...@apache.org.
Cancelled the disk usage collection if no longer needed.
We actively cancel the disk usage collection if its result is no
longer needed. This is a performance optimization.
Review: https://reviews.apache.org/r/58642
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/c0cfacfb
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/c0cfacfb
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/c0cfacfb
Branch: refs/heads/master
Commit: c0cfacfb134f74ff35f48e36af2f914e5530057d
Parents: d680ff6
Author: Jie Yu <yu...@gmail.com>
Authored: Fri Apr 21 15:24:10 2017 -0700
Committer: Jie Yu <yu...@gmail.com>
Committed: Wed Apr 26 13:41:53 2017 -0700
----------------------------------------------------------------------
src/slave/containerizer/mesos/isolators/posix/disk.cpp | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/c0cfacfb/src/slave/containerizer/mesos/isolators/posix/disk.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/isolators/posix/disk.cpp b/src/slave/containerizer/mesos/isolators/posix/disk.cpp
index 127f490..126344e 100644
--- a/src/slave/containerizer/mesos/isolators/posix/disk.cpp
+++ b/src/slave/containerizer/mesos/isolators/posix/disk.cpp
@@ -261,6 +261,8 @@ Future<Nothing> PosixDiskIsolatorProcess::update(
// Remove paths that we no longer interested in.
foreach (const string& path, info->paths.keys()) {
if (!quotas.contains(path)) {
+ // Cancel the usage collection as we are no longer interested.
+ info->paths[path].usage.discard();
info->paths.erase(path);
}
}
[2/6] mesos git commit: Fixed a style issue in mesos.proto.
Posted by ji...@apache.org.
Fixed a style issue in mesos.proto.
Review: https://reviews.apache.org/r/58641
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/d680ff65
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/d680ff65
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/d680ff65
Branch: refs/heads/master
Commit: d680ff65e79d1d31de2cdc3ee70401c44fb2ac63
Parents: 54e47b4
Author: Jie Yu <yu...@gmail.com>
Authored: Fri Apr 21 13:28:47 2017 -0700
Committer: Jie Yu <yu...@gmail.com>
Committed: Wed Apr 26 13:41:53 2017 -0700
----------------------------------------------------------------------
include/mesos/mesos.proto | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/d680ff65/include/mesos/mesos.proto
----------------------------------------------------------------------
diff --git a/include/mesos/mesos.proto b/include/mesos/mesos.proto
index 0087343..3ed147d 100644
--- a/include/mesos/mesos.proto
+++ b/include/mesos/mesos.proto
@@ -1052,6 +1052,7 @@ message Resource {
optional SharedInfo shared = 10;
}
+
/**
* When the network bandwidth caps are enabled and the container
* is over its limit, outbound packets may be either delayed or
[5/6] mesos git commit: Updated the tests to verify persistent volume
usage.
Posted by ji...@apache.org.
Updated the tests to verify persistent volume usage.
Review: https://reviews.apache.org/r/58645
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/df5038f2
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/df5038f2
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/df5038f2
Branch: refs/heads/master
Commit: df5038f254d34e9bac16372e4d091811ffb52f64
Parents: 62ea5ad
Author: Jie Yu <yu...@gmail.com>
Authored: Fri Apr 21 19:45:52 2017 -0700
Committer: Jie Yu <yu...@gmail.com>
Committed: Wed Apr 26 13:52:22 2017 -0700
----------------------------------------------------------------------
src/tests/disk_quota_tests.cpp | 79 ++++++++++++++++++++++++++++++++-----
1 file changed, 70 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/df5038f2/src/tests/disk_quota_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/disk_quota_tests.cpp b/src/tests/disk_quota_tests.cpp
index d67867e..f758bee 100644
--- a/src/tests/disk_quota_tests.cpp
+++ b/src/tests/disk_quota_tests.cpp
@@ -449,6 +449,8 @@ TEST_F(DiskQuotaTest, ResourceStatistics)
slave::Flags flags = CreateSlaveFlags();
flags.isolation = "posix/cpu,posix/mem,disk/du";
+ flags.resources = strings::format("disk(%s):10", DEFAULT_TEST_ROLE).get();
+
// NOTE: We can't pause the clock because we need the reaper to reap
// the 'du' subprocess.
flags.container_disk_watch_interval = Milliseconds(1);
@@ -469,10 +471,16 @@ TEST_F(DiskQuotaTest, ResourceStatistics)
ASSERT_SOME(slave);
+ FrameworkInfo frameworkInfo = DEFAULT_FRAMEWORK_INFO;
+ frameworkInfo.set_role(DEFAULT_TEST_ROLE);
+
MockScheduler sched;
MesosSchedulerDriver driver(
- &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL);
+ &sched,
+ frameworkInfo,
+ master.get()->pid,
+ DEFAULT_CREDENTIAL);
EXPECT_CALL(sched, registered(_, _, _));
@@ -488,22 +496,48 @@ TEST_F(DiskQuotaTest, ResourceStatistics)
const Offer& offer = offers.get()[0];
+ Resource volume = createPersistentVolume(
+ Megabytes(4),
+ DEFAULT_TEST_ROLE,
+ "id1",
+ "path1",
+ None(),
+ None(),
+ DEFAULT_CREDENTIAL.principal());
+
+ Resources taskResources = Resources::parse("cpus:1;mem:128").get();
+
+ taskResources += createDiskResource(
+ "3",
+ DEFAULT_TEST_ROLE,
+ None(),
+ None());
+
+ taskResources += volume;
+
// Create a task that uses 2MB disk.
TaskInfo task = createTask(
offer.slave_id(),
- Resources::parse("cpus:1;mem:128;disk:3").get(),
- "dd if=/dev/zero of=file bs=1048576 count=2 && sleep 1000");
+ taskResources,
+ "dd if=/dev/zero of=file bs=1048576 count=2 && "
+ "dd if=/dev/zero of=path1/file bs=1048576 count=2 && "
+ "sleep 1000");
- Future<TaskStatus> status;
+ Future<TaskStatus> status1;
+ Future<TaskStatus> status2;
EXPECT_CALL(sched, statusUpdate(&driver, _))
- .WillOnce(FutureArg<1>(&status))
+ .WillOnce(FutureArg<1>(&status1))
+ .WillOnce(FutureArg<1>(&status2))
.WillRepeatedly(Return()); // Ignore subsequent updates.
- driver.launchTasks(offer.id(), {task});
+ driver.acceptOffers(
+ {offer.id()},
+ {CREATE(volume),
+ LAUNCH({task})});
- AWAIT_READY(status);
- EXPECT_EQ(task.task_id(), status->task_id());
- EXPECT_EQ(TASK_RUNNING, status->state());
+ AWAIT_READY(status1);
+ EXPECT_EQ(task.task_id(), status1->task_id());
+ EXPECT_EQ(TASK_RUNNING, status1->state());
Future<hashset<ContainerID>> containers = containerizer->containers();
@@ -523,6 +557,27 @@ TEST_F(DiskQuotaTest, ResourceStatistics)
if (usage->has_disk_used_bytes()) {
EXPECT_LE(usage->disk_used_bytes(), usage->disk_limit_bytes());
+ }
+
+ ASSERT_EQ(2u, usage->disk_statistics().size());
+
+ bool done = true;
+ foreach (const DiskStatistics& statistics, usage->disk_statistics()) {
+ ASSERT_TRUE(statistics.has_limit_bytes());
+ EXPECT_EQ(
+ statistics.has_persistence() ? Megabytes(4) : Megabytes(3),
+ statistics.limit_bytes());
+
+ if (!statistics.has_used_bytes()) {
+ done = false;
+ } else {
+ EXPECT_GT(
+ statistics.has_persistence() ? Megabytes(4) : Megabytes(3),
+ statistics.used_bytes());
+ }
+ }
+
+ if (done) {
break;
}
@@ -532,6 +587,12 @@ TEST_F(DiskQuotaTest, ResourceStatistics)
elapsed += Milliseconds(1);
}
+ driver.killTask(task.task_id());
+
+ AWAIT_READY(status2);
+ EXPECT_EQ(task.task_id(), status2->task_id());
+ EXPECT_EQ(TASK_KILLED, status2->state());
+
driver.stop();
driver.join();
}
[4/6] mesos git commit: Improved POSIX disk isolator to report usage
for persistent volumes.
Posted by ji...@apache.org.
Improved POSIX disk isolator to report usage for persistent volumes.
Review: https://reviews.apache.org/r/58644
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/62ea5ad9
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/62ea5ad9
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/62ea5ad9
Branch: refs/heads/master
Commit: 62ea5ad976ce243ea6f9989d2fda1b633934744f
Parents: dff76aa
Author: Jie Yu <yu...@gmail.com>
Authored: Fri Apr 21 19:44:43 2017 -0700
Committer: Jie Yu <yu...@gmail.com>
Committed: Wed Apr 26 13:52:16 2017 -0700
----------------------------------------------------------------------
include/mesos/mesos.proto | 11 ++++++
include/mesos/v1/mesos.proto | 11 ++++++
.../mesos/isolators/posix/disk.cpp | 38 ++++++++++++++++----
3 files changed, 53 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/62ea5ad9/include/mesos/mesos.proto
----------------------------------------------------------------------
diff --git a/include/mesos/mesos.proto b/include/mesos/mesos.proto
index 3ed147d..65e01b2 100644
--- a/include/mesos/mesos.proto
+++ b/include/mesos/mesos.proto
@@ -1185,6 +1185,14 @@ message SNMPStatistics {
}
+message DiskStatistics {
+ optional Resource.DiskInfo.Source source = 1;
+ optional Resource.DiskInfo.Persistence persistence = 2;
+ optional uint64 limit_bytes = 3;
+ optional uint64 used_bytes = 4;
+}
+
+
/**
* A snapshot of resource usage statistics.
*/
@@ -1259,6 +1267,9 @@ message ResourceStatistics {
optional uint64 disk_limit_bytes = 26;
optional uint64 disk_used_bytes = 27;
+ // Per disk (resource) statistics.
+ repeated DiskStatistics disk_statistics = 43;
+
// Perf statistics.
optional PerfStatistics perf = 13;
http://git-wip-us.apache.org/repos/asf/mesos/blob/62ea5ad9/include/mesos/v1/mesos.proto
----------------------------------------------------------------------
diff --git a/include/mesos/v1/mesos.proto b/include/mesos/v1/mesos.proto
index a4426c4..7423da9 100644
--- a/include/mesos/v1/mesos.proto
+++ b/include/mesos/v1/mesos.proto
@@ -1178,6 +1178,14 @@ message SNMPStatistics {
}
+message DiskStatistics {
+ optional Resource.DiskInfo.Source source = 1;
+ optional Resource.DiskInfo.Persistence persistence = 2;
+ optional uint64 limit_bytes = 3;
+ optional uint64 used_bytes = 4;
+}
+
+
/**
* A snapshot of resource usage statistics.
*/
@@ -1252,6 +1260,9 @@ message ResourceStatistics {
optional uint64 disk_limit_bytes = 26;
optional uint64 disk_used_bytes = 27;
+ // Per disk (resource) statistics.
+ repeated DiskStatistics disk_statistics = 43;
+
// Perf statistics.
optional PerfStatistics perf = 13;
http://git-wip-us.apache.org/repos/asf/mesos/blob/62ea5ad9/src/slave/containerizer/mesos/isolators/posix/disk.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/isolators/posix/disk.cpp b/src/slave/containerizer/mesos/isolators/posix/disk.cpp
index 335e9f7..eb23025 100644
--- a/src/slave/containerizer/mesos/isolators/posix/disk.cpp
+++ b/src/slave/containerizer/mesos/isolators/posix/disk.cpp
@@ -388,22 +388,46 @@ Future<ResourceStatistics> PosixDiskIsolatorProcess::usage(
return Failure("Unknown container");
}
- // TODO(hartem): Report volume usage as well (MESOS-4263).
ResourceStatistics result;
const Owned<Info>& info = infos[containerId];
- if (info->paths.contains(info->directory)) {
- Option<Bytes> quota = info->paths[info->directory].quota.disk();
+ foreachpair (const string& path,
+ const Info::PathInfo& pathInfo,
+ info->paths) {
+ DiskStatistics *statistics = result.add_disk_statistics();
+
+ Option<Bytes> quota = pathInfo.quota.disk();
CHECK_SOME(quota);
- result.set_disk_limit_bytes(quota.get().bytes());
+ statistics->set_limit_bytes(quota->bytes());
+ if (path == info->directory) {
+ result.set_disk_limit_bytes(quota->bytes());
+ }
// NOTE: There may be a large delay (# of containers * interval)
// until an initial cached value is returned here!
- if (info->paths[info->directory].lastUsage.isSome()) {
- result.set_disk_used_bytes(
- info->paths[info->directory].lastUsage.get().bytes());
+ if (pathInfo.lastUsage.isSome()) {
+ statistics->set_used_bytes(pathInfo.lastUsage->bytes());
+ if (path == info->directory) {
+ result.set_disk_used_bytes(pathInfo.lastUsage->bytes());
+ }
+ }
+
+ // Set meta information for persistent volumes.
+ if (path != info->directory) {
+ // TODO(jieyu): For persistent volumes, validate that there is
+ // only one Resource object associated with it.
+ Resource resource = *pathInfo.quota.begin();
+
+ if (resource.has_disk() && resource.disk().has_source()) {
+ statistics->mutable_source()->CopyFrom(resource.disk().source());
+ }
+
+ if (resource.has_disk() && resource.disk().has_persistence()) {
+ statistics->mutable_persistence()->CopyFrom(
+ resource.disk().persistence());
+ }
}
}