You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by ji...@apache.org on 2016/01/04 18:21:03 UTC
[1/2] mesos git commit: Added support for checking whether a given
path is absolute.
Repository: mesos
Updated Branches:
refs/heads/master 440eb96ee -> 4706504c5
Added support for checking whether a given path is absolute.
Review: https://reviews.apache.org/r/41704/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/5682052b
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/5682052b
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/5682052b
Branch: refs/heads/master
Commit: 5682052be45d67dc2bcdf969ee38bb55cb4e2019
Parents: 440eb96
Author: Artem Harutyunyan <ar...@mesosphere.io>
Authored: Mon Jan 4 09:20:30 2016 -0800
Committer: Jie Yu <yu...@gmail.com>
Committed: Mon Jan 4 09:20:30 2016 -0800
----------------------------------------------------------------------
.../3rdparty/stout/include/stout/path.hpp | 92 ++++++++++++--------
.../3rdparty/stout/tests/path_tests.cpp | 18 ++++
2 files changed, 72 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/5682052b/3rdparty/libprocess/3rdparty/stout/include/stout/path.hpp
----------------------------------------------------------------------
diff --git a/3rdparty/libprocess/3rdparty/stout/include/stout/path.hpp b/3rdparty/libprocess/3rdparty/stout/include/stout/path.hpp
index 386188b..ef53804 100644
--- a/3rdparty/libprocess/3rdparty/stout/include/stout/path.hpp
+++ b/3rdparty/libprocess/3rdparty/stout/include/stout/path.hpp
@@ -19,6 +19,52 @@
#include <stout/strings.hpp>
+namespace path {
+
+// Base case.
+inline std::string join(const std::string& path1, const std::string& path2)
+{
+ return strings::remove(path1, "/", strings::SUFFIX) + "/" +
+ strings::remove(path2, "/", strings::PREFIX);
+}
+
+
+template <typename... Paths>
+inline std::string join(
+ const std::string& path1,
+ const std::string& path2,
+ Paths&&... paths)
+{
+ return join(path1, join(path2, std::forward<Paths>(paths)...));
+}
+
+
+inline std::string join(const std::vector<std::string>& paths)
+{
+ if (paths.empty()) {
+ return "";
+ }
+
+ std::string result = paths[0];
+ for (size_t i = 1; i < paths.size(); ++i) {
+ result = join(result, paths[i]);
+ }
+ return result;
+}
+
+
+inline bool absolute(const std::string& path)
+{
+ if (path.empty() || path[0] != '/') {
+ return false;
+ }
+
+ return true;
+}
+
+} // namespace path {
+
+
/**
* Represents a POSIX file systems path and offers common path
* manipulations.
@@ -29,8 +75,8 @@ public:
explicit Path(const std::string& path)
: value(strings::remove(path, "file://", strings::PREFIX)) {}
- // TODO(cmaloney): Add more useful operations such as 'absolute()',
- // 'directoryname()', 'filename()', etc.
+ // TODO(cmaloney): Add more useful operations such as 'directoryname()',
+ // 'filename()', etc.
/**
* Extracts the component following the final '/'. Trailing '/'
@@ -176,6 +222,12 @@ public:
return _basename.substr(index);
}
+ // Checks whether the path is absolute.
+ inline bool absolute() const
+ {
+ return path::absolute(value);
+ }
+
// Implicit conversion from Path to string.
operator std::string() const
{
@@ -193,40 +245,4 @@ inline std::ostream& operator<<(
return stream << path.value;
}
-
-namespace path {
-
-// Base case.
-inline std::string join(const std::string& path1, const std::string& path2)
-{
- return strings::remove(path1, "/", strings::SUFFIX) + "/" +
- strings::remove(path2, "/", strings::PREFIX);
-}
-
-
-template <typename... Paths>
-inline std::string join(
- const std::string& path1,
- const std::string& path2,
- Paths&&... paths)
-{
- return join(path1, join(path2, std::forward<Paths>(paths)...));
-}
-
-
-inline std::string join(const std::vector<std::string>& paths)
-{
- if (paths.empty()) {
- return "";
- }
-
- std::string result = paths[0];
- for (size_t i = 1; i < paths.size(); ++i) {
- result = join(result, paths[i]);
- }
- return result;
-}
-
-} // namespace path {
-
#endif // __STOUT_PATH_HPP__
http://git-wip-us.apache.org/repos/asf/mesos/blob/5682052b/3rdparty/libprocess/3rdparty/stout/tests/path_tests.cpp
----------------------------------------------------------------------
diff --git a/3rdparty/libprocess/3rdparty/stout/tests/path_tests.cpp b/3rdparty/libprocess/3rdparty/stout/tests/path_tests.cpp
index 4374e64..79cda84 100644
--- a/3rdparty/libprocess/3rdparty/stout/tests/path_tests.cpp
+++ b/3rdparty/libprocess/3rdparty/stout/tests/path_tests.cpp
@@ -143,6 +143,24 @@ TEST(PathTest, Join)
}
+TEST(PathTest, Absolute)
+{
+ // Check absolute paths.
+ EXPECT_TRUE(path::absolute("/"));
+ EXPECT_TRUE(path::absolute("/foo"));
+ EXPECT_TRUE(path::absolute("/foo/bar"));
+ EXPECT_TRUE(path::absolute("/foo/bar/../baz"));
+
+ // Check relative paths.
+ EXPECT_FALSE(path::absolute(""));
+ EXPECT_FALSE(path::absolute("."));
+ EXPECT_FALSE(path::absolute(".."));
+ EXPECT_FALSE(path::absolute("../"));
+ EXPECT_FALSE(path::absolute("./foo"));
+ EXPECT_FALSE(path::absolute("../foo"));
+}
+
+
class PathFileTest : public TemporaryDirectoryTest {};
[2/2] mesos git commit: Added support for enforcing quota on
(persistent) volumes (MESOS-4198).
Posted by ji...@apache.org.
Added support for enforcing quota on (persistent) volumes (MESOS-4198).
Review: https://reviews.apache.org/r/41705/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/4706504c
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/4706504c
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/4706504c
Branch: refs/heads/master
Commit: 4706504c51446f31253a88a733d7aa30e9fea842
Parents: 5682052
Author: Artem Harutyunyan <ar...@mesosphere.io>
Authored: Mon Jan 4 09:20:36 2016 -0800
Committer: Jie Yu <yu...@gmail.com>
Committed: Mon Jan 4 09:20:36 2016 -0800
----------------------------------------------------------------------
.../mesos/isolators/posix/disk.cpp | 22 +++--
src/tests/disk_quota_tests.cpp | 94 +++++++++++++++++++-
2 files changed, 110 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/4706504c/src/slave/containerizer/mesos/isolators/posix/disk.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/isolators/posix/disk.cpp b/src/slave/containerizer/mesos/isolators/posix/disk.cpp
index d971db0..248c34a 100644
--- a/src/slave/containerizer/mesos/isolators/posix/disk.cpp
+++ b/src/slave/containerizer/mesos/isolators/posix/disk.cpp
@@ -38,6 +38,7 @@
#include <stout/lambda.hpp>
#include <stout/numify.hpp>
#include <stout/strings.hpp>
+#include <stout/path.hpp>
#include <stout/os/exists.hpp>
#include <stout/os/killtree.hpp>
@@ -167,13 +168,23 @@ Future<Nothing> PosixDiskIsolatorProcess::update(
// NOTE: We do not allow the case where has_disk() is true but
// with nothing set inside DiskInfo. The master will enforce it.
- if (!resource.has_disk()) {
- // Regular disk used for executor working directory.
+ if (!resource.has_disk() || !resource.disk().has_volume()) {
+ // If either DiskInfo or DiskInfo.Volume are not set we're dealing
+ // with the working directory of the executor (aka the sanbox).
path = info->directory;
} else {
- // TODO(jieyu): Support persistent volmes as well.
- LOG(ERROR) << "Enforcing disk quota unsupported for " << resource;
- continue;
+ // Otherwise it is a disk resource (such as a persistent volume) and
+ // we extract the path from the protobuf.
+ path = resource.disk().volume().container_path();
+
+ // In case the path in the protobuf is not an absolute path it is
+ // relative to the working directory of the executor. We always store
+ // the absolute path.
+ if (!path::absolute(path)) {
+ // We prepend "/" at the end to make sure that 'du' runs on actual
+ // directory pointed by the symlink (and not the symlink itself).
+ path = path::join(info->directory, path, "");
+ }
}
quotas[path] += resource;
@@ -272,6 +283,7 @@ Future<ResourceStatistics> PosixDiskIsolatorProcess::usage(
return Failure("Unknown container");
}
+ // TODO(hartem): Report volume usage as well (MESOS-4263).
ResourceStatistics result;
const Owned<Info>& info = infos[containerId];
http://git-wip-us.apache.org/repos/asf/mesos/blob/4706504c/src/tests/disk_quota_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/disk_quota_tests.cpp b/src/tests/disk_quota_tests.cpp
index ce736c3..1577cf7 100644
--- a/src/tests/disk_quota_tests.cpp
+++ b/src/tests/disk_quota_tests.cpp
@@ -178,7 +178,7 @@ TEST_F(DiskQuotaTest, DiskUsageExceedsQuota)
AWAIT_READY(offers);
EXPECT_FALSE(offers.get().empty());
- Offer offer = offers.get()[0];
+ const Offer& offer = offers.get()[0];
// Create a task which requests 1MB disk, but actually uses more
// than 2MB disk.
@@ -210,6 +210,98 @@ TEST_F(DiskQuotaTest, DiskUsageExceedsQuota)
}
+// This test verifies that the container will be killed if the volume
+// usage exceeds its quota.
+TEST_F(DiskQuotaTest, VolumeUsageExceedsQuota)
+{
+ FrameworkInfo frameworkInfo = DEFAULT_FRAMEWORK_INFO;
+ frameworkInfo.set_role("role1");
+
+ master::Flags masterFlags = CreateMasterFlags();
+
+ Try<PID<Master>> master = StartMaster(masterFlags);
+ ASSERT_SOME(master);
+
+ slave::Flags slaveFlags = CreateSlaveFlags();
+ slaveFlags.isolation = "posix/cpu,posix/mem,posix/disk";
+
+ // NOTE: We can't pause the clock because we need the reaper to reap
+ // the 'du' subprocess.
+ slaveFlags.container_disk_watch_interval = Milliseconds(1);
+ slaveFlags.enforce_container_disk_quota = true;
+ slaveFlags.resources = "cpus:2;mem:128;disk(role1):128";
+
+ Try<PID<Slave>> slave = StartSlave(slaveFlags);
+ ASSERT_SOME(slave);
+
+ MockScheduler sched;
+ MesosSchedulerDriver driver(
+ &sched, frameworkInfo, master.get(), DEFAULT_CREDENTIAL);
+
+ Future<FrameworkID> frameworkId;
+ EXPECT_CALL(sched, registered(&driver, _, _))
+ .WillOnce(FutureArg<1>(&frameworkId));
+
+ Future<vector<Offer>> offers;
+ EXPECT_CALL(sched, resourceOffers(&driver, _))
+ .WillOnce(FutureArg<1>(&offers))
+ .WillRepeatedly(Return()); // Ignore subsequent offers.
+
+ driver.start();
+
+ AWAIT_READY(frameworkId);
+
+ AWAIT_READY(offers);
+ EXPECT_FALSE(offers.get().empty());
+
+ Offer offer = offers.get()[0];
+
+ // Create a task that requests a 1 MB persistent volume but atempts
+ // to use 2MB.
+ Resources volume = createPersistentVolume(
+ Megabytes(1),
+ "role1",
+ "id1",
+ "volume_path");
+
+ // We intentionally request a sandbox that is much bugger (16MB) than
+ // the file the task writes (2MB) to the persistent volume (1MB). This
+ // makes sure that the quota is indeed enforced on the persistent volume.
+ Resources taskResources =
+ Resources::parse("cpus:1;mem:64;disk(role1):16").get() + volume;
+
+ TaskInfo task = createTask(
+ offer.slave_id(),
+ taskResources,
+ "dd if=/dev/zero of=volume_path/file bs=1048576 count=2 && sleep 1000");
+
+ Future<TaskStatus> status1;
+ Future<TaskStatus> status2;
+ EXPECT_CALL(sched, statusUpdate(&driver, _))
+ .WillOnce(FutureArg<1>(&status1))
+ .WillOnce(FutureArg<1>(&status2));
+
+ // Create the volume and launch the task.
+ driver.acceptOffers(
+ {offer.id()},
+ {CREATE(volume),
+ LAUNCH({task})});
+
+ AWAIT_READY(status1);
+ EXPECT_EQ(task.task_id(), status1.get().task_id());
+ EXPECT_EQ(TASK_RUNNING, status1.get().state());
+
+ AWAIT_READY(status2);
+ EXPECT_EQ(task.task_id(), status1.get().task_id());
+ EXPECT_EQ(TASK_FAILED, status2.get().state());
+
+ driver.stop();
+ driver.join();
+
+ Shutdown();
+}
+
+
// This test verifies that the container will not be killed if
// disk_enforce_quota flag is false (even if the disk usage exceeds
// its quota).