You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by ji...@apache.org on 2016/04/11 18:19:40 UTC
mesos git commit: Improved overlay backend to make the rootfs
writable.
Repository: mesos
Updated Branches:
refs/heads/master d353bcb16 -> 4dfa91fc2
Improved overlay backend to make the rootfs writable.
Review: https://reviews.apache.org/r/45358/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/4dfa91fc
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/4dfa91fc
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/4dfa91fc
Branch: refs/heads/master
Commit: 4dfa91fc21f80204f5125b2e2f35c489f8fb41d8
Parents: d353bcb
Author: Shuai Lin <li...@gmail.com>
Authored: Mon Apr 11 08:51:31 2016 -0700
Committer: Jie Yu <yu...@gmail.com>
Committed: Mon Apr 11 09:17:38 2016 -0700
----------------------------------------------------------------------
docs/container-image.md | 10 ---
.../containerizer/mesos/provisioner/backend.hpp | 3 +-
.../mesos/provisioner/backends/bind.cpp | 3 +-
.../mesos/provisioner/backends/bind.hpp | 3 +-
.../mesos/provisioner/backends/copy.cpp | 3 +-
.../mesos/provisioner/backends/copy.hpp | 3 +-
.../mesos/provisioner/backends/overlay.cpp | 69 ++++++++++++++------
.../mesos/provisioner/backends/overlay.hpp | 21 ++++--
.../containerizer/mesos/provisioner/paths.cpp | 13 ++++
.../containerizer/mesos/provisioner/paths.hpp | 7 ++
.../mesos/provisioner/provisioner.cpp | 10 ++-
.../containerizer/provisioner_backend_tests.cpp | 48 ++++++++------
12 files changed, 130 insertions(+), 63 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/4dfa91fc/docs/container-image.md
----------------------------------------------------------------------
diff --git a/docs/container-image.md b/docs/container-image.md
index c4abfa6..843881e 100644
--- a/docs/container-image.md
+++ b/docs/container-image.md
@@ -298,16 +298,6 @@ for more detail.
For more information of overlayfs, please refer to
[here](https://www.kernel.org/doc/Documentation/filesystems/overlayfs.txt).
-The overlay backend currently has these two limitations:
-
-1. The overlay backend only supports more than one single layer. One
-single layer images will fail to provision and the container will fail
-to launch! This limitation will be solved soon.
-
-2. Similar to the bind backend, the filesystem will be read-only.
-Please refer to the second limitation of the bind backend for more
-details. We will resolve this limitation soon.
-
## Executor Dependencies in a Container Image
http://git-wip-us.apache.org/repos/asf/mesos/blob/4dfa91fc/src/slave/containerizer/mesos/provisioner/backend.hpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/backend.hpp b/src/slave/containerizer/mesos/provisioner/backend.hpp
index c6cca81..93819c8 100644
--- a/src/slave/containerizer/mesos/provisioner/backend.hpp
+++ b/src/slave/containerizer/mesos/provisioner/backend.hpp
@@ -50,7 +50,8 @@ public:
// another layer earlier in the list.
virtual process::Future<Nothing> provision(
const std::vector<std::string>& layers,
- const std::string& rootfs) = 0;
+ const std::string& rootfs,
+ const std::string& backendDir) = 0;
// Destroy the root filesystem provisioned at the specified 'rootfs'
// directory. Return false if there is no provisioned root filesystem
http://git-wip-us.apache.org/repos/asf/mesos/blob/4dfa91fc/src/slave/containerizer/mesos/provisioner/backends/bind.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/backends/bind.cpp b/src/slave/containerizer/mesos/provisioner/backends/bind.cpp
index 9b9f0b9..ea7829b 100644
--- a/src/slave/containerizer/mesos/provisioner/backends/bind.cpp
+++ b/src/slave/containerizer/mesos/provisioner/backends/bind.cpp
@@ -90,7 +90,8 @@ BindBackend::BindBackend(Owned<BindBackendProcess> _process)
Future<Nothing> BindBackend::provision(
const vector<string>& layers,
- const string& rootfs)
+ const string& rootfs,
+ const string& backendDir)
{
return dispatch(
process.get(), &BindBackendProcess::provision, layers, rootfs);
http://git-wip-us.apache.org/repos/asf/mesos/blob/4dfa91fc/src/slave/containerizer/mesos/provisioner/backends/bind.hpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/backends/bind.hpp b/src/slave/containerizer/mesos/provisioner/backends/bind.hpp
index 9eda944..2b2fcdc 100644
--- a/src/slave/containerizer/mesos/provisioner/backends/bind.hpp
+++ b/src/slave/containerizer/mesos/provisioner/backends/bind.hpp
@@ -53,7 +53,8 @@ public:
virtual process::Future<Nothing> provision(
const std::vector<std::string>& layers,
- const std::string& rootfs);
+ const std::string& rootfs,
+ const std::string& backendDir);
virtual process::Future<bool> destroy(const std::string& rootfs);
http://git-wip-us.apache.org/repos/asf/mesos/blob/4dfa91fc/src/slave/containerizer/mesos/provisioner/backends/copy.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/backends/copy.cpp b/src/slave/containerizer/mesos/provisioner/backends/copy.cpp
index f353c89..b9f6d7a 100644
--- a/src/slave/containerizer/mesos/provisioner/backends/copy.cpp
+++ b/src/slave/containerizer/mesos/provisioner/backends/copy.cpp
@@ -77,7 +77,8 @@ CopyBackend::CopyBackend(Owned<CopyBackendProcess> _process)
Future<Nothing> CopyBackend::provision(
const vector<string>& layers,
- const string& rootfs)
+ const string& rootfs,
+ const string& backendDir)
{
return dispatch(
process.get(), &CopyBackendProcess::provision, layers, rootfs);
http://git-wip-us.apache.org/repos/asf/mesos/blob/4dfa91fc/src/slave/containerizer/mesos/provisioner/backends/copy.hpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/backends/copy.hpp b/src/slave/containerizer/mesos/provisioner/backends/copy.hpp
index b62507f..5bd6a52 100644
--- a/src/slave/containerizer/mesos/provisioner/backends/copy.hpp
+++ b/src/slave/containerizer/mesos/provisioner/backends/copy.hpp
@@ -47,7 +47,8 @@ public:
// path.
virtual process::Future<Nothing> provision(
const std::vector<std::string>& layers,
- const std::string& rootfs);
+ const std::string& rootfs,
+ const std::string& backendDir);
virtual process::Future<bool> destroy(const std::string& rootfs);
http://git-wip-us.apache.org/repos/asf/mesos/blob/4dfa91fc/src/slave/containerizer/mesos/provisioner/backends/overlay.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/backends/overlay.cpp b/src/slave/containerizer/mesos/provisioner/backends/overlay.cpp
index 93892a7..ebe1fc5 100644
--- a/src/slave/containerizer/mesos/provisioner/backends/overlay.cpp
+++ b/src/slave/containerizer/mesos/provisioner/backends/overlay.cpp
@@ -45,7 +45,10 @@ namespace slave {
class OverlayBackendProcess : public Process<OverlayBackendProcess>
{
public:
- Future<Nothing> provision(const vector<string>& layers, const string& rootfs);
+ Future<Nothing> provision(
+ const vector<string>& layers,
+ const string& rootfs,
+ const string& backendDir);
Future<bool> destroy(const string& rootfs);
};
@@ -55,15 +58,15 @@ Try<Owned<Backend>> OverlayBackend::create(const Flags&)
{
Result<string> user = os::user();
if (!user.isSome()) {
- return Error("Failed to determine user: " +
- (user.isError() ? user.error() : "username not found"));
+ return Error(
+ "Failed to determine user: " +
+ (user.isError() ? user.error() : "username not found"));
}
if (user.get() != "root") {
return Error(
"OverlayBackend requires root privileges, "
- "but is running as user " +
- user.get());
+ "but is running as user " + user.get());
}
Try<bool> supported = fs::overlay::supported();
@@ -96,13 +99,17 @@ OverlayBackend::OverlayBackend(Owned<OverlayBackendProcess> _process)
Future<Nothing> OverlayBackend::provision(
const vector<string>& layers,
- const string& rootfs)
+ const string& rootfs,
+ const string& backendDir)
{
return dispatch(
- process.get(), &OverlayBackendProcess::provision, layers, rootfs);
+ process.get(),
+ &OverlayBackendProcess::provision,
+ layers,
+ rootfs,
+ backendDir);
}
-
Future<bool> OverlayBackend::destroy(const string& rootfs)
{
return dispatch(process.get(), &OverlayBackendProcess::destroy, rootfs);
@@ -111,34 +118,54 @@ Future<bool> OverlayBackend::destroy(const string& rootfs)
Future<Nothing> OverlayBackendProcess::provision(
const vector<string>& layers,
- const string& rootfs)
+ const string& rootfs,
+ const string& backendDir)
{
if (layers.size() == 0) {
return Failure("No filesystem layer provided");
}
- if (layers.size() == 1) {
- return Failure("Need more than one layer for overlay");
+ Try<Nothing> mkdir = os::mkdir(rootfs);
+ if (mkdir.isError()) {
+ return Failure(
+ "Failed to create container rootfs at '" +
+ rootfs + "': " + mkdir.error());
}
- Try<Nothing> mkdir = os::mkdir(rootfs);
+ const string scratchDirId = Path(rootfs).basename();
+ const string scratchDir = path::join(backendDir, "scratch", scratchDirId);
+ const string upperdir = path::join(scratchDir, "upperdir");
+ const string workdir = path::join(scratchDir, "workdir");
+
+ mkdir = os::mkdir(upperdir);
if (mkdir.isError()) {
- return Failure("Failed to create container rootfs at '" + rootfs + "': " +
- mkdir.error());
+ return Failure(
+ "Failed to create overlay upperdir at '" +
+ upperdir + "': " + mkdir.error());
}
- // For overlayfs, the specified lower directories will be stacked beginning
- // from the rightmost one and going left. But we need the first layer in the
- // vector to be the the bottom most layer.
- std::string lowerDir = "lowerdir=";
- lowerDir += strings::join(":", adaptor::reverse(layers));
+ mkdir = os::mkdir(workdir);
+ if (mkdir.isError()) {
+ return Failure(
+ "Failed to create overlay workdir at '" +
+ workdir + "': " + mkdir.error());
+ }
+
+ // For overlayfs, the specified lower directories will be stacked
+ // beginning from the rightmost one and going left. But we need the
+ // first layer in the vector to be the the bottom most layer.
+ string options = "lowerdir=" + strings::join(":", adaptor::reverse(layers));
+ options += ",upperdir=" + upperdir;
+ options += ",workdir=" + workdir;
+
+ VLOG(1) << "Provisioning image rootfs with overlayfs: '" << options << "'";
Try<Nothing> mount = fs::mount(
"overlay",
rootfs,
"overlay",
- MS_RDONLY,
- lowerDir);
+ 0,
+ options);
if (mount.isError()) {
return Failure(
http://git-wip-us.apache.org/repos/asf/mesos/blob/4dfa91fc/src/slave/containerizer/mesos/provisioner/backends/overlay.hpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/backends/overlay.hpp b/src/slave/containerizer/mesos/provisioner/backends/overlay.hpp
index 85cc737..387f28a 100644
--- a/src/slave/containerizer/mesos/provisioner/backends/overlay.hpp
+++ b/src/slave/containerizer/mesos/provisioner/backends/overlay.hpp
@@ -27,10 +27,20 @@ namespace slave {
class OverlayBackendProcess;
-// This is a specialized backend that is useful for deploying multi-layer
-// images using the overlayfs-based backend.
-// 1) OverlayBackend does not support images with a single layer.
-// 2) The filesystem is read-only.
+// This backend mounts the images layers to the rootfs using the overlay file
+// system. The directory layout is as follows:
+// <work_dir> ('--work_dir' flag)
+// |-- provisioner
+// |-- containers
+// |-- <container-id>
+// |-- backends
+// |-- overlay
+// |-- rootfses
+// |-- <rootfs_id> (the rootfs)
+// |-- scratch
+// |-- <rootfs_id> (the scratch space)
+// |-- upperdir
+// |-- workdir
class OverlayBackend : public Backend
{
public:
@@ -40,7 +50,8 @@ public:
virtual process::Future<Nothing> provision(
const std::vector<std::string>& layers,
- const std::string& rootfs);
+ const std::string& rootfs,
+ const std::string& backendDir);
virtual process::Future<bool> destroy(const std::string& rootfs);
http://git-wip-us.apache.org/repos/asf/mesos/blob/4dfa91fc/src/slave/containerizer/mesos/provisioner/paths.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/paths.cpp b/src/slave/containerizer/mesos/provisioner/paths.cpp
index 07581f6..86a45f3 100644
--- a/src/slave/containerizer/mesos/provisioner/paths.cpp
+++ b/src/slave/containerizer/mesos/provisioner/paths.cpp
@@ -183,6 +183,19 @@ Try<hashmap<string, hashset<string>>> listContainerRootfses(
return results;
}
+
+string getBackendDir(
+ const string& provisionerDir,
+ const ContainerID& containerId,
+ const string& backend)
+{
+ return getBackendDir(
+ getBackendsDir(
+ getContainerDir(
+ provisionerDir, containerId)),
+ backend);
+}
+
} // namespace paths {
} // namespace provisioner {
} // namespace slave {
http://git-wip-us.apache.org/repos/asf/mesos/blob/4dfa91fc/src/slave/containerizer/mesos/provisioner/paths.hpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/paths.hpp b/src/slave/containerizer/mesos/provisioner/paths.hpp
index 2ea38ac..9829d6b 100644
--- a/src/slave/containerizer/mesos/provisioner/paths.hpp
+++ b/src/slave/containerizer/mesos/provisioner/paths.hpp
@@ -45,6 +45,7 @@ namespace paths {
// Under each backend a rootfs is identified by the 'rootfs_id' which
// is a UUID.
+
std::string getContainerDir(
const std::string& provisionerDir,
const ContainerID& containerId);
@@ -69,6 +70,12 @@ listContainerRootfses(
Try<hashset<ContainerID>> listContainers(
const std::string& provisionerDir);
+
+std::string getBackendDir(
+ const std::string& provisionerDir,
+ const ContainerID& containerId,
+ const std::string& backend);
+
} // namespace paths {
} // namespace provisioner {
} // namespace slave {
http://git-wip-us.apache.org/repos/asf/mesos/blob/4dfa91fc/src/slave/containerizer/mesos/provisioner/provisioner.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/provisioner.cpp b/src/slave/containerizer/mesos/provisioner/provisioner.cpp
index 8a4938e..dcbbbaf 100644
--- a/src/slave/containerizer/mesos/provisioner/provisioner.cpp
+++ b/src/slave/containerizer/mesos/provisioner/provisioner.cpp
@@ -293,7 +293,15 @@ Future<ProvisionInfo> ProvisionerProcess::_provision(
infos[containerId]->rootfses[backend].insert(rootfsId);
- return backends.get(backend).get()->provision(ImageInfo.layers, rootfs)
+ string backendDir = provisioner::paths::getBackendDir(
+ rootDir,
+ containerId,
+ backend);
+
+ return backends.get(backend).get()->provision(
+ ImageInfo.layers,
+ rootfs,
+ backendDir)
.then([rootfs, ImageInfo]() -> Future<ProvisionInfo> {
return ProvisionInfo{rootfs, ImageInfo.dockerManifest};
});
http://git-wip-us.apache.org/repos/asf/mesos/blob/4dfa91fc/src/tests/containerizer/provisioner_backend_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/containerizer/provisioner_backend_tests.cpp b/src/tests/containerizer/provisioner_backend_tests.cpp
index d49204f..bc04760 100644
--- a/src/tests/containerizer/provisioner_backend_tests.cpp
+++ b/src/tests/containerizer/provisioner_backend_tests.cpp
@@ -91,20 +91,27 @@ TEST_F(OverlayBackendTest, ROOT_OVERLAYFS_OverlayFSBackend)
hashmap<string, Owned<Backend>> backends = Backend::create(slave::Flags());
ASSERT_TRUE(backends.contains("overlay"));
- AWAIT_READY(backends["overlay"]->provision({layer1, layer2}, rootfs));
+ AWAIT_READY(backends["overlay"]->provision(
+ {layer1, layer2},
+ rootfs,
+ sandbox.get()));
EXPECT_TRUE(os::exists(path::join(rootfs, "dir1", "1")));
- Try<string> read = os::read(path::join(rootfs, "dir1", "1"));
- EXPECT_SOME_EQ("1", read);
+ EXPECT_SOME_EQ("1", os::read(path::join(rootfs, "dir1", "1")));
EXPECT_TRUE(os::exists(path::join(rootfs, "dir2", "2")));
- read = os::read(path::join(rootfs, "dir2", "2"));
- EXPECT_SOME_EQ("2", read);
+ EXPECT_SOME_EQ("2", os::read(path::join(rootfs, "dir2", "2")));
- EXPECT_TRUE(os::exists(path::join(rootfs, "file")));
- read = os::read(path::join(rootfs, "file"));
// Last layer should overwrite existing file of earlier layers.
- EXPECT_SOME_EQ("test2", read);
+ EXPECT_TRUE(os::exists(path::join(rootfs, "file")));
+ EXPECT_SOME_EQ("test2", os::read(path::join(rootfs, "file")));
+
+ // Rootfs should be writable.
+ ASSERT_SOME(os::write(path::join(rootfs, "file"), "test3"));
+
+ // Files created in rootfs should shadow the files of lower dirs.
+ EXPECT_SOME_EQ("test3", os::read(path::join(rootfs, "file")));
+ EXPECT_SOME_EQ("test2", os::read(path::join(layer2, "file")));
AWAIT_READY(backends["overlay"]->destroy(rootfs));
@@ -130,7 +137,10 @@ TEST_F(BindBackendTest, ROOT_BindBackend)
string target = path::join(os::getcwd(), "target");
- AWAIT_READY(backends["bind"]->provision({rootfs}, target));
+ AWAIT_READY(backends["bind"]->provision(
+ {rootfs},
+ target,
+ sandbox.get()));
EXPECT_TRUE(os::stat::isdir(path::join(target, "tmp")));
@@ -170,24 +180,20 @@ TEST_F(CopyBackendTest, ROOT_CopyBackend)
hashmap<string, Owned<Backend>> backends = Backend::create(slave::Flags());
ASSERT_TRUE(backends.contains("copy"));
- AWAIT_READY(backends["copy"]->provision({layer1, layer2}, rootfs));
+ AWAIT_READY(backends["copy"]->provision(
+ {layer1, layer2},
+ rootfs,
+ sandbox.get()));
EXPECT_TRUE(os::exists(path::join(rootfs, "dir1", "1")));
- Try<string> read = os::read(path::join(rootfs, "dir1", "1"));
- ASSERT_SOME(read);
- EXPECT_EQ("1", read.get());
+ EXPECT_SOME_EQ("1", os::read(path::join(rootfs, "dir1", "1")));
EXPECT_TRUE(os::exists(path::join(rootfs, "dir2", "2")));
- read = os::read(path::join(rootfs, "dir2", "2"));
- ASSERT_SOME(read);
- EXPECT_EQ("2", read.get());
-
- EXPECT_TRUE(os::exists(path::join(rootfs, "file")));
- read = os::read(path::join(rootfs, "file"));
- ASSERT_SOME(read);
+ EXPECT_SOME_EQ("2", os::read(path::join(rootfs, "dir2", "2")));
// Last layer should overwrite existing file.
- EXPECT_EQ("test2", read.get());
+ EXPECT_TRUE(os::exists(path::join(rootfs, "file")));
+ EXPECT_SOME_EQ("test2", os::read(path::join(rootfs, "file")));
AWAIT_READY(backends["copy"]->destroy(rootfs));