You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by qi...@apache.org on 2016/11/24 01:42:58 UTC
[1/4] mesos git commit: Added backport of MESOS-6360 (Mesos part) to
1.1.x.
Repository: mesos
Updated Branches:
refs/heads/1.1.x de3a1b58a -> b810e22d3
Added backport of MESOS-6360 (Mesos part) to 1.1.x.
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/b810e22d
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/b810e22d
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/b810e22d
Branch: refs/heads/1.1.x
Commit: b810e22d37cffd4431599ea75ec509db56d10460
Parents: 2d65576
Author: Qian Zhang <zh...@gmail.com>
Authored: Wed Nov 23 17:39:33 2016 +0800
Committer: Qian Zhang <zh...@gmail.com>
Committed: Thu Nov 24 09:37:56 2016 +0800
----------------------------------------------------------------------
src/CMakeLists.txt | 2 +
src/Makefile.am | 3 +
.../mesos/provisioner/backends/copy.cpp | 108 ++++++++++++++-
.../mesos/provisioner/docker/store.cpp | 24 +++-
.../mesos/provisioner/provisioner.cpp | 118 +---------------
.../mesos/provisioner/provisioner.hpp | 5 -
.../containerizer/mesos/provisioner/utils.cpp | 133 +++++++++++++++++++
.../containerizer/mesos/provisioner/utils.hpp | 37 ++++++
src/slave/containerizer/mesos/utils.cpp | 40 ++++++
src/slave/containerizer/mesos/utils.hpp | 15 +--
.../containerizer/provisioner_docker_tests.cpp | 95 +++++++++----
11 files changed, 411 insertions(+), 169 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/b810e22d/src/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index e60d34c..85acbbe 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -325,9 +325,11 @@ set(AGENT_SRC
slave/containerizer/mesos/launcher.cpp
slave/containerizer/mesos/mount.cpp
slave/containerizer/mesos/paths.cpp
+ slave/containerizer/mesos/utils.cpp
slave/containerizer/mesos/provisioner/paths.cpp
slave/containerizer/mesos/provisioner/provisioner.cpp
slave/containerizer/mesos/provisioner/store.cpp
+ slave/containerizer/mesos/provisioner/utils.cpp
slave/containerizer/mesos/provisioner/appc/cache.cpp
slave/containerizer/mesos/provisioner/appc/fetcher.cpp
slave/containerizer/mesos/provisioner/appc/paths.cpp
http://git-wip-us.apache.org/repos/asf/mesos/blob/b810e22d/src/Makefile.am
----------------------------------------------------------------------
diff --git a/src/Makefile.am b/src/Makefile.am
index 73d337d..f82c985 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -828,6 +828,7 @@ libmesos_no_3rdparty_la_SOURCES += \
slave/containerizer/mesos/launcher.cpp \
slave/containerizer/mesos/mount.cpp \
slave/containerizer/mesos/paths.cpp \
+ slave/containerizer/mesos/utils.cpp \
slave/containerizer/mesos/isolators/docker/volume/driver.cpp \
slave/containerizer/mesos/isolators/docker/volume/paths.cpp \
slave/containerizer/mesos/isolators/filesystem/posix.cpp \
@@ -839,6 +840,7 @@ libmesos_no_3rdparty_la_SOURCES += \
slave/containerizer/mesos/provisioner/paths.cpp \
slave/containerizer/mesos/provisioner/provisioner.cpp \
slave/containerizer/mesos/provisioner/store.cpp \
+ slave/containerizer/mesos/provisioner/utils.cpp \
slave/containerizer/mesos/provisioner/appc/cache.cpp \
slave/containerizer/mesos/provisioner/appc/fetcher.cpp \
slave/containerizer/mesos/provisioner/appc/paths.cpp \
@@ -971,6 +973,7 @@ libmesos_no_3rdparty_la_SOURCES += \
slave/containerizer/mesos/provisioner/paths.hpp \
slave/containerizer/mesos/provisioner/provisioner.hpp \
slave/containerizer/mesos/provisioner/store.hpp \
+ slave/containerizer/mesos/provisioner/utils.hpp \
slave/containerizer/mesos/provisioner/appc/cache.hpp \
slave/containerizer/mesos/provisioner/appc/fetcher.hpp \
slave/containerizer/mesos/provisioner/appc/paths.hpp \
http://git-wip-us.apache.org/repos/asf/mesos/blob/b810e22d/src/slave/containerizer/mesos/provisioner/backends/copy.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/backends/copy.cpp b/src/slave/containerizer/mesos/provisioner/backends/copy.cpp
index 9c5354e..0ce3e1e 100644
--- a/src/slave/containerizer/mesos/provisioner/backends/copy.cpp
+++ b/src/slave/containerizer/mesos/provisioner/backends/copy.cpp
@@ -16,6 +16,8 @@
#include <list>
+#include <mesos/docker/spec.hpp>
+
#include <process/collect.hpp>
#include <process/defer.hpp>
#include <process/dispatch.hpp>
@@ -24,7 +26,6 @@
#include <process/process.hpp>
#include <process/subprocess.hpp>
-
#include <stout/foreach.hpp>
#include <stout/os.hpp>
@@ -32,7 +33,6 @@
#include "slave/containerizer/mesos/provisioner/backends/copy.hpp"
-
using namespace process;
using std::string;
@@ -128,9 +128,93 @@ Future<Nothing> CopyBackendProcess::provision(
Future<Nothing> CopyBackendProcess::_provision(
- string layer,
- const string& rootfs)
+ string layer,
+ const string& rootfs)
{
+#ifndef __WINDOWS__
+ // Traverse the layer to check if there is any whiteout files, if
+ // yes, remove the corresponding files/directories from the rootfs.
+ // Note: We assume all image types use AUFS whiteout format.
+ char* source[] = {const_cast<char*>(layer.c_str()), nullptr};
+
+ FTS* tree = ::fts_open(source, FTS_NOCHDIR | FTS_PHYSICAL, nullptr);
+ if (tree == nullptr) {
+ return Failure("Failed to open '" + layer + "': " + os::strerror(errno));
+ }
+
+ vector<string> whiteouts;
+ for (FTSENT *node = ::fts_read(tree);
+ node != nullptr; node = ::fts_read(tree)) {
+ if (node->fts_info != FTS_F) {
+ continue;
+ }
+
+ if (!strings::startsWith(node->fts_name, docker::spec::WHITEOUT_PREFIX)) {
+ continue;
+ }
+
+ string ftsPath = string(node->fts_path);
+ Path whiteout = Path(ftsPath.substr(layer.length() + 1));
+
+ // Keep the relative paths of the whiteout files, we will
+ // remove them from rootfs after layer is copied to rootfs.
+ whiteouts.push_back(whiteout.string());
+
+ if (node->fts_name == string(docker::spec::WHITEOUT_OPAQUE_PREFIX)) {
+ const string path = path::join(rootfs, Path(whiteout).dirname());
+
+ // Remove the entries under the directory labeled
+ // as opaque whiteout from rootfs.
+ Try<Nothing> rmdir = os::rmdir(path, true, false);
+ if (rmdir.isError()) {
+ ::fts_close(tree);
+ return Failure(
+ "Failed to remove the entries under the directory labeled as"
+ " opaque whiteout '" + path + "': " + rmdir.error());
+ }
+ } else {
+ const string path = path::join(
+ rootfs,
+ whiteout.dirname(),
+ whiteout.basename().substr(strlen(docker::spec::WHITEOUT_PREFIX)));
+
+ // The file/directory labeled as whiteout may have already been
+ // removed with the code above due to its parent directory labeled
+ // as opaque whiteout, so here we need to check if it still exists
+ // before trying to remove it.
+ if (os::exists(path)) {
+ if (os::stat::isdir(path)) {
+ Try<Nothing> rmdir = os::rmdir(path);
+ if (rmdir.isError()) {
+ ::fts_close(tree);
+ return Failure(
+ "Failed to remove the directory labeled as whiteout '" +
+ path + "': " + rmdir.error());
+ }
+ } else {
+ Try<Nothing> rm = os::rm(path);
+ if (rm.isError()) {
+ ::fts_close(tree);
+ return Failure(
+ "Failed to remove the file labeled as whiteout '" +
+ path + "': " + rm.error());
+ }
+ }
+ }
+ }
+ }
+
+ if (errno != 0) {
+ Error error = ErrnoError();
+ ::fts_close(tree);
+ return Failure(error);
+ }
+
+ if (::fts_close(tree) != 0) {
+ return Failure(
+ "Failed to stop traversing file system: " + os::strerror(errno));
+ }
+
VLOG(1) << "Copying layer path '" << layer << "' to rootfs '" << rootfs
<< "'";
@@ -160,7 +244,7 @@ Future<Nothing> CopyBackendProcess::_provision(
Subprocess cp = s.get();
return cp.status()
- .then([cp](const Option<int>& status) -> Future<Nothing> {
+ .then([=](const Option<int>& status) -> Future<Nothing> {
if (status.isNone()) {
return Failure("Failed to reap subprocess to copy image");
} else if (status.get() != 0) {
@@ -170,8 +254,22 @@ Future<Nothing> CopyBackendProcess::_provision(
});
}
+ // Remove the whiteout files from rootfs.
+ foreach (const string whiteout, whiteouts) {
+ Try<Nothing> rm = os::rm(path::join(rootfs, whiteout));
+ if (rm.isError()) {
+ return Failure(
+ "Failed to remove whiteout file '" +
+ whiteout + "': " + rm.error());
+ }
+ }
+
return Nothing();
});
+#else
+ return Failure(
+ "Provisioning a rootfs from an image is not supported on Windows");
+#endif // __WINDOWS__
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/b810e22d/src/slave/containerizer/mesos/provisioner/docker/store.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/docker/store.cpp b/src/slave/containerizer/mesos/provisioner/docker/store.cpp
index e192f86..9dccd06 100644
--- a/src/slave/containerizer/mesos/provisioner/docker/store.cpp
+++ b/src/slave/containerizer/mesos/provisioner/docker/store.cpp
@@ -30,6 +30,9 @@
#include <mesos/docker/spec.hpp>
+#include "slave/containerizer/mesos/provisioner/constants.hpp"
+#include "slave/containerizer/mesos/provisioner/utils.hpp"
+
#include "slave/containerizer/mesos/provisioner/docker/metadata_manager.hpp"
#include "slave/containerizer/mesos/provisioner/docker/paths.hpp"
#include "slave/containerizer/mesos/provisioner/docker/puller.hpp"
@@ -331,6 +334,23 @@ Future<Nothing> StoreProcess::moveLayer(
flags.docker_store_dir,
layerId);
+ const string sourceRootfs = paths::getImageLayerRootfsPath(
+ source,
+ flags.image_provisioner_backend);
+
+#ifdef __linux__
+ // If the backend is "overlay", we need to convert
+ // AUFS whiteout files to OverlayFS whiteout files.
+ if (flags.image_provisioner_backend == OVERLAY_BACKEND) {
+ Try<Nothing> convert = convertWhiteouts(sourceRootfs);
+ if (convert.isError()) {
+ return Failure(
+ "Failed to convert the whiteout files under '" +
+ sourceRootfs + "': " + convert.error());
+ }
+ }
+#endif
+
if (!os::exists(target)) {
// This is the case that we pull the layer for the first time.
Try<Nothing> mkdir = os::mkdir(target);
@@ -349,10 +369,6 @@ Future<Nothing> StoreProcess::moveLayer(
} else {
// This is the case where the layer has already been pulled with a
// different backend.
- const string sourceRootfs = paths::getImageLayerRootfsPath(
- source,
- flags.image_provisioner_backend);
-
Try<Nothing> rename = os::rename(sourceRootfs, targetRootfs);
if (rename.isError()) {
return Failure(
http://git-wip-us.apache.org/repos/asf/mesos/blob/b810e22d/src/slave/containerizer/mesos/provisioner/provisioner.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/provisioner.cpp b/src/slave/containerizer/mesos/provisioner/provisioner.cpp
index de2c121..db2d267 100644
--- a/src/slave/containerizer/mesos/provisioner/provisioner.cpp
+++ b/src/slave/containerizer/mesos/provisioner/provisioner.cpp
@@ -311,124 +311,10 @@ Future<ProvisionInfo> ProvisionerProcess::_provision(
imageInfo.layers,
rootfs,
backendDir)
- .then(defer(self(), &Self::__provision, rootfs, image, imageInfo));
-}
-
-
-// This function is currently docker image specific. Depending
-// on docker v1 spec, a docker image may include filesystem
-// changeset, which may need to delete directories or files.
-// The file/directory to be deleted will be labeled by creating
-// a 'whiteout' file, which is at the same location and with the
-// basename of the deleted file or directory prefixed with '.wh.'.
-// For the directory which has an opaque whiteout file '.wh..wh..opq'
-// under it, we need to delete all the files/directories under it.
-// Please see:
-// https://github.com/docker/docker/blob/master/image/spec/v1.md
-// https://github.com/docker/docker/blob/master/pkg/archive/whiteouts.go
-// And OCI image spec also has the concepts 'whiteout' and 'opaque whiteout':
-// https://github.com/opencontainers/image-spec/blob/master/layer.md#whiteouts
-Future<ProvisionInfo> ProvisionerProcess::__provision(
- const string& rootfs,
- const Image& image,
- const ImageInfo& imageInfo)
-{
-#ifdef __WINDOWS__
- return ProvisionInfo{
- rootfs, imageInfo.dockerManifest, imageInfo.appcManifest};
-#else
- // Skip single-layered images since no 'whiteout' files needs
- // to be handled, and this excludes any image using the bind
- // backend.
- if (imageInfo.layers.size() == 1 || image.type() != Image::DOCKER) {
+ .then([=]() -> Future<ProvisionInfo> {
return ProvisionInfo{
rootfs, imageInfo.dockerManifest, imageInfo.appcManifest};
- }
-
- // TODO(hausdorff): The FTS API is not available on some platforms, such as
- // Windows. We will need to either (1) prove that this is not necessary for
- // Windows Containers, which use much of the Docker spec themselves, or (2)
- // make this code compatible with Windows, as we did with other code that
- // depended on FTS, such as `os::rmdir`. See MESOS-5610.
- char* _rootfs[] = {const_cast<char*>(rootfs.c_str()), nullptr};
-
- FTS* tree = ::fts_open(_rootfs, FTS_NOCHDIR | FTS_PHYSICAL, nullptr);
- if (tree == nullptr) {
- return Failure("Failed to open '" + rootfs + "': " + os::strerror(errno));
- }
-
- vector<string> whiteout;
- vector<string> whiteoutOpaque;
-
- for (FTSENT *node = ::fts_read(tree);
- node != nullptr; node = ::fts_read(tree)) {
- if (node->fts_info == FTS_F &&
- strings::startsWith(node->fts_name, string(spec::WHITEOUT_PREFIX))) {
- Path path = Path(node->fts_path);
- if (node->fts_name == string(spec::WHITEOUT_OPAQUE_PREFIX)) {
- whiteoutOpaque.push_back(path.dirname());
- } else {
- whiteout.push_back(path::join(
- path.dirname(),
- path.basename().substr(strlen(spec::WHITEOUT_PREFIX))));
- }
-
- Try<Nothing> rm = os::rm(path.string());
- if (rm.isError()) {
- ::fts_close(tree);
- return Failure(
- "Failed to remove whiteout file '" +
- path.string() + "': " + rm.error());
- }
- }
- }
-
- if (errno != 0) {
- Error error = ErrnoError();
- ::fts_close(tree);
- return Failure(error);
- }
-
- if (::fts_close(tree) != 0) {
- return Failure(
- "Failed to stop traversing file system: " + os::strerror(errno));
- }
-
- foreach (const string& path, whiteoutOpaque) {
- Try<Nothing> rmdir = os::rmdir(path, true, false);
- if (rmdir.isError()) {
- return Failure(
- "Failed to remove the entries under the directory labeled as"
- " opaque whiteout '" + path + "': " + rmdir.error());
- }
- }
-
- foreach (const string& path, whiteout) {
- // The file/directory labeled as whiteout may have already been
- // removed with the code above due to its parent directory labeled
- // as opaque whiteout, so here we need to check if it still exists
- // before trying to remove it.
- if (os::exists(path)) {
- if (os::stat::isdir(path)) {
- Try<Nothing> rmdir = os::rmdir(path);
- if (rmdir.isError()) {
- return Failure(
- "Failed to remove the directory labeled as whiteout '" +
- path + "': " + rmdir.error());
- }
- } else {
- Try<Nothing> rm = os::rm(path);
- if (rm.isError()) {
- return Failure(
- "Failed to remove the file labeled as whiteout '" +
- path + "': " + rm.error());
- }
- }
- }
- }
-
- return ProvisionInfo{rootfs, imageInfo.dockerManifest, None()};
-#endif // __WINDOWS__
+ });
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/b810e22d/src/slave/containerizer/mesos/provisioner/provisioner.hpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/provisioner.hpp b/src/slave/containerizer/mesos/provisioner/provisioner.hpp
index c5481c5..0a48617 100644
--- a/src/slave/containerizer/mesos/provisioner/provisioner.hpp
+++ b/src/slave/containerizer/mesos/provisioner/provisioner.hpp
@@ -134,11 +134,6 @@ private:
const Image& image,
const ImageInfo& imageInfo);
- process::Future<ProvisionInfo> __provision(
- const std::string& rootfs,
- const Image& image,
- const ImageInfo& imageInfo);
-
process::Future<bool> _destroy(const ContainerID& containerId);
const Flags flags;
http://git-wip-us.apache.org/repos/asf/mesos/blob/b810e22d/src/slave/containerizer/mesos/provisioner/utils.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/utils.cpp b/src/slave/containerizer/mesos/provisioner/utils.cpp
new file mode 100644
index 0000000..f480ee5
--- /dev/null
+++ b/src/slave/containerizer/mesos/provisioner/utils.cpp
@@ -0,0 +1,133 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef __WINDOWS__
+#include <fts.h>
+
+#include <sys/stat.h>
+#endif // __WINDOWS__
+
+#include <string>
+#include <string.h>
+
+#include <mesos/docker/spec.hpp>
+
+#include <stout/error.hpp>
+#include <stout/nothing.hpp>
+#include <stout/os.hpp>
+#include <stout/path.hpp>
+#include <stout/strings.hpp>
+#include <stout/try.hpp>
+
+#include <stout/os/rm.hpp>
+#include <stout/os/strerror.hpp>
+#include <stout/os/xattr.hpp>
+
+#include "slave/containerizer/mesos/provisioner/utils.hpp"
+
+using std::string;
+
+namespace mesos {
+namespace internal {
+namespace slave {
+
+#ifdef __linux__
+// This function is used to convert AUFS whiteouts to OverlayFS whiteouts and
+// the AUFS whiteouts will be removed after the conversion is done. Whiteout is
+// for hiding directories and files in the lower level layers. AUFS whiteout is
+// the whiteout standard in Docker, and it relies on '.wh..wh..opq' to hide
+// directory and '.wh.<filename>' to hide file, whereas OverlayFS relies on
+// setting an extended attribute 'trusted.overlay.opaque' on the directory to
+// hide it and creating character device with 0/0 device number to hide file.
+// See the links below for more details:
+// http://aufs.sourceforge.net/aufs.html
+// https://www.kernel.org/doc/Documentation/filesystems/overlayfs.txt
+Try<Nothing> convertWhiteouts(const string& directory)
+{
+ char* rootfs[] = {const_cast<char*>(directory.c_str()), nullptr};
+
+ FTS* tree = ::fts_open(rootfs, FTS_NOCHDIR | FTS_PHYSICAL, nullptr);
+ if (tree == nullptr) {
+ return Error("Failed to open '" + directory + "': " + os::strerror(errno));
+ }
+
+ for (FTSENT *node = ::fts_read(tree);
+ node != nullptr; node = ::fts_read(tree)) {
+ if (node->fts_info != FTS_F) {
+ continue;
+ }
+
+ if (!strings::startsWith(node->fts_name, docker::spec::WHITEOUT_PREFIX)) {
+ continue;
+ }
+
+ const Path path = Path(node->fts_path);
+ if (node->fts_name == string(docker::spec::WHITEOUT_OPAQUE_PREFIX)) {
+ Try<Nothing> setxattr = os::setxattr(
+ path.dirname(),
+ "trusted.overlay.opaque",
+ "y",
+ 0);
+
+ if (setxattr.isError()) {
+ ::fts_close(tree);
+ return Error(
+ "Failed to set extended attribute 'trusted.overlay.opaque'"
+ " for the directory '" + path.dirname() + "': " +
+ setxattr.error());
+ }
+ } else {
+ const string originalPath = path::join(
+ path.dirname(),
+ path.basename().substr(strlen(docker::spec::WHITEOUT_PREFIX)));
+
+ Try<Nothing> mknod = os::mknod(originalPath, S_IFCHR, 0);
+ if (mknod.isError()) {
+ ::fts_close(tree);
+ return Error(
+ "Failed to create character device '" +
+ originalPath + "': " + mknod.error());
+ }
+ }
+
+ // Remove the AUFS whiteout file.
+ Try<Nothing> rm = os::rm(path.string());
+ if (rm.isError()) {
+ ::fts_close(tree);
+ return Error(
+ "Failed to remove AUFS whiteout file '" +
+ path.string() + "': " + rm.error());
+ }
+ }
+
+ if (errno != 0) {
+ Error error = ErrnoError();
+ ::fts_close(tree);
+ return error;
+ }
+
+ if (::fts_close(tree) != 0) {
+ return Error(
+ "Failed to stop traversing file system: " + os::strerror(errno));
+ }
+
+ return Nothing();
+}
+#endif
+
+} // namespace slave {
+} // namespace internal {
+} // namespace mesos {
http://git-wip-us.apache.org/repos/asf/mesos/blob/b810e22d/src/slave/containerizer/mesos/provisioner/utils.hpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/utils.hpp b/src/slave/containerizer/mesos/provisioner/utils.hpp
new file mode 100644
index 0000000..5b6c162
--- /dev/null
+++ b/src/slave/containerizer/mesos/provisioner/utils.hpp
@@ -0,0 +1,37 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef __MESOS_CONTAINERIZER_PROVISIONER_UTILS_HPP__
+#define __MESOS_CONTAINERIZER_PROVISIONER_UTILS_HPP__
+
+#include <string>
+
+#include <stout/nothing.hpp>
+#include <stout/try.hpp>
+
+namespace mesos {
+namespace internal {
+namespace slave {
+
+#ifdef __linux__
+Try<Nothing> convertWhiteouts(const std::string& directory);
+#endif
+
+} // namespace slave {
+} // namespace internal {
+} // namespace mesos {
+
+#endif // __MESOS_CONTAINERIZER_PROVISIONER_UTILS_HPP__
http://git-wip-us.apache.org/repos/asf/mesos/blob/b810e22d/src/slave/containerizer/mesos/utils.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/utils.cpp b/src/slave/containerizer/mesos/utils.cpp
new file mode 100644
index 0000000..237aea4
--- /dev/null
+++ b/src/slave/containerizer/mesos/utils.cpp
@@ -0,0 +1,40 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "slave/containerizer/mesos/utils.hpp"
+
+namespace mesos {
+namespace internal {
+namespace slave {
+
+ContainerID getRootContainerId(const ContainerID& containerId)
+{
+ ContainerID rootContainerId = containerId;
+ while (rootContainerId.has_parent()) {
+ // NOTE: Looks like protobuf does not handle copying well when
+ // nesting message is involved, because the source and the target
+ // point to the same object. Therefore, we create a temporary
+ // variable and use an extra copy here.
+ ContainerID id = rootContainerId.parent();
+ rootContainerId = id;
+ }
+
+ return rootContainerId;
+}
+
+} // namespace slave {
+} // namespace internal {
+} // namespace mesos {
http://git-wip-us.apache.org/repos/asf/mesos/blob/b810e22d/src/slave/containerizer/mesos/utils.hpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/utils.hpp b/src/slave/containerizer/mesos/utils.hpp
index 178ebf3..f24215b 100644
--- a/src/slave/containerizer/mesos/utils.hpp
+++ b/src/slave/containerizer/mesos/utils.hpp
@@ -23,20 +23,7 @@ namespace mesos {
namespace internal {
namespace slave {
-static ContainerID getRootContainerId(const ContainerID& containerId)
-{
- ContainerID rootContainerId = containerId;
- while (rootContainerId.has_parent()) {
- // NOTE: Looks like protobuf does not handle copying well when
- // nesting message is involved, because the source and the target
- // point to the same object. Therefore, we create a temporary
- // variable and use an extra copy here.
- ContainerID id = rootContainerId.parent();
- rootContainerId = id;
- }
-
- return rootContainerId;
-}
+ContainerID getRootContainerId(const ContainerID& containerId);
} // namespace slave {
} // namespace internal {
http://git-wip-us.apache.org/repos/asf/mesos/blob/b810e22d/src/tests/containerizer/provisioner_docker_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/containerizer/provisioner_docker_tests.cpp b/src/tests/containerizer/provisioner_docker_tests.cpp
index d8fb7ab..9682169 100644
--- a/src/tests/containerizer/provisioner_docker_tests.cpp
+++ b/src/tests/containerizer/provisioner_docker_tests.cpp
@@ -28,6 +28,12 @@
#include <mesos/docker/spec.hpp>
+#ifdef __linux__
+#include "linux/fs.hpp"
+#endif
+
+#include "slave/containerizer/mesos/provisioner/constants.hpp"
+
#include "slave/containerizer/mesos/provisioner/docker/metadata_manager.hpp"
#include "slave/containerizer/mesos/provisioner/docker/paths.hpp"
#include "slave/containerizer/mesos/provisioner/docker/puller.hpp"
@@ -55,6 +61,10 @@ using process::Promise;
using master::Master;
+using mesos::internal::slave::AUFS_BACKEND;
+using mesos::internal::slave::COPY_BACKEND;
+using mesos::internal::slave::OVERLAY_BACKEND;
+
using mesos::master::detector::MasterDetector;
using slave::ImageInfo;
@@ -64,6 +74,8 @@ using slave::docker::Puller;
using slave::docker::RegistryPuller;
using slave::docker::Store;
+using testing::WithParamInterface;
+
namespace mesos {
namespace internal {
namespace tests {
@@ -552,9 +564,10 @@ TEST_F(ProvisionerDockerPullerTest, ROOT_INTERNET_CURL_Normalize)
}
-// This test verifies that any docker image containing whiteout files
-// will be processed correctly.
-TEST_F(ProvisionerDockerPullerTest, ROOT_INTERNET_CURL_Whiteout)
+// This test verifies that the scratch based docker image (that
+// only contain a single binary and its dependencies) can be
+// launched correctly.
+TEST_F(ProvisionerDockerPullerTest, ROOT_INTERNET_CURL_ScratchImage)
{
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
@@ -585,17 +598,8 @@ TEST_F(ProvisionerDockerPullerTest, ROOT_INTERNET_CURL_Whiteout)
const Offer& offer = offers.get()[0];
- // We are using the docker image 'cirros' to verify that
- // the symlink /etc/rc3.d/S40-network does not exist in
- // container's rootfs because it is labeled by a '.wh.'
- // whiteout file, and both should be removed.
CommandInfo command;
command.set_shell(false);
- command.set_value("/usr/bin/test");
- command.add_arguments("test");
- command.add_arguments("!");
- command.add_arguments("-f");
- command.add_arguments("/etc/rc3.d/S40-network");
TaskInfo task = createTask(
offer.slave_id(),
@@ -604,7 +608,10 @@ TEST_F(ProvisionerDockerPullerTest, ROOT_INTERNET_CURL_Whiteout)
Image image;
image.set_type(Image::DOCKER);
- image.mutable_docker()->set_name("cirros");
+
+ // 'hello-world' is a scratch image. It contains only one
+ // binary 'hello' in its rootfs.
+ image.mutable_docker()->set_name("hello-world");
ContainerInfo* container = task.mutable_container();
container->set_type(ContainerInfo::MESOS);
@@ -631,10 +638,40 @@ TEST_F(ProvisionerDockerPullerTest, ROOT_INTERNET_CURL_Whiteout)
}
-// This test verifies that the scratch based docker image (that
-// only contain a single binary and its dependencies) can be
-// launched correctly.
-TEST_F(ProvisionerDockerPullerTest, ROOT_INTERNET_CURL_ScratchImage)
+class ProvisionerDockerWhiteoutTest
+ : public MesosTest,
+ public WithParamInterface<string>
+{
+public:
+ // Returns the supported backends.
+ static vector<string> parameters()
+ {
+ vector<string> backends = {COPY_BACKEND};
+
+ Try<bool> aufsSupported = fs::supported("aufs");
+ if (aufsSupported.isSome() && aufsSupported.get()) {
+ backends.push_back(AUFS_BACKEND);
+ }
+
+ Try<bool> overlayfsSupported = fs::supported("overlayfs");
+ if (overlayfsSupported.isSome() && overlayfsSupported.get()) {
+ backends.push_back(OVERLAY_BACKEND);
+ }
+
+ return backends;
+ }
+};
+
+
+INSTANTIATE_TEST_CASE_P(
+ BackendFlag,
+ ProvisionerDockerWhiteoutTest,
+ ::testing::ValuesIn(ProvisionerDockerWhiteoutTest::parameters()));
+
+
+// This test verifies that a docker image containing whiteout files
+// will be processed correctly by copy, aufs and overlay backends.
+TEST_P(ProvisionerDockerWhiteoutTest, ROOT_INTERNET_CURL_Whiteout)
{
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
@@ -642,6 +679,7 @@ TEST_F(ProvisionerDockerPullerTest, ROOT_INTERNET_CURL_ScratchImage)
slave::Flags flags = CreateSlaveFlags();
flags.isolation = "docker/runtime,filesystem/linux";
flags.image_providers = "docker";
+ flags.image_provisioner_backend = GetParam();
Owned<MasterDetector> detector = master.get()->createDetector();
Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), flags);
@@ -665,20 +703,27 @@ TEST_F(ProvisionerDockerPullerTest, ROOT_INTERNET_CURL_ScratchImage)
const Offer& offer = offers.get()[0];
- CommandInfo command;
- command.set_shell(false);
+ // We are using the docker image 'zhq527725/whiteout' to verify that the
+ // files '/dir1/file1' and '/dir1/dir2/file2' do not exist in container's
+ // rootfs because of the following two whiteout files in the image:
+ // '/dir1/.wh.file1'
+ // '/dir1/dir2/.wh..wh..opq'
+ // And we also verify that the file '/dir1/dir2/file3' exists in container's
+ // rootfs which will NOT be applied by '/dir1/dir2/.wh..wh..opq' since they
+ // are in the same layer.
+ // See more details about this docker image in the link below:
+ // https://hub.docker.com/r/zhq527725/whiteout/
+ CommandInfo command = createCommandInfo(
+ "test ! -f /dir1/file1 && "
+ "test ! -f /dir1/dir2/file2 && "
+ "test -f /dir1/dir2/file3");
TaskInfo task = createTask(
offer.slave_id(),
Resources::parse("cpus:1;mem:128").get(),
command);
- Image image;
- image.set_type(Image::DOCKER);
-
- // 'hello-world' is a scratch image. It contains only one
- // binary 'hello' in its rootfs.
- image.mutable_docker()->set_name("hello-world");
+ Image image = createDockerImage("zhq527725/whiteout");
ContainerInfo* container = task.mutable_container();
container->set_type(ContainerInfo::MESOS);
[3/4] mesos git commit: Added backport of MESOS-6360 (stout part) to
1.1.x.
Posted by qi...@apache.org.
Added backport of MESOS-6360 (stout part) to 1.1.x.
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/2d655760
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/2d655760
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/2d655760
Branch: refs/heads/1.1.x
Commit: 2d6557601ead04d51b9d5ca461c2c20c3f5f65a4
Parents: 14355b7
Author: Qian Zhang <zh...@gmail.com>
Authored: Wed Nov 23 17:38:11 2016 +0800
Committer: Qian Zhang <zh...@gmail.com>
Committed: Thu Nov 24 09:37:56 2016 +0800
----------------------------------------------------------------------
3rdparty/stout/include/Makefile.am | 3 +
3rdparty/stout/include/stout/os.hpp | 1 +
3rdparty/stout/include/stout/os/posix/xattr.hpp | 99 ++++++++++++++++++++
.../stout/include/stout/os/windows/xattr.hpp | 38 ++++++++
3rdparty/stout/include/stout/os/xattr.hpp | 26 +++++
3rdparty/stout/tests/os/filesystem_tests.cpp | 38 ++++++++
6 files changed, 205 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/2d655760/3rdparty/stout/include/Makefile.am
----------------------------------------------------------------------
diff --git a/3rdparty/stout/include/Makefile.am b/3rdparty/stout/include/Makefile.am
index 1eb9c14..d5bc728 100644
--- a/3rdparty/stout/include/Makefile.am
+++ b/3rdparty/stout/include/Makefile.am
@@ -104,6 +104,7 @@ nobase_include_HEADERS = \
stout/os/utime.hpp \
stout/os/wait.hpp \
stout/os/write.hpp \
+ stout/os/xattr.hpp \
stout/os/posix/bootid.hpp \
stout/os/posix/chown.hpp \
stout/os/posix/chroot.hpp \
@@ -128,6 +129,7 @@ nobase_include_HEADERS = \
stout/os/posix/stat.hpp \
stout/os/posix/su.hpp \
stout/os/posix/write.hpp \
+ stout/os/posix/xattr.hpp \
stout/os/raw/argv.hpp \
stout/os/raw/environment.hpp \
stout/os/windows/bootid.hpp \
@@ -153,6 +155,7 @@ nobase_include_HEADERS = \
stout/os/windows/stat.hpp \
stout/os/windows/su.hpp \
stout/os/windows/write.hpp \
+ stout/os/windows/xattr.hpp \
stout/path.hpp \
stout/preprocessor.hpp \
stout/proc.hpp \
http://git-wip-us.apache.org/repos/asf/mesos/blob/2d655760/3rdparty/stout/include/stout/os.hpp
----------------------------------------------------------------------
diff --git a/3rdparty/stout/include/stout/os.hpp b/3rdparty/stout/include/stout/os.hpp
index 96e8621..af43bbf 100644
--- a/3rdparty/stout/include/stout/os.hpp
+++ b/3rdparty/stout/include/stout/os.hpp
@@ -64,6 +64,7 @@
#include <stout/os/touch.hpp>
#include <stout/os/utime.hpp>
#include <stout/os/wait.hpp>
+#include <stout/os/xattr.hpp>
#include <stout/os/raw/argv.hpp>
#include <stout/os/raw/environment.hpp>
http://git-wip-us.apache.org/repos/asf/mesos/blob/2d655760/3rdparty/stout/include/stout/os/posix/xattr.hpp
----------------------------------------------------------------------
diff --git a/3rdparty/stout/include/stout/os/posix/xattr.hpp b/3rdparty/stout/include/stout/os/posix/xattr.hpp
new file mode 100644
index 0000000..518940f
--- /dev/null
+++ b/3rdparty/stout/include/stout/os/posix/xattr.hpp
@@ -0,0 +1,99 @@
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef __STOUT_OS_POSIX_XATTR_HPP__
+#define __STOUT_OS_POSIX_XATTR_HPP__
+
+#include <sys/xattr.h>
+
+namespace os {
+
+inline Try<Nothing> setxattr(
+ const std::string& path,
+ const std::string& name,
+ const std::string& value,
+ int flags)
+{
+#ifdef __APPLE__
+ if (::setxattr(
+ path.c_str(),
+ name.c_str(),
+ value.c_str(),
+ value.length(),
+ 0,
+ flags) < 0) {
+#else
+ if (::setxattr(
+ path.c_str(),
+ name.c_str(),
+ value.c_str(),
+ value.length(),
+ flags) < 0) {
+#endif
+ return ErrnoError();
+ }
+
+ return Nothing();
+}
+
+
+inline Try<std::string> getxattr(
+ const std::string& path,
+ const std::string& name)
+{
+ // Get the current size of the attribute.
+#ifdef __APPLE__
+ ssize_t size = ::getxattr(path.c_str(), name.c_str(), nullptr, 0, 0, 0);
+#else
+ ssize_t size = ::getxattr(path.c_str(), name.c_str(), nullptr, 0);
+#endif
+ if (size < 0) {
+ return ErrnoError();
+ }
+
+ char* temp = new char[size + 1];
+ ::memset(temp, 0, (size_t)size + 1);
+
+#ifdef __APPLE__
+ if (::getxattr(path.c_str(), name.c_str(), temp, (size_t)size, 0, 0) < 0) {
+#else
+ if (::getxattr(path.c_str(), name.c_str(), temp, (size_t)size) < 0) {
+#endif
+ delete[] temp;
+ return ErrnoError();
+ }
+
+ std::string result(temp);
+ delete[] temp;
+
+ return result;
+}
+
+
+inline Try<Nothing> removexattr(
+ const std::string& path,
+ const std::string& name)
+{
+#ifdef __APPLE__
+ if (::removexattr(path.c_str(), name.c_str(), 0) < 0) {
+#else
+ if (::removexattr(path.c_str(), name.c_str()) < 0) {
+#endif
+ return ErrnoError();
+ }
+
+ return Nothing();
+}
+
+} // namespace os {
+
+#endif /* __STOUT_OS_POSIX_XATTR_HPP__ */
http://git-wip-us.apache.org/repos/asf/mesos/blob/2d655760/3rdparty/stout/include/stout/os/windows/xattr.hpp
----------------------------------------------------------------------
diff --git a/3rdparty/stout/include/stout/os/windows/xattr.hpp b/3rdparty/stout/include/stout/os/windows/xattr.hpp
new file mode 100644
index 0000000..3157f72
--- /dev/null
+++ b/3rdparty/stout/include/stout/os/windows/xattr.hpp
@@ -0,0 +1,38 @@
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef __STOUT_OS_WINDOWS_XATTR_HPP__
+#define __STOUT_OS_WINDOWS_XATTR_HPP__
+
+namespace os {
+
+// NOTE: These functions are deleted because Windows does
+// not support POSIX extended attribute semantics.
+inline Try<Nothing> setxattr(
+ const std::string& path,
+ const std::string& name,
+ const std::string& value,
+ int flags) = delete;
+
+
+inline Try<std::string> getxattr(
+ const std::string& path,
+ const std::string& name) = delete;
+
+
+inline Try<std::string> removexattr(
+ const std::string& path,
+ const std::string& name) = delete;
+
+} // namespace os {
+
+#endif /* __STOUT_OS_WINDOWS_XATTR_HPP__ */
http://git-wip-us.apache.org/repos/asf/mesos/blob/2d655760/3rdparty/stout/include/stout/os/xattr.hpp
----------------------------------------------------------------------
diff --git a/3rdparty/stout/include/stout/os/xattr.hpp b/3rdparty/stout/include/stout/os/xattr.hpp
new file mode 100644
index 0000000..b86b50a
--- /dev/null
+++ b/3rdparty/stout/include/stout/os/xattr.hpp
@@ -0,0 +1,26 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef __STOUT_OS_XATTR_HPP__
+#define __STOUT_OS_XATTR_HPP__
+
+#ifdef __WINDOWS__
+#include <stout/os/windows/xattr.hpp>
+#else
+#include <stout/os/posix/xattr.hpp>
+#endif // __WINDOWS__
+
+#endif // __STOUT_OS_XATTR_HPP__
http://git-wip-us.apache.org/repos/asf/mesos/blob/2d655760/3rdparty/stout/tests/os/filesystem_tests.cpp
----------------------------------------------------------------------
diff --git a/3rdparty/stout/tests/os/filesystem_tests.cpp b/3rdparty/stout/tests/os/filesystem_tests.cpp
index e3894ef..2246084 100644
--- a/3rdparty/stout/tests/os/filesystem_tests.cpp
+++ b/3rdparty/stout/tests/os/filesystem_tests.cpp
@@ -31,6 +31,7 @@
#include <stout/os/rm.hpp>
#include <stout/os/touch.hpp>
#include <stout/os/write.hpp>
+#include <stout/os/xattr.hpp>
#include <stout/tests/utils.hpp>
@@ -436,3 +437,40 @@ TEST_F(FsTest, Close)
_CrtSetReportMode(_CRT_ASSERT, previous_report_mode);
#endif // __WINDOWS__
}
+
+
+#if defined(__linux__) || defined(__APPLE__)
+TEST_F(FsTest, Xattr)
+{
+ const string file = path::join(os::getcwd(), UUID::random().toString());
+
+ // Create file.
+ ASSERT_SOME(os::touch(file));
+ ASSERT_TRUE(os::exists(file));
+
+ // Set an extended attribute.
+ Try<Nothing> setxattr = os::setxattr(
+ file,
+ "user.mesos.test",
+ "y",
+ 0);
+
+ // Only run this test if extended attribute is supported.
+ if (setxattr.isError() && setxattr.error() == os::strerror(ENOTSUP)) {
+ return;
+ }
+
+ ASSERT_SOME(setxattr);
+
+ // Get the extended attribute.
+ Try<string> value = os::getxattr(file, "user.mesos.test");
+ ASSERT_SOME(value);
+ EXPECT_EQ(value.get(), "y");
+
+ // Remove the extended attribute.
+ ASSERT_SOME(os::removexattr(file, "user.mesos.test"));
+
+ // Get the extended attribute again which should not exist.
+ ASSERT_ERROR(os::getxattr(file, "user.mesos.test"));
+}
+#endif // __linux__ || __APPLE__
[4/4] mesos git commit: Added backport of r52910 to 1.1.x.
Posted by qi...@apache.org.
Added backport of r52910 to 1.1.x.
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/b7f0bc19
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/b7f0bc19
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/b7f0bc19
Branch: refs/heads/1.1.x
Commit: b7f0bc1943bb5cbe5830df987a9888d90bbcd18e
Parents: de3a1b5
Author: Qian Zhang <zh...@gmail.com>
Authored: Wed Nov 23 17:49:17 2016 +0800
Committer: Qian Zhang <zh...@gmail.com>
Committed: Thu Nov 24 09:37:56 2016 +0800
----------------------------------------------------------------------
src/Makefile.am | 1 +
.../containerizer/mesos/provisioner/backend.cpp | 9 ++---
.../mesos/provisioner/constants.hpp | 34 ++++++++++++++++++
.../mesos/provisioner/docker/paths.cpp | 4 ++-
src/slave/flags.cpp | 4 ++-
.../containerizer/provisioner_appc_tests.cpp | 13 ++++---
.../containerizer/provisioner_backend_tests.cpp | 37 ++++++++++++--------
7 files changed, 76 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/b7f0bc19/src/Makefile.am
----------------------------------------------------------------------
diff --git a/src/Makefile.am b/src/Makefile.am
index 3bcc0f2..73d337d 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -967,6 +967,7 @@ libmesos_no_3rdparty_la_SOURCES += \
slave/containerizer/mesos/isolators/volume/sandbox_path.hpp \
slave/containerizer/mesos/isolators/windows.hpp \
slave/containerizer/mesos/provisioner/backend.hpp \
+ slave/containerizer/mesos/provisioner/constants.hpp \
slave/containerizer/mesos/provisioner/paths.hpp \
slave/containerizer/mesos/provisioner/provisioner.hpp \
slave/containerizer/mesos/provisioner/store.hpp \
http://git-wip-us.apache.org/repos/asf/mesos/blob/b7f0bc19/src/slave/containerizer/mesos/provisioner/backend.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/backend.cpp b/src/slave/containerizer/mesos/provisioner/backend.cpp
index 9c258ed..157967f 100644
--- a/src/slave/containerizer/mesos/provisioner/backend.cpp
+++ b/src/slave/containerizer/mesos/provisioner/backend.cpp
@@ -23,6 +23,7 @@
#endif
#include "slave/containerizer/mesos/provisioner/backend.hpp"
+#include "slave/containerizer/mesos/provisioner/constants.hpp"
#ifdef __linux__
#include "slave/containerizer/mesos/provisioner/backends/aufs.hpp"
@@ -46,14 +47,14 @@ hashmap<string, Owned<Backend>> Backend::create(const Flags& flags)
hashmap<string, Try<Owned<Backend>>(*)(const Flags&)> creators;
#ifdef __linux__
- creators.put("bind", &BindBackend::create);
+ creators.put(BIND_BACKEND, &BindBackend::create);
Try<bool> aufsSupported = fs::supported("aufs");
if (aufsSupported.isError()) {
LOG(WARNING) << "Failed to check aufs availability: '"
<< aufsSupported.error();
} else if (aufsSupported.get()) {
- creators.put("aufs", &AufsBackend::create);
+ creators.put(AUFS_BACKEND, &AufsBackend::create);
}
Try<bool> overlayfsSupported = fs::supported("overlayfs");
@@ -61,11 +62,11 @@ hashmap<string, Owned<Backend>> Backend::create(const Flags& flags)
LOG(WARNING) << "Failed to check overlayfs availability: '"
<< overlayfsSupported.error();
} else if (overlayfsSupported.get()) {
- creators.put("overlay", &OverlayBackend::create);
+ creators.put(OVERLAY_BACKEND, &OverlayBackend::create);
}
#endif // __linux__
- creators.put("copy", &CopyBackend::create);
+ creators.put(COPY_BACKEND, &CopyBackend::create);
hashmap<string, Owned<Backend>> backends;
http://git-wip-us.apache.org/repos/asf/mesos/blob/b7f0bc19/src/slave/containerizer/mesos/provisioner/constants.hpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/constants.hpp b/src/slave/containerizer/mesos/provisioner/constants.hpp
new file mode 100644
index 0000000..ab488eb
--- /dev/null
+++ b/src/slave/containerizer/mesos/provisioner/constants.hpp
@@ -0,0 +1,34 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef __MESOS_PROVISIONER_CONSTANTS_HPP__
+#define __MESOS_PROVISIONER_CONSTANTS_HPP__
+
+namespace mesos {
+namespace internal {
+namespace slave {
+
+// Provisioner backends.
+constexpr char AUFS_BACKEND[] = "aufs";
+constexpr char BIND_BACKEND[] = "bind";
+constexpr char COPY_BACKEND[] = "copy";
+constexpr char OVERLAY_BACKEND[] = "overlay";
+
+} // namespace slave {
+} // namespace internal {
+} // namespace mesos {
+
+#endif // __MESOS_PROVISIONER_CONSTANTS_HPP__
http://git-wip-us.apache.org/repos/asf/mesos/blob/b7f0bc19/src/slave/containerizer/mesos/provisioner/docker/paths.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/docker/paths.cpp b/src/slave/containerizer/mesos/provisioner/docker/paths.cpp
index dea3756..cd684b3 100644
--- a/src/slave/containerizer/mesos/provisioner/docker/paths.cpp
+++ b/src/slave/containerizer/mesos/provisioner/docker/paths.cpp
@@ -14,6 +14,8 @@
// See the License for the specific language governing permissions and
// limitations under the License.
+#include "slave/containerizer/mesos/provisioner/constants.hpp"
+
#include "slave/containerizer/mesos/provisioner/docker/paths.hpp"
#include <stout/path.hpp>
@@ -58,7 +60,7 @@ string getImageLayerManifestPath(const string& storeDir, const string& layerId)
string getImageLayerRootfsPath(const string& layerPath, const string& backend)
{
- if (backend == "overlay") {
+ if (backend == OVERLAY_BACKEND) {
return path::join(layerPath, "rootfs." + backend);
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/b7f0bc19/src/slave/flags.cpp
----------------------------------------------------------------------
diff --git a/src/slave/flags.cpp b/src/slave/flags.cpp
index 87d9e46..e837cfc 100644
--- a/src/slave/flags.cpp
+++ b/src/slave/flags.cpp
@@ -34,6 +34,8 @@
#include "slave/containerizer/mesos/linux_launcher.hpp"
#endif // __linux__
+#include "slave/containerizer/mesos/provisioner/constants.hpp"
+
using std::string;
mesos::internal::slave::Flags::Flags()
@@ -128,7 +130,7 @@ mesos::internal::slave::Flags::Flags()
"image_provisioner_backend",
"Strategy for provisioning container rootfs from images,\n"
"e.g., `aufs`, `bind`, `copy`, `overlay`.",
- "copy");
+ COPY_BACKEND);
add(&Flags::appc_simple_discovery_uri_prefix,
"appc_simple_discovery_uri_prefix",
http://git-wip-us.apache.org/repos/asf/mesos/blob/b7f0bc19/src/tests/containerizer/provisioner_appc_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/containerizer/provisioner_appc_tests.cpp b/src/tests/containerizer/provisioner_appc_tests.cpp
index a999fc7..8b4b832 100644
--- a/src/tests/containerizer/provisioner_appc_tests.cpp
+++ b/src/tests/containerizer/provisioner_appc_tests.cpp
@@ -37,6 +37,7 @@
#include "slave/paths.hpp"
+#include "slave/containerizer/mesos/provisioner/constants.hpp"
#include "slave/containerizer/mesos/provisioner/paths.hpp"
#include "slave/containerizer/mesos/provisioner/provisioner.hpp"
@@ -63,6 +64,8 @@ using process::Process;
using testing::_;
using testing::Return;
+using mesos::internal::slave::BIND_BACKEND;
+using mesos::internal::slave::COPY_BACKEND;
using mesos::internal::slave::Fetcher;
using mesos::internal::slave::Provisioner;
using mesos::internal::slave::appc::Store;
@@ -284,7 +287,7 @@ TEST_F(ProvisionerAppcTest, ROOT_Provision)
slave::Flags flags;
flags.image_providers = "APPC";
flags.appc_store_dir = path::join(os::getcwd(), "store");
- flags.image_provisioner_backend = "bind";
+ flags.image_provisioner_backend = BIND_BACKEND;
flags.work_dir = path::join(sandbox.get(), "work_dir");
Try<Owned<Provisioner>> provisioner = Provisioner::create(flags);
@@ -348,7 +351,7 @@ TEST_F(ProvisionerAppcTest, ROOT_ProvisionNestedContainer)
slave::Flags flags;
flags.image_providers = "APPC";
flags.appc_store_dir = path::join(os::getcwd(), "store");
- flags.image_provisioner_backend = "bind";
+ flags.image_provisioner_backend = BIND_BACKEND;
flags.work_dir = path::join(sandbox.get(), "work_dir");
Try<Owned<Provisioner>> provisioner = Provisioner::create(flags);
@@ -418,7 +421,7 @@ TEST_F(ProvisionerAppcTest, Recover)
slave::Flags flags;
flags.image_providers = "APPC";
flags.appc_store_dir = path::join(os::getcwd(), "store");
- flags.image_provisioner_backend = "copy";
+ flags.image_provisioner_backend = COPY_BACKEND;
flags.work_dir = path::join(sandbox.get(), "work_dir");
Try<Owned<Provisioner>> provisioner = Provisioner::create(flags);
@@ -489,7 +492,7 @@ TEST_F(ProvisionerAppcTest, RecoverNestedContainer)
slave::Flags flags;
flags.image_providers = "APPC";
flags.appc_store_dir = path::join(os::getcwd(), "store");
- flags.image_provisioner_backend = "copy";
+ flags.image_provisioner_backend = COPY_BACKEND;
flags.work_dir = path::join(sandbox.get(), "work_dir");
Try<Owned<Provisioner>> provisioner = Provisioner::create(flags);
@@ -558,7 +561,7 @@ TEST_F(ProvisionerAppcTest, RecoverNestedContainerNoParentImage)
slave::Flags flags;
flags.image_providers = "APPC";
flags.appc_store_dir = path::join(os::getcwd(), "store");
- flags.image_provisioner_backend = "copy";
+ flags.image_provisioner_backend = COPY_BACKEND;
flags.work_dir = path::join(sandbox.get(), "work_dir");
Try<Owned<Provisioner>> provisioner = Provisioner::create(flags);
http://git-wip-us.apache.org/repos/asf/mesos/blob/b7f0bc19/src/tests/containerizer/provisioner_backend_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/containerizer/provisioner_backend_tests.cpp b/src/tests/containerizer/provisioner_backend_tests.cpp
index 661977b..727bf64 100644
--- a/src/tests/containerizer/provisioner_backend_tests.cpp
+++ b/src/tests/containerizer/provisioner_backend_tests.cpp
@@ -33,12 +33,19 @@
#include "slave/containerizer/mesos/provisioner/backends/copy.hpp"
#include "slave/containerizer/mesos/provisioner/backends/overlay.hpp"
+#include "slave/containerizer/mesos/provisioner/constants.hpp"
+
#include "tests/flags.hpp"
using namespace process;
using namespace mesos::internal::slave;
+using mesos::internal::slave::AUFS_BACKEND;
+using mesos::internal::slave::BIND_BACKEND;
+using mesos::internal::slave::COPY_BACKEND;
+using mesos::internal::slave::OVERLAY_BACKEND;
+
using std::string;
using std::vector;
@@ -89,9 +96,9 @@ TEST_F(OverlayBackendTest, ROOT_OVERLAYFS_OverlayFSBackend)
string rootfs = path::join(os::getcwd(), "rootfs");
hashmap<string, Owned<Backend>> backends = Backend::create(slave::Flags());
- ASSERT_TRUE(backends.contains("overlay"));
+ ASSERT_TRUE(backends.contains(OVERLAY_BACKEND));
- AWAIT_READY(backends["overlay"]->provision(
+ AWAIT_READY(backends[OVERLAY_BACKEND]->provision(
{layer1, layer2},
rootfs,
sandbox.get()));
@@ -113,7 +120,7 @@ TEST_F(OverlayBackendTest, ROOT_OVERLAYFS_OverlayFSBackend)
EXPECT_SOME_EQ("test3", os::read(path::join(rootfs, "file")));
EXPECT_SOME_EQ("test2", os::read(path::join(layer2, "file")));
- AWAIT_READY(backends["overlay"]->destroy(rootfs, sandbox.get()));
+ AWAIT_READY(backends[OVERLAY_BACKEND]->destroy(rootfs, sandbox.get()));
EXPECT_FALSE(os::exists(rootfs));
}
@@ -144,9 +151,9 @@ TEST_F(OverlayBackendTest, ROOT_OVERLAYFS_OverlayFSBackendWithManyLayers)
string rootfs = path::join(sandbox.get(), "rootfs");
hashmap<string, Owned<Backend>> backends = Backend::create(slave::Flags());
- ASSERT_TRUE(backends.contains("overlay"));
+ ASSERT_TRUE(backends.contains(OVERLAY_BACKEND));
- AWAIT_READY(backends["overlay"]->provision(
+ AWAIT_READY(backends[OVERLAY_BACKEND]->provision(
layers,
rootfs,
sandbox.get()));
@@ -158,7 +165,7 @@ TEST_F(OverlayBackendTest, ROOT_OVERLAYFS_OverlayFSBackendWithManyLayers)
strings::format("dir%d", i).get())));
}
- AWAIT_READY(backends["overlay"]->destroy(rootfs, sandbox.get()));
+ AWAIT_READY(backends[OVERLAY_BACKEND]->destroy(rootfs, sandbox.get()));
}
@@ -176,11 +183,11 @@ TEST_F(BindBackendTest, ROOT_BindBackend)
ASSERT_SOME(mkdir);
hashmap<string, Owned<Backend>> backends = Backend::create(slave::Flags());
- ASSERT_TRUE(backends.contains("bind"));
+ ASSERT_TRUE(backends.contains(BIND_BACKEND));
string target = path::join(os::getcwd(), "target");
- AWAIT_READY(backends["bind"]->provision(
+ AWAIT_READY(backends[BIND_BACKEND]->provision(
{rootfs},
target,
sandbox.get()));
@@ -193,7 +200,7 @@ TEST_F(BindBackendTest, ROOT_BindBackend)
EXPECT_TRUE(os::Permissions(mode.get()).owner.w);
EXPECT_ERROR(os::write(path::join(target, "tmp", "test"), "data"));
- AWAIT_READY(backends["bind"]->destroy(target, sandbox.get()));
+ AWAIT_READY(backends[BIND_BACKEND]->destroy(target, sandbox.get()));
EXPECT_FALSE(os::exists(target));
}
@@ -220,9 +227,9 @@ TEST_F(AufsBackendTest, ROOT_AUFS_AufsBackend)
string rootfs = path::join(os::getcwd(), "rootfs");
hashmap<string, Owned<Backend>> backends = Backend::create(slave::Flags());
- ASSERT_TRUE(backends.contains("aufs"));
+ ASSERT_TRUE(backends.contains(AUFS_BACKEND));
- AWAIT_READY(backends["aufs"]->provision(
+ AWAIT_READY(backends[AUFS_BACKEND]->provision(
{layer1, layer2},
rootfs,
sandbox.get()));
@@ -244,7 +251,7 @@ TEST_F(AufsBackendTest, ROOT_AUFS_AufsBackend)
EXPECT_SOME_EQ("test3", os::read(path::join(rootfs, "file")));
EXPECT_SOME_EQ("test2", os::read(path::join(layer2, "file")));
- AWAIT_READY(backends["aufs"]->destroy(rootfs, sandbox.get()));
+ AWAIT_READY(backends[AUFS_BACKEND]->destroy(rootfs, sandbox.get()));
EXPECT_FALSE(os::exists(rootfs));
}
@@ -272,9 +279,9 @@ TEST_F(CopyBackendTest, ROOT_CopyBackend)
string rootfs = path::join(os::getcwd(), "rootfs");
hashmap<string, Owned<Backend>> backends = Backend::create(slave::Flags());
- ASSERT_TRUE(backends.contains("copy"));
+ ASSERT_TRUE(backends.contains(COPY_BACKEND));
- AWAIT_READY(backends["copy"]->provision(
+ AWAIT_READY(backends[COPY_BACKEND]->provision(
{layer1, layer2},
rootfs,
sandbox.get()));
@@ -289,7 +296,7 @@ TEST_F(CopyBackendTest, ROOT_CopyBackend)
EXPECT_TRUE(os::exists(path::join(rootfs, "file")));
EXPECT_SOME_EQ("test2", os::read(path::join(rootfs, "file")));
- AWAIT_READY(backends["copy"]->destroy(rootfs, sandbox.get()));
+ AWAIT_READY(backends[COPY_BACKEND]->destroy(rootfs, sandbox.get()));
EXPECT_FALSE(os::exists(rootfs));
}
[2/4] mesos git commit: Added backport of MESOS-6002 to 1.1.x.
Posted by qi...@apache.org.
Added backport of MESOS-6002 to 1.1.x.
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/14355b75
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/14355b75
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/14355b75
Branch: refs/heads/1.1.x
Commit: 14355b757ac0dd39db1928c00cb9dad5fd2deb65
Parents: b7f0bc1
Author: Qian Zhang <zh...@gmail.com>
Authored: Wed Nov 23 20:41:10 2016 +0800
Committer: Qian Zhang <zh...@gmail.com>
Committed: Thu Nov 24 09:37:56 2016 +0800
----------------------------------------------------------------------
.../mesos/provisioner/backends/aufs.cpp | 16 ++++++++++------
1 file changed, 10 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/14355b75/src/slave/containerizer/mesos/provisioner/backends/aufs.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/backends/aufs.cpp b/src/slave/containerizer/mesos/provisioner/backends/aufs.cpp
index c69dc42..c92c6c7 100644
--- a/src/slave/containerizer/mesos/provisioner/backends/aufs.cpp
+++ b/src/slave/containerizer/mesos/provisioner/backends/aufs.cpp
@@ -145,12 +145,16 @@ Future<Nothing> AufsBackendProcess::provision(
// See http://aufs.sourceforge.net/aufs2/man.html
// for the mount syntax for aufs.
- string options = "dirs=" + workdir + ":";
-
- // For aufs, the specified lower directories will be stacked
- // beginning from the rightmost one and going left. But we need the
- // first layer in the vector to be the bottom most layer.
- options += strings::join(":", adaptor::reverse(layers));
+ string options = "dirs=" + workdir + "=rw";
+
+ // For aufs, the specified lower directories will be stacked beginning
+ // from the rightmost one and going left. But we need the first layer
+ // in the vector to be the bottom most layer. And for each layer, we
+ // need to mount it with the option 'ro+wh' so that it will be read-only
+ // and its whiteout files (if any) will be well handled.
+ foreach (const string& layer, adaptor::reverse(layers)) {
+ options += ":" + layer + "=ro+wh";
+ }
VLOG(1) << "Provisioning image rootfs with aufs: '" << options << "'";