You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by qi...@apache.org on 2016/11/05 08:40:08 UTC
[1/6] mesos git commit: Divided utils.hpp to utils.hpp and utils.cpp.
Repository: mesos
Updated Branches:
refs/heads/master 61de69849 -> 3b6af1d8d
Divided utils.hpp to utils.hpp and utils.cpp.
Review: https://reviews.apache.org/r/53053
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/ac62ee27
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/ac62ee27
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/ac62ee27
Branch: refs/heads/master
Commit: ac62ee27f3f2d226732da439a74ed33aed7c071f
Parents: 50f6c11
Author: Qian Zhang <zh...@cn.ibm.com>
Authored: Thu Oct 20 17:34:43 2016 +0800
Committer: Qian Zhang <zh...@gmail.com>
Committed: Sat Nov 5 16:18:29 2016 +0800
----------------------------------------------------------------------
src/Makefile.am | 1 +
src/slave/containerizer/mesos/utils.cpp | 40 ++++++++++++++++++++++++++++
src/slave/containerizer/mesos/utils.hpp | 15 +----------
3 files changed, 42 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/ac62ee27/src/Makefile.am
----------------------------------------------------------------------
diff --git a/src/Makefile.am b/src/Makefile.am
index a5569ec..f08b986 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -829,6 +829,7 @@ libmesos_no_3rdparty_la_SOURCES += \
slave/containerizer/mesos/launcher.cpp \
slave/containerizer/mesos/mount.cpp \
slave/containerizer/mesos/paths.cpp \
+ slave/containerizer/mesos/utils.cpp \
slave/containerizer/mesos/isolators/docker/volume/driver.cpp \
slave/containerizer/mesos/isolators/docker/volume/paths.cpp \
slave/containerizer/mesos/isolators/filesystem/posix.cpp \
http://git-wip-us.apache.org/repos/asf/mesos/blob/ac62ee27/src/slave/containerizer/mesos/utils.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/utils.cpp b/src/slave/containerizer/mesos/utils.cpp
new file mode 100644
index 0000000..237aea4
--- /dev/null
+++ b/src/slave/containerizer/mesos/utils.cpp
@@ -0,0 +1,40 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "slave/containerizer/mesos/utils.hpp"
+
+namespace mesos {
+namespace internal {
+namespace slave {
+
+ContainerID getRootContainerId(const ContainerID& containerId)
+{
+ ContainerID rootContainerId = containerId;
+ while (rootContainerId.has_parent()) {
+ // NOTE: Looks like protobuf does not handle copying well when
+ // nesting message is involved, because the source and the target
+ // point to the same object. Therefore, we create a temporary
+ // variable and use an extra copy here.
+ ContainerID id = rootContainerId.parent();
+ rootContainerId = id;
+ }
+
+ return rootContainerId;
+}
+
+} // namespace slave {
+} // namespace internal {
+} // namespace mesos {
http://git-wip-us.apache.org/repos/asf/mesos/blob/ac62ee27/src/slave/containerizer/mesos/utils.hpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/utils.hpp b/src/slave/containerizer/mesos/utils.hpp
index 178ebf3..f24215b 100644
--- a/src/slave/containerizer/mesos/utils.hpp
+++ b/src/slave/containerizer/mesos/utils.hpp
@@ -23,20 +23,7 @@ namespace mesos {
namespace internal {
namespace slave {
-static ContainerID getRootContainerId(const ContainerID& containerId)
-{
- ContainerID rootContainerId = containerId;
- while (rootContainerId.has_parent()) {
- // NOTE: Looks like protobuf does not handle copying well when
- // nesting message is involved, because the source and the target
- // point to the same object. Therefore, we create a temporary
- // variable and use an extra copy here.
- ContainerID id = rootContainerId.parent();
- rootContainerId = id;
- }
-
- return rootContainerId;
-}
+ContainerID getRootContainerId(const ContainerID& containerId);
} // namespace slave {
} // namespace internal {
[5/6] mesos git commit: Implemented handling AUFS whiteouts for copy
backend.
Posted by qi...@apache.org.
Implemented handling AUFS whiteouts for copy backend.
Review: https://reviews.apache.org/r/53115
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/0f4c15ad
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/0f4c15ad
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/0f4c15ad
Branch: refs/heads/master
Commit: 0f4c15ad735e437a8e8b9bb6846025aea02c765b
Parents: 3800cfd
Author: Qian Zhang <zh...@cn.ibm.com>
Authored: Sat Oct 22 21:45:44 2016 +0800
Committer: Qian Zhang <zh...@gmail.com>
Committed: Sat Nov 5 16:18:30 2016 +0800
----------------------------------------------------------------------
.../mesos/provisioner/backends/copy.cpp | 103 ++++++++++++++++++-
1 file changed, 98 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/0f4c15ad/src/slave/containerizer/mesos/provisioner/backends/copy.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/backends/copy.cpp b/src/slave/containerizer/mesos/provisioner/backends/copy.cpp
index 9c5354e..7e98110 100644
--- a/src/slave/containerizer/mesos/provisioner/backends/copy.cpp
+++ b/src/slave/containerizer/mesos/provisioner/backends/copy.cpp
@@ -16,6 +16,8 @@
#include <list>
+#include <mesos/docker/spec.hpp>
+
#include <process/collect.hpp>
#include <process/defer.hpp>
#include <process/dispatch.hpp>
@@ -24,7 +26,6 @@
#include <process/process.hpp>
#include <process/subprocess.hpp>
-
#include <stout/foreach.hpp>
#include <stout/os.hpp>
@@ -32,7 +33,6 @@
#include "slave/containerizer/mesos/provisioner/backends/copy.hpp"
-
using namespace process;
using std::string;
@@ -128,9 +128,92 @@ Future<Nothing> CopyBackendProcess::provision(
Future<Nothing> CopyBackendProcess::_provision(
- string layer,
- const string& rootfs)
+ string layer,
+ const string& rootfs)
{
+ // Traverse the layer to check if there is any whiteout files, if
+ // yes, remove the corresponding files/directories from the rootfs.
+ // Note: We assume all image types use AUFS whiteout format.
+ char* source[] = {const_cast<char*>(layer.c_str()), nullptr};
+
+ FTS* tree = ::fts_open(source, FTS_NOCHDIR | FTS_PHYSICAL, nullptr);
+ if (tree == nullptr) {
+ return Failure("Failed to open '" + layer + "': " + os::strerror(errno));
+ }
+
+ vector<string> whiteouts;
+ for (FTSENT *node = ::fts_read(tree);
+ node != nullptr; node = ::fts_read(tree)) {
+ if (node->fts_info != FTS_F) {
+ continue;
+ }
+
+ if (!strings::startsWith(node->fts_name, docker::spec::WHITEOUT_PREFIX)) {
+ continue;
+ }
+
+ string ftsPath = string(node->fts_path);
+ Path whiteout = Path(ftsPath.substr(layer.length() + 1));
+
+ // Keep the relative paths of the whiteout files, we will
+ // remove them from rootfs after layer is copied to rootfs.
+ whiteouts.push_back(whiteout.string());
+
+ if (node->fts_name == string(docker::spec::WHITEOUT_OPAQUE_PREFIX)) {
+ const string path = path::join(rootfs, Path(whiteout).dirname());
+
+ // Remove the entries under the directory labeled
+ // as opaque whiteout from rootfs.
+ Try<Nothing> rmdir = os::rmdir(path, true, false);
+ if (rmdir.isError()) {
+ ::fts_close(tree);
+ return Failure(
+ "Failed to remove the entries under the directory labeled as"
+ " opaque whiteout '" + path + "': " + rmdir.error());
+ }
+ } else {
+ const string path = path::join(
+ rootfs,
+ whiteout.dirname(),
+ whiteout.basename().substr(strlen(docker::spec::WHITEOUT_PREFIX)));
+
+ // The file/directory labeled as whiteout may have already been
+ // removed with the code above due to its parent directory labeled
+ // as opaque whiteout, so here we need to check if it still exists
+ // before trying to remove it.
+ if (os::exists(path)) {
+ if (os::stat::isdir(path)) {
+ Try<Nothing> rmdir = os::rmdir(path);
+ if (rmdir.isError()) {
+ ::fts_close(tree);
+ return Failure(
+ "Failed to remove the directory labeled as whiteout '" +
+ path + "': " + rmdir.error());
+ }
+ } else {
+ Try<Nothing> rm = os::rm(path);
+ if (rm.isError()) {
+ ::fts_close(tree);
+ return Failure(
+ "Failed to remove the file labeled as whiteout '" +
+ path + "': " + rm.error());
+ }
+ }
+ }
+ }
+ }
+
+ if (errno != 0) {
+ Error error = ErrnoError();
+ ::fts_close(tree);
+ return Failure(error);
+ }
+
+ if (::fts_close(tree) != 0) {
+ return Failure(
+ "Failed to stop traversing file system: " + os::strerror(errno));
+ }
+
VLOG(1) << "Copying layer path '" << layer << "' to rootfs '" << rootfs
<< "'";
@@ -160,7 +243,7 @@ Future<Nothing> CopyBackendProcess::_provision(
Subprocess cp = s.get();
return cp.status()
- .then([cp](const Option<int>& status) -> Future<Nothing> {
+ .then([=](const Option<int>& status) -> Future<Nothing> {
if (status.isNone()) {
return Failure("Failed to reap subprocess to copy image");
} else if (status.get() != 0) {
@@ -170,6 +253,16 @@ Future<Nothing> CopyBackendProcess::_provision(
});
}
+ // Remove the whiteout files from rootfs.
+ foreach (const string whiteout, whiteouts) {
+ Try<Nothing> rm = os::rm(path::join(rootfs, whiteout));
+ if (rm.isError()) {
+ return Failure(
+ "Failed to remove whiteout file '" +
+ whiteout + "': " + rm.error());
+ }
+ }
+
return Nothing();
});
}
[2/6] mesos git commit: Added `setxattr()` and `getxattr()` in stout.
Posted by qi...@apache.org.
Added `setxattr()` and `getxattr()` in stout.
Review: https://reviews.apache.org/r/53041
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/e3c12c4c
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/e3c12c4c
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/e3c12c4c
Branch: refs/heads/master
Commit: e3c12c4c3fabc64c8a3de9e514340bfb52237d69
Parents: 61de698
Author: Qian Zhang <zh...@cn.ibm.com>
Authored: Wed Oct 19 16:19:59 2016 +0800
Committer: Qian Zhang <zh...@gmail.com>
Committed: Sat Nov 5 16:18:29 2016 +0800
----------------------------------------------------------------------
3rdparty/stout/include/Makefile.am | 3 +
3rdparty/stout/include/stout/os.hpp | 1 +
3rdparty/stout/include/stout/os/posix/xattr.hpp | 99 ++++++++++++++++++++
.../stout/include/stout/os/windows/xattr.hpp | 38 ++++++++
3rdparty/stout/include/stout/os/xattr.hpp | 26 +++++
5 files changed, 167 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/e3c12c4c/3rdparty/stout/include/Makefile.am
----------------------------------------------------------------------
diff --git a/3rdparty/stout/include/Makefile.am b/3rdparty/stout/include/Makefile.am
index 1eb9c14..d5bc728 100644
--- a/3rdparty/stout/include/Makefile.am
+++ b/3rdparty/stout/include/Makefile.am
@@ -104,6 +104,7 @@ nobase_include_HEADERS = \
stout/os/utime.hpp \
stout/os/wait.hpp \
stout/os/write.hpp \
+ stout/os/xattr.hpp \
stout/os/posix/bootid.hpp \
stout/os/posix/chown.hpp \
stout/os/posix/chroot.hpp \
@@ -128,6 +129,7 @@ nobase_include_HEADERS = \
stout/os/posix/stat.hpp \
stout/os/posix/su.hpp \
stout/os/posix/write.hpp \
+ stout/os/posix/xattr.hpp \
stout/os/raw/argv.hpp \
stout/os/raw/environment.hpp \
stout/os/windows/bootid.hpp \
@@ -153,6 +155,7 @@ nobase_include_HEADERS = \
stout/os/windows/stat.hpp \
stout/os/windows/su.hpp \
stout/os/windows/write.hpp \
+ stout/os/windows/xattr.hpp \
stout/path.hpp \
stout/preprocessor.hpp \
stout/proc.hpp \
http://git-wip-us.apache.org/repos/asf/mesos/blob/e3c12c4c/3rdparty/stout/include/stout/os.hpp
----------------------------------------------------------------------
diff --git a/3rdparty/stout/include/stout/os.hpp b/3rdparty/stout/include/stout/os.hpp
index 96e8621..af43bbf 100644
--- a/3rdparty/stout/include/stout/os.hpp
+++ b/3rdparty/stout/include/stout/os.hpp
@@ -64,6 +64,7 @@
#include <stout/os/touch.hpp>
#include <stout/os/utime.hpp>
#include <stout/os/wait.hpp>
+#include <stout/os/xattr.hpp>
#include <stout/os/raw/argv.hpp>
#include <stout/os/raw/environment.hpp>
http://git-wip-us.apache.org/repos/asf/mesos/blob/e3c12c4c/3rdparty/stout/include/stout/os/posix/xattr.hpp
----------------------------------------------------------------------
diff --git a/3rdparty/stout/include/stout/os/posix/xattr.hpp b/3rdparty/stout/include/stout/os/posix/xattr.hpp
new file mode 100644
index 0000000..518940f
--- /dev/null
+++ b/3rdparty/stout/include/stout/os/posix/xattr.hpp
@@ -0,0 +1,99 @@
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef __STOUT_OS_POSIX_XATTR_HPP__
+#define __STOUT_OS_POSIX_XATTR_HPP__
+
+#include <sys/xattr.h>
+
+namespace os {
+
+inline Try<Nothing> setxattr(
+ const std::string& path,
+ const std::string& name,
+ const std::string& value,
+ int flags)
+{
+#ifdef __APPLE__
+ if (::setxattr(
+ path.c_str(),
+ name.c_str(),
+ value.c_str(),
+ value.length(),
+ 0,
+ flags) < 0) {
+#else
+ if (::setxattr(
+ path.c_str(),
+ name.c_str(),
+ value.c_str(),
+ value.length(),
+ flags) < 0) {
+#endif
+ return ErrnoError();
+ }
+
+ return Nothing();
+}
+
+
+inline Try<std::string> getxattr(
+ const std::string& path,
+ const std::string& name)
+{
+ // Get the current size of the attribute.
+#ifdef __APPLE__
+ ssize_t size = ::getxattr(path.c_str(), name.c_str(), nullptr, 0, 0, 0);
+#else
+ ssize_t size = ::getxattr(path.c_str(), name.c_str(), nullptr, 0);
+#endif
+ if (size < 0) {
+ return ErrnoError();
+ }
+
+ char* temp = new char[size + 1];
+ ::memset(temp, 0, (size_t)size + 1);
+
+#ifdef __APPLE__
+ if (::getxattr(path.c_str(), name.c_str(), temp, (size_t)size, 0, 0) < 0) {
+#else
+ if (::getxattr(path.c_str(), name.c_str(), temp, (size_t)size) < 0) {
+#endif
+ delete[] temp;
+ return ErrnoError();
+ }
+
+ std::string result(temp);
+ delete[] temp;
+
+ return result;
+}
+
+
+inline Try<Nothing> removexattr(
+ const std::string& path,
+ const std::string& name)
+{
+#ifdef __APPLE__
+ if (::removexattr(path.c_str(), name.c_str(), 0) < 0) {
+#else
+ if (::removexattr(path.c_str(), name.c_str()) < 0) {
+#endif
+ return ErrnoError();
+ }
+
+ return Nothing();
+}
+
+} // namespace os {
+
+#endif /* __STOUT_OS_POSIX_XATTR_HPP__ */
http://git-wip-us.apache.org/repos/asf/mesos/blob/e3c12c4c/3rdparty/stout/include/stout/os/windows/xattr.hpp
----------------------------------------------------------------------
diff --git a/3rdparty/stout/include/stout/os/windows/xattr.hpp b/3rdparty/stout/include/stout/os/windows/xattr.hpp
new file mode 100644
index 0000000..3157f72
--- /dev/null
+++ b/3rdparty/stout/include/stout/os/windows/xattr.hpp
@@ -0,0 +1,38 @@
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef __STOUT_OS_WINDOWS_XATTR_HPP__
+#define __STOUT_OS_WINDOWS_XATTR_HPP__
+
+namespace os {
+
+// NOTE: These functions are deleted because Windows does
+// not support POSIX extended attribute semantics.
+inline Try<Nothing> setxattr(
+ const std::string& path,
+ const std::string& name,
+ const std::string& value,
+ int flags) = delete;
+
+
+inline Try<std::string> getxattr(
+ const std::string& path,
+ const std::string& name) = delete;
+
+
+inline Try<std::string> removexattr(
+ const std::string& path,
+ const std::string& name) = delete;
+
+} // namespace os {
+
+#endif /* __STOUT_OS_WINDOWS_XATTR_HPP__ */
http://git-wip-us.apache.org/repos/asf/mesos/blob/e3c12c4c/3rdparty/stout/include/stout/os/xattr.hpp
----------------------------------------------------------------------
diff --git a/3rdparty/stout/include/stout/os/xattr.hpp b/3rdparty/stout/include/stout/os/xattr.hpp
new file mode 100644
index 0000000..b86b50a
--- /dev/null
+++ b/3rdparty/stout/include/stout/os/xattr.hpp
@@ -0,0 +1,26 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef __STOUT_OS_XATTR_HPP__
+#define __STOUT_OS_XATTR_HPP__
+
+#ifdef __WINDOWS__
+#include <stout/os/windows/xattr.hpp>
+#else
+#include <stout/os/posix/xattr.hpp>
+#endif // __WINDOWS__
+
+#endif // __STOUT_OS_XATTR_HPP__
[6/6] mesos git commit: Removed `ProvisionerProcess::__provision()`.
Posted by qi...@apache.org.
Removed `ProvisionerProcess::__provision()`.
Review: https://reviews.apache.org/r/53116
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/3b6af1d8
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/3b6af1d8
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/3b6af1d8
Branch: refs/heads/master
Commit: 3b6af1d8d336edeefb14f5a86e028c3b099371f5
Parents: 0f4c15a
Author: Qian Zhang <zh...@cn.ibm.com>
Authored: Sun Oct 23 09:03:14 2016 +0800
Committer: Qian Zhang <zh...@gmail.com>
Committed: Sat Nov 5 16:18:30 2016 +0800
----------------------------------------------------------------------
.../mesos/provisioner/provisioner.cpp | 118 +------------------
.../mesos/provisioner/provisioner.hpp | 5 -
2 files changed, 2 insertions(+), 121 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/3b6af1d8/src/slave/containerizer/mesos/provisioner/provisioner.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/provisioner.cpp b/src/slave/containerizer/mesos/provisioner/provisioner.cpp
index de2c121..db2d267 100644
--- a/src/slave/containerizer/mesos/provisioner/provisioner.cpp
+++ b/src/slave/containerizer/mesos/provisioner/provisioner.cpp
@@ -311,124 +311,10 @@ Future<ProvisionInfo> ProvisionerProcess::_provision(
imageInfo.layers,
rootfs,
backendDir)
- .then(defer(self(), &Self::__provision, rootfs, image, imageInfo));
-}
-
-
-// This function is currently docker image specific. Depending
-// on docker v1 spec, a docker image may include filesystem
-// changeset, which may need to delete directories or files.
-// The file/directory to be deleted will be labeled by creating
-// a 'whiteout' file, which is at the same location and with the
-// basename of the deleted file or directory prefixed with '.wh.'.
-// For the directory which has an opaque whiteout file '.wh..wh..opq'
-// under it, we need to delete all the files/directories under it.
-// Please see:
-// https://github.com/docker/docker/blob/master/image/spec/v1.md
-// https://github.com/docker/docker/blob/master/pkg/archive/whiteouts.go
-// And OCI image spec also has the concepts 'whiteout' and 'opaque whiteout':
-// https://github.com/opencontainers/image-spec/blob/master/layer.md#whiteouts
-Future<ProvisionInfo> ProvisionerProcess::__provision(
- const string& rootfs,
- const Image& image,
- const ImageInfo& imageInfo)
-{
-#ifdef __WINDOWS__
- return ProvisionInfo{
- rootfs, imageInfo.dockerManifest, imageInfo.appcManifest};
-#else
- // Skip single-layered images since no 'whiteout' files needs
- // to be handled, and this excludes any image using the bind
- // backend.
- if (imageInfo.layers.size() == 1 || image.type() != Image::DOCKER) {
+ .then([=]() -> Future<ProvisionInfo> {
return ProvisionInfo{
rootfs, imageInfo.dockerManifest, imageInfo.appcManifest};
- }
-
- // TODO(hausdorff): The FTS API is not available on some platforms, such as
- // Windows. We will need to either (1) prove that this is not necessary for
- // Windows Containers, which use much of the Docker spec themselves, or (2)
- // make this code compatible with Windows, as we did with other code that
- // depended on FTS, such as `os::rmdir`. See MESOS-5610.
- char* _rootfs[] = {const_cast<char*>(rootfs.c_str()), nullptr};
-
- FTS* tree = ::fts_open(_rootfs, FTS_NOCHDIR | FTS_PHYSICAL, nullptr);
- if (tree == nullptr) {
- return Failure("Failed to open '" + rootfs + "': " + os::strerror(errno));
- }
-
- vector<string> whiteout;
- vector<string> whiteoutOpaque;
-
- for (FTSENT *node = ::fts_read(tree);
- node != nullptr; node = ::fts_read(tree)) {
- if (node->fts_info == FTS_F &&
- strings::startsWith(node->fts_name, string(spec::WHITEOUT_PREFIX))) {
- Path path = Path(node->fts_path);
- if (node->fts_name == string(spec::WHITEOUT_OPAQUE_PREFIX)) {
- whiteoutOpaque.push_back(path.dirname());
- } else {
- whiteout.push_back(path::join(
- path.dirname(),
- path.basename().substr(strlen(spec::WHITEOUT_PREFIX))));
- }
-
- Try<Nothing> rm = os::rm(path.string());
- if (rm.isError()) {
- ::fts_close(tree);
- return Failure(
- "Failed to remove whiteout file '" +
- path.string() + "': " + rm.error());
- }
- }
- }
-
- if (errno != 0) {
- Error error = ErrnoError();
- ::fts_close(tree);
- return Failure(error);
- }
-
- if (::fts_close(tree) != 0) {
- return Failure(
- "Failed to stop traversing file system: " + os::strerror(errno));
- }
-
- foreach (const string& path, whiteoutOpaque) {
- Try<Nothing> rmdir = os::rmdir(path, true, false);
- if (rmdir.isError()) {
- return Failure(
- "Failed to remove the entries under the directory labeled as"
- " opaque whiteout '" + path + "': " + rmdir.error());
- }
- }
-
- foreach (const string& path, whiteout) {
- // The file/directory labeled as whiteout may have already been
- // removed with the code above due to its parent directory labeled
- // as opaque whiteout, so here we need to check if it still exists
- // before trying to remove it.
- if (os::exists(path)) {
- if (os::stat::isdir(path)) {
- Try<Nothing> rmdir = os::rmdir(path);
- if (rmdir.isError()) {
- return Failure(
- "Failed to remove the directory labeled as whiteout '" +
- path + "': " + rmdir.error());
- }
- } else {
- Try<Nothing> rm = os::rm(path);
- if (rm.isError()) {
- return Failure(
- "Failed to remove the file labeled as whiteout '" +
- path + "': " + rm.error());
- }
- }
- }
- }
-
- return ProvisionInfo{rootfs, imageInfo.dockerManifest, None()};
-#endif // __WINDOWS__
+ });
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/3b6af1d8/src/slave/containerizer/mesos/provisioner/provisioner.hpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/provisioner.hpp b/src/slave/containerizer/mesos/provisioner/provisioner.hpp
index c5481c5..0a48617 100644
--- a/src/slave/containerizer/mesos/provisioner/provisioner.hpp
+++ b/src/slave/containerizer/mesos/provisioner/provisioner.hpp
@@ -134,11 +134,6 @@ private:
const Image& image,
const ImageInfo& imageInfo);
- process::Future<ProvisionInfo> __provision(
- const std::string& rootfs,
- const Image& image,
- const ImageInfo& imageInfo);
-
process::Future<bool> _destroy(const ContainerID& containerId);
const Flags flags;
[3/6] mesos git commit: Added `FsTest.Xattr` test.
Posted by qi...@apache.org.
Added `FsTest.Xattr` test.
Review: https://reviews.apache.org/r/53042
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/50f6c117
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/50f6c117
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/50f6c117
Branch: refs/heads/master
Commit: 50f6c117e3874efbefe605b51cd3d3efb2cf64ba
Parents: e3c12c4
Author: Qian Zhang <zh...@cn.ibm.com>
Authored: Wed Oct 19 21:39:02 2016 +0800
Committer: Qian Zhang <zh...@gmail.com>
Committed: Sat Nov 5 16:18:29 2016 +0800
----------------------------------------------------------------------
3rdparty/stout/tests/os/filesystem_tests.cpp | 36 +++++++++++++++++++++++
1 file changed, 36 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/50f6c117/3rdparty/stout/tests/os/filesystem_tests.cpp
----------------------------------------------------------------------
diff --git a/3rdparty/stout/tests/os/filesystem_tests.cpp b/3rdparty/stout/tests/os/filesystem_tests.cpp
index e3894ef..5c8f422 100644
--- a/3rdparty/stout/tests/os/filesystem_tests.cpp
+++ b/3rdparty/stout/tests/os/filesystem_tests.cpp
@@ -31,6 +31,7 @@
#include <stout/os/rm.hpp>
#include <stout/os/touch.hpp>
#include <stout/os/write.hpp>
+#include <stout/os/xattr.hpp>
#include <stout/tests/utils.hpp>
@@ -436,3 +437,38 @@ TEST_F(FsTest, Close)
_CrtSetReportMode(_CRT_ASSERT, previous_report_mode);
#endif // __WINDOWS__
}
+
+
+TEST_F(FsTest, Xattr)
+{
+ const string file = path::join(os::getcwd(), UUID::random().toString());
+
+ // Create file.
+ ASSERT_SOME(os::touch(file));
+ ASSERT_TRUE(os::exists(file));
+
+ // Set an extended attribute.
+ Try<Nothing> setxattr = os::setxattr(
+ file,
+ "user.mesos.test",
+ "y",
+ 0);
+
+ // Only run this test if extended attribute is supported.
+ if (setxattr.isError() && setxattr.error() == os::strerror(ENOTSUP)) {
+ return;
+ }
+
+ ASSERT_SOME(setxattr);
+
+ // Get the extended attribute.
+ Try<string> value = os::getxattr(file, "user.mesos.test");
+ ASSERT_SOME(value);
+ EXPECT_EQ(value.get(), "y");
+
+ // Remove the extended attribute.
+ ASSERT_SOME(os::removexattr(file, "user.mesos.test"));
+
+ // Get the extended attribute again which should not exist.
+ ASSERT_ERROR(os::getxattr(file, "user.mesos.test"));
+}
[4/6] mesos git commit: Implemented the conversion from AUFS
whiteouts to OverlayFS whiteouts.
Posted by qi...@apache.org.
Implemented the conversion from AUFS whiteouts to OverlayFS whiteouts.
Review: https://reviews.apache.org/r/53161
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/3800cfd0
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/3800cfd0
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/3800cfd0
Branch: refs/heads/master
Commit: 3800cfd0d2d65e649b21d079e1c1a4ce743558dd
Parents: ac62ee2
Author: Qian Zhang <zh...@cn.ibm.com>
Authored: Tue Oct 25 11:20:26 2016 +0800
Committer: Qian Zhang <zh...@gmail.com>
Committed: Sat Nov 5 16:18:30 2016 +0800
----------------------------------------------------------------------
src/Makefile.am | 2 +
.../mesos/provisioner/docker/store.cpp | 24 +++-
.../containerizer/mesos/provisioner/utils.cpp | 111 +++++++++++++++++++
.../containerizer/mesos/provisioner/utils.hpp | 36 ++++++
4 files changed, 169 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/3800cfd0/src/Makefile.am
----------------------------------------------------------------------
diff --git a/src/Makefile.am b/src/Makefile.am
index f08b986..5a47c93 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -842,6 +842,7 @@ libmesos_no_3rdparty_la_SOURCES += \
slave/containerizer/mesos/provisioner/paths.cpp \
slave/containerizer/mesos/provisioner/provisioner.cpp \
slave/containerizer/mesos/provisioner/store.cpp \
+ slave/containerizer/mesos/provisioner/utils.cpp \
slave/containerizer/mesos/provisioner/appc/cache.cpp \
slave/containerizer/mesos/provisioner/appc/fetcher.cpp \
slave/containerizer/mesos/provisioner/appc/paths.cpp \
@@ -976,6 +977,7 @@ libmesos_no_3rdparty_la_SOURCES += \
slave/containerizer/mesos/provisioner/paths.hpp \
slave/containerizer/mesos/provisioner/provisioner.hpp \
slave/containerizer/mesos/provisioner/store.hpp \
+ slave/containerizer/mesos/provisioner/utils.hpp \
slave/containerizer/mesos/provisioner/appc/cache.hpp \
slave/containerizer/mesos/provisioner/appc/fetcher.hpp \
slave/containerizer/mesos/provisioner/appc/paths.hpp \
http://git-wip-us.apache.org/repos/asf/mesos/blob/3800cfd0/src/slave/containerizer/mesos/provisioner/docker/store.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/docker/store.cpp b/src/slave/containerizer/mesos/provisioner/docker/store.cpp
index e192f86..9dccd06 100644
--- a/src/slave/containerizer/mesos/provisioner/docker/store.cpp
+++ b/src/slave/containerizer/mesos/provisioner/docker/store.cpp
@@ -30,6 +30,9 @@
#include <mesos/docker/spec.hpp>
+#include "slave/containerizer/mesos/provisioner/constants.hpp"
+#include "slave/containerizer/mesos/provisioner/utils.hpp"
+
#include "slave/containerizer/mesos/provisioner/docker/metadata_manager.hpp"
#include "slave/containerizer/mesos/provisioner/docker/paths.hpp"
#include "slave/containerizer/mesos/provisioner/docker/puller.hpp"
@@ -331,6 +334,23 @@ Future<Nothing> StoreProcess::moveLayer(
flags.docker_store_dir,
layerId);
+ const string sourceRootfs = paths::getImageLayerRootfsPath(
+ source,
+ flags.image_provisioner_backend);
+
+#ifdef __linux__
+ // If the backend is "overlay", we need to convert
+ // AUFS whiteout files to OverlayFS whiteout files.
+ if (flags.image_provisioner_backend == OVERLAY_BACKEND) {
+ Try<Nothing> convert = convertWhiteouts(sourceRootfs);
+ if (convert.isError()) {
+ return Failure(
+ "Failed to convert the whiteout files under '" +
+ sourceRootfs + "': " + convert.error());
+ }
+ }
+#endif
+
if (!os::exists(target)) {
// This is the case that we pull the layer for the first time.
Try<Nothing> mkdir = os::mkdir(target);
@@ -349,10 +369,6 @@ Future<Nothing> StoreProcess::moveLayer(
} else {
// This is the case where the layer has already been pulled with a
// different backend.
- const string sourceRootfs = paths::getImageLayerRootfsPath(
- source,
- flags.image_provisioner_backend);
-
Try<Nothing> rename = os::rename(sourceRootfs, targetRootfs);
if (rename.isError()) {
return Failure(
http://git-wip-us.apache.org/repos/asf/mesos/blob/3800cfd0/src/slave/containerizer/mesos/provisioner/utils.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/utils.cpp b/src/slave/containerizer/mesos/provisioner/utils.cpp
new file mode 100644
index 0000000..340cf48
--- /dev/null
+++ b/src/slave/containerizer/mesos/provisioner/utils.cpp
@@ -0,0 +1,111 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "slave/containerizer/mesos/provisioner/utils.hpp"
+
+using std::string;
+
+namespace mesos {
+namespace internal {
+namespace slave {
+
+#ifdef __linux__
+// This function is used to convert AUFS whiteouts to OverlayFS whiteouts and
+// the AUFS whiteouts will be removed after the conversion is done. Whiteout is
+// for hiding directories and files in the lower level layers. AUFS whiteout is
+// the whiteout standard in Docker, and it relies on '.wh..wh..opq' to hide
+// directory and '.wh.<filename>' to hide file, whereas OverlayFS relies on
+// setting an extended attribute 'trusted.overlay.opaque' on the directory to
+// hide it and creating character device with 0/0 device number to hide file.
+// See the links below for more details:
+// http://aufs.sourceforge.net/aufs.html
+// https://www.kernel.org/doc/Documentation/filesystems/overlayfs.txt
+Try<Nothing> convertWhiteouts(const string& directory)
+{
+ char* rootfs[] = {const_cast<char*>(directory.c_str()), nullptr};
+
+ FTS* tree = ::fts_open(rootfs, FTS_NOCHDIR | FTS_PHYSICAL, nullptr);
+ if (tree == nullptr) {
+ return Error("Failed to open '" + directory + "': " + os::strerror(errno));
+ }
+
+ for (FTSENT *node = ::fts_read(tree);
+ node != nullptr; node = ::fts_read(tree)) {
+ if (node->fts_info != FTS_F) {
+ continue;
+ }
+
+ if (!strings::startsWith(node->fts_name, docker::spec::WHITEOUT_PREFIX)) {
+ continue;
+ }
+
+ const Path path = Path(node->fts_path);
+ if (node->fts_name == string(docker::spec::WHITEOUT_OPAQUE_PREFIX)) {
+ Try<Nothing> setxattr = os::setxattr(
+ path.dirname(),
+ "trusted.overlay.opaque",
+ "y",
+ 0);
+
+ if (setxattr.isError()) {
+ ::fts_close(tree);
+ return Error(
+ "Failed to set extended attribute 'trusted.overlay.opaque'"
+ " for the directory '" + path.dirname() + "': " +
+ setxattr.error());
+ }
+ } else {
+ const string originalPath = path::join(
+ path.dirname(),
+ path.basename().substr(strlen(docker::spec::WHITEOUT_PREFIX)));
+
+ Try<Nothing> mknod = os::mknod(originalPath, S_IFCHR, 0);
+ if (mknod.isError()) {
+ ::fts_close(tree);
+ return Error(
+ "Failed to create character device '" +
+ originalPath + "': " + mknod.error());
+ }
+ }
+
+ // Remove the AUFS whiteout file.
+ Try<Nothing> rm = os::rm(path.string());
+ if (rm.isError()) {
+ ::fts_close(tree);
+ return Error(
+ "Failed to remove AUFS whiteout file '" +
+ path.string() + "': " + rm.error());
+ }
+ }
+
+ if (errno != 0) {
+ Error error = ErrnoError();
+ ::fts_close(tree);
+ return error;
+ }
+
+ if (::fts_close(tree) != 0) {
+ return Error(
+ "Failed to stop traversing file system: " + os::strerror(errno));
+ }
+
+ return Nothing();
+}
+#endif
+
+} // namespace slave {
+} // namespace internal {
+} // namespace mesos {
http://git-wip-us.apache.org/repos/asf/mesos/blob/3800cfd0/src/slave/containerizer/mesos/provisioner/utils.hpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/utils.hpp b/src/slave/containerizer/mesos/provisioner/utils.hpp
new file mode 100644
index 0000000..4efce0e
--- /dev/null
+++ b/src/slave/containerizer/mesos/provisioner/utils.hpp
@@ -0,0 +1,36 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef __MESOS_CONTAINERIZER_UTILS_HPP__
+#define __MESOS_CONTAINERIZER_UTILS_HPP__
+
+#include <stout/os.hpp>
+
+#include <mesos/docker/spec.hpp>
+
+namespace mesos {
+namespace internal {
+namespace slave {
+
+#ifdef __linux__
+Try<Nothing> convertWhiteouts(const std::string& directory);
+#endif
+
+} // namespace slave {
+} // namespace internal {
+} // namespace mesos {
+
+#endif // __MESOS_CONTAINERIZER_UTILS_HPP__