You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by tn...@apache.org on 2015/09/03 03:31:49 UTC
mesos git commit: Add Copy backend for provisioners.
Repository: mesos
Updated Branches:
refs/heads/master bcc45e3f3 -> fb631516c
Add Copy backend for provisioners.
Review: https://reviews.apache.org/r/37921
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/fb631516
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/fb631516
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/fb631516
Branch: refs/heads/master
Commit: fb631516cc9878ae953fe481e498c3ae96b27852
Parents: bcc45e3
Author: Timothy Chen <tn...@apache.org>
Authored: Thu Aug 27 17:22:34 2015 -0700
Committer: Timothy Chen <tn...@gmail.com>
Committed: Wed Sep 2 18:31:40 2015 -0700
----------------------------------------------------------------------
src/Makefile.am | 2 +
.../containerizer/provisioners/backend.cpp | 2 +
.../provisioners/backends/copy.cpp | 203 +++++++++++++++++++
.../provisioners/backends/copy.hpp | 61 ++++++
.../containerizer/provisioner_backend_tests.cpp | 49 +++++
5 files changed, 317 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/fb631516/src/Makefile.am
----------------------------------------------------------------------
diff --git a/src/Makefile.am b/src/Makefile.am
index 7b4d9f6..4b643a3 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -486,6 +486,7 @@ libmesos_no_3rdparty_la_SOURCES = \
slave/containerizer/provisioners/appc/spec.cpp \
slave/containerizer/provisioners/appc/store.cpp \
slave/containerizer/provisioners/backend.cpp \
+ slave/containerizer/provisioners/backends/copy.cpp \
slave/resource_estimators/noop.cpp \
usage/usage.cpp \
v1/attributes.cpp \
@@ -764,6 +765,7 @@ libmesos_no_3rdparty_la_SOURCES += \
slave/containerizer/provisioners/appc/store.hpp \
slave/containerizer/provisioners/backend.hpp \
slave/containerizer/provisioners/backends/bind.hpp \
+ slave/containerizer/provisioners/backends/copy.hpp \
slave/containerizer/isolators/posix.hpp \
slave/containerizer/isolators/posix/disk.hpp \
slave/containerizer/isolators/cgroups/constants.hpp \
http://git-wip-us.apache.org/repos/asf/mesos/blob/fb631516/src/slave/containerizer/provisioners/backend.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/provisioners/backend.cpp b/src/slave/containerizer/provisioners/backend.cpp
index 2f7c335..6560ece 100644
--- a/src/slave/containerizer/provisioners/backend.cpp
+++ b/src/slave/containerizer/provisioners/backend.cpp
@@ -23,6 +23,7 @@
#include "slave/containerizer/provisioners/backend.hpp"
#include "slave/containerizer/provisioners/backends/bind.hpp"
+#include "slave/containerizer/provisioners/backends/copy.hpp"
using namespace process;
@@ -39,6 +40,7 @@ hashmap<string, Owned<Backend>> Backend::create(const Flags& flags)
#ifdef __linux__
creators.put("bind", &BindBackend::create);
#endif // __linux__
+ creators.put("copy", &CopyBackend::create);
hashmap<string, Owned<Backend>> backends;
http://git-wip-us.apache.org/repos/asf/mesos/blob/fb631516/src/slave/containerizer/provisioners/backends/copy.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/provisioners/backends/copy.cpp b/src/slave/containerizer/provisioners/backends/copy.cpp
new file mode 100644
index 0000000..b569465
--- /dev/null
+++ b/src/slave/containerizer/provisioners/backends/copy.cpp
@@ -0,0 +1,203 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <list>
+
+#include <process/collect.hpp>
+#include <process/defer.hpp>
+#include <process/dispatch.hpp>
+#include <process/io.hpp>
+#include <process/process.hpp>
+#include <process/subprocess.hpp>
+
+
+#include <stout/foreach.hpp>
+#include <stout/os.hpp>
+
+#include "common/status_utils.hpp"
+
+#include "slave/containerizer/provisioners/backends/copy.hpp"
+
+
+using namespace process;
+
+using std::string;
+using std::list;
+using std::vector;
+
+namespace mesos {
+namespace internal {
+namespace slave {
+
+class CopyBackendProcess : public Process<CopyBackendProcess>
+{
+public:
+ Future<Nothing> provision(const vector<string>& layers, const string& rootfs);
+
+ Future<bool> destroy(const string& rootfs);
+
+private:
+ Future<Nothing> _provision(string layer, const string& rootfs);
+};
+
+
+Try<Owned<Backend>> CopyBackend::create(const Flags&)
+{
+ return Owned<Backend>(new CopyBackend(
+ Owned<CopyBackendProcess>(new CopyBackendProcess())));
+}
+
+
+CopyBackend::~CopyBackend()
+{
+ terminate(process.get());
+ wait(process.get());
+}
+
+
+CopyBackend::CopyBackend(Owned<CopyBackendProcess> _process)
+ : process(_process)
+{
+ spawn(CHECK_NOTNULL(process.get()));
+}
+
+
+Future<Nothing> CopyBackend::provision(
+ const vector<string>& layers,
+ const string& rootfs)
+{
+ return dispatch(
+ process.get(), &CopyBackendProcess::provision, layers, rootfs);
+}
+
+
+Future<bool> CopyBackend::destroy(const string& rootfs)
+{
+ return dispatch(process.get(), &CopyBackendProcess::destroy, rootfs);
+}
+
+
+Future<Nothing> CopyBackendProcess::provision(
+ const vector<string>& layers,
+ const string& rootfs)
+{
+ if (layers.size() == 0) {
+ return Failure("No filesystem layers provided");
+ }
+
+ if (os::exists(rootfs)) {
+ return Failure("Rootfs is already provisioned");
+ }
+
+ Try<Nothing> mkdir = os::mkdir(rootfs);
+ if (mkdir.isError()) {
+ return Failure("Failed to create rootfs directory: " + mkdir.error());
+ }
+
+ list<Future<Nothing>> futures{Nothing()};
+
+ foreach (const string layer, layers) {
+ futures.push_back(
+ futures.back().then(
+ defer(self(), &Self::_provision, layer, rootfs)));
+ }
+
+ return collect(futures)
+ .then([]() -> Future<Nothing> { return Nothing(); });
+}
+
+
+Future<Nothing> CopyBackendProcess::_provision(
+ string layer,
+ const string& rootfs)
+{
+ VLOG(1) << "Copying layer path '" << layer << "' to rootfs '" << rootfs
+ << "'";
+
+#ifdef __APPLE__
+ if (!strings::endsWith(layer, "/")) {
+ layer += "/";
+ }
+
+ // OSX cp doesn't support -T flag, but supports source trailing
+ // slash so we only copy the content but not the folder.
+ vector<string> args{"cp", "-a", layer, rootfs};
+#else
+ vector<string> args{"cp", "-aT", layer, rootfs};
+#endif // __APPLE__
+
+ Try<Subprocess> s = subprocess(
+ "cp",
+ args,
+ Subprocess::PATH("/dev/null"),
+ Subprocess::PATH("/dev/null"),
+ Subprocess::PIPE());
+
+ if (s.isError()) {
+ return Failure("Failed to create 'cp' subprocess: " + s.error());
+ }
+
+ Subprocess cp = s.get();
+
+ return cp.status()
+ .then([cp](const Option<int>& status) -> Future<Nothing> {
+ if (status.isNone()) {
+ return Failure("Failed to reap subprocess to copy image");
+ } else if (status.get() != 0) {
+ return io::read(cp.err().get())
+ .then([](const string& err) -> Future<Nothing> {
+ return Failure("Failed to copy layer: " + err);
+ });
+ }
+
+ return Nothing();
+ });
+}
+
+
+Future<bool> CopyBackendProcess::destroy(const string& rootfs)
+{
+ vector<string> argv{"rm", "-rf", rootfs};
+
+ Try<Subprocess> s = subprocess(
+ "rm",
+ argv,
+ Subprocess::PATH("/dev/null"),
+ Subprocess::FD(STDOUT_FILENO),
+ Subprocess::FD(STDERR_FILENO));
+
+ if (s.isError()) {
+ return Failure("Failed to create 'rm' subprocess: " + s.error());
+ }
+
+ return s.get().status()
+ .then([](const Option<int>& status) -> Future<bool> {
+ if (status.isNone()) {
+ return Failure("Failed to reap subprocess to destroy rootfs");
+ } else if (status.get() != 0) {
+ return Failure("Failed to destroy rootfs, exit status: " +
+ WSTRINGIFY(status.get()));
+ }
+
+ return true;
+ });
+}
+
+} // namespace slave {
+} // namespace internal {
+} // namespace mesos {
http://git-wip-us.apache.org/repos/asf/mesos/blob/fb631516/src/slave/containerizer/provisioners/backends/copy.hpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/provisioners/backends/copy.hpp b/src/slave/containerizer/provisioners/backends/copy.hpp
new file mode 100644
index 0000000..2abca37
--- /dev/null
+++ b/src/slave/containerizer/provisioners/backends/copy.hpp
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __MESOS_PROVISIONER_COPY_HPP__
+#define __MESOS_PROVISIONER_COPY_HPP__
+
+#include "slave/containerizer/provisioners/backend.hpp"
+
+namespace mesos {
+namespace internal {
+namespace slave {
+
+// Forward declaration.
+class CopyBackendProcess;
+
+
+class CopyBackend : public Backend
+{
+public:
+ virtual ~CopyBackend();
+
+ // CopyBackend doesn't use any flag.
+ static Try<process::Owned<Backend>> create(const Flags&);
+
+ // Provisions a rootfs given the layers' paths and target rootfs
+ // path.
+ virtual process::Future<Nothing> provision(
+ const std::vector<std::string>& layers,
+ const std::string& rootfs);
+
+ virtual process::Future<bool> destroy(const std::string& rootfs);
+
+private:
+ explicit CopyBackend(process::Owned<CopyBackendProcess> process);
+
+ CopyBackend(const CopyBackend&); // Not copyable.
+ CopyBackend& operator=(const CopyBackend&); // Not assignable.
+
+ process::Owned<CopyBackendProcess> process;
+};
+
+} // namespace slave {
+} // namespace internal {
+} // namespace mesos {
+
+#endif // __MESOS_PROVISIONER_COPY_HPP__
http://git-wip-us.apache.org/repos/asf/mesos/blob/fb631516/src/tests/containerizer/provisioner_backend_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/containerizer/provisioner_backend_tests.cpp b/src/tests/containerizer/provisioner_backend_tests.cpp
index d321850..f2498b1 100644
--- a/src/tests/containerizer/provisioner_backend_tests.cpp
+++ b/src/tests/containerizer/provisioner_backend_tests.cpp
@@ -30,6 +30,7 @@
#endif // __linux__
#include "slave/containerizer/provisioners/backends/bind.hpp"
+#include "slave/containerizer/provisioners/backends/copy.hpp"
#include "tests/flags.hpp"
#include "tests/utils.hpp"
@@ -99,6 +100,54 @@ TEST_F(BindBackendTest, ROOT_BindBackend)
}
#endif // __linux__
+
+class CopyBackendTest : public TemporaryDirectoryTest {};
+
+
+// Provision a rootfs using multiple layers with the copy backend.
+TEST_F(CopyBackendTest, ROOT_CopyBackend)
+{
+ string layer1 = path::join(os::getcwd(), "source1");
+ ASSERT_SOME(os::mkdir(layer1));
+ ASSERT_SOME(os::mkdir(path::join(layer1, "dir1")));
+ ASSERT_SOME(os::write(path::join(layer1, "dir1", "1"), "1"));
+ ASSERT_SOME(os::write(path::join(layer1, "file"), "test1"));
+
+ string layer2 = path::join(os::getcwd(), "source2");
+ ASSERT_SOME(os::mkdir(layer2));
+ ASSERT_SOME(os::mkdir(path::join(layer2, "dir2")));
+ ASSERT_SOME(os::write(path::join(layer2, "dir2", "2"), "2"));
+ ASSERT_SOME(os::write(path::join(layer2, "file"), "test2"));
+
+ string rootfs = path::join(os::getcwd(), "rootfs");
+
+ hashmap<string, Owned<Backend>> backends = Backend::create(slave::Flags());
+ ASSERT_TRUE(backends.contains("copy"));
+
+ AWAIT_READY(backends["copy"]->provision({layer1, layer2}, rootfs));
+
+ EXPECT_TRUE(os::exists(path::join(rootfs, "dir1", "1")));
+ Try<string> read = os::read(path::join(rootfs, "dir1", "1"));
+ ASSERT_SOME(read);
+ EXPECT_EQ(read.get(), "1");
+
+ EXPECT_TRUE(os::exists(path::join(rootfs, "dir2", "2")));
+ read = os::read(path::join(rootfs, "dir2", "2"));
+ ASSERT_SOME(read);
+ EXPECT_EQ(read.get(), "2");
+
+ EXPECT_TRUE(os::exists(path::join(rootfs, "file")));
+ read = os::read(path::join(rootfs, "file"));
+ ASSERT_SOME(read);
+
+ // Last layer should overwrite existing file.
+ EXPECT_EQ(read.get(), "test2");
+
+ AWAIT_READY(backends["copy"]->destroy(rootfs));
+
+ EXPECT_FALSE(os::exists(rootfs));
+}
+
} // namespace tests {
} // namespace internal {
} // namespace mesos {