You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by tn...@apache.org on 2015/09/25 18:32:57 UTC
[05/17] mesos git commit: Added Docker image reference store.
Added Docker image reference store.
Review: https://reviews.apache.org/r/37247
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/e8906a13
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/e8906a13
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/e8906a13
Branch: refs/heads/master
Commit: e8906a1339acdf55eb4c03061fbac872f991f447
Parents: 86d87aa
Author: Lily Chen <li...@mesosphere.io>
Authored: Tue Jul 28 16:13:00 2015 -0700
Committer: Timothy Chen <tn...@apache.org>
Committed: Fri Sep 25 09:02:04 2015 -0700
----------------------------------------------------------------------
src/Makefile.am | 10 +
src/messages/docker_provisioner.hpp | 24 ++
src/messages/docker_provisioner.proto | 35 +++
.../provisioners/docker/reference_store.cpp | 218 +++++++++++++++++++
.../provisioners/docker/reference_store.hpp | 137 ++++++++++++
.../containerizer/provisioners/docker/store.cpp | 31 +--
.../containerizer/provisioners/docker/store.hpp | 7 +-
7 files changed, 430 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/e8906a13/src/Makefile.am
----------------------------------------------------------------------
diff --git a/src/Makefile.am b/src/Makefile.am
index 916be39..0b49a3b 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -249,6 +249,11 @@ STATE_PROTOS = messages/state.pb.cc messages/state.pb.h
BUILT_SOURCES += $(STATE_PROTOS)
CLEANFILES += $(STATE_PROTOS)
+DOCKER_PROVISIONER_PROTOS = messages/docker_provisioner.pb.cc messages/docker_provisioner.pb.h
+
+BUILT_SOURCES += $(DOCKER_PROVISIONER_PROTOS)
+CLEANFILES += $(DOCKER_PROVISIONER_PROTOS)
+
REGISTRY_PROTOS = master/registry.pb.cc master/registry.pb.h
BUILT_SOURCES += $(REGISTRY_PROTOS)
@@ -435,6 +440,7 @@ nodist_libmesos_no_3rdparty_la_SOURCES = \
$(CXX_PROTOS) \
$(FLAGS_PROTOS) \
$(MESSAGES_PROTOS) \
+ $(DOCKER_PROVISIONER_PROTOS) \
$(REGISTRY_PROTOS)
# TODO(tillt): Remove authentication/cram_md5/* which will enable us to
@@ -479,6 +485,7 @@ libmesos_no_3rdparty_la_SOURCES = \
master/validation.cpp \
master/allocator/allocator.cpp \
master/allocator/sorter/drf/sorter.cpp \
+ messages/docker_provisioner.proto \
messages/flags.proto \
messages/messages.cpp \
messages/messages.proto \
@@ -693,6 +700,7 @@ if OS_LINUX
libmesos_no_3rdparty_la_SOURCES += slave/containerizer/linux_launcher.cpp
libmesos_no_3rdparty_la_SOURCES += slave/containerizer/provisioner/backends/bind.cpp
libmesos_no_3rdparty_la_SOURCES += slave/containerizer/provisioner/docker.cpp
+ libmesos_no_3rdparty_la_SOURCES += slave/containerizer/provisioner/docker/reference_store.cpp
libmesos_no_3rdparty_la_SOURCES += slave/containerizer/provisioner/docker/store.cpp
else
EXTRA_DIST += linux/cgroups.cpp
@@ -788,6 +796,7 @@ libmesos_no_3rdparty_la_SOURCES += \
master/allocator/mesos/hierarchical.hpp \
master/allocator/sorter/drf/sorter.hpp \
master/allocator/sorter/sorter.hpp \
+ messages/docker_provisioner.hpp \
messages/flags.hpp \
messages/messages.hpp \
module/manager.hpp \
@@ -818,6 +827,7 @@ libmesos_no_3rdparty_la_SOURCES += \
slave/containerizer/provisioner/backends/bind.hpp \
slave/containerizer/provisioner/backends/copy.hpp \
slave/containerizer/provisioner/docker.hpp \
+ slave/containerizer/provisioner/docker/reference_store.hpp \
slave/containerizer/provisioner/docker/registry_client.hpp \
slave/containerizer/provisioner/docker/store.hpp \
slave/containerizer/provisioner/docker/token_manager.hpp \
http://git-wip-us.apache.org/repos/asf/mesos/blob/e8906a13/src/messages/docker_provisioner.hpp
----------------------------------------------------------------------
diff --git a/src/messages/docker_provisioner.hpp b/src/messages/docker_provisioner.hpp
new file mode 100644
index 0000000..b076fdb
--- /dev/null
+++ b/src/messages/docker_provisioner.hpp
@@ -0,0 +1,24 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __DOCKER_PROVISIONER_HPP__
+#define __DOCKER_PROVISIONER_HPP__
+
+#include "messages/docker_provisioner.pb.h"
+
+#endif // __DOCKER_PROVISIONER_HPP__
http://git-wip-us.apache.org/repos/asf/mesos/blob/e8906a13/src/messages/docker_provisioner.proto
----------------------------------------------------------------------
diff --git a/src/messages/docker_provisioner.proto b/src/messages/docker_provisioner.proto
new file mode 100644
index 0000000..9de6707
--- /dev/null
+++ b/src/messages/docker_provisioner.proto
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import "mesos/mesos.proto";
+
+package mesos.internal;
+
+/**
+ * A Docker Image name and the layer ids of the layers that comprise the image.
+ * The layerIds are ordered, with the root layer id (no parent layer id) first
+ * and the leaf layer id last.
+ */
+message DockerProvisionerImages {
+ message Image {
+ required string name = 1;
+ repeated string layer_ids = 2;
+ }
+
+ repeated Image images = 1;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/mesos/blob/e8906a13/src/slave/containerizer/provisioners/docker/reference_store.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/provisioners/docker/reference_store.cpp b/src/slave/containerizer/provisioners/docker/reference_store.cpp
new file mode 100644
index 0000000..b435ed4
--- /dev/null
+++ b/src/slave/containerizer/provisioners/docker/reference_store.cpp
@@ -0,0 +1,218 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <vector>
+
+#include <glog/logging.h>
+
+#include <stout/foreach.hpp>
+#include <stout/hashset.hpp>
+#include <stout/os.hpp>
+#include <stout/protobuf.hpp>
+
+#include <process/defer.hpp>
+#include <process/dispatch.hpp>
+#include <process/owned.hpp>
+
+#include "common/status_utils.hpp"
+
+#include "messages/docker_provisioner.hpp"
+
+#include "slave/containerizer/provisioners/docker/reference_store.hpp"
+#include "slave/state.hpp"
+
+using namespace process;
+
+using std::list;
+using std::string;
+using std::vector;
+
+namespace mesos {
+namespace internal {
+namespace slave {
+namespace docker {
+
+Try<Owned<ReferenceStore>> ReferenceStore::create(const Flags& flags)
+{
+ Try<Owned<ReferenceStoreProcess>> process =
+ ReferenceStoreProcess::create(flags);
+ if (process.isError()) {
+ return Error("Failed to create reference store: " + process.error());
+ }
+ return Owned<ReferenceStore>(new ReferenceStore(process.get()));
+}
+
+
+ReferenceStore::ReferenceStore(Owned<ReferenceStoreProcess> process)
+ : process(process)
+{
+ process::spawn(CHECK_NOTNULL(process.get()));
+}
+
+
+ReferenceStore::~ReferenceStore()
+{
+ process::terminate(process.get());
+ process::wait(process.get());
+}
+
+
+void ReferenceStore::initialize()
+{
+ process::dispatch(process.get(), &ReferenceStoreProcess::initialize);
+}
+
+
+Future<DockerImage> ReferenceStore::put(
+ const string& name,
+ const list<string>& layers)
+{
+ return dispatch(
+ process.get(), &ReferenceStoreProcess::put, name, layers);
+}
+
+
+Future<Option<DockerImage>> ReferenceStore::get(const string& name)
+{
+ return dispatch(process.get(), &ReferenceStoreProcess::get, name);
+}
+
+
+ReferenceStoreProcess::ReferenceStoreProcess(const Flags& flags)
+ : flags(flags) {}
+
+
+Try<Owned<ReferenceStoreProcess>> ReferenceStoreProcess::create(
+ const Flags& flags)
+{
+ Owned<ReferenceStoreProcess> referenceStore =
+ Owned<ReferenceStoreProcess>(new ReferenceStoreProcess(flags));
+
+ return referenceStore;
+}
+
+
+Future<DockerImage> ReferenceStoreProcess::put(
+ const string& name,
+ const list<string>& layers)
+{
+ storedImages[name] = DockerImage(name, layers);
+
+ Try<Nothing> status = persist();
+ if (status.isError()) {
+ return Failure("Failed to save state of Docker images" + status.error());
+ }
+
+ return storedImages[name];
+}
+
+
+Future<Option<DockerImage>> ReferenceStoreProcess::get(const string& name)
+{
+ if (!storedImages.contains(name)) {
+ return None();
+ }
+
+ return storedImages[name];
+}
+
+
+Try<Nothing> ReferenceStoreProcess::persist()
+{
+ DockerProvisionerImages images;
+
+ foreachpair(
+ const string& name, const DockerImage& dockerImage, storedImages) {
+ DockerProvisionerImages::Image* image = images.add_images();
+
+ image->set_name(name);
+
+ foreach (const string& layer, dockerImage.layers) {
+ image->add_layer_ids(layer);
+ }
+ }
+
+ Try<string> path = path::join(flags.docker_store_dir, "storedImages");
+ if (path.isError()) {
+ return Error("Failure to construct path to repositories lookup: " +
+ path.error());
+ }
+
+ Try<Nothing> status =
+ mesos::internal::slave::state::checkpoint(path.get(), images);
+ if (status.isError()) {
+ return Error("Failed to perform checkpoint: " + status.error());
+ }
+
+ return Nothing();
+}
+
+
+void ReferenceStoreProcess::initialize()
+{
+ Try<string> path = path::join(flags.docker_store_dir, "storedImages");
+
+ storedImages.clear();
+ if (!os::exists(path.get())) {
+ LOG(INFO) << "No images to load from disk. Docker provisioner image "
+ << "storage path: " << path.get() << " does not exist.";
+ return;
+ }
+
+ Result<DockerProvisionerImages> images =
+ ::protobuf::read<DockerProvisionerImages>(path.get());
+ if (images.isError()) {
+ LOG(ERROR) << "Failed to read protobuf for Docker provisioner image: "
+ << images.error();
+ return;
+ }
+
+ for (int i = 0; i < images.get().images_size(); i++) {
+ string imageName = images.get().images(i).name();
+
+ list<string> layers;
+ vector<string> missingLayers;
+ for (int j = 0; j < images.get().images(i).layer_ids_size(); j++) {
+ string layerId = images.get().images(i).layer_ids(j);
+
+ layers.push_back(layerId);
+
+ if (!os::exists(path::join(flags.docker_store_dir, layerId))) {
+ missingLayers.push_back(layerId);
+ }
+ }
+
+ if (!missingLayers.empty()) {
+ foreach (const string& layer, missingLayers) {
+ LOG(WARNING) << "Image layer: " << layer << " required for Docker "
+ << "image: " << imageName << " is not on disk.";
+ }
+ LOG(WARNING) << "Skipped loading image: " << imageName
+ << " due to missing layers.";
+ continue;
+ }
+
+ VLOG(1) << "Loaded Docker image: " << imageName << " from disk.";
+ storedImages[imageName] = DockerImage(imageName, layers);
+ }
+}
+
+} // namespace docker {
+} // namespace slave {
+} // namespace internal {
+} // namespace mesos {
http://git-wip-us.apache.org/repos/asf/mesos/blob/e8906a13/src/slave/containerizer/provisioners/docker/reference_store.hpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/provisioners/docker/reference_store.hpp b/src/slave/containerizer/provisioners/docker/reference_store.hpp
new file mode 100644
index 0000000..d9f7070
--- /dev/null
+++ b/src/slave/containerizer/provisioners/docker/reference_store.hpp
@@ -0,0 +1,137 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __MESOS_DOCKER_REFERENCE_STORE__
+#define __MESOS_DOCKER_REFERENCE_STORE__
+
+#include <list>
+#include <string>
+
+#include <stout/hashmap.hpp>
+#include <stout/json.hpp>
+#include <stout/option.hpp>
+#include <stout/protobuf.hpp>
+#include <stout/try.hpp>
+
+#include <process/future.hpp>
+#include <process/owned.hpp>
+#include <process/process.hpp>
+
+#include "slave/containerizer/provisioners/docker.hpp"
+#include "slave/flags.hpp"
+
+namespace mesos {
+namespace internal {
+namespace slave {
+namespace docker {
+
+// Forward Declaration.
+class ReferenceStoreProcess;
+
+/**
+ * The Reference Store is a way to track the Docker images used by the
+ * provisioner that are stored in on disk. It keeps track of the layers
+ * that Docker images are composed of and recovers DockerImage objects upon
+ * initialization by checking for dependent layers stored on disk.
+ * Currently, image layers are stored indefinitely, with no garbage collection
+ * of unreferenced image layers.
+ */
+class ReferenceStore
+{
+public:
+ ~ReferenceStore();
+
+ /**
+ * Recover all Docker Images that are on disk by checking if all
+ * layer dependencies for that layer are present on disk.
+ */
+ void initialize();
+
+ static Try<process::Owned<ReferenceStore>> create(const Flags& flags);
+
+ /**
+ * Create a DockerImage, put it in reference store and persist the reference
+ * store state to disk.
+ *
+ * @param name the name of the Docker image to place in the reference store.
+ * @param layers the list of layer ids that comprise the Docker image in
+ * order where the root layer's id (no parent layer) is first
+ * and the leaf layer's id is last.
+ */
+ process::Future<DockerImage> put(
+ const std::string& name,
+ const std::list<std::string>& layers);
+
+ /**
+ * Retrieve DockerImage based on image name if it is among the DockerImages
+ * stored in memory.
+ *
+ * @param name the name of the Docker image to retrieve
+ */
+ process::Future<Option<DockerImage>> get(const std::string& name);
+
+private:
+ explicit ReferenceStore(process::Owned<ReferenceStoreProcess> process);
+
+ ReferenceStore(const ReferenceStore&); // Not copyable.
+ ReferenceStore& operator=(const ReferenceStore&); // Not assignable.
+
+ process::Owned<ReferenceStoreProcess> process;
+};
+
+
+class ReferenceStoreProcess : public process::Process<ReferenceStoreProcess>
+{
+public:
+ ~ReferenceStoreProcess() {}
+
+ // Explicitly use 'initialize' since we are overloading below.
+ using process::ProcessBase::initialize;
+
+ void initialize();
+
+ static Try<process::Owned<ReferenceStoreProcess>> create(const Flags& flags);
+
+ process::Future<DockerImage> put(
+ const std::string& name,
+ const std::list<std::string>& layers);
+
+ process::Future<Option<DockerImage>> get(const std::string& name);
+
+ // TODO(chenlily): Implement removal of unreferenced images.
+
+private:
+ ReferenceStoreProcess(const Flags& flags);
+
+ // Write out reference store state to persistent store.
+ Try<Nothing> persist();
+
+ const Flags flags;
+
+ // This is a lookup table for images that are stored in memory. It is keyed
+ // by the name of the DockerImage.
+ // For example, "ubuntu:14.04" -> ubuntu14:04 DockerImage.
+ hashmap<std::string, DockerImage> storedImages;
+};
+
+} // namespace docker {
+} // namespace slave {
+} // namespace internal {
+} // namespace mesos {
+
+#endif // __MESOS_DOCKER_REFERENCE_STORE__
http://git-wip-us.apache.org/repos/asf/mesos/blob/e8906a13/src/slave/containerizer/provisioners/docker/store.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/provisioners/docker/store.cpp b/src/slave/containerizer/provisioners/docker/store.cpp
index 9453d6f..b902f8d 100644
--- a/src/slave/containerizer/provisioners/docker/store.cpp
+++ b/src/slave/containerizer/provisioners/docker/store.cpp
@@ -107,20 +107,12 @@ Try<Owned<LocalStoreProcess>> LocalStoreProcess::create(
const Flags& flags,
Fetcher* fetcher)
{
- Owned<LocalStoreProcess> store =
- Owned<LocalStoreProcess>(new LocalStoreProcess(flags));
-
- Try<Nothing> restore = store->restore(flags);
- if (restore.isError()) {
- return Error("Failed to restore store: " + restore.error());
- }
-
- return store;
+ return Owned<LocalStoreProcess>(new LocalStoreProcess(flags));
}
LocalStoreProcess::LocalStoreProcess(const Flags& flags)
- : flags(flags) {}
+ : flags(flags), refStore(ReferenceStore::create(flags).get()) {}
Future<DockerImage> LocalStoreProcess::put(
@@ -263,10 +255,7 @@ Future<DockerImage> LocalStoreProcess::putImage(
return putLayers(staging, layers, sandbox)
.then([=]() -> Future<DockerImage> {
- images[name] = DockerImage(name, layers);
-
- // TODO(chenlily): update reference store or replace with reference store
- return images[name];
+ return refStore->put(name, layers);
});
}
@@ -415,19 +404,7 @@ Future<Nothing> LocalStoreProcess::moveLayer(
Future<Option<DockerImage>> LocalStoreProcess::get(const string& name)
{
- if (!images.contains(name)) {
- return None();
- }
-
- return images[name];
-}
-
-
-// Recover stored image layers and update layers map.
-// TODO(chenlily): Implement restore.
-Try<Nothing> LocalStoreProcess::restore(const Flags& flags)
-{
- return Nothing();
+ return refStore->get(name);
}
} // namespace docker {
http://git-wip-us.apache.org/repos/asf/mesos/blob/e8906a13/src/slave/containerizer/provisioners/docker/store.hpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/provisioners/docker/store.hpp b/src/slave/containerizer/provisioners/docker/store.hpp
index 043f2d7..256e146 100644
--- a/src/slave/containerizer/provisioners/docker/store.hpp
+++ b/src/slave/containerizer/provisioners/docker/store.hpp
@@ -35,6 +35,7 @@
#include "slave/containerizer/fetcher.hpp"
#include "slave/containerizer/provisioners/docker.hpp"
+#include "slave/containerizer/provisioners/docker/reference_store.hpp"
#include "slave/flags.hpp"
namespace mesos {
@@ -133,8 +134,6 @@ public:
private:
LocalStoreProcess(const Flags& flags);
- Try<Nothing> restore(const Flags& flags);
-
process::Future<Nothing> untarImage(
const std::string& tarPath,
const std::string& staging);
@@ -165,9 +164,7 @@ private:
const Flags flags;
- // This hashmap maps a Docker image by name to its corresponding DockerImage
- // object.
- hashmap<std::string, DockerImage> images;
+ process::Owned<ReferenceStore> refStore;
};
} // namespace docker {