You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by gi...@apache.org on 2018/06/07 19:13:00 UTC
[6/6] mesos git commit: Renamed local_puller to image_tar_puller.
Renamed local_puller to image_tar_puller.
Review: https://reviews.apache.org/r/66652
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/c185752b
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/c185752b
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/c185752b
Branch: refs/heads/master
Commit: c185752b71850ac79901bbe441851f8b5bcc3f7c
Parents: af531b6
Author: Gilbert Song <so...@gmail.com>
Authored: Mon Apr 16 22:22:05 2018 -0700
Committer: Gilbert Song <so...@gmail.com>
Committed: Thu Jun 7 12:11:38 2018 -0700
----------------------------------------------------------------------
src/CMakeLists.txt | 2 +-
src/Makefile.am | 4 +-
.../provisioner/docker/image_tar_puller.cpp | 402 +++++++++++++++++++
.../provisioner/docker/image_tar_puller.hpp | 73 ++++
.../mesos/provisioner/docker/local_puller.cpp | 402 -------------------
.../mesos/provisioner/docker/local_puller.hpp | 73 ----
.../mesos/provisioner/docker/puller.cpp | 6 +-
.../containerizer/provisioner_docker_tests.cpp | 6 +-
.../containerizer/runtime_isolator_tests.cpp | 12 +-
9 files changed, 490 insertions(+), 490 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/c185752b/src/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 773771b..10b0946 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -175,7 +175,7 @@ set(AGENT_SRC
slave/containerizer/mesos/provisioner/appc/paths.cpp
slave/containerizer/mesos/provisioner/appc/store.cpp
slave/containerizer/mesos/provisioner/backends/copy.cpp
- slave/containerizer/mesos/provisioner/docker/local_puller.cpp
+ slave/containerizer/mesos/provisioner/docker/image_tar_puller.cpp
slave/containerizer/mesos/provisioner/docker/metadata_manager.cpp
slave/containerizer/mesos/provisioner/docker/paths.cpp
slave/containerizer/mesos/provisioner/docker/puller.cpp
http://git-wip-us.apache.org/repos/asf/mesos/blob/c185752b/src/Makefile.am
----------------------------------------------------------------------
diff --git a/src/Makefile.am b/src/Makefile.am
index 9032453..2bcee1e 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -1096,7 +1096,7 @@ libmesos_no_3rdparty_la_SOURCES += \
slave/containerizer/mesos/provisioner/appc/paths.cpp \
slave/containerizer/mesos/provisioner/appc/store.cpp \
slave/containerizer/mesos/provisioner/backends/copy.cpp \
- slave/containerizer/mesos/provisioner/docker/local_puller.cpp \
+ slave/containerizer/mesos/provisioner/docker/image_tar_puller.cpp \
slave/containerizer/mesos/provisioner/docker/metadata_manager.cpp \
slave/containerizer/mesos/provisioner/docker/paths.cpp \
slave/containerizer/mesos/provisioner/docker/puller.cpp \
@@ -1259,7 +1259,7 @@ libmesos_no_3rdparty_la_SOURCES += \
slave/containerizer/mesos/provisioner/appc/paths.hpp \
slave/containerizer/mesos/provisioner/appc/store.hpp \
slave/containerizer/mesos/provisioner/backends/copy.hpp \
- slave/containerizer/mesos/provisioner/docker/local_puller.hpp \
+ slave/containerizer/mesos/provisioner/docker/image_tar_puller.hpp \
slave/containerizer/mesos/provisioner/docker/message.hpp \
slave/containerizer/mesos/provisioner/docker/metadata_manager.hpp \
slave/containerizer/mesos/provisioner/docker/paths.hpp \
http://git-wip-us.apache.org/repos/asf/mesos/blob/c185752b/src/slave/containerizer/mesos/provisioner/docker/image_tar_puller.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/docker/image_tar_puller.cpp b/src/slave/containerizer/mesos/provisioner/docker/image_tar_puller.cpp
new file mode 100644
index 0000000..6549bf5
--- /dev/null
+++ b/src/slave/containerizer/mesos/provisioner/docker/image_tar_puller.cpp
@@ -0,0 +1,402 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <string>
+#include <vector>
+
+#include <glog/logging.h>
+
+#include <stout/json.hpp>
+#include <stout/os.hpp>
+#include <stout/result.hpp>
+#include <stout/strings.hpp>
+
+#include <process/collect.hpp>
+#include <process/defer.hpp>
+#include <process/dispatch.hpp>
+#include <process/id.hpp>
+#include <process/process.hpp>
+
+#include "common/command_utils.hpp"
+
+#include "hdfs/hdfs.hpp"
+
+#include "uri/schemes/file.hpp"
+#include "uri/schemes/hdfs.hpp"
+
+#include "slave/containerizer/mesos/provisioner/docker/image_tar_puller.hpp"
+#include "slave/containerizer/mesos/provisioner/docker/paths.hpp"
+
+using namespace process;
+
+namespace spec = docker::spec;
+
+using std::string;
+using std::vector;
+
+namespace mesos {
+namespace internal {
+namespace slave {
+namespace docker {
+
+class ImageTarPullerProcess : public Process<ImageTarPullerProcess>
+{
+public:
+ ImageTarPullerProcess(
+ const string& _storeDir,
+ const URI& _archivesUri,
+ const Shared<uri::Fetcher>& _fetcher)
+ : ProcessBase(process::ID::generate("docker-provisioner-local-puller")),
+ storeDir(_storeDir),
+ archivesUri(_archivesUri),
+ fetcher(_fetcher) {}
+
+ ~ImageTarPullerProcess() {}
+
+ Future<vector<string>> pull(
+ const spec::ImageReference& reference,
+ const string& directory,
+ const string& backend);
+
+private:
+ Future<vector<string>> _pull(
+ const spec::ImageReference& reference,
+ const string& directory,
+ const string& backend);
+
+ Result<string> getParentLayerId(
+ const string& directory,
+ const string& layerId);
+
+ Future<Nothing> extractLayers(
+ const string& directory,
+ const vector<string>& layerIds,
+ const string& backend);
+
+ Future<Nothing> extractLayer(
+ const string& directory,
+ const string& layerId,
+ const string& backend);
+
+ const string storeDir;
+ const URI archivesUri;
+
+ Shared<uri::Fetcher> fetcher;
+};
+
+
+static Try<URI> parseUri(const string& uri)
+{
+ if (strings::startsWith(uri, "/")) {
+ return uri::file(uri);
+ }
+
+ return HDFS::parse(uri);
+}
+
+
+Try<Owned<Puller>> ImageTarPuller::create(
+ const Flags& flags,
+ const Shared<uri::Fetcher>& fetcher)
+{
+ // This should already been verified at puller.cpp.
+ if (!strings::startsWith(flags.docker_registry, "/") &&
+ !strings::startsWith(flags.docker_registry, "hdfs://")) {
+ return Error("Expecting registry url starting with '/' or 'hdfs'");
+ }
+
+ Try<URI> uri = parseUri(flags.docker_registry);
+ if (uri.isError()) {
+ return Error(
+ "Failed to parse the agent flag --docker_registry '" +
+ flags.docker_registry + "': " + uri.error());
+ }
+
+ VLOG(1) << "Creating image tar puller with docker registry '"
+ << flags.docker_registry << "'";
+
+ Owned<ImageTarPullerProcess> process(
+ new ImageTarPullerProcess(
+ flags.docker_store_dir,
+ uri.get(),
+ fetcher));
+
+ return Owned<Puller>(new ImageTarPuller(process));
+}
+
+
+ImageTarPuller::ImageTarPuller(Owned<ImageTarPullerProcess> _process)
+ : process(_process)
+{
+ spawn(process.get());
+}
+
+
+ImageTarPuller::~ImageTarPuller()
+{
+ terminate(process.get());
+ wait(process.get());
+}
+
+
+Future<vector<string>> ImageTarPuller::pull(
+ const spec::ImageReference& reference,
+ const string& directory,
+ const string& backend,
+ const Option<Secret>& config)
+{
+ return dispatch(
+ process.get(),
+ &ImageTarPullerProcess::pull,
+ reference,
+ directory,
+ backend);
+}
+
+
+Future<vector<string>> ImageTarPullerProcess::pull(
+ const spec::ImageReference& reference,
+ const string& directory,
+ const string& backend)
+{
+ // TODO(jieyu): We need to handle the case where the image reference
+ // contains a slash '/'.
+ const string image = stringify(reference);
+
+ // TODO(gilbert): Support 'http' and 'https'.
+ if (archivesUri.scheme() == "hdfs") {
+ URI uri = archivesUri;
+ uri.set_path(paths::getImageArchiveTarPath(archivesUri.path(), image));
+
+ VLOG(1) << "Fetching image '" << reference
+ << "' from '" << uri
+ << "' to '" << directory << "' using HDFS uri fetcher";
+
+ return fetcher->fetch(uri, directory)
+ .then(defer(self(), [=]() -> Future<vector<string>> {
+ const string source = paths::getImageArchiveTarPath(directory, image);
+
+ VLOG(1) << "Untarring image '" << reference
+ << "' from '" << source
+ << "' to '" << directory << "'";
+
+ return command::untar(Path(source), Path(directory))
+ .then(defer(self(), &Self::_pull, reference, directory, backend));
+ }));
+ }
+
+ const string tarPath = paths::getImageArchiveTarPath(
+ archivesUri.path(), image);
+
+ if (!os::exists(tarPath)) {
+ return Failure(
+ "Failed to find archive for image '" +
+ image + "' at '" + tarPath + "'");
+ }
+
+ VLOG(1) << "Untarring image '" << reference
+ << "' from '" << tarPath
+ << "' to '" << directory << "'";
+
+ return command::untar(Path(tarPath), Path(directory))
+ .then(defer(self(), &Self::_pull, reference, directory, backend));
+}
+
+
+Future<vector<string>> ImageTarPullerProcess::_pull(
+ const spec::ImageReference& reference,
+ const string& directory,
+ const string& backend)
+{
+ // We first parse the 'repositories' JSON file to get the top most
+ // layer id for the image.
+ Try<string> _repositories = os::read(path::join(directory, "repositories"));
+ if (_repositories.isError()) {
+ return Failure("Failed to read 'repositories': " + _repositories.error());
+ }
+
+ VLOG(1) << "The repositories JSON file for image '" << reference
+ << "' is '" << _repositories.get() << "'";
+
+ Try<JSON::Object> repositories =
+ JSON::parse<JSON::Object>(_repositories.get());
+
+ if (repositories.isError()) {
+ return Failure("Failed to parse 'repositories': " + repositories.error());
+ }
+
+ // We are looking for the topmost layer, so we know that is it OK to
+ // use at() rather than find() on the JSON object.
+ Result<JSON::Object> repository =
+ repositories->at<JSON::Object>(reference.repository());
+
+ // If we didn't find the bare repository name, try
+ // with the registry-qualified name. This would look like
+ // "registry.example.com/image".
+ if (repository.isNone() && reference.has_registry()) {
+ repository = repositories->at<JSON::Object>(
+ path::join(reference.registry(), reference.repository()));
+ }
+
+ if (repository.isError()) {
+ return Failure(
+ "Failed to find repository '" + reference.repository() +
+ "' in 'repositories': " + repository.error());
+ } else if (repository.isNone()) {
+ return Failure(
+ "Repository '" + reference.repository() + "' does not "
+ "exist in 'repositories'");
+ }
+
+ const string tag = reference.has_tag()
+ ? reference.tag()
+ : "latest";
+
+ // NOTE: We don't use JSON find here since a tag might contain '.'.
+ Result<JSON::String> layerId = repository->at<JSON::String>(tag);
+
+ if (layerId.isError()) {
+ return Failure(
+ "Failed to access layer id '" + tag + "': " + layerId.error());
+ } else if (layerId.isNone()) {
+ return Failure("Layer id '" + tag + "' is not found");
+ }
+
+ // Do a traverse to find all parent image layer ids. Here, we assume
+ // that all the parent layers are part of the archive tar, thus are
+ // already extracted under 'directory'.
+ vector<string> layerIds = { layerId->value };
+ Result<string> parentLayerId = getParentLayerId(directory, layerId->value);
+ while (parentLayerId.isSome()) {
+ // NOTE: We put parent layer ids in front because that's what the
+ // provisioner backends assume.
+ layerIds.insert(layerIds.begin(), parentLayerId.get());
+ parentLayerId = getParentLayerId(directory, parentLayerId.get());
+ }
+
+ if (parentLayerId.isError()) {
+ return Failure(
+ "Failed to find parent layer id for layer '" + layerId->value +
+ "': " + parentLayerId.error());
+ }
+
+ return extractLayers(directory, layerIds, backend)
+ .then([layerIds]() -> vector<string> { return layerIds; });
+}
+
+
+Result<string> ImageTarPullerProcess::getParentLayerId(
+ const string& directory,
+ const string& layerId)
+{
+ const string path =
+ paths::getImageLayerManifestPath(path::join(directory, layerId));
+
+ Try<string> _manifest = os::read(path);
+ if (_manifest.isError()) {
+ return Error(
+ "Failed to read manifest from '" + path + "': " + _manifest.error());
+ }
+
+ Try<JSON::Object> manifest = JSON::parse<JSON::Object>(_manifest.get());
+ if (manifest.isError()) {
+ return Error(
+ "Failed to parse manifest from '" + path + "': " + manifest.error());
+ }
+
+ Result<JSON::Value> parentLayerId = manifest->find<JSON::Value>("parent");
+ if (parentLayerId.isError()) {
+ return Error(
+ "Failed to parse 'parent' key in manifest from '" + path + "': " +
+ parentLayerId.error());
+ } else if (parentLayerId.isNone()) {
+ return None();
+ } else if (parentLayerId->is<JSON::Null>()) {
+ return None();
+ } else if (!parentLayerId->is<JSON::String>()) {
+ return Error("Unexpected 'parent' type in manifest from '" + path + "'");
+ }
+
+ const string id = parentLayerId->as<JSON::String>().value;
+ if (id == "") {
+ return None();
+ } else {
+ return id;
+ }
+}
+
+
+Future<Nothing> ImageTarPullerProcess::extractLayers(
+ const string& directory,
+ const vector<string>& layerIds,
+ const string& backend)
+{
+ vector<Future<Nothing>> futures;
+ foreach (const string& layerId, layerIds) {
+ // Check if the layer is already in the store. If yes, skip the
+ // unnecessary extracting.
+ if (os::exists(paths::getImageLayerRootfsPath(
+ storeDir,
+ layerId,
+ backend))) {
+ continue;
+ }
+
+ futures.push_back(extractLayer(directory, layerId, backend));
+ }
+
+ return collect(futures)
+ .then([]() { return Nothing(); });
+}
+
+
+Future<Nothing> ImageTarPullerProcess::extractLayer(
+ const string& directory,
+ const string& layerId,
+ const string& backend)
+{
+ const string layerPath = path::join(directory, layerId);
+ const string tar = paths::getImageLayerTarPath(layerPath);
+ const string rootfs = paths::getImageLayerRootfsPath(layerPath, backend);
+
+ VLOG(1) << "Extracting layer tar ball '" << tar
+ << " to rootfs '" << rootfs << "'";
+
+ Try<Nothing> mkdir = os::mkdir(rootfs);
+ if (mkdir.isError()) {
+ return Failure(
+ "Failed to create directory '" + rootfs + "'"
+ ": " + mkdir.error());
+ }
+
+ return command::untar(Path(tar), Path(rootfs))
+ .then([tar]() -> Future<Nothing> {
+ // Remove the tar after the extraction.
+ Try<Nothing> rm = os::rm(tar);
+ if (rm.isError()) {
+ return Failure(
+ "Failed to remove '" + tar + "' "
+ "after extraction: " + rm.error());
+ }
+
+ return Nothing();
+ });
+}
+
+} // namespace docker {
+} // namespace slave {
+} // namespace internal {
+} // namespace mesos {
http://git-wip-us.apache.org/repos/asf/mesos/blob/c185752b/src/slave/containerizer/mesos/provisioner/docker/image_tar_puller.hpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/docker/image_tar_puller.hpp b/src/slave/containerizer/mesos/provisioner/docker/image_tar_puller.hpp
new file mode 100644
index 0000000..f18a758
--- /dev/null
+++ b/src/slave/containerizer/mesos/provisioner/docker/image_tar_puller.hpp
@@ -0,0 +1,73 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef __PROVISIONER_DOCKER_IMAGE_TAR_PULLER_HPP__
+#define __PROVISIONER_DOCKER_IMAGE_TAR_PULLER_HPP__
+
+#include <process/owned.hpp>
+
+#include <mesos/docker/spec.hpp>
+
+#include <mesos/uri/fetcher.hpp>
+
+#include "slave/containerizer/mesos/provisioner/docker/puller.hpp"
+
+#include "slave/flags.hpp"
+
+namespace mesos {
+namespace internal {
+namespace slave {
+namespace docker {
+
+// Forward declaration.
+class ImageTarPullerProcess;
+
+
+/**
+ * ImageTarPuller assumes Docker images are stored in a local directory
+ * (configured with flags.docker_registry), with all the
+ * images saved as tars with file names in the form of <repo>:<tag>.tar.
+ */
+class ImageTarPuller : public Puller
+{
+public:
+ static Try<process::Owned<Puller>> create(
+ const Flags& flags,
+ const process::Shared<uri::Fetcher>& fetcher);
+
+ ~ImageTarPuller();
+
+ process::Future<std::vector<std::string>> pull(
+ const ::docker::spec::ImageReference& reference,
+ const std::string& directory,
+ const std::string& backend,
+ const Option<Secret>& config = None());
+
+private:
+ explicit ImageTarPuller(process::Owned<ImageTarPullerProcess> _process);
+
+ ImageTarPuller(const ImageTarPuller&) = delete; // Not copyable.
+ ImageTarPuller& operator=(const ImageTarPuller&) = delete; // Not assignable.
+
+ process::Owned<ImageTarPullerProcess> process;
+};
+
+} // namespace docker {
+} // namespace slave {
+} // namespace internal {
+} // namespace mesos {
+
+#endif // __PROVISIONER_DOCKER_IMAGE_TAR_PULLER_HPP__
http://git-wip-us.apache.org/repos/asf/mesos/blob/c185752b/src/slave/containerizer/mesos/provisioner/docker/local_puller.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/docker/local_puller.cpp b/src/slave/containerizer/mesos/provisioner/docker/local_puller.cpp
deleted file mode 100644
index df715e2..0000000
--- a/src/slave/containerizer/mesos/provisioner/docker/local_puller.cpp
+++ /dev/null
@@ -1,402 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include <string>
-#include <vector>
-
-#include <glog/logging.h>
-
-#include <stout/json.hpp>
-#include <stout/os.hpp>
-#include <stout/result.hpp>
-#include <stout/strings.hpp>
-
-#include <process/collect.hpp>
-#include <process/defer.hpp>
-#include <process/dispatch.hpp>
-#include <process/id.hpp>
-#include <process/process.hpp>
-
-#include "common/command_utils.hpp"
-
-#include "hdfs/hdfs.hpp"
-
-#include "uri/schemes/file.hpp"
-#include "uri/schemes/hdfs.hpp"
-
-#include "slave/containerizer/mesos/provisioner/docker/local_puller.hpp"
-#include "slave/containerizer/mesos/provisioner/docker/paths.hpp"
-
-using namespace process;
-
-namespace spec = docker::spec;
-
-using std::string;
-using std::vector;
-
-namespace mesos {
-namespace internal {
-namespace slave {
-namespace docker {
-
-class LocalPullerProcess : public Process<LocalPullerProcess>
-{
-public:
- LocalPullerProcess(
- const string& _storeDir,
- const URI& _archivesUri,
- const Shared<uri::Fetcher>& _fetcher)
- : ProcessBase(process::ID::generate("docker-provisioner-local-puller")),
- storeDir(_storeDir),
- archivesUri(_archivesUri),
- fetcher(_fetcher) {}
-
- ~LocalPullerProcess() {}
-
- Future<vector<string>> pull(
- const spec::ImageReference& reference,
- const string& directory,
- const string& backend);
-
-private:
- Future<vector<string>> _pull(
- const spec::ImageReference& reference,
- const string& directory,
- const string& backend);
-
- Result<string> getParentLayerId(
- const string& directory,
- const string& layerId);
-
- Future<Nothing> extractLayers(
- const string& directory,
- const vector<string>& layerIds,
- const string& backend);
-
- Future<Nothing> extractLayer(
- const string& directory,
- const string& layerId,
- const string& backend);
-
- const string storeDir;
- const URI archivesUri;
-
- Shared<uri::Fetcher> fetcher;
-};
-
-
-static Try<URI> parseUri(const string& uri)
-{
- if (strings::startsWith(uri, "/")) {
- return uri::file(uri);
- }
-
- return HDFS::parse(uri);
-}
-
-
-Try<Owned<Puller>> LocalPuller::create(
- const Flags& flags,
- const Shared<uri::Fetcher>& fetcher)
-{
- // This should already been verified at puller.cpp.
- if (!strings::startsWith(flags.docker_registry, "/") &&
- !strings::startsWith(flags.docker_registry, "hdfs://")) {
- return Error("Expecting registry url starting with '/' or 'hdfs'");
- }
-
- Try<URI> uri = parseUri(flags.docker_registry);
- if (uri.isError()) {
- return Error(
- "Failed to parse the agent flag --docker_registry '" +
- flags.docker_registry + "': " + uri.error());
- }
-
- VLOG(1) << "Creating local puller with docker registry '"
- << flags.docker_registry << "'";
-
- Owned<LocalPullerProcess> process(
- new LocalPullerProcess(
- flags.docker_store_dir,
- uri.get(),
- fetcher));
-
- return Owned<Puller>(new LocalPuller(process));
-}
-
-
-LocalPuller::LocalPuller(Owned<LocalPullerProcess> _process)
- : process(_process)
-{
- spawn(process.get());
-}
-
-
-LocalPuller::~LocalPuller()
-{
- terminate(process.get());
- wait(process.get());
-}
-
-
-Future<vector<string>> LocalPuller::pull(
- const spec::ImageReference& reference,
- const string& directory,
- const string& backend,
- const Option<Secret>& config)
-{
- return dispatch(
- process.get(),
- &LocalPullerProcess::pull,
- reference,
- directory,
- backend);
-}
-
-
-Future<vector<string>> LocalPullerProcess::pull(
- const spec::ImageReference& reference,
- const string& directory,
- const string& backend)
-{
- // TODO(jieyu): We need to handle the case where the image reference
- // contains a slash '/'.
- const string image = stringify(reference);
-
- // TODO(gilbert): Support 'http' and 'https'.
- if (archivesUri.scheme() == "hdfs") {
- URI uri = archivesUri;
- uri.set_path(paths::getImageArchiveTarPath(archivesUri.path(), image));
-
- VLOG(1) << "Fetching image '" << reference
- << "' from '" << uri
- << "' to '" << directory << "' using HDFS uri fetcher";
-
- return fetcher->fetch(uri, directory)
- .then(defer(self(), [=]() -> Future<vector<string>> {
- const string source = paths::getImageArchiveTarPath(directory, image);
-
- VLOG(1) << "Untarring image '" << reference
- << "' from '" << source
- << "' to '" << directory << "'";
-
- return command::untar(Path(source), Path(directory))
- .then(defer(self(), &Self::_pull, reference, directory, backend));
- }));
- }
-
- const string tarPath = paths::getImageArchiveTarPath(
- archivesUri.path(), image);
-
- if (!os::exists(tarPath)) {
- return Failure(
- "Failed to find archive for image '" +
- image + "' at '" + tarPath + "'");
- }
-
- VLOG(1) << "Untarring image '" << reference
- << "' from '" << tarPath
- << "' to '" << directory << "'";
-
- return command::untar(Path(tarPath), Path(directory))
- .then(defer(self(), &Self::_pull, reference, directory, backend));
-}
-
-
-Future<vector<string>> LocalPullerProcess::_pull(
- const spec::ImageReference& reference,
- const string& directory,
- const string& backend)
-{
- // We first parse the 'repositories' JSON file to get the top most
- // layer id for the image.
- Try<string> _repositories = os::read(path::join(directory, "repositories"));
- if (_repositories.isError()) {
- return Failure("Failed to read 'repositories': " + _repositories.error());
- }
-
- VLOG(1) << "The repositories JSON file for image '" << reference
- << "' is '" << _repositories.get() << "'";
-
- Try<JSON::Object> repositories =
- JSON::parse<JSON::Object>(_repositories.get());
-
- if (repositories.isError()) {
- return Failure("Failed to parse 'repositories': " + repositories.error());
- }
-
- // We are looking for the topmost layer, so we know that is it OK to
- // use at() rather than find() on the JSON object.
- Result<JSON::Object> repository =
- repositories->at<JSON::Object>(reference.repository());
-
- // If we didn't find the bare repository name, try
- // with the registry-qualified name. This would look like
- // "registry.example.com/image".
- if (repository.isNone() && reference.has_registry()) {
- repository = repositories->at<JSON::Object>(
- path::join(reference.registry(), reference.repository()));
- }
-
- if (repository.isError()) {
- return Failure(
- "Failed to find repository '" + reference.repository() +
- "' in 'repositories': " + repository.error());
- } else if (repository.isNone()) {
- return Failure(
- "Repository '" + reference.repository() + "' does not "
- "exist in 'repositories'");
- }
-
- const string tag = reference.has_tag()
- ? reference.tag()
- : "latest";
-
- // NOTE: We don't use JSON find here since a tag might contain '.'.
- Result<JSON::String> layerId = repository->at<JSON::String>(tag);
-
- if (layerId.isError()) {
- return Failure(
- "Failed to access layer id '" + tag + "': " + layerId.error());
- } else if (layerId.isNone()) {
- return Failure("Layer id '" + tag + "' is not found");
- }
-
- // Do a traverse to find all parent image layer ids. Here, we assume
- // that all the parent layers are part of the archive tar, thus are
- // already extracted under 'directory'.
- vector<string> layerIds = { layerId->value };
- Result<string> parentLayerId = getParentLayerId(directory, layerId->value);
- while (parentLayerId.isSome()) {
- // NOTE: We put parent layer ids in front because that's what the
- // provisioner backends assume.
- layerIds.insert(layerIds.begin(), parentLayerId.get());
- parentLayerId = getParentLayerId(directory, parentLayerId.get());
- }
-
- if (parentLayerId.isError()) {
- return Failure(
- "Failed to find parent layer id for layer '" + layerId->value +
- "': " + parentLayerId.error());
- }
-
- return extractLayers(directory, layerIds, backend)
- .then([layerIds]() -> vector<string> { return layerIds; });
-}
-
-
-Result<string> LocalPullerProcess::getParentLayerId(
- const string& directory,
- const string& layerId)
-{
- const string path =
- paths::getImageLayerManifestPath(path::join(directory, layerId));
-
- Try<string> _manifest = os::read(path);
- if (_manifest.isError()) {
- return Error(
- "Failed to read manifest from '" + path + "': " + _manifest.error());
- }
-
- Try<JSON::Object> manifest = JSON::parse<JSON::Object>(_manifest.get());
- if (manifest.isError()) {
- return Error(
- "Failed to parse manifest from '" + path + "': " + manifest.error());
- }
-
- Result<JSON::Value> parentLayerId = manifest->find<JSON::Value>("parent");
- if (parentLayerId.isError()) {
- return Error(
- "Failed to parse 'parent' key in manifest from '" + path + "': " +
- parentLayerId.error());
- } else if (parentLayerId.isNone()) {
- return None();
- } else if (parentLayerId->is<JSON::Null>()) {
- return None();
- } else if (!parentLayerId->is<JSON::String>()) {
- return Error("Unexpected 'parent' type in manifest from '" + path + "'");
- }
-
- const string id = parentLayerId->as<JSON::String>().value;
- if (id == "") {
- return None();
- } else {
- return id;
- }
-}
-
-
-Future<Nothing> LocalPullerProcess::extractLayers(
- const string& directory,
- const vector<string>& layerIds,
- const string& backend)
-{
- vector<Future<Nothing>> futures;
- foreach (const string& layerId, layerIds) {
- // Check if the layer is already in the store. If yes, skip the
- // unnecessary extracting.
- if (os::exists(paths::getImageLayerRootfsPath(
- storeDir,
- layerId,
- backend))) {
- continue;
- }
-
- futures.push_back(extractLayer(directory, layerId, backend));
- }
-
- return collect(futures)
- .then([]() { return Nothing(); });
-}
-
-
-Future<Nothing> LocalPullerProcess::extractLayer(
- const string& directory,
- const string& layerId,
- const string& backend)
-{
- const string layerPath = path::join(directory, layerId);
- const string tar = paths::getImageLayerTarPath(layerPath);
- const string rootfs = paths::getImageLayerRootfsPath(layerPath, backend);
-
- VLOG(1) << "Extracting layer tar ball '" << tar
- << " to rootfs '" << rootfs << "'";
-
- Try<Nothing> mkdir = os::mkdir(rootfs);
- if (mkdir.isError()) {
- return Failure(
- "Failed to create directory '" + rootfs + "'"
- ": " + mkdir.error());
- }
-
- return command::untar(Path(tar), Path(rootfs))
- .then([tar]() -> Future<Nothing> {
- // Remove the tar after the extraction.
- Try<Nothing> rm = os::rm(tar);
- if (rm.isError()) {
- return Failure(
- "Failed to remove '" + tar + "' "
- "after extraction: " + rm.error());
- }
-
- return Nothing();
- });
-}
-
-} // namespace docker {
-} // namespace slave {
-} // namespace internal {
-} // namespace mesos {
http://git-wip-us.apache.org/repos/asf/mesos/blob/c185752b/src/slave/containerizer/mesos/provisioner/docker/local_puller.hpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/docker/local_puller.hpp b/src/slave/containerizer/mesos/provisioner/docker/local_puller.hpp
deleted file mode 100644
index 37f2510..0000000
--- a/src/slave/containerizer/mesos/provisioner/docker/local_puller.hpp
+++ /dev/null
@@ -1,73 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#ifndef __PROVISIONER_DOCKER_LOCAL_PULLER_HPP__
-#define __PROVISIONER_DOCKER_LOCAL_PULLER_HPP__
-
-#include <process/owned.hpp>
-
-#include <mesos/docker/spec.hpp>
-
-#include <mesos/uri/fetcher.hpp>
-
-#include "slave/containerizer/mesos/provisioner/docker/puller.hpp"
-
-#include "slave/flags.hpp"
-
-namespace mesos {
-namespace internal {
-namespace slave {
-namespace docker {
-
-// Forward declaration.
-class LocalPullerProcess;
-
-
-/**
- * LocalPuller assumes Docker images are stored in a local directory
- * (configured with flags.docker_registry), with all the
- * images saved as tars with file names in the form of <repo>:<tag>.tar.
- */
-class LocalPuller : public Puller
-{
-public:
- static Try<process::Owned<Puller>> create(
- const Flags& flags,
- const process::Shared<uri::Fetcher>& fetcher);
-
- ~LocalPuller();
-
- process::Future<std::vector<std::string>> pull(
- const ::docker::spec::ImageReference& reference,
- const std::string& directory,
- const std::string& backend,
- const Option<Secret>& config = None());
-
-private:
- explicit LocalPuller(process::Owned<LocalPullerProcess> _process);
-
- LocalPuller(const LocalPuller&) = delete; // Not copyable.
- LocalPuller& operator=(const LocalPuller&) = delete; // Not assignable.
-
- process::Owned<LocalPullerProcess> process;
-};
-
-} // namespace docker {
-} // namespace slave {
-} // namespace internal {
-} // namespace mesos {
-
-#endif // __PROVISIONER_DOCKER_LOCAL_PULLER_HPP__
http://git-wip-us.apache.org/repos/asf/mesos/blob/c185752b/src/slave/containerizer/mesos/provisioner/docker/puller.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/docker/puller.cpp b/src/slave/containerizer/mesos/provisioner/docker/puller.cpp
index cb4248b..d013c9d 100644
--- a/src/slave/containerizer/mesos/provisioner/docker/puller.cpp
+++ b/src/slave/containerizer/mesos/provisioner/docker/puller.cpp
@@ -19,7 +19,7 @@
#include <stout/strings.hpp>
#include <stout/try.hpp>
-#include "slave/containerizer/mesos/provisioner/docker/local_puller.hpp"
+#include "slave/containerizer/mesos/provisioner/docker/image_tar_puller.hpp"
#include "slave/containerizer/mesos/provisioner/docker/puller.hpp"
#include "slave/containerizer/mesos/provisioner/docker/registry_puller.hpp"
@@ -45,9 +45,9 @@ Try<Owned<Puller>> Puller::create(
// image tarballs or the remote docker registry.
if (strings::startsWith(flags.docker_registry, "/") ||
strings::startsWith(flags.docker_registry, "hdfs://")) {
- Try<Owned<Puller>> puller = LocalPuller::create(flags, fetcher);
+ Try<Owned<Puller>> puller = ImageTarPuller::create(flags, fetcher);
if (puller.isError()) {
- return Error("Failed to create local puller: " + puller.error());
+ return Error("Failed to create image tar puller " + puller.error());
}
return puller.get();
http://git-wip-us.apache.org/repos/asf/mesos/blob/c185752b/src/tests/containerizer/provisioner_docker_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/containerizer/provisioner_docker_tests.cpp b/src/tests/containerizer/provisioner_docker_tests.cpp
index bb84f3f..71247c3 100644
--- a/src/tests/containerizer/provisioner_docker_tests.cpp
+++ b/src/tests/containerizer/provisioner_docker_tests.cpp
@@ -378,7 +378,7 @@ class ProvisionerDockerTest
// This test verifies that local docker image can be pulled and
// provisioned correctly, and shell command should be executed.
-TEST_F(ProvisionerDockerTest, ROOT_LocalPullerSimpleCommand)
+TEST_F(ProvisionerDockerTest, ROOT_ImageTarPullerSimpleCommand)
{
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
@@ -473,9 +473,9 @@ INSTANTIATE_TEST_CASE_P(
"hdfs://"})));
-// This test verifies that the local puller could pull image
+// This test verifies that the image tar puller could pull image
// with the hdfs uri fetcher plugin.
-TEST_P(ProvisionerDockerHdfsTest, ROOT_LocalPullerHdfsFetcherSimpleCommand)
+TEST_P(ProvisionerDockerHdfsTest, ROOT_ImageTarPullerHdfsFetcherSimpleCommand)
{
string hadoopPath = os::getcwd();
ASSERT_TRUE(os::exists(hadoopPath));
http://git-wip-us.apache.org/repos/asf/mesos/blob/c185752b/src/tests/containerizer/runtime_isolator_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/containerizer/runtime_isolator_tests.cpp b/src/tests/containerizer/runtime_isolator_tests.cpp
index b703b82..539ae68 100644
--- a/src/tests/containerizer/runtime_isolator_tests.cpp
+++ b/src/tests/containerizer/runtime_isolator_tests.cpp
@@ -84,7 +84,7 @@ class DockerRuntimeIsolatorTest : public MesosTest {};
// This test verifies that docker image default cmd is executed correctly.
// This corresponds to the case in runtime isolator logic table: sh=0,
// value=0, argv=1, entrypoint=0, cmd=1.
-TEST_F(DockerRuntimeIsolatorTest, ROOT_DockerDefaultCmdLocalPuller)
+TEST_F(DockerRuntimeIsolatorTest, ROOT_DockerDefaultCmdImageTarPuller)
{
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
@@ -176,9 +176,9 @@ TEST_F(DockerRuntimeIsolatorTest, ROOT_DockerDefaultCmdLocalPuller)
// This test verifies that docker image default entrypoint is executed
-// correctly using local puller. This corresponds to the case in runtime
-// isolator logic table: sh=0, value=0, argv=1, entrypoint=1, cmd=0.
-TEST_F(DockerRuntimeIsolatorTest, ROOT_DockerDefaultEntryptLocalPuller)
+// correctly using image tar puller. This corresponds to the case in
+// runtime isolator logic table: sh=0, value=0, argv=1, entrypoint=1, cmd=0.
+TEST_F(DockerRuntimeIsolatorTest, ROOT_DockerDefaultEntryptImageTarPuller)
{
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
@@ -456,7 +456,7 @@ TEST_F(DockerRuntimeIsolatorTest, ROOT_INTERNET_CURL_NestedSimpleCommand)
// This is a regression test for MESOS-6852. It corresponds to the
// following case in runtime isolator logic table for nested
// container: sh=0, value=0, argv=1, entrypoint=0, cmd=1.
-TEST_F(DockerRuntimeIsolatorTest, ROOT_NestedDockerDefaultCmdLocalPuller)
+TEST_F(DockerRuntimeIsolatorTest, ROOT_NestedDockerDefaultCmdImageTarPuller)
{
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);
@@ -579,7 +579,7 @@ TEST_F(DockerRuntimeIsolatorTest, ROOT_NestedDockerDefaultCmdLocalPuller)
// It corresponds to the following case in runtime isolator
// logic table for nested container: sh=0, value=0, argv=1,
// entrypoint=1, cmd=0.
-TEST_F(DockerRuntimeIsolatorTest, ROOT_NestedDockerDefaultEntryptLocalPuller)
+TEST_F(DockerRuntimeIsolatorTest, ROOT_NestedDockerDefaultEntryptImageTarPuller)
{
Try<Owned<cluster::Master>> master = StartMaster();
ASSERT_SOME(master);