You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by tn...@apache.org on 2015/11/09 08:34:19 UTC
[2/3] mesos git commit: Renamed RemotePuller to RegistryPuller.
Renamed RemotePuller to RegistryPuller.
Review: https://reviews.apache.org/r/40069
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/04361416
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/04361416
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/04361416
Branch: refs/heads/master
Commit: 0436141660701c240a8d2fafd8216ceb4c1fffb0
Parents: fff91d7
Author: Timothy Chen <tn...@gmail.com>
Authored: Sat Nov 7 08:58:23 2015 -0800
Committer: Timothy Chen <tn...@apache.org>
Committed: Sun Nov 8 23:34:07 2015 -0800
----------------------------------------------------------------------
src/CMakeLists.txt | 2 +-
src/Makefile.am | 4 +-
.../mesos/provisioner/docker/puller.cpp | 8 +-
.../provisioner/docker/registry_client.hpp | 4 +-
.../provisioner/docker/registry_puller.cpp | 404 +++++++++++++++++++
.../provisioner/docker/registry_puller.hpp | 85 ++++
.../mesos/provisioner/docker/remote_puller.cpp | 402 ------------------
.../mesos/provisioner/docker/remote_puller.hpp | 85 ----
.../containerizer/provisioner_docker_tests.cpp | 42 +-
9 files changed, 519 insertions(+), 517 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/04361416/src/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 74bfa91..cf655b2 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -208,7 +208,7 @@ if (NOT WIN32)
slave/containerizer/mesos/provisioner/backends/copy.cpp
slave/containerizer/mesos/provisioner/docker/message.proto
slave/containerizer/mesos/provisioner/docker/registry_client.cpp
- slave/containerizer/mesos/provisioner/docker/remote_puller.cpp
+ slave/containerizer/mesos/provisioner/docker/registry_puller.cpp
slave/containerizer/mesos/provisioner/docker/spec.cpp
slave/containerizer/mesos/provisioner/docker/token_manager.cpp
slave/resource_estimators/noop.cpp
http://git-wip-us.apache.org/repos/asf/mesos/blob/04361416/src/Makefile.am
----------------------------------------------------------------------
diff --git a/src/Makefile.am b/src/Makefile.am
index 9e6eb7c..3263991 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -562,7 +562,7 @@ libmesos_no_3rdparty_la_SOURCES = \
slave/containerizer/mesos/provisioner/docker/paths.cpp \
slave/containerizer/mesos/provisioner/docker/puller.cpp \
slave/containerizer/mesos/provisioner/docker/registry_client.cpp \
- slave/containerizer/mesos/provisioner/docker/remote_puller.cpp \
+ slave/containerizer/mesos/provisioner/docker/registry_puller.cpp \
slave/containerizer/mesos/provisioner/docker/spec.cpp \
slave/containerizer/mesos/provisioner/docker/store.cpp \
slave/containerizer/mesos/provisioner/docker/token_manager.cpp \
@@ -884,7 +884,7 @@ libmesos_no_3rdparty_la_SOURCES += \
slave/containerizer/mesos/provisioner/docker/paths.hpp \
slave/containerizer/mesos/provisioner/docker/puller.hpp \
slave/containerizer/mesos/provisioner/docker/registry_client.hpp \
- slave/containerizer/mesos/provisioner/docker/remote_puller.hpp \
+ slave/containerizer/mesos/provisioner/docker/registry_puller.hpp \
slave/containerizer/mesos/provisioner/docker/spec.hpp \
slave/containerizer/mesos/provisioner/docker/store.hpp \
slave/containerizer/mesos/provisioner/docker/token_manager.hpp \
http://git-wip-us.apache.org/repos/asf/mesos/blob/04361416/src/slave/containerizer/mesos/provisioner/docker/puller.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/docker/puller.cpp b/src/slave/containerizer/mesos/provisioner/docker/puller.cpp
index ae644ec..70501f1 100644
--- a/src/slave/containerizer/mesos/provisioner/docker/puller.cpp
+++ b/src/slave/containerizer/mesos/provisioner/docker/puller.cpp
@@ -19,7 +19,7 @@
#include "slave/containerizer/mesos/provisioner/docker/puller.hpp"
#include "slave/containerizer/mesos/provisioner/docker/local_puller.hpp"
-#include "slave/containerizer/mesos/provisioner/docker/remote_puller.hpp"
+#include "slave/containerizer/mesos/provisioner/docker/registry_puller.hpp"
using std::string;
@@ -38,10 +38,10 @@ Try<Owned<Puller>> Puller::create(const Flags& flags)
return Owned<Puller>(new LocalPuller(flags));
}
- if (puller == "remote") {
- Try<Owned<Puller>> puller = RemotePuller::create(flags);
+ if (puller == "registry") {
+ Try<Owned<Puller>> puller = RegistryPuller::create(flags);
if (puller.isError()) {
- return Error("Failed to create remote puller: " + puller.error());
+ return Error("Failed to create registry puller: " + puller.error());
}
return puller.get();
http://git-wip-us.apache.org/repos/asf/mesos/blob/04361416/src/slave/containerizer/mesos/provisioner/docker/registry_client.hpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/docker/registry_client.hpp b/src/slave/containerizer/mesos/provisioner/docker/registry_client.hpp
index 7bba371..3f04747 100644
--- a/src/slave/containerizer/mesos/provisioner/docker/registry_client.hpp
+++ b/src/slave/containerizer/mesos/provisioner/docker/registry_client.hpp
@@ -99,7 +99,7 @@ public:
const Option<Credentials>& credentials);
/**
- * Fetches manifest for a repository from the client's remote registry server.
+ * Fetches manifest for a repository from the client's docker registry server.
*
* @param imageName Image information(Name, tag).
* @return Manifest on success.
@@ -110,7 +110,7 @@ public:
/**
- * Fetches blob for a repository from the client's remote registry server.
+ * Fetches blob for a repository from the client's docker registry server.
*
* @param path path of the repository on the registry.
* @param digest digest of the blob (from manifest).
http://git-wip-us.apache.org/repos/asf/mesos/blob/04361416/src/slave/containerizer/mesos/provisioner/docker/registry_puller.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/docker/registry_puller.cpp b/src/slave/containerizer/mesos/provisioner/docker/registry_puller.cpp
new file mode 100644
index 0000000..6bd9838
--- /dev/null
+++ b/src/slave/containerizer/mesos/provisioner/docker/registry_puller.cpp
@@ -0,0 +1,404 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "slave/containerizer/mesos/provisioner/docker/registry_puller.hpp"
+
+#include <list>
+
+#include <process/collect.hpp>
+#include <process/defer.hpp>
+#include <process/dispatch.hpp>
+#include <process/subprocess.hpp>
+
+#include "common/status_utils.hpp"
+
+#include "slave/containerizer/mesos/provisioner/docker/registry_client.hpp"
+
+using std::list;
+using std::string;
+using std::vector;
+
+using process::Failure;
+using process::Future;
+using process::Owned;
+using process::Process;
+using process::Promise;
+using process::Subprocess;
+
+namespace http = process::http;
+
+namespace mesos {
+namespace internal {
+namespace slave {
+namespace docker {
+
+using FileSystemLayerInfo = registry::FileSystemLayerInfo;
+using Manifest = registry::Manifest;
+using RegistryClient = registry::RegistryClient;
+
+
+class RegistryPullerProcess : public Process<RegistryPullerProcess>
+{
+public:
+ static Try<Owned<RegistryPullerProcess>> create(const Flags& flags);
+
+ process::Future<PulledImageInfo> pull(
+ const Image::Name& imageName,
+ const Path& downloadDir);
+
+ Duration getTimeout() const
+ {
+ return pullTimeout_;
+ }
+
+private:
+ explicit RegistryPullerProcess(
+ const Owned<RegistryClient>& registry,
+ const Duration& timeout);
+
+ Try<Image::Name> getRemoteNameFromLocalName(const Image::Name& name);
+
+ Future<PulledLayerInfo> downloadLayer(
+ const string& remoteName,
+ const Path& downloadDir,
+ const FileSystemLayerInfo& layer);
+
+ Owned<RegistryClient> registryClient_;
+ const Duration pullTimeout_;
+ hashmap<string, Owned<Promise<PulledLayerInfo>>> downloadTracker_;
+
+ RegistryPullerProcess(const RegistryPullerProcess&) = delete;
+ RegistryPullerProcess& operator=(const RegistryPullerProcess&) = delete;
+};
+
+
+Try<Owned<Puller>> RegistryPuller::create(const Flags& flags)
+{
+ Try<Owned<RegistryPullerProcess>> process =
+ RegistryPullerProcess::create(flags);
+
+ if (process.isError()) {
+ return Error(process.error());
+ }
+
+ return Owned<Puller>(new RegistryPuller(process.get()));
+}
+
+
+RegistryPuller::RegistryPuller(const Owned<RegistryPullerProcess>& process)
+ : process_(process)
+{
+ spawn(CHECK_NOTNULL(process_.get()));
+}
+
+
+RegistryPuller::~RegistryPuller()
+{
+ terminate(process_.get());
+ process::wait(process_.get());
+}
+
+
+Future<PulledImageInfo> RegistryPuller::pull(
+ const Image::Name& imageName,
+ const Path& downloadDir)
+{
+ return dispatch(
+ process_.get(),
+ &RegistryPullerProcess::pull,
+ imageName,
+ downloadDir)
+ // We have 'after' here because the local puller need not have a timeout.
+ // Timeout is a flag for the remote puller and needs to be handled here.
+ .after(process_->getTimeout(), [imageName](
+ Future<PulledImageInfo> future) {
+ LOG(WARNING) << "Failed to download image '"
+ << imageName.repository()
+ << "': timeout";
+ future.discard();
+
+ return Failure(
+ "Failed to download image '" + imageName.repository() + "': timeout");
+ });
+}
+
+
+Try<Owned<RegistryPullerProcess>> RegistryPullerProcess::create(
+ const Flags& flags)
+{
+ Result<double> timeoutSecs = numify<double>(flags.docker_puller_timeout_secs);
+ if ((timeoutSecs.isError()) || (timeoutSecs.get() <= 0)) {
+ return Error(
+ "Failed to create remote puller - invalid timeout value: " +
+ flags.docker_puller_timeout_secs);
+ }
+
+ auto getPort = [](
+ const string& stringVal) -> Try<uint16_t> {
+ Result<uint16_t> port = numify<uint16_t>(stringVal);
+ if (port.isError()) {
+ return Error("Failed to parse '" + stringVal + "' as port");
+ }
+
+ return port.get();
+ };
+
+ // TODO(jojy): Maybe do a nslookup/tcp dial of these servers.
+ Try<uint16_t> registryPort = getPort(flags.docker_registry_port);
+ if (registryPort.isError()) {
+ return Error("Failed to parse registry port: " + registryPort.error());
+ }
+
+ Try<uint16_t> authServerPort = getPort(flags.docker_auth_server_port);
+ if (authServerPort.isError()) {
+ return Error(
+ "Failed to parse authentication server port: " +
+ authServerPort.error());
+ }
+
+ Try<Owned<RegistryClient>> registry = RegistryClient::create(
+ http::URL("https", flags.docker_registry, registryPort.get()),
+ http::URL("https", flags.docker_auth_server, authServerPort.get()),
+ None());
+
+ if (registry.isError()) {
+ return Error("Failed to create registry client: " + registry.error());
+ }
+
+ return Owned<RegistryPullerProcess>(new RegistryPullerProcess(
+ registry.get(),
+ Seconds(timeoutSecs.get())));
+}
+
+
+RegistryPullerProcess::RegistryPullerProcess(
+ const Owned<RegistryClient>& registry,
+ const Duration& timeout)
+ : registryClient_(registry),
+ pullTimeout_(timeout) {}
+
+
+Future<PulledLayerInfo> RegistryPullerProcess::downloadLayer(
+ const string& remoteName,
+ const Path& downloadDir,
+ const FileSystemLayerInfo& layer)
+{
+ VLOG(1) << "Downloading layer '" << layer.layerId
+ << "' for image '" << remoteName << "'";
+
+ Owned<Promise<PulledLayerInfo>> downloadPromise;
+
+ if (!downloadTracker_.contains(layer.layerId)) {
+ downloadPromise =
+ Owned<Promise<PulledLayerInfo>>(new Promise<PulledLayerInfo>());
+
+ downloadTracker_.insert({layer.layerId, downloadPromise});
+ } else {
+ VLOG(1) << "Download already in progress for image '" << remoteName
+ << "''s layer '" << layer.layerId << "'";
+
+ return downloadTracker_.at(layer.layerId)->future();
+ }
+
+ const Path downloadFile(path::join(downloadDir, layer.layerId));
+
+ registryClient_->getBlob(
+ remoteName,
+ layer.checksumInfo,
+ downloadFile)
+ .onAny(defer(
+ self(),
+ [this, remoteName, downloadDir, layer, downloadPromise, downloadFile](
+ const Future<size_t>& blobSizeFuture) {
+ downloadTracker_.erase(layer.layerId);
+
+ if(!blobSizeFuture.isReady()) {
+ if (blobSizeFuture.isDiscarded()) {
+ downloadPromise->fail(
+ "Failed to download layer '" + layer.layerId +
+ "': future discarded");
+ } else {
+ downloadPromise->fail(
+ "Failed to download layer '" + layer.layerId + "': " +
+ blobSizeFuture.failure());
+ }
+
+ return;
+ }
+
+ if (blobSizeFuture.get() <= 0) {
+ downloadPromise->fail(
+ "Failed to download layer '" + layer.layerId + "': no content");
+ return;
+ }
+
+ downloadPromise->set(PulledLayerInfo(layer.layerId, downloadFile));
+ }));
+
+ return downloadPromise->future();
+}
+
+
+Try<Image::Name> RegistryPullerProcess::getRemoteNameFromLocalName(
+ const Image::Name& imageName)
+{
+ //TODO(jojy): Canonical names could be something like "ubuntu14.04" but the
+ //remote name for it could be library/ubuntu14.04. This mapping has to come
+ //from a configuration in the near future. Docker has different namings
+ //depending on whether its an "official" repo or not.
+
+ // From the docker github code documentation:
+ // RepositoryInfo Examples:
+ // {
+ // "Index" : {
+ // "Name" : "docker.io",
+ // "Mirrors" : ["https://registry-2.docker.io/v1/",
+ // "https://registry-3.docker.io/v1/"],
+ // "Secure" : true,
+ // "Official" : true,
+ // },
+ // "RemoteName" : "library/debian",
+ // "LocalName" : "debian",
+ // "CanonicalName" : "docker.io/debian"
+ // "Official" : true,
+ // }
+ //
+ // {
+ // "Index" : {
+ // "Name" : "127.0.0.1:5000",
+ // "Mirrors" : [],
+ // "Secure" : false,
+ // "Official" : false,
+ // },
+ // "RemoteName" : "user/repo",
+ // "LocalName" : "127.0.0.1:5000/user/repo",
+ // "CanonicalName" : "127.0.0.1:5000/user/repo",
+ // "Official" : false,
+ // }
+
+ // For now, assuming that its an "official" repo.
+ if (strings::contains(imageName.repository(), "/")) {
+ return Error("Unexpected '/' in local image name");
+ }
+
+ // Create a remote Image::Name from the given input name.
+ Image::Name remoteImageName = imageName;
+ remoteImageName.set_repository("library/" + imageName.repository());
+
+ return remoteImageName;
+}
+
+
+Future<Nothing> untar(const string& file, const string& directory)
+{
+ const vector<string> argv = {
+ "tar",
+ "-C",
+ directory,
+ "-x",
+ "-f",
+ file
+ };
+
+ Try<Subprocess> s = subprocess(
+ "tar",
+ argv,
+ Subprocess::PATH("/dev/null"),
+ Subprocess::PATH("/dev/null"),
+ Subprocess::PATH("/dev/null"));
+
+ if (s.isError()) {
+ return Failure(
+ "Failed to create untar subprocess for file '" +
+ file + "': " + s.error());
+ }
+
+ return s.get().status()
+ .then([file](const Option<int>& status) -> Future<Nothing> {
+ if (status.isNone()) {
+ return Failure(
+ "Failed to reap untar subprocess for file '" + file + "'");
+ }
+
+ if (!WIFEXITED(status.get()) ||
+ WEXITSTATUS(status.get()) != 0) {
+ return Failure(
+ "Untar process for file '" + file + "' failed with exit code: " +
+ WSTRINGIFY(status.get()));
+ }
+
+ return Nothing();
+ });
+}
+
+
+Future<PulledImageInfo> RegistryPullerProcess::pull(
+ const Image::Name& imageName,
+ const Path& downloadDir)
+{
+ Try<Image::Name> remoteImageName = getRemoteNameFromLocalName(imageName);
+ if (remoteImageName.isError()) {
+ return Failure(
+ "Failed to get remote name from local name for '" +
+ imageName.repository() + "': " + remoteImageName.error());
+ }
+
+
+ // TODO(jojy): Delete downloaded files in the directory on discard and
+ // failure?
+ // TODO(jojy): Iterate through the futures and log the failed future.
+ return registryClient_->getManifest(remoteImageName.get())
+ .then(defer(self(), [this, downloadDir, remoteImageName](
+ const Manifest& manifest) {
+ list<Future<PulledLayerInfo>> downloadFutures;
+
+ foreach (const FileSystemLayerInfo& layer, manifest.fsLayerInfos) {
+ downloadFutures.push_back(downloadLayer(
+ remoteImageName.get().repository(),
+ downloadDir,
+ layer));
+ }
+
+ return collect(downloadFutures);
+ }))
+ .then([downloadDir]
+ (const Future<list<PulledLayerInfo>>& layerFutures)
+ -> Future<PulledImageInfo> {
+ list<Future<Nothing>> untarFutures;
+
+ foreach (const PulledLayerInfo layerInfo, layerFutures.get()) {
+ untarFutures.emplace_back(untar(layerInfo.second, downloadDir));
+ }
+
+ return collect(untarFutures)
+ .then([layerFutures]() {
+ PulledImageInfo layers;
+
+ foreach (const PulledLayerInfo layerInfo, layerFutures.get()) {
+ layers.emplace_back(layerInfo);
+ }
+
+ return layers;
+ });
+ });
+}
+
+} // namespace docker {
+} // namespace slave {
+} // namespace internal {
+} // namespace mesos {
http://git-wip-us.apache.org/repos/asf/mesos/blob/04361416/src/slave/containerizer/mesos/provisioner/docker/registry_puller.hpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/docker/registry_puller.hpp b/src/slave/containerizer/mesos/provisioner/docker/registry_puller.hpp
new file mode 100644
index 0000000..a664955
--- /dev/null
+++ b/src/slave/containerizer/mesos/provisioner/docker/registry_puller.hpp
@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __PROVISIONER_DOCKER_REGISTRY_PULLER_HPP__
+#define __PROVISIONER_DOCKER_REGISTRY_PULLER_HPP__
+
+#include <list>
+#include <string>
+#include <utility>
+
+#include <stout/duration.hpp>
+#include <stout/path.hpp>
+
+#include <process/future.hpp>
+#include <process/http.hpp>
+#include <process/process.hpp>
+
+#include "slave/containerizer/mesos/provisioner/docker/message.hpp"
+#include "slave/containerizer/mesos/provisioner/docker/puller.hpp"
+
+namespace mesos {
+namespace internal {
+namespace slave {
+namespace docker {
+
+// Forward declarations.
+class RegistryPullerProcess;
+
+/*
+ * Pulls an image from remote registry.
+ */
+class RegistryPuller : public Puller
+{
+public:
+ typedef std::pair<std::string, std::string> PulledLayerInfo;
+ typedef std::list<PulledLayerInfo> PulledImageInfo;
+
+ /**
+ * Factory method for creating RegistryPuller.
+ */
+ static Try<process::Owned<Puller>> create(const Flags& flags);
+
+ ~RegistryPuller();
+
+ /**
+ * Pulls an image into a download directory.
+ *
+ * @param imageName local name of the image.
+ * @param downloadDir path to which the layers should be downloaded.
+ */
+ process::Future<PulledImageInfo> pull(
+ const Image::Name& imageName,
+ const Path& downloadDir);
+
+
+private:
+ RegistryPuller(const process::Owned<RegistryPullerProcess>& process);
+
+ process::Owned<RegistryPullerProcess> process_;
+
+ RegistryPuller(const RegistryPuller&) = delete;
+ RegistryPuller& operator=(const RegistryPuller&) = delete;
+};
+
+} // namespace docker {
+} // namespace slave {
+} // namespace internal {
+} // namespace mesos {
+
+#endif // __PROVISIONER_DOCKER_REGISTRY_PULLER_HPP__
http://git-wip-us.apache.org/repos/asf/mesos/blob/04361416/src/slave/containerizer/mesos/provisioner/docker/remote_puller.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/docker/remote_puller.cpp b/src/slave/containerizer/mesos/provisioner/docker/remote_puller.cpp
deleted file mode 100644
index daee2f5..0000000
--- a/src/slave/containerizer/mesos/provisioner/docker/remote_puller.cpp
+++ /dev/null
@@ -1,402 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "slave/containerizer/mesos/provisioner/docker/remote_puller.hpp"
-
-#include <list>
-
-#include <process/collect.hpp>
-#include <process/defer.hpp>
-#include <process/dispatch.hpp>
-#include <process/subprocess.hpp>
-
-#include "common/status_utils.hpp"
-
-#include "slave/containerizer/mesos/provisioner/docker/registry_client.hpp"
-
-using std::list;
-using std::string;
-using std::vector;
-
-using process::Failure;
-using process::Future;
-using process::Owned;
-using process::Process;
-using process::Promise;
-using process::Subprocess;
-
-namespace http = process::http;
-
-namespace mesos {
-namespace internal {
-namespace slave {
-namespace docker {
-
-using FileSystemLayerInfo = registry::FileSystemLayerInfo;
-using Manifest = registry::Manifest;
-using RegistryClient = registry::RegistryClient;
-
-
-class RemotePullerProcess : public Process<RemotePullerProcess>
-{
-public:
- static Try<Owned<RemotePullerProcess>> create(const Flags& flags);
-
- process::Future<PulledImageInfo> pull(
- const Image::Name& imageName,
- const Path& downloadDir);
-
- Duration getTimeout() const
- {
- return pullTimeout_;
- }
-
-private:
- explicit RemotePullerProcess(
- const Owned<RegistryClient>& registry,
- const Duration& timeout);
-
- Try<Image::Name> getRemoteNameFromLocalName(const Image::Name& name);
-
- Future<PulledLayerInfo> downloadLayer(
- const string& remoteName,
- const Path& downloadDir,
- const FileSystemLayerInfo& layer);
-
- Owned<RegistryClient> registryClient_;
- const Duration pullTimeout_;
- hashmap<string, Owned<Promise<PulledLayerInfo>>> downloadTracker_;
-
- RemotePullerProcess(const RemotePullerProcess&) = delete;
- RemotePullerProcess& operator=(const RemotePullerProcess&) = delete;
-};
-
-
-Try<Owned<Puller>> RemotePuller::create(const Flags& flags)
-{
- Try<Owned<RemotePullerProcess>> process = RemotePullerProcess::create(flags);
-
- if (process.isError()) {
- return Error(process.error());
- }
-
- return Owned<Puller>(new RemotePuller(process.get()));
-}
-
-
-RemotePuller::RemotePuller(const Owned<RemotePullerProcess>& process)
- : process_(process)
-{
- spawn(CHECK_NOTNULL(process_.get()));
-}
-
-
-RemotePuller::~RemotePuller()
-{
- terminate(process_.get());
- process::wait(process_.get());
-}
-
-
-Future<PulledImageInfo> RemotePuller::pull(
- const Image::Name& imageName,
- const Path& downloadDir)
-{
- return dispatch(
- process_.get(),
- &RemotePullerProcess::pull,
- imageName,
- downloadDir)
- // We have 'after' here because the local puller need not have a timeout.
- // Timeout is a flag for the remote puller and needs to be handled here.
- .after(process_->getTimeout(), [imageName](
- Future<PulledImageInfo> future) {
- LOG(WARNING) << "Failed to download image '"
- << imageName.repository()
- << "': timeout";
- future.discard();
-
- return Failure(
- "Failed to download image '" + imageName.repository() + "': timeout");
- });
-}
-
-
-Try<Owned<RemotePullerProcess>> RemotePullerProcess::create(const Flags& flags)
-{
- Result<double> timeoutSecs = numify<double>(flags.docker_puller_timeout_secs);
- if ((timeoutSecs.isError()) || (timeoutSecs.get() <= 0)) {
- return Error(
- "Failed to create remote puller - invalid timeout value: " +
- flags.docker_puller_timeout_secs);
- }
-
- auto getPort = [](
- const string& stringVal) -> Try<uint16_t> {
- Result<uint16_t> port = numify<uint16_t>(stringVal);
- if (port.isError()) {
- return Error("Failed to parse '" + stringVal + "' as port");
- }
-
- return port.get();
- };
-
- // TODO(jojy): Maybe do a nslookup/tcp dial of these servers.
- Try<uint16_t> registryPort = getPort(flags.docker_registry_port);
- if (registryPort.isError()) {
- return Error("Failed to parse registry port: " + registryPort.error());
- }
-
- Try<uint16_t> authServerPort = getPort(flags.docker_auth_server_port);
- if (authServerPort.isError()) {
- return Error(
- "Failed to parse authentication server port: " +
- authServerPort.error());
- }
-
- Try<Owned<RegistryClient>> registry = RegistryClient::create(
- http::URL("https", flags.docker_registry, registryPort.get()),
- http::URL("https", flags.docker_auth_server, authServerPort.get()),
- None());
-
- if (registry.isError()) {
- return Error("Failed to create registry client: " + registry.error());
- }
-
- return Owned<RemotePullerProcess>(new RemotePullerProcess(
- registry.get(),
- Seconds(timeoutSecs.get())));
-}
-
-
-RemotePullerProcess::RemotePullerProcess(
- const Owned<RegistryClient>& registry,
- const Duration& timeout)
- : registryClient_(registry),
- pullTimeout_(timeout) {}
-
-
-Future<PulledLayerInfo> RemotePullerProcess::downloadLayer(
- const string& remoteName,
- const Path& downloadDir,
- const FileSystemLayerInfo& layer)
-{
- VLOG(1) << "Downloading layer '" << layer.layerId
- << "' for image '" << remoteName << "'";
-
- Owned<Promise<PulledLayerInfo>> downloadPromise;
-
- if (!downloadTracker_.contains(layer.layerId)) {
- downloadPromise =
- Owned<Promise<PulledLayerInfo>>(new Promise<PulledLayerInfo>());
-
- downloadTracker_.insert({layer.layerId, downloadPromise});
- } else {
- VLOG(1) << "Download already in progress for image '" << remoteName
- << "''s layer '" << layer.layerId << "'";
-
- return downloadTracker_.at(layer.layerId)->future();
- }
-
- const Path downloadFile(path::join(downloadDir, layer.layerId));
-
- registryClient_->getBlob(
- remoteName,
- layer.checksumInfo,
- downloadFile)
- .onAny(defer(
- self(),
- [this, remoteName, downloadDir, layer, downloadPromise, downloadFile](
- const Future<size_t>& blobSizeFuture) {
- downloadTracker_.erase(layer.layerId);
-
- if(!blobSizeFuture.isReady()) {
- if (blobSizeFuture.isDiscarded()) {
- downloadPromise->fail(
- "Failed to download layer '" + layer.layerId +
- "': future discarded");
- } else {
- downloadPromise->fail(
- "Failed to download layer '" + layer.layerId + "': " +
- blobSizeFuture.failure());
- }
-
- return;
- }
-
- if (blobSizeFuture.get() <= 0) {
- downloadPromise->fail(
- "Failed to download layer '" + layer.layerId + "': no content");
- return;
- }
-
- downloadPromise->set(PulledLayerInfo(layer.layerId, downloadFile));
- }));
-
- return downloadPromise->future();
-}
-
-
-Try<Image::Name> RemotePullerProcess::getRemoteNameFromLocalName(
- const Image::Name& imageName)
-{
- //TODO(jojy): Canonical names could be something like "ubuntu14.04" but the
- //remote name for it could be library/ubuntu14.04. This mapping has to come
- //from a configuration in the near future. Docker has different namings
- //depending on whether its an "official" repo or not.
-
- // From the docker github code documentation:
- // RepositoryInfo Examples:
- // {
- // "Index" : {
- // "Name" : "docker.io",
- // "Mirrors" : ["https://registry-2.docker.io/v1/",
- // "https://registry-3.docker.io/v1/"],
- // "Secure" : true,
- // "Official" : true,
- // },
- // "RemoteName" : "library/debian",
- // "LocalName" : "debian",
- // "CanonicalName" : "docker.io/debian"
- // "Official" : true,
- // }
- //
- // {
- // "Index" : {
- // "Name" : "127.0.0.1:5000",
- // "Mirrors" : [],
- // "Secure" : false,
- // "Official" : false,
- // },
- // "RemoteName" : "user/repo",
- // "LocalName" : "127.0.0.1:5000/user/repo",
- // "CanonicalName" : "127.0.0.1:5000/user/repo",
- // "Official" : false,
- // }
-
- // For now, assuming that its an "official" repo.
- if (strings::contains(imageName.repository(), "/")) {
- return Error("Unexpected '/' in local image name");
- }
-
- // Create a remote Image::Name from the given input name.
- Image::Name remoteImageName = imageName;
- remoteImageName.set_repository("library/" + imageName.repository());
-
- return remoteImageName;
-}
-
-
-Future<Nothing> untar(const string& file, const string& directory)
-{
- const vector<string> argv = {
- "tar",
- "-C",
- directory,
- "-x",
- "-f",
- file
- };
-
- Try<Subprocess> s = subprocess(
- "tar",
- argv,
- Subprocess::PATH("/dev/null"),
- Subprocess::PATH("/dev/null"),
- Subprocess::PATH("/dev/null"));
-
- if (s.isError()) {
- return Failure(
- "Failed to create untar subprocess for file '" +
- file + "': " + s.error());
- }
-
- return s.get().status()
- .then([file](const Option<int>& status) -> Future<Nothing> {
- if (status.isNone()) {
- return Failure(
- "Failed to reap untar subprocess for file '" + file + "'");
- }
-
- if (!WIFEXITED(status.get()) ||
- WEXITSTATUS(status.get()) != 0) {
- return Failure(
- "Untar process for file '" + file + "' failed with exit code: " +
- WSTRINGIFY(status.get()));
- }
-
- return Nothing();
- });
-}
-
-
-Future<PulledImageInfo> RemotePullerProcess::pull(
- const Image::Name& imageName,
- const Path& downloadDir)
-{
- Try<Image::Name> remoteImageName = getRemoteNameFromLocalName(imageName);
- if (remoteImageName.isError()) {
- return Failure(
- "Failed to get remote name from local name for '" +
- imageName.repository() + "': " + remoteImageName.error());
- }
-
-
- // TODO(jojy): Delete downloaded files in the directory on discard and
- // failure?
- // TODO(jojy): Iterate through the futures and log the failed future.
- return registryClient_->getManifest(remoteImageName.get())
- .then(defer(self(), [this, downloadDir, remoteImageName](
- const Manifest& manifest) {
- list<Future<PulledLayerInfo>> downloadFutures;
-
- foreach (const FileSystemLayerInfo& layer, manifest.fsLayerInfos) {
- downloadFutures.push_back(downloadLayer(
- remoteImageName.get().repository(),
- downloadDir,
- layer));
- }
-
- return collect(downloadFutures);
- }))
- .then([downloadDir]
- (const Future<list<PulledLayerInfo>>& layerFutures)
- -> Future<PulledImageInfo> {
- list<Future<Nothing>> untarFutures;
-
- foreach (const PulledLayerInfo layerInfo, layerFutures.get()) {
- untarFutures.emplace_back(untar(layerInfo.second, downloadDir));
- }
-
- return collect(untarFutures)
- .then([layerFutures]() {
- PulledImageInfo layers;
-
- foreach (const PulledLayerInfo layerInfo, layerFutures.get()) {
- layers.emplace_back(layerInfo);
- }
-
- return layers;
- });
- });
-}
-
-} // namespace docker {
-} // namespace slave {
-} // namespace internal {
-} // namespace mesos {
http://git-wip-us.apache.org/repos/asf/mesos/blob/04361416/src/slave/containerizer/mesos/provisioner/docker/remote_puller.hpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/docker/remote_puller.hpp b/src/slave/containerizer/mesos/provisioner/docker/remote_puller.hpp
deleted file mode 100644
index 5154ba5..0000000
--- a/src/slave/containerizer/mesos/provisioner/docker/remote_puller.hpp
+++ /dev/null
@@ -1,85 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef __PROVISIONER_DOCKER_REMOTE_PULLER_HPP__
-#define __PROVISIONER_DOCKER_REMOTE_PULLER_HPP__
-
-#include <list>
-#include <string>
-#include <utility>
-
-#include <stout/duration.hpp>
-#include <stout/path.hpp>
-
-#include <process/future.hpp>
-#include <process/http.hpp>
-#include <process/process.hpp>
-
-#include "slave/containerizer/mesos/provisioner/docker/message.hpp"
-#include "slave/containerizer/mesos/provisioner/docker/puller.hpp"
-
-namespace mesos {
-namespace internal {
-namespace slave {
-namespace docker {
-
-// Forward declarations.
-class RemotePullerProcess;
-
-/*
- * Pulls an image from remote registry.
- */
-class RemotePuller : public Puller
-{
-public:
- typedef std::pair<std::string, std::string> PulledLayerInfo;
- typedef std::list<PulledLayerInfo> PulledImageInfo;
-
- /**
- * Factory method for creating RemotePuller.
- */
- static Try<process::Owned<Puller>> create(const Flags& flags);
-
- ~RemotePuller();
-
- /**
- * Pulls an image into a download directory.
- *
- * @param imageName local name of the image.
- * @param downloadDir path to which the layers should be downloaded.
- */
- process::Future<PulledImageInfo> pull(
- const Image::Name& imageName,
- const Path& downloadDir);
-
-
-private:
- RemotePuller(const process::Owned<RemotePullerProcess>& process);
-
- process::Owned<RemotePullerProcess> process_;
-
- RemotePuller(const RemotePuller&) = delete;
- RemotePuller& operator=(const RemotePuller&) = delete;
-};
-
-} // namespace docker {
-} // namespace slave {
-} // namespace internal {
-} // namespace mesos {
-
-#endif // __PROVISIONER_DOCKER_REMOTE_PULLER_HPP__
http://git-wip-us.apache.org/repos/asf/mesos/blob/04361416/src/tests/containerizer/provisioner_docker_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/containerizer/provisioner_docker_tests.cpp b/src/tests/containerizer/provisioner_docker_tests.cpp
index fd6444a..27052b6 100644
--- a/src/tests/containerizer/provisioner_docker_tests.cpp
+++ b/src/tests/containerizer/provisioner_docker_tests.cpp
@@ -42,7 +42,7 @@
#include "slave/containerizer/mesos/provisioner/docker/metadata_manager.hpp"
#include "slave/containerizer/mesos/provisioner/docker/paths.hpp"
#include "slave/containerizer/mesos/provisioner/docker/registry_client.hpp"
-#include "slave/containerizer/mesos/provisioner/docker/remote_puller.hpp"
+#include "slave/containerizer/mesos/provisioner/docker/registry_puller.hpp"
#include "slave/containerizer/mesos/provisioner/docker/spec.hpp"
#include "slave/containerizer/mesos/provisioner/docker/store.hpp"
#include "slave/containerizer/mesos/provisioner/docker/token_manager.hpp"
@@ -988,9 +988,9 @@ TEST_F(RegistryClientTest, BadRequest)
}
-// Tests docker RemotePuller component. It simulates pulling an image layer from
-// remote registry and then verifies the content saved on disk.
-TEST_F(RegistryClientTest, SimpleRemotePuller)
+// Tests Docker RegistryPuller component. It simulates pulling an image layer
+// from Docker registry and then verifies the content saved on disk.
+TEST_F(RegistryClientTest, SimpleRegistryPuller)
{
Try<Socket> server = getServer();
@@ -1006,19 +1006,19 @@ TEST_F(RegistryClientTest, SimpleRemotePuller)
flags.docker_auth_server = server.get().address().get().hostname().get();
flags.docker_auth_server_port = stringify(server.get().address().get().port);
- Try<Owned<Puller>> remotePuller = RemotePuller::create(flags);
+ Try<Owned<Puller>> registryPuller = RegistryPuller::create(flags);
- ASSERT_SOME(remotePuller);
+ ASSERT_SOME(registryPuller);
- const Path remotePullerPath(RegistryClientTest::OUTPUT_DIR);
+ const Path registryPullerPath(RegistryClientTest::OUTPUT_DIR);
Try<slave::docker::Image::Name> imageName =
parseImageName("busybox");
ASSERT_SOME(imageName);
- Future<list<pair<string, string>>> remotePullerFuture =
- remotePuller.get()->pull(imageName.get(), remotePullerPath);
+ Future<list<pair<string, string>>> registryPullerFuture =
+ registryPuller.get()->pull(imageName.get(), registryPullerPath);
const string unauthResponseHeaders = "WWW-Authenticate: Bearer"
" realm=\"https://auth.docker.io/token\","
@@ -1033,8 +1033,8 @@ TEST_F(RegistryClientTest, SimpleRemotePuller)
AWAIT_ASSERT_READY(socket);
// Send 401 Unauthorized response for a manifest request.
- Future<string> remotePullerHttpRequestFuture = Socket(socket.get()).recv();
- AWAIT_ASSERT_READY(remotePullerHttpRequestFuture);
+ Future<string> registryPullerHttpRequestFuture = Socket(socket.get()).recv();
+ AWAIT_ASSERT_READY(registryPullerHttpRequestFuture);
AWAIT_ASSERT_READY(Socket(socket.get()).send(unauthHttpResponse));
// Token response.
@@ -1060,8 +1060,8 @@ TEST_F(RegistryClientTest, SimpleRemotePuller)
socket = server.get().accept();
AWAIT_ASSERT_READY(socket);
- remotePullerHttpRequestFuture = Socket(socket.get()).recv();
- AWAIT_ASSERT_READY(remotePullerHttpRequestFuture);
+ registryPullerHttpRequestFuture = Socket(socket.get()).recv();
+ AWAIT_ASSERT_READY(registryPullerHttpRequestFuture);
const string manifestResponse = " \
{ \
@@ -1124,8 +1124,8 @@ TEST_F(RegistryClientTest, SimpleRemotePuller)
socket = server.get().accept();
AWAIT_ASSERT_READY(socket);
- remotePullerHttpRequestFuture = Socket(socket.get()).recv();
- AWAIT_ASSERT_READY(remotePullerHttpRequestFuture);
+ registryPullerHttpRequestFuture = Socket(socket.get()).recv();
+ AWAIT_ASSERT_READY(registryPullerHttpRequestFuture);
const string redirectHttpResponse =
string("HTTP/1.1 307 Temporary Redirect\r\n") +
@@ -1140,15 +1140,15 @@ TEST_F(RegistryClientTest, SimpleRemotePuller)
const string BLOB_FILE = "blob";
const string blobResponse = "hello docker";
- Path blobPath(path::join(remotePullerPath, BLOB_FILE));
+ Path blobPath(path::join(registryPullerPath, BLOB_FILE));
ASSERT_SOME(os::write(blobPath, blobResponse));
- Path blobTarPath(path::join(remotePullerPath, BLOB_FILE + ".tar"));
+ Path blobTarPath(path::join(registryPullerPath, BLOB_FILE + ".tar"));
vector<string> argv = {
"tar",
"-C",
- remotePullerPath,
+ registryPullerPath,
"-c",
"-f",
blobTarPath,
@@ -1212,14 +1212,14 @@ TEST_F(RegistryClientTest, SimpleRemotePuller)
socket = server.get().accept();
AWAIT_ASSERT_READY(socket);
- remotePullerHttpRequestFuture = Socket(socket.get()).recv();
- AWAIT_ASSERT_READY(remotePullerHttpRequestFuture);
+ registryPullerHttpRequestFuture = Socket(socket.get()).recv();
+ AWAIT_ASSERT_READY(registryPullerHttpRequestFuture);
AWAIT_ASSERT_READY(Socket(socket.get()).send(
responseBuffer.get(),
blobResponseSize));
- AWAIT_ASSERT_READY(remotePullerFuture);
+ AWAIT_ASSERT_READY(registryPullerFuture);
Try<string> blob = os::read(blobPath);
ASSERT_SOME(blob);