You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by jo...@apache.org on 2015/11/09 14:54:03 UTC
[2/2] mesos git commit: Revert "Added Docker RemotePuller."
Revert "Added Docker RemotePuller."
This reverts commit fff91d7ac1157b71ca5173b1f50cd05216ae59c0.
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/cee49589
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/cee49589
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/cee49589
Branch: refs/heads/master
Commit: cee49589840d58b27921f88f23db3ef70d91c08d
Parents: a74ebf2
Author: Joris Van Remoortere <jo...@gmail.com>
Authored: Mon Nov 9 14:46:56 2015 +0100
Committer: Joris Van Remoortere <jo...@gmail.com>
Committed: Mon Nov 9 14:46:56 2015 +0100
----------------------------------------------------------------------
src/CMakeLists.txt | 1 -
src/Makefile.am | 2 -
.../mesos/provisioner/docker/local_puller.cpp | 2 +-
.../mesos/provisioner/docker/local_puller.hpp | 2 +-
.../mesos/provisioner/docker/puller.cpp | 10 -
.../mesos/provisioner/docker/puller.hpp | 5 +-
.../provisioner/docker/registry_client.hpp | 2 +
.../mesos/provisioner/docker/remote_puller.cpp | 402 -------------------
.../mesos/provisioner/docker/remote_puller.hpp | 85 ----
.../mesos/provisioner/docker/store.cpp | 2 +-
src/slave/flags.cpp | 25 --
src/slave/flags.hpp | 5 -
.../containerizer/provisioner_docker_tests.cpp | 277 +------------
13 files changed, 28 insertions(+), 792 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/cee49589/src/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 74bfa91..cbc25e3 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -208,7 +208,6 @@ if (NOT WIN32)
slave/containerizer/mesos/provisioner/backends/copy.cpp
slave/containerizer/mesos/provisioner/docker/message.proto
slave/containerizer/mesos/provisioner/docker/registry_client.cpp
- slave/containerizer/mesos/provisioner/docker/remote_puller.cpp
slave/containerizer/mesos/provisioner/docker/spec.cpp
slave/containerizer/mesos/provisioner/docker/token_manager.cpp
slave/resource_estimators/noop.cpp
http://git-wip-us.apache.org/repos/asf/mesos/blob/cee49589/src/Makefile.am
----------------------------------------------------------------------
diff --git a/src/Makefile.am b/src/Makefile.am
index 004ab31..938b8d4 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -562,7 +562,6 @@ libmesos_no_3rdparty_la_SOURCES = \
slave/containerizer/mesos/provisioner/docker/paths.cpp \
slave/containerizer/mesos/provisioner/docker/puller.cpp \
slave/containerizer/mesos/provisioner/docker/registry_client.cpp \
- slave/containerizer/mesos/provisioner/docker/remote_puller.cpp \
slave/containerizer/mesos/provisioner/docker/spec.cpp \
slave/containerizer/mesos/provisioner/docker/store.cpp \
slave/containerizer/mesos/provisioner/docker/token_manager.cpp \
@@ -884,7 +883,6 @@ libmesos_no_3rdparty_la_SOURCES += \
slave/containerizer/mesos/provisioner/docker/paths.hpp \
slave/containerizer/mesos/provisioner/docker/puller.hpp \
slave/containerizer/mesos/provisioner/docker/registry_client.hpp \
- slave/containerizer/mesos/provisioner/docker/remote_puller.hpp \
slave/containerizer/mesos/provisioner/docker/spec.hpp \
slave/containerizer/mesos/provisioner/docker/store.hpp \
slave/containerizer/mesos/provisioner/docker/token_manager.hpp \
http://git-wip-us.apache.org/repos/asf/mesos/blob/cee49589/src/slave/containerizer/mesos/provisioner/docker/local_puller.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/docker/local_puller.cpp b/src/slave/containerizer/mesos/provisioner/docker/local_puller.cpp
index 9b2b812..f314f20 100644
--- a/src/slave/containerizer/mesos/provisioner/docker/local_puller.cpp
+++ b/src/slave/containerizer/mesos/provisioner/docker/local_puller.cpp
@@ -96,7 +96,7 @@ LocalPuller::~LocalPuller()
Future<list<pair<string, string>>> LocalPuller::pull(
const Image::Name& name,
- const Path& directory)
+ const string& directory)
{
return dispatch(process.get(), &LocalPullerProcess::pull, name, directory);
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/cee49589/src/slave/containerizer/mesos/provisioner/docker/local_puller.hpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/docker/local_puller.hpp b/src/slave/containerizer/mesos/provisioner/docker/local_puller.hpp
index 103fe4b..87d8002 100644
--- a/src/slave/containerizer/mesos/provisioner/docker/local_puller.hpp
+++ b/src/slave/containerizer/mesos/provisioner/docker/local_puller.hpp
@@ -49,7 +49,7 @@ public:
process::Future<std::list<std::pair<std::string, std::string>>> pull(
const Image::Name& name,
- const Path& directory);
+ const std::string& directory);
private:
LocalPuller& operator=(const LocalPuller&) = delete; // Not assignable.
http://git-wip-us.apache.org/repos/asf/mesos/blob/cee49589/src/slave/containerizer/mesos/provisioner/docker/puller.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/docker/puller.cpp b/src/slave/containerizer/mesos/provisioner/docker/puller.cpp
index ae644ec..f61f9e5 100644
--- a/src/slave/containerizer/mesos/provisioner/docker/puller.cpp
+++ b/src/slave/containerizer/mesos/provisioner/docker/puller.cpp
@@ -19,7 +19,6 @@
#include "slave/containerizer/mesos/provisioner/docker/puller.hpp"
#include "slave/containerizer/mesos/provisioner/docker/local_puller.hpp"
-#include "slave/containerizer/mesos/provisioner/docker/remote_puller.hpp"
using std::string;
@@ -38,15 +37,6 @@ Try<Owned<Puller>> Puller::create(const Flags& flags)
return Owned<Puller>(new LocalPuller(flags));
}
- if (puller == "remote") {
- Try<Owned<Puller>> puller = RemotePuller::create(flags);
- if (puller.isError()) {
- return Error("Failed to create remote puller: " + puller.error());
- }
-
- return puller.get();
- }
-
return Error("Unknown or unsupported docker puller: " + puller);
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/cee49589/src/slave/containerizer/mesos/provisioner/docker/puller.hpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/docker/puller.hpp b/src/slave/containerizer/mesos/provisioner/docker/puller.hpp
index e420255..8010b8a 100644
--- a/src/slave/containerizer/mesos/provisioner/docker/puller.hpp
+++ b/src/slave/containerizer/mesos/provisioner/docker/puller.hpp
@@ -22,9 +22,6 @@
#include <list>
#include <utility>
-#include <stout/duration.hpp>
-#include <stout/option.hpp>
-#include <stout/path.hpp>
#include <stout/try.hpp>
#include <process/future.hpp>
@@ -59,7 +56,7 @@ public:
*/
virtual process::Future<std::list<std::pair<std::string, std::string>>> pull(
const docker::Image::Name& name,
- const Path& directory) = 0;
+ const std::string& directory) = 0;
};
} // namespace docker {
http://git-wip-us.apache.org/repos/asf/mesos/blob/cee49589/src/slave/containerizer/mesos/provisioner/docker/registry_client.hpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/docker/registry_client.hpp b/src/slave/containerizer/mesos/provisioner/docker/registry_client.hpp
index 7bba371..bc487d3 100644
--- a/src/slave/containerizer/mesos/provisioner/docker/registry_client.hpp
+++ b/src/slave/containerizer/mesos/provisioner/docker/registry_client.hpp
@@ -115,6 +115,8 @@ public:
* @param path path of the repository on the registry.
* @param digest digest of the blob (from manifest).
* @param filePath file path to store the fetched blob.
+ * @param maxSize Maximum size of the response thats acceptable. Will default
+ * to MAX_RESPONSE_SIZE.
* @return size of downloaded blob on success.
* Failure in case of any errors.
*/
http://git-wip-us.apache.org/repos/asf/mesos/blob/cee49589/src/slave/containerizer/mesos/provisioner/docker/remote_puller.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/docker/remote_puller.cpp b/src/slave/containerizer/mesos/provisioner/docker/remote_puller.cpp
deleted file mode 100644
index daee2f5..0000000
--- a/src/slave/containerizer/mesos/provisioner/docker/remote_puller.cpp
+++ /dev/null
@@ -1,402 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "slave/containerizer/mesos/provisioner/docker/remote_puller.hpp"
-
-#include <list>
-
-#include <process/collect.hpp>
-#include <process/defer.hpp>
-#include <process/dispatch.hpp>
-#include <process/subprocess.hpp>
-
-#include "common/status_utils.hpp"
-
-#include "slave/containerizer/mesos/provisioner/docker/registry_client.hpp"
-
-using std::list;
-using std::string;
-using std::vector;
-
-using process::Failure;
-using process::Future;
-using process::Owned;
-using process::Process;
-using process::Promise;
-using process::Subprocess;
-
-namespace http = process::http;
-
-namespace mesos {
-namespace internal {
-namespace slave {
-namespace docker {
-
-using FileSystemLayerInfo = registry::FileSystemLayerInfo;
-using Manifest = registry::Manifest;
-using RegistryClient = registry::RegistryClient;
-
-
-class RemotePullerProcess : public Process<RemotePullerProcess>
-{
-public:
- static Try<Owned<RemotePullerProcess>> create(const Flags& flags);
-
- process::Future<PulledImageInfo> pull(
- const Image::Name& imageName,
- const Path& downloadDir);
-
- Duration getTimeout() const
- {
- return pullTimeout_;
- }
-
-private:
- explicit RemotePullerProcess(
- const Owned<RegistryClient>& registry,
- const Duration& timeout);
-
- Try<Image::Name> getRemoteNameFromLocalName(const Image::Name& name);
-
- Future<PulledLayerInfo> downloadLayer(
- const string& remoteName,
- const Path& downloadDir,
- const FileSystemLayerInfo& layer);
-
- Owned<RegistryClient> registryClient_;
- const Duration pullTimeout_;
- hashmap<string, Owned<Promise<PulledLayerInfo>>> downloadTracker_;
-
- RemotePullerProcess(const RemotePullerProcess&) = delete;
- RemotePullerProcess& operator=(const RemotePullerProcess&) = delete;
-};
-
-
-Try<Owned<Puller>> RemotePuller::create(const Flags& flags)
-{
- Try<Owned<RemotePullerProcess>> process = RemotePullerProcess::create(flags);
-
- if (process.isError()) {
- return Error(process.error());
- }
-
- return Owned<Puller>(new RemotePuller(process.get()));
-}
-
-
-RemotePuller::RemotePuller(const Owned<RemotePullerProcess>& process)
- : process_(process)
-{
- spawn(CHECK_NOTNULL(process_.get()));
-}
-
-
-RemotePuller::~RemotePuller()
-{
- terminate(process_.get());
- process::wait(process_.get());
-}
-
-
-Future<PulledImageInfo> RemotePuller::pull(
- const Image::Name& imageName,
- const Path& downloadDir)
-{
- return dispatch(
- process_.get(),
- &RemotePullerProcess::pull,
- imageName,
- downloadDir)
- // We have 'after' here because the local puller need not have a timeout.
- // Timeout is a flag for the remote puller and needs to be handled here.
- .after(process_->getTimeout(), [imageName](
- Future<PulledImageInfo> future) {
- LOG(WARNING) << "Failed to download image '"
- << imageName.repository()
- << "': timeout";
- future.discard();
-
- return Failure(
- "Failed to download image '" + imageName.repository() + "': timeout");
- });
-}
-
-
-Try<Owned<RemotePullerProcess>> RemotePullerProcess::create(const Flags& flags)
-{
- Result<double> timeoutSecs = numify<double>(flags.docker_puller_timeout_secs);
- if ((timeoutSecs.isError()) || (timeoutSecs.get() <= 0)) {
- return Error(
- "Failed to create remote puller - invalid timeout value: " +
- flags.docker_puller_timeout_secs);
- }
-
- auto getPort = [](
- const string& stringVal) -> Try<uint16_t> {
- Result<uint16_t> port = numify<uint16_t>(stringVal);
- if (port.isError()) {
- return Error("Failed to parse '" + stringVal + "' as port");
- }
-
- return port.get();
- };
-
- // TODO(jojy): Maybe do a nslookup/tcp dial of these servers.
- Try<uint16_t> registryPort = getPort(flags.docker_registry_port);
- if (registryPort.isError()) {
- return Error("Failed to parse registry port: " + registryPort.error());
- }
-
- Try<uint16_t> authServerPort = getPort(flags.docker_auth_server_port);
- if (authServerPort.isError()) {
- return Error(
- "Failed to parse authentication server port: " +
- authServerPort.error());
- }
-
- Try<Owned<RegistryClient>> registry = RegistryClient::create(
- http::URL("https", flags.docker_registry, registryPort.get()),
- http::URL("https", flags.docker_auth_server, authServerPort.get()),
- None());
-
- if (registry.isError()) {
- return Error("Failed to create registry client: " + registry.error());
- }
-
- return Owned<RemotePullerProcess>(new RemotePullerProcess(
- registry.get(),
- Seconds(timeoutSecs.get())));
-}
-
-
-RemotePullerProcess::RemotePullerProcess(
- const Owned<RegistryClient>& registry,
- const Duration& timeout)
- : registryClient_(registry),
- pullTimeout_(timeout) {}
-
-
-Future<PulledLayerInfo> RemotePullerProcess::downloadLayer(
- const string& remoteName,
- const Path& downloadDir,
- const FileSystemLayerInfo& layer)
-{
- VLOG(1) << "Downloading layer '" << layer.layerId
- << "' for image '" << remoteName << "'";
-
- Owned<Promise<PulledLayerInfo>> downloadPromise;
-
- if (!downloadTracker_.contains(layer.layerId)) {
- downloadPromise =
- Owned<Promise<PulledLayerInfo>>(new Promise<PulledLayerInfo>());
-
- downloadTracker_.insert({layer.layerId, downloadPromise});
- } else {
- VLOG(1) << "Download already in progress for image '" << remoteName
- << "''s layer '" << layer.layerId << "'";
-
- return downloadTracker_.at(layer.layerId)->future();
- }
-
- const Path downloadFile(path::join(downloadDir, layer.layerId));
-
- registryClient_->getBlob(
- remoteName,
- layer.checksumInfo,
- downloadFile)
- .onAny(defer(
- self(),
- [this, remoteName, downloadDir, layer, downloadPromise, downloadFile](
- const Future<size_t>& blobSizeFuture) {
- downloadTracker_.erase(layer.layerId);
-
- if(!blobSizeFuture.isReady()) {
- if (blobSizeFuture.isDiscarded()) {
- downloadPromise->fail(
- "Failed to download layer '" + layer.layerId +
- "': future discarded");
- } else {
- downloadPromise->fail(
- "Failed to download layer '" + layer.layerId + "': " +
- blobSizeFuture.failure());
- }
-
- return;
- }
-
- if (blobSizeFuture.get() <= 0) {
- downloadPromise->fail(
- "Failed to download layer '" + layer.layerId + "': no content");
- return;
- }
-
- downloadPromise->set(PulledLayerInfo(layer.layerId, downloadFile));
- }));
-
- return downloadPromise->future();
-}
-
-
-Try<Image::Name> RemotePullerProcess::getRemoteNameFromLocalName(
- const Image::Name& imageName)
-{
- //TODO(jojy): Canonical names could be something like "ubuntu14.04" but the
- //remote name for it could be library/ubuntu14.04. This mapping has to come
- //from a configuration in the near future. Docker has different namings
- //depending on whether its an "official" repo or not.
-
- // From the docker github code documentation:
- // RepositoryInfo Examples:
- // {
- // "Index" : {
- // "Name" : "docker.io",
- // "Mirrors" : ["https://registry-2.docker.io/v1/",
- // "https://registry-3.docker.io/v1/"],
- // "Secure" : true,
- // "Official" : true,
- // },
- // "RemoteName" : "library/debian",
- // "LocalName" : "debian",
- // "CanonicalName" : "docker.io/debian"
- // "Official" : true,
- // }
- //
- // {
- // "Index" : {
- // "Name" : "127.0.0.1:5000",
- // "Mirrors" : [],
- // "Secure" : false,
- // "Official" : false,
- // },
- // "RemoteName" : "user/repo",
- // "LocalName" : "127.0.0.1:5000/user/repo",
- // "CanonicalName" : "127.0.0.1:5000/user/repo",
- // "Official" : false,
- // }
-
- // For now, assuming that its an "official" repo.
- if (strings::contains(imageName.repository(), "/")) {
- return Error("Unexpected '/' in local image name");
- }
-
- // Create a remote Image::Name from the given input name.
- Image::Name remoteImageName = imageName;
- remoteImageName.set_repository("library/" + imageName.repository());
-
- return remoteImageName;
-}
-
-
-Future<Nothing> untar(const string& file, const string& directory)
-{
- const vector<string> argv = {
- "tar",
- "-C",
- directory,
- "-x",
- "-f",
- file
- };
-
- Try<Subprocess> s = subprocess(
- "tar",
- argv,
- Subprocess::PATH("/dev/null"),
- Subprocess::PATH("/dev/null"),
- Subprocess::PATH("/dev/null"));
-
- if (s.isError()) {
- return Failure(
- "Failed to create untar subprocess for file '" +
- file + "': " + s.error());
- }
-
- return s.get().status()
- .then([file](const Option<int>& status) -> Future<Nothing> {
- if (status.isNone()) {
- return Failure(
- "Failed to reap untar subprocess for file '" + file + "'");
- }
-
- if (!WIFEXITED(status.get()) ||
- WEXITSTATUS(status.get()) != 0) {
- return Failure(
- "Untar process for file '" + file + "' failed with exit code: " +
- WSTRINGIFY(status.get()));
- }
-
- return Nothing();
- });
-}
-
-
-Future<PulledImageInfo> RemotePullerProcess::pull(
- const Image::Name& imageName,
- const Path& downloadDir)
-{
- Try<Image::Name> remoteImageName = getRemoteNameFromLocalName(imageName);
- if (remoteImageName.isError()) {
- return Failure(
- "Failed to get remote name from local name for '" +
- imageName.repository() + "': " + remoteImageName.error());
- }
-
-
- // TODO(jojy): Delete downloaded files in the directory on discard and
- // failure?
- // TODO(jojy): Iterate through the futures and log the failed future.
- return registryClient_->getManifest(remoteImageName.get())
- .then(defer(self(), [this, downloadDir, remoteImageName](
- const Manifest& manifest) {
- list<Future<PulledLayerInfo>> downloadFutures;
-
- foreach (const FileSystemLayerInfo& layer, manifest.fsLayerInfos) {
- downloadFutures.push_back(downloadLayer(
- remoteImageName.get().repository(),
- downloadDir,
- layer));
- }
-
- return collect(downloadFutures);
- }))
- .then([downloadDir]
- (const Future<list<PulledLayerInfo>>& layerFutures)
- -> Future<PulledImageInfo> {
- list<Future<Nothing>> untarFutures;
-
- foreach (const PulledLayerInfo layerInfo, layerFutures.get()) {
- untarFutures.emplace_back(untar(layerInfo.second, downloadDir));
- }
-
- return collect(untarFutures)
- .then([layerFutures]() {
- PulledImageInfo layers;
-
- foreach (const PulledLayerInfo layerInfo, layerFutures.get()) {
- layers.emplace_back(layerInfo);
- }
-
- return layers;
- });
- });
-}
-
-} // namespace docker {
-} // namespace slave {
-} // namespace internal {
-} // namespace mesos {
http://git-wip-us.apache.org/repos/asf/mesos/blob/cee49589/src/slave/containerizer/mesos/provisioner/docker/remote_puller.hpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/docker/remote_puller.hpp b/src/slave/containerizer/mesos/provisioner/docker/remote_puller.hpp
deleted file mode 100644
index 5154ba5..0000000
--- a/src/slave/containerizer/mesos/provisioner/docker/remote_puller.hpp
+++ /dev/null
@@ -1,85 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef __PROVISIONER_DOCKER_REMOTE_PULLER_HPP__
-#define __PROVISIONER_DOCKER_REMOTE_PULLER_HPP__
-
-#include <list>
-#include <string>
-#include <utility>
-
-#include <stout/duration.hpp>
-#include <stout/path.hpp>
-
-#include <process/future.hpp>
-#include <process/http.hpp>
-#include <process/process.hpp>
-
-#include "slave/containerizer/mesos/provisioner/docker/message.hpp"
-#include "slave/containerizer/mesos/provisioner/docker/puller.hpp"
-
-namespace mesos {
-namespace internal {
-namespace slave {
-namespace docker {
-
-// Forward declarations.
-class RemotePullerProcess;
-
-/*
- * Pulls an image from remote registry.
- */
-class RemotePuller : public Puller
-{
-public:
- typedef std::pair<std::string, std::string> PulledLayerInfo;
- typedef std::list<PulledLayerInfo> PulledImageInfo;
-
- /**
- * Factory method for creating RemotePuller.
- */
- static Try<process::Owned<Puller>> create(const Flags& flags);
-
- ~RemotePuller();
-
- /**
- * Pulls an image into a download directory.
- *
- * @param imageName local name of the image.
- * @param downloadDir path to which the layers should be downloaded.
- */
- process::Future<PulledImageInfo> pull(
- const Image::Name& imageName,
- const Path& downloadDir);
-
-
-private:
- RemotePuller(const process::Owned<RemotePullerProcess>& process);
-
- process::Owned<RemotePullerProcess> process_;
-
- RemotePuller(const RemotePuller&) = delete;
- RemotePuller& operator=(const RemotePuller&) = delete;
-};
-
-} // namespace docker {
-} // namespace slave {
-} // namespace internal {
-} // namespace mesos {
-
-#endif // __PROVISIONER_DOCKER_REMOTE_PULLER_HPP__
http://git-wip-us.apache.org/repos/asf/mesos/blob/cee49589/src/slave/containerizer/mesos/provisioner/docker/store.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/docker/store.cpp b/src/slave/containerizer/mesos/provisioner/docker/store.cpp
index a4a4d13..bb02d65 100644
--- a/src/slave/containerizer/mesos/provisioner/docker/store.cpp
+++ b/src/slave/containerizer/mesos/provisioner/docker/store.cpp
@@ -180,7 +180,7 @@ Future<Image> StoreProcess::_get(
return Failure("Failed to create a staging directory");
}
- return puller->pull(name, Path(staging.get()))
+ return puller->pull(name, staging.get())
.then(defer(self(), &Self::moveLayers, staging.get(), lambda::_1))
.then(defer(self(), &Self::storeImage, name, lambda::_1))
.onAny([staging]() {
http://git-wip-us.apache.org/repos/asf/mesos/blob/cee49589/src/slave/flags.cpp
----------------------------------------------------------------------
diff --git a/src/slave/flags.cpp b/src/slave/flags.cpp
index 70aaeaa..825b612 100644
--- a/src/slave/flags.cpp
+++ b/src/slave/flags.cpp
@@ -93,16 +93,6 @@ mesos::internal::slave::Flags::Flags()
"Directory the appc provisioner will store images in.",
"/tmp/mesos/store/appc");
- add(&Flags::docker_auth_server,
- "docker_auth_server",
- "Docker authentication server",
- "auth.docker.io");
-
- add(&Flags::docker_auth_server_port,
- "docker_auth_server_port",
- "Docker authentication server port",
- "443");
-
add(&Flags::docker_local_archives_dir,
"docker_local_archives_dir",
"Directory for docker local puller to look in for image archives",
@@ -113,21 +103,6 @@ mesos::internal::slave::Flags::Flags()
"Strategy for docker puller to fetch images",
"local");
- add(&Flags::docker_puller_timeout_secs,
- "docker_puller_timeout",
- "Timeout value in seconds for pulling images from Docker registry",
- "60");
-
- add(&Flags::docker_registry,
- "docker_registry",
- "Default Docker image registry server host",
- "registry-1.docker.io");
-
- add(&Flags::docker_registry_port,
- "docker_registry_port",
- "Default Docker registry server port",
- "443");
-
add(&Flags::docker_store_dir,
"docker_store_dir",
"Directory the docker provisioner will store images in",
http://git-wip-us.apache.org/repos/asf/mesos/blob/cee49589/src/slave/flags.hpp
----------------------------------------------------------------------
diff --git a/src/slave/flags.hpp b/src/slave/flags.hpp
index 6ae7c94..3e93b52 100644
--- a/src/slave/flags.hpp
+++ b/src/slave/flags.hpp
@@ -53,13 +53,8 @@ public:
std::string image_provisioner_backend;
std::string appc_store_dir;
- std::string docker_auth_server;
- std::string docker_auth_server_port;
std::string docker_local_archives_dir;
std::string docker_puller;
- std::string docker_puller_timeout_secs;
- std::string docker_registry;
- std::string docker_registry_port;
std::string docker_store_dir;
std::string default_role;
http://git-wip-us.apache.org/repos/asf/mesos/blob/cee49589/src/tests/containerizer/provisioner_docker_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/containerizer/provisioner_docker_tests.cpp b/src/tests/containerizer/provisioner_docker_tests.cpp
index fd6444a..fe6a90f 100644
--- a/src/tests/containerizer/provisioner_docker_tests.cpp
+++ b/src/tests/containerizer/provisioner_docker_tests.cpp
@@ -16,8 +16,6 @@
* limitations under the License.
*/
-#include <utility>
-
#include <gmock/gmock.h>
#include <gtest/gtest.h>
@@ -42,7 +40,6 @@
#include "slave/containerizer/mesos/provisioner/docker/metadata_manager.hpp"
#include "slave/containerizer/mesos/provisioner/docker/paths.hpp"
#include "slave/containerizer/mesos/provisioner/docker/registry_client.hpp"
-#include "slave/containerizer/mesos/provisioner/docker/remote_puller.hpp"
#include "slave/containerizer/mesos/provisioner/docker/spec.hpp"
#include "slave/containerizer/mesos/provisioner/docker/store.hpp"
#include "slave/containerizer/mesos/provisioner/docker/token_manager.hpp"
@@ -52,7 +49,6 @@
using std::list;
using std::map;
-using std::pair;
using std::string;
using std::vector;
@@ -515,13 +511,6 @@ class RegistryClientTest : public virtual SSLTest, public TokenHelper
protected:
RegistryClientTest() {}
- Try<Socket> getServer() {
- return setup_server({
- {"SSL_ENABLED", "true"},
- {"SSL_KEY_FILE", key_path().value},
- {"SSL_CERT_FILE", certificate_path().value}});
- }
-
static void SetUpTestCase()
{
if (os::mkdir(RegistryClientTest::OUTPUT_DIR).isError()) {
@@ -543,7 +532,10 @@ const string RegistryClientTest::OUTPUT_DIR = "output_dir";
// Tests TokenManager for a simple token request.
TEST_F(RegistryClientTest, SimpleGetToken)
{
- Try<Socket> server = getServer();
+ Try<Socket> server = setup_server({
+ {"SSL_ENABLED", "true"},
+ {"SSL_KEY_FILE", key_path().value},
+ {"SSL_CERT_FILE", certificate_path().value}});
ASSERT_SOME(server);
ASSERT_SOME(server.get().address());
@@ -607,7 +599,10 @@ TEST_F(RegistryClientTest, SimpleGetToken)
// Tests TokenManager for bad token response from server.
TEST_F(RegistryClientTest, BadTokenResponse)
{
- Try<Socket> server = getServer();
+ Try<Socket> server = setup_server({
+ {"SSL_ENABLED", "true"},
+ {"SSL_KEY_FILE", key_path().value},
+ {"SSL_CERT_FILE", certificate_path().value}});
ASSERT_SOME(server);
ASSERT_SOME(server.get().address());
@@ -670,7 +665,10 @@ TEST_F(RegistryClientTest, BadTokenServerAddress)
// Tests docker registry's getManifest API.
TEST_F(RegistryClientTest, SimpleGetManifest)
{
- Try<Socket> server = getServer();
+ Try<Socket> server = setup_server({
+ {"SSL_ENABLED", "true"},
+ {"SSL_KEY_FILE", key_path().value},
+ {"SSL_CERT_FILE", certificate_path().value}});
ASSERT_SOME(server);
ASSERT_SOME(server.get().address());
@@ -832,9 +830,13 @@ TEST_F(RegistryClientTest, SimpleGetManifest)
// Tests docker registry's getBlob API.
-TEST_F(RegistryClientTest, SimpleGetBlob)
+// TODO(jojy): Re-enable this test when MESOS-3798 is resolved.
+TEST_F(RegistryClientTest, DISABLED_SimpleGetBlob)
{
- Try<Socket> server = getServer();
+ Try<Socket> server = setup_server({
+ {"SSL_ENABLED", "true"},
+ {"SSL_KEY_FILE", key_path().value},
+ {"SSL_CERT_FILE", certificate_path().value}});
ASSERT_SOME(server);
ASSERT_SOME(server.get().address());
@@ -939,7 +941,10 @@ TEST_F(RegistryClientTest, SimpleGetBlob)
TEST_F(RegistryClientTest, BadRequest)
{
- Try<Socket> server = getServer();
+ Try<Socket> server = setup_server({
+ {"SSL_ENABLED", "true"},
+ {"SSL_KEY_FILE", key_path().value},
+ {"SSL_CERT_FILE", certificate_path().value}});
ASSERT_SOME(server);
ASSERT_SOME(server.get().address());
@@ -988,244 +993,6 @@ TEST_F(RegistryClientTest, BadRequest)
}
-// Tests docker RemotePuller component. It simulates pulling an image layer from
-// remote registry and then verifies the content saved on disk.
-TEST_F(RegistryClientTest, SimpleRemotePuller)
-{
- Try<Socket> server = getServer();
-
- ASSERT_SOME(server);
- ASSERT_SOME(server.get().address());
- ASSERT_SOME(server.get().address().get().hostname());
-
- Future<Socket> socket = server.get().accept();
-
- Flags flags;
- flags.docker_registry = server.get().address().get().hostname().get();
- flags.docker_registry_port = stringify(server.get().address().get().port);
- flags.docker_auth_server = server.get().address().get().hostname().get();
- flags.docker_auth_server_port = stringify(server.get().address().get().port);
-
- Try<Owned<Puller>> remotePuller = RemotePuller::create(flags);
-
- ASSERT_SOME(remotePuller);
-
- const Path remotePullerPath(RegistryClientTest::OUTPUT_DIR);
-
- Try<slave::docker::Image::Name> imageName =
- parseImageName("busybox");
-
- ASSERT_SOME(imageName);
-
- Future<list<pair<string, string>>> remotePullerFuture =
- remotePuller.get()->pull(imageName.get(), remotePullerPath);
-
- const string unauthResponseHeaders = "WWW-Authenticate: Bearer"
- " realm=\"https://auth.docker.io/token\","
- "service=" + stringify(server.get().address().get()) + ","
- "scope=\"repository:library/busybox:pull\"";
-
- const string unauthHttpResponse =
- string("HTTP/1.1 401 Unauthorized\r\n") +
- unauthResponseHeaders + "\r\n" +
- "\r\n";
-
- AWAIT_ASSERT_READY(socket);
-
- // Send 401 Unauthorized response for a manifest request.
- Future<string> remotePullerHttpRequestFuture = Socket(socket.get()).recv();
- AWAIT_ASSERT_READY(remotePullerHttpRequestFuture);
- AWAIT_ASSERT_READY(Socket(socket.get()).send(unauthHttpResponse));
-
- // Token response.
- socket = server.get().accept();
- AWAIT_ASSERT_READY(socket);
-
- Future<string> tokenRequestFuture = Socket(socket.get()).recv();
- AWAIT_ASSERT_READY(tokenRequestFuture);
-
- const string tokenResponse =
- "{\"token\":\"" + getDefaultTokenString() + "\"}";
-
- const string tokenHttpResponse =
- string("HTTP/1.1 200 OK\r\n") +
- "Content-Length : " +
- stringify(tokenResponse.length()) + "\r\n" +
- "\r\n" +
- tokenResponse;
-
- AWAIT_ASSERT_READY(Socket(socket.get()).send(tokenHttpResponse));
-
- // Manifest response.
- socket = server.get().accept();
- AWAIT_ASSERT_READY(socket);
-
- remotePullerHttpRequestFuture = Socket(socket.get()).recv();
- AWAIT_ASSERT_READY(remotePullerHttpRequestFuture);
-
- const string manifestResponse = " \
- { \
- \"schemaVersion\": 1, \
- \"name\": \"library/busybox\", \
- \"tag\": \"latest\", \
- \"architecture\": \"amd64\", \
- \"fsLayers\": [ \
- { \
- \"blobSum\": \
- \"sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4\" \
- } \
- ], \
- \"history\": [ \
- { \
- \"v1Compatibility\": \
- \"{\\\"id\\\": \
- \\\"1ce2e90b0bc7224de3db1f0d646fe8e2c4dd37f1793928287f6074bc451a57ea\\\", \
- \\\"parent\\\": \
- \\\"cf2616975b4a3cba083ca99bc3f0bf25f5f528c3c52be1596b30f60b0b1c37ff\\\" \
- }\" \
- } \
- ], \
- \"signatures\": [ \
- { \
- \"header\": { \
- \"jwk\": { \
- \"crv\": \"P-256\", \
- \"kid\": \
- \"OOI5:SI3T:LC7D:O7DX:FY6S:IAYW:WDRN:VQEM:BCFL:OIST:Q3LO:GTQQ\", \
- \"kty\": \"EC\", \
- \"x\": \"J2N5ePGhlblMI2cdsR6NrAG_xbNC_X7s1HRtk5GXvzM\", \
- \"y\": \"Idr-tEBjnNnfq6_71aeXBi3Z9ah_rrE209l4wiaohk0\" \
- }, \
- \"alg\": \"ES256\" \
- }, \
- \"signature\": \
-\"65vq57TakC_yperuhfefF4uvTbKO2L45gYGDs5bIEgOEarAs7_"
-"4dbEV5u-W7uR8gF6EDKfowUCmTq3a5vEOJ3w\", \
- \"protected\": \
- \"eyJmb3JtYXRMZW5ndGgiOjUwNTgsImZvcm1hdFRhaWwiOiJDbjAiLCJ0aW1lIjoiMjAxNS"
- "0wOC0xMVQwMzo0Mjo1OVoifQ\" \
- } \
- ] \
- }";
-
- const string manifestHttpResponse =
- string("HTTP/1.1 200 OK\r\n") +
- "Content-Length : " +
- stringify(manifestResponse.length()) + "\r\n" +
- "Docker-Content-Digest: "
- "sha256:df9e13f36d2d5b30c16bfbf2a6110c45ebed0bfa1ea42d357651bc6c736d5322"
- + "\r\n" +
- "\r\n" +
- manifestResponse;
-
- AWAIT_ASSERT_READY(Socket(socket.get()).send(manifestHttpResponse));
-
- // Redirect response.
- socket = server.get().accept();
- AWAIT_ASSERT_READY(socket);
-
- remotePullerHttpRequestFuture = Socket(socket.get()).recv();
- AWAIT_ASSERT_READY(remotePullerHttpRequestFuture);
-
- const string redirectHttpResponse =
- string("HTTP/1.1 307 Temporary Redirect\r\n") +
- "Location: https://" +
- stringify(server.get().address().get()) + "\r\n" +
- "\r\n";
-
- AWAIT_ASSERT_READY(Socket(socket.get()).send(redirectHttpResponse));
-
- // Prepare the blob response from the server. The blob response buffer is a
- // tarball. So we create a tarball of our test response and send that.
- const string BLOB_FILE = "blob";
- const string blobResponse = "hello docker";
-
- Path blobPath(path::join(remotePullerPath, BLOB_FILE));
- ASSERT_SOME(os::write(blobPath, blobResponse));
-
- Path blobTarPath(path::join(remotePullerPath, BLOB_FILE + ".tar"));
-
- vector<string> argv = {
- "tar",
- "-C",
- remotePullerPath,
- "-c",
- "-f",
- blobTarPath,
- BLOB_FILE
- };
-
- Try<Subprocess> s = subprocess(
- "tar",
- argv,
- Subprocess::PATH("/dev/null"),
- Subprocess::PATH("/dev/null"),
- Subprocess::PATH("/dev/null"));
- ASSERT_SOME(s);
- AWAIT_ASSERT_READY(s.get().status());
-
- Try<Bytes> tarSize = os::stat::size(blobTarPath);
- ASSERT_SOME(tarSize);
-
- ASSERT_SOME(os::rm(blobPath));
-
- std::unique_ptr<char[]> tarBuffer(new char[tarSize.get().bytes()]);
- ASSERT_NE(tarBuffer.get(), nullptr);
-
- Try<int> fd = os::open(
- blobTarPath,
- O_RDONLY,
- S_IRUSR | S_IRGRP | S_IROTH);
- ASSERT_SOME(fd);
-
- ASSERT_SOME(os::nonblock(fd.get()));
-
- AWAIT_ASSERT_READY(io::read(
- fd.get(),
- tarBuffer.get(),
- tarSize.get().bytes()));
-
- const string blobHttpResponse =
- string("HTTP/1.1 200 OK\r\n") +
- "Content-type : application/octet-stream\r\n" +
- "Content-Length : " +
- stringify(tarSize.get().bytes()) + "\r\n" +
- "\r\n";
-
- const size_t blobResponseSize =
- blobHttpResponse.length() + tarSize.get().bytes();
-
- std::unique_ptr<char[]> responseBuffer(
- new char[blobResponseSize]);
-
- ASSERT_NE(responseBuffer.get(), nullptr);
-
- memcpy(
- responseBuffer.get(),
- blobHttpResponse.c_str(),
- blobHttpResponse.length());
- memcpy(
- responseBuffer.get() + blobHttpResponse.length(),
- tarBuffer.get(),
- tarSize.get().bytes());
-
- socket = server.get().accept();
- AWAIT_ASSERT_READY(socket);
-
- remotePullerHttpRequestFuture = Socket(socket.get()).recv();
- AWAIT_ASSERT_READY(remotePullerHttpRequestFuture);
-
- AWAIT_ASSERT_READY(Socket(socket.get()).send(
- responseBuffer.get(),
- blobResponseSize));
-
- AWAIT_ASSERT_READY(remotePullerFuture);
-
- Try<string> blob = os::read(blobPath);
- ASSERT_SOME(blob);
- ASSERT_EQ(blob.get(), blobResponse);
-}
-
#endif // USE_SSL_SOCKET