You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by ji...@apache.org on 2016/02/24 00:11:26 UTC
[2/3] mesos git commit: Used uri::Fetcher to pull docker images in
docker registry puller.
Used uri::Fetcher to pull docker images in docker registry puller.
Review: https://reviews.apache.org/r/43860
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/66d0f449
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/66d0f449
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/66d0f449
Branch: refs/heads/master
Commit: 66d0f4496f146fb0c68d23e08d25a9af108c6218
Parents: caf6c02
Author: Jie Yu <yu...@gmail.com>
Authored: Mon Feb 22 16:52:01 2016 -0800
Committer: Jie Yu <yu...@gmail.com>
Committed: Tue Feb 23 15:08:08 2016 -0800
----------------------------------------------------------------------
docs/configuration.md | 17 -
docs/endpoints/slave/state.json.md | 2 -
docs/endpoints/slave/state.md | 2 -
.../mesos/provisioner/docker/puller.cpp | 7 +-
.../mesos/provisioner/docker/puller.hpp | 7 +-
.../provisioner/docker/registry_puller.cpp | 319 +++---
.../provisioner/docker/registry_puller.hpp | 7 +-
.../mesos/provisioner/docker/store.cpp | 11 +-
src/slave/flags.cpp | 10 -
src/slave/flags.hpp | 2 -
src/slave/http.cpp | 2 -
.../containerizer/provisioner_docker_tests.cpp | 978 +------------------
12 files changed, 220 insertions(+), 1144 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/66d0f449/docs/configuration.md
----------------------------------------------------------------------
diff --git a/docs/configuration.md b/docs/configuration.md
index b04e873..2353e78 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -1057,15 +1057,6 @@ containerizer.
</tr>
<tr>
<td>
- --docker_auth_server=VALUE
- </td>
- <td>
-Docker authentication server used to authenticate with Docker registry
-(default: https://auth.docker.io)
- </td>
-</tr>
-<tr>
- <td>
--[no-]docker_kill_orphans
</td>
<td>
@@ -1090,14 +1081,6 @@ recovers.
</tr>
<tr>
<td>
- --docker_puller_timeout=VALUE
- </td>
- <td>
-Timeout in seconds for pulling images from the Docker registry (default: 60secs)
- </td>
-</tr>
-<tr>
- <td>
--docker_registry=VALUE
</td>
<td>
http://git-wip-us.apache.org/repos/asf/mesos/blob/66d0f449/docs/endpoints/slave/state.json.md
----------------------------------------------------------------------
diff --git a/docs/endpoints/slave/state.json.md b/docs/endpoints/slave/state.json.md
index 0a31159..a6403ee 100644
--- a/docs/endpoints/slave/state.json.md
+++ b/docs/endpoints/slave/state.json.md
@@ -80,12 +80,10 @@ Example (**Note**: this is not exhaustive):
"work_dir" : "/tmp/mesos",
"launcher_dir" : "/path/to/mesos/build/src",
"registration_backoff_factor" : "1secs",
- "docker_auth_server" : "https://auth.docker.io",
"oversubscribed_resources_interval" : "15secs",
"enforce_container_disk_quota" : "false",
"container_disk_watch_interval" : "15secs",
"disk_watch_interval" : "1mins",
- "docker_puller_timeout" : "60",
"cgroups_limit_swap" : "false",
"hostname_lookup" : "true",
"perf_duration" : "10secs",
http://git-wip-us.apache.org/repos/asf/mesos/blob/66d0f449/docs/endpoints/slave/state.md
----------------------------------------------------------------------
diff --git a/docs/endpoints/slave/state.md b/docs/endpoints/slave/state.md
index 1077f69..bdd555e 100644
--- a/docs/endpoints/slave/state.md
+++ b/docs/endpoints/slave/state.md
@@ -80,12 +80,10 @@ Example (**Note**: this is not exhaustive):
"work_dir" : "/tmp/mesos",
"launcher_dir" : "/path/to/mesos/build/src",
"registration_backoff_factor" : "1secs",
- "docker_auth_server" : "https://auth.docker.io",
"oversubscribed_resources_interval" : "15secs",
"enforce_container_disk_quota" : "false",
"container_disk_watch_interval" : "15secs",
"disk_watch_interval" : "1mins",
- "docker_puller_timeout" : "60",
"cgroups_limit_swap" : "false",
"hostname_lookup" : "true",
"perf_duration" : "10secs",
http://git-wip-us.apache.org/repos/asf/mesos/blob/66d0f449/src/slave/containerizer/mesos/provisioner/docker/puller.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/docker/puller.cpp b/src/slave/containerizer/mesos/provisioner/docker/puller.cpp
index d012ae4..ac9dae8 100644
--- a/src/slave/containerizer/mesos/provisioner/docker/puller.cpp
+++ b/src/slave/containerizer/mesos/provisioner/docker/puller.cpp
@@ -22,13 +22,16 @@
#include "slave/containerizer/mesos/provisioner/docker/registry_puller.hpp"
using process::Owned;
+using process::Shared;
namespace mesos {
namespace internal {
namespace slave {
namespace docker {
-Try<Owned<Puller>> Puller::create(const Flags& flags)
+Try<Owned<Puller>> Puller::create(
+ const Flags& flags,
+ const Shared<uri::Fetcher>& fetcher)
{
// TODO(tnachen): Support multiple registries in the puller.
if (strings::startsWith(flags.docker_registry, "/")) {
@@ -40,7 +43,7 @@ Try<Owned<Puller>> Puller::create(const Flags& flags)
return puller.get();
}
- Try<Owned<Puller>> puller = RegistryPuller::create(flags);
+ Try<Owned<Puller>> puller = RegistryPuller::create(flags, fetcher);
if (puller.isError()) {
return Error("Failed to create registry puller: " + puller.error());
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/66d0f449/src/slave/containerizer/mesos/provisioner/docker/puller.hpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/docker/puller.hpp b/src/slave/containerizer/mesos/provisioner/docker/puller.hpp
index 51894dd..191f3fc 100644
--- a/src/slave/containerizer/mesos/provisioner/docker/puller.hpp
+++ b/src/slave/containerizer/mesos/provisioner/docker/puller.hpp
@@ -24,9 +24,12 @@
#include <process/future.hpp>
#include <process/owned.hpp>
+#include <process/shared.hpp>
#include <mesos/docker/spec.hpp>
+#include <mesos/uri/fetcher.hpp>
+
#include "slave/flags.hpp"
namespace mesos {
@@ -37,7 +40,9 @@ namespace docker {
class Puller
{
public:
- static Try<process::Owned<Puller>> create(const Flags& flags);
+ static Try<process::Owned<Puller>> create(
+ const Flags& flags,
+ const process::Shared<uri::Fetcher>& fetcher);
virtual ~Puller() {}
http://git-wip-us.apache.org/repos/asf/mesos/blob/66d0f449/src/slave/containerizer/mesos/provisioner/docker/registry_puller.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/docker/registry_puller.cpp b/src/slave/containerizer/mesos/provisioner/docker/registry_puller.cpp
index d09e2ec..e08a959 100644
--- a/src/slave/containerizer/mesos/provisioner/docker/registry_puller.cpp
+++ b/src/slave/containerizer/mesos/provisioner/docker/registry_puller.cpp
@@ -14,17 +14,23 @@
// See the License for the specific language governing permissions and
// limitations under the License.
+#include <glog/logging.h>
+
#include <process/collect.hpp>
#include <process/defer.hpp>
#include <process/dispatch.hpp>
+#include <process/http.hpp>
+#include <stout/os/exists.hpp>
#include <stout/os/mkdir.hpp>
#include <stout/os/rm.hpp>
+#include <stout/os/write.hpp>
#include "common/command_utils.hpp"
+#include "uri/schemes/docker.hpp"
+
#include "slave/containerizer/mesos/provisioner/docker/paths.hpp"
-#include "slave/containerizer/mesos/provisioner/docker/registry_client.hpp"
#include "slave/containerizer/mesos/provisioner/docker/registry_puller.hpp"
namespace http = process::http;
@@ -38,68 +44,77 @@ using process::Failure;
using process::Future;
using process::Owned;
using process::Process;
-using process::Promise;
+using process::Shared;
+
+using process::defer;
+using process::dispatch;
+using process::spawn;
+using process::wait;
namespace mesos {
namespace internal {
namespace slave {
namespace docker {
-using RegistryClient = registry::RegistryClient;
-
-
class RegistryPullerProcess : public Process<RegistryPullerProcess>
{
public:
- static Try<Owned<RegistryPullerProcess>> create(const Flags& flags);
+ RegistryPullerProcess(
+ const string& _storeDir,
+ const http::URL& _defaultRegistryUrl,
+ const Shared<uri::Fetcher>& _fetcher);
- process::Future<vector<string>> pull(
+ Future<vector<string>> pull(
const spec::ImageReference& reference,
const string& directory);
private:
- RegistryPullerProcess(
- const Owned<RegistryClient>& registry,
- const Duration& timeout);
-
- Future<vector<string>> downloadLayers(
- const spec::v2::ImageManifest& manifest,
+ Future<vector<string>> _pull(
const spec::ImageReference& reference,
const string& directory);
- Future<Nothing> downloadLayer(
+ Future<Nothing> fetchLayer(
const spec::ImageReference& reference,
const string& directory,
const string& blobSum,
- const string& layerId);
+ const spec::v1::ImageManifest& v1);
- Future<vector<string>> untarLayers(
+ Future<Nothing> _fetchLayer(
const string& directory,
- const vector<string>& layerIds);
-
- Future<Nothing> untarLayer(
- const string& directory,
- const string& layerId);
-
- Owned<RegistryClient> registryClient_;
- const Duration pullTimeout_;
- hashmap<string, Owned<Promise<Nothing>>> downloadTracker_;
+ const string& blobSum,
+ const spec::v1::ImageManifest& v1);
RegistryPullerProcess(const RegistryPullerProcess&) = delete;
RegistryPullerProcess& operator=(const RegistryPullerProcess&) = delete;
+
+ const string storeDir;
+
+ // If the user does not specify the registry url in the image
+ // reference, this registry url will be used as the default.
+ const http::URL defaultRegistryUrl;
+
+ Shared<uri::Fetcher> fetcher;
};
-Try<Owned<Puller>> RegistryPuller::create(const Flags& flags)
+Try<Owned<Puller>> RegistryPuller::create(
+ const Flags& flags,
+ const Shared<uri::Fetcher>& fetcher)
{
- Try<Owned<RegistryPullerProcess>> process =
- RegistryPullerProcess::create(flags);
-
- if (process.isError()) {
- return Error(process.error());
+ Try<http::URL> defaultRegistryUrl = http::URL::parse(flags.docker_registry);
+ if (defaultRegistryUrl.isError()) {
+ return Error(
+ "Failed to parse the default Docker registry: " +
+ defaultRegistryUrl.error());
}
- return Owned<Puller>(new RegistryPuller(process.get()));
+ Owned<RegistryPullerProcess> process(
+ new RegistryPullerProcess(
+ flags.docker_store_dir,
+ defaultRegistryUrl.get(),
+ fetcher));
+
+ return Owned<Puller>(new RegistryPuller(process));
}
@@ -113,7 +128,7 @@ RegistryPuller::RegistryPuller(Owned<RegistryPullerProcess> _process)
RegistryPuller::~RegistryPuller()
{
terminate(process.get());
- process::wait(process.get());
+ wait(process.get());
}
@@ -129,177 +144,159 @@ Future<vector<string>> RegistryPuller::pull(
}
-Try<Owned<RegistryPullerProcess>> RegistryPullerProcess::create(
- const Flags& flags)
-{
- Result<double> timeoutSecs = numify<double>(flags.docker_puller_timeout_secs);
- if ((timeoutSecs.isError()) || (timeoutSecs.get() <= 0)) {
- return Error(
- "Failed to create registry puller - invalid timeout value: " +
- flags.docker_puller_timeout_secs);
- }
-
- Try<http::URL> registryUrl = http::URL::parse(flags.docker_registry);
- if (registryUrl.isError()) {
- return Error("Failed to parse Docker registry: " + registryUrl.error());
- }
-
- Try<http::URL> authServerUrl = http::URL::parse(flags.docker_auth_server);
- if (authServerUrl.isError()) {
- return Error("Failed to parse Docker auth server: " +
- authServerUrl.error());
- }
-
- Try<Owned<RegistryClient>> registry = RegistryClient::create(
- registryUrl.get(), authServerUrl.get());
-
- if (registry.isError()) {
- return Error("Failed to create registry client: " + registry.error());
- }
-
- return Owned<RegistryPullerProcess>(new RegistryPullerProcess(
- registry.get(),
- Seconds(timeoutSecs.get())));
-}
-
-
RegistryPullerProcess::RegistryPullerProcess(
- const Owned<RegistryClient>& registry,
- const Duration& timeout)
- : registryClient_(registry),
- pullTimeout_(timeout) {}
+ const string& _storeDir,
+ const http::URL& _defaultRegistryUrl,
+ const Shared<uri::Fetcher>& _fetcher)
+ : storeDir(_storeDir),
+ defaultRegistryUrl(_defaultRegistryUrl),
+ fetcher(_fetcher) {}
Future<vector<string>> RegistryPullerProcess::pull(
const spec::ImageReference& reference,
const string& directory)
{
- // TODO(jojy): Have one outgoing manifest request per image.
- return registryClient_->getManifest(reference)
- .then(process::defer(self(), [=](const spec::v2::ImageManifest& manifest) {
- return downloadLayers(manifest, reference, directory);
- }))
- .then(process::defer(self(), [=](const vector<string>& layerIds) {
- return untarLayers(directory, layerIds);
- }))
- .after(pullTimeout_, [reference](Future<vector<string>> future) {
- future.discard();
- return Failure("Timed out");
- });
+ // TODO(jieyu): Consider introducing a 'normalize' function to
+ // normalize 'reference' here. For instance, we need to add
+ // 'library/' prefix if the user does not specify a repository.
+ // Also consider merging the registry generation logic below into
+ // 'normalize'.
+
+ URI manifestUri;
+ if (reference.has_registry()) {
+ // TODO(jieyu): The user specified registry might contain port. We
+ // need to parse it and set the 'scheme' and 'port' accordingly.
+ manifestUri = uri::docker::manifest(
+ reference.repository(),
+ (reference.has_tag() ? reference.tag() : "latest"),
+ reference.registry());
+ } else {
+ const string registry = defaultRegistryUrl.domain.isSome()
+ ? defaultRegistryUrl.domain.get()
+ : stringify(defaultRegistryUrl.ip.get());
+
+ const Option<int> port = defaultRegistryUrl.port.isSome()
+ ? static_cast<int>(defaultRegistryUrl.port.get())
+ : Option<int>();
+
+ manifestUri = uri::docker::manifest(
+ reference.repository(),
+ (reference.has_tag() ? reference.tag() : "latest"),
+ registry,
+ defaultRegistryUrl.scheme,
+ port);
+ }
+
+ return fetcher->fetch(manifestUri, directory)
+ .then(defer(self(), &Self::_pull, reference, directory));
}
-Future<vector<string>> RegistryPullerProcess::downloadLayers(
- const spec::v2::ImageManifest& manifest,
+Future<vector<string>> RegistryPullerProcess::_pull(
const spec::ImageReference& reference,
const string& directory)
{
+ Try<string> _manifest = os::read(path::join(directory, "manifest"));
+ if (_manifest.isError()) {
+ return Failure("Failed to read the manifest: " + _manifest.error());
+ }
+
+ Try<spec::v2::ImageManifest> manifest = spec::v2::parse(_manifest.get());
+ if (manifest.isError()) {
+ return Failure("Failed to parse the manifest: " + manifest.error());
+ }
+
list<Future<Nothing>> futures;
vector<string> layerIds;
- CHECK_EQ(manifest.fslayers_size(), manifest.history_size());
+ CHECK_EQ(manifest->fslayers_size(), manifest->history_size());
- for (int i = 0; i < manifest.fslayers_size(); i++) {
- CHECK(manifest.history(i).has_v1());
+ for (int i = 0; i < manifest->fslayers_size(); i++) {
+ CHECK(manifest->history(i).has_v1());
- layerIds.push_back(manifest.history(i).v1().id());
+ layerIds.push_back(manifest->history(i).v1().id());
- futures.push_back(downloadLayer(
+ futures.push_back(fetchLayer(
reference,
directory,
- manifest.fslayers(i).blobsum(),
- manifest.history(i).v1().id()));
+ manifest->fslayers(i).blobsum(),
+ manifest->history(i).v1()));
}
- // TODO(jojy): Delete downloaded files in the directory on discard and
- // failure?
- // TODO(jojy): Iterate through the futures and log the failed future.
return collect(futures)
.then([layerIds]() { return layerIds; });
}
-Future<Nothing> RegistryPullerProcess::downloadLayer(
+Future<Nothing> RegistryPullerProcess::fetchLayer(
const spec::ImageReference& reference,
const string& directory,
const string& blobSum,
- const string& layerId)
+ const spec::v1::ImageManifest& v1)
{
- VLOG(1) << "Downloading layer '" << layerId
- << "' for image '" << stringify(reference) << "'";
-
- if (downloadTracker_.contains(layerId)) {
- VLOG(1) << "Download already in progress for image '"
- << stringify(reference) << "', layer '" << layerId << "'";
-
- return downloadTracker_.at(layerId)->future();
+ // Check if the layer is in the store or not. If yes, skip the
+ // unnecessary fetching.
+ if (os::exists(paths::getImageLayerPath(storeDir, v1.id()))) {
+ return Nothing();
}
- Owned<Promise<Nothing>> downloadPromise(new Promise<Nothing>());
-
- downloadTracker_.insert({layerId, downloadPromise});
-
- const Path downloadFile(path::join(directory, layerId + ".tar"));
-
- registryClient_->getBlob(
- reference,
- blobSum,
- downloadFile)
- .onAny(process::defer(
- self(),
- [this, layerId, downloadPromise, downloadFile](
- const Future<size_t>& future) {
- downloadTracker_.erase(layerId);
-
- if (!future.isReady()) {
- downloadPromise->fail(
- "Failed to download layer '" + layerId + "': " +
- (future.isFailed() ? future.failure() : "future discarded"));
- } else if (future.get() == 0) {
- // We don't expect Docker registry to return empty response
- // even with empty layers.
- downloadPromise->fail(
- "Failed to download layer '" + layerId + "': no content");
- } else {
- downloadPromise->set(Nothing());
- }
- }));
-
- return downloadPromise->future();
-}
-
-
-Future<vector<string>> RegistryPullerProcess::untarLayers(
- const string& directory,
- const vector<string>& layerIds)
-{
- list<Future<Nothing>> futures;
- foreach (const string& layerId, layerIds) {
- VLOG(1) << "Untarring layer '" << layerId
- << "' downloaded from registry to directory '"
- << directory << "'";
-
- futures.emplace_back(untarLayer(directory, layerId));
+ VLOG(1) << "Fetching layer '" << v1.id() << "' for image '"
+ << reference << "'";
+
+ URI blobUri;
+ if (reference.has_registry()) {
+ // TODO(jieyu): The user specified registry might contain port. We
+ // need to parse it and set the 'scheme' and 'port' accordingly.
+ blobUri = uri::docker::blob(
+ reference.repository(),
+ blobSum,
+ reference.registry());
+ } else {
+ const string registry = defaultRegistryUrl.domain.isSome()
+ ? defaultRegistryUrl.domain.get()
+ : stringify(defaultRegistryUrl.ip.get());
+
+ const Option<int> port = defaultRegistryUrl.port.isSome()
+ ? static_cast<int>(defaultRegistryUrl.port.get())
+ : Option<int>();
+
+ blobUri = uri::docker::blob(
+ reference.repository(),
+ blobSum,
+ registry,
+ defaultRegistryUrl.scheme,
+ port);
}
- return collect(futures)
- .then([layerIds]() { return layerIds; });
+ return fetcher->fetch(blobUri, directory)
+ .then(defer(self(), &Self::_fetchLayer, directory, blobSum, v1));
}
-Future<Nothing> RegistryPullerProcess::untarLayer(
+Future<Nothing> RegistryPullerProcess::_fetchLayer(
const string& directory,
- const string& layerId)
+ const string& blobSum,
+ const spec::v1::ImageManifest& v1)
{
- const string layerPath = path::join(directory, layerId);
- const string tar = path::join(directory, layerId + ".tar");
+ const string layerPath = path::join(directory, v1.id());
+ const string tar = path::join(directory, blobSum);
const string rootfs = paths::getImageLayerRootfsPath(layerPath);
+ const string manifest = paths::getImageLayerManifestPath(layerPath);
- Try<Nothing> mkdir = os::mkdir(rootfs);
+ // NOTE: This will create 'layerPath' as well.
+ Try<Nothing> mkdir = os::mkdir(rootfs, true);
if (mkdir.isError()) {
return Failure(
- "Failed to create directory '" + rootfs + "'"
- ": " + mkdir.error());
+ "Failed to create rootfs directory '" + rootfs + "' "
+ "for layer '" + v1.id() + "': " + mkdir.error());
+ }
+
+ Try<Nothing> write = os::write(manifest, stringify(JSON::protobuf(v1)));
+ if (write.isError()) {
+ return Failure(
+ "Failed to save the layer manifest for layer '" +
+ v1.id() + "': " + write.error());
}
return command::untar(Path(tar), Path(rootfs))
@@ -308,8 +305,8 @@ Future<Nothing> RegistryPullerProcess::untarLayer(
Try<Nothing> rm = os::rm(tar);
if (rm.isError()) {
return Failure(
- "Failed to remove '" + tar + "' "
- "after extraction: " + rm.error());
+ "Failed to remove '" + tar + "' "
+ "after extraction: " + rm.error());
}
return Nothing();
http://git-wip-us.apache.org/repos/asf/mesos/blob/66d0f449/src/slave/containerizer/mesos/provisioner/docker/registry_puller.hpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/docker/registry_puller.hpp b/src/slave/containerizer/mesos/provisioner/docker/registry_puller.hpp
index c429df9..bbd6005 100644
--- a/src/slave/containerizer/mesos/provisioner/docker/registry_puller.hpp
+++ b/src/slave/containerizer/mesos/provisioner/docker/registry_puller.hpp
@@ -18,9 +18,12 @@
#define __PROVISIONER_DOCKER_REGISTRY_PULLER_HPP__
#include <process/owned.hpp>
+#include <process/shared.hpp>
#include <stout/try.hpp>
+#include <mesos/uri/fetcher.hpp>
+
#include "slave/containerizer/mesos/provisioner/docker/puller.hpp"
#include "slave/flags.hpp"
@@ -39,7 +42,9 @@ class RegistryPullerProcess;
class RegistryPuller : public Puller
{
public:
- static Try<process::Owned<Puller>> create(const Flags& flags);
+ static Try<process::Owned<Puller>> create(
+ const Flags& flags,
+ const process::Shared<uri::Fetcher>& fetcher);
~RegistryPuller();
http://git-wip-us.apache.org/repos/asf/mesos/blob/66d0f449/src/slave/containerizer/mesos/provisioner/docker/store.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/docker/store.cpp b/src/slave/containerizer/mesos/provisioner/docker/store.cpp
index d802a79..47d3dc1 100644
--- a/src/slave/containerizer/mesos/provisioner/docker/store.cpp
+++ b/src/slave/containerizer/mesos/provisioner/docker/store.cpp
@@ -34,6 +34,8 @@
#include "slave/containerizer/mesos/provisioner/docker/paths.hpp"
#include "slave/containerizer/mesos/provisioner/docker/puller.hpp"
+#include "uri/fetcher.hpp"
+
using namespace process;
namespace spec = docker::spec;
@@ -89,7 +91,14 @@ private:
Try<Owned<slave::Store>> Store::create(const Flags& flags)
{
- Try<Owned<Puller>> puller = Puller::create(flags);
+ // TODO(jieyu): We should inject URI fetcher from top level, instead
+ // of creating it here.
+ Try<Owned<uri::Fetcher>> fetcher = uri::fetcher::create();
+ if (fetcher.isError()) {
+ return Error("Failed to create the URI fetcher: " + fetcher.error());
+ }
+
+ Try<Owned<Puller>> puller = Puller::create(flags, fetcher->share());
if (puller.isError()) {
return Error("Failed to create Docker puller: " + puller.error());
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/66d0f449/src/slave/flags.cpp
----------------------------------------------------------------------
diff --git a/src/slave/flags.cpp b/src/slave/flags.cpp
index 855812e..1c6a87b 100644
--- a/src/slave/flags.cpp
+++ b/src/slave/flags.cpp
@@ -124,16 +124,6 @@ mesos::internal::slave::Flags::Flags()
"Directory the appc provisioner will store images in.\n",
"/tmp/mesos/store/appc");
- add(&Flags::docker_auth_server,
- "docker_auth_server",
- "Docker authentication server used to authenticate with Docker registry",
- "https://auth.docker.io");
-
- add(&Flags::docker_puller_timeout_secs,
- "docker_puller_timeout",
- "Timeout in seconds for pulling images from the Docker registry",
- "60");
-
add(&Flags::docker_registry,
"docker_registry",
"The default url for pulling Docker images. It could either be a Docker\n"
http://git-wip-us.apache.org/repos/asf/mesos/blob/66d0f449/src/slave/flags.hpp
----------------------------------------------------------------------
diff --git a/src/slave/flags.hpp b/src/slave/flags.hpp
index 54c1a69..c079321 100644
--- a/src/slave/flags.hpp
+++ b/src/slave/flags.hpp
@@ -53,8 +53,6 @@ public:
std::string appc_simple_discovery_uri_prefix;
std::string appc_store_dir;
- std::string docker_auth_server;
- std::string docker_puller_timeout_secs;
std::string docker_registry;
std::string docker_store_dir;
http://git-wip-us.apache.org/repos/asf/mesos/blob/66d0f449/src/slave/http.cpp
----------------------------------------------------------------------
diff --git a/src/slave/http.cpp b/src/slave/http.cpp
index a18085e..4eb1faf 100644
--- a/src/slave/http.cpp
+++ b/src/slave/http.cpp
@@ -465,12 +465,10 @@ string Slave::Http::STATE_HELP() {
" \"work_dir\" : \"/tmp/mesos\",",
" \"launcher_dir\" : \"/path/to/mesos/build/src\",",
" \"registration_backoff_factor\" : \"1secs\",",
- " \"docker_auth_server\" : \"https://auth.docker.io\",",
" \"oversubscribed_resources_interval\" : \"15secs\",",
" \"enforce_container_disk_quota\" : \"false\",",
" \"container_disk_watch_interval\" : \"15secs\",",
" \"disk_watch_interval\" : \"1mins\",",
- " \"docker_puller_timeout\" : \"60\",",
" \"cgroups_limit_swap\" : \"false\",",
" \"hostname_lookup\" : \"true\",",
" \"perf_duration\" : \"10secs\",",
http://git-wip-us.apache.org/repos/asf/mesos/blob/66d0f449/src/tests/containerizer/provisioner_docker_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/containerizer/provisioner_docker_tests.cpp b/src/tests/containerizer/provisioner_docker_tests.cpp
index c6b1e69..b3c6f88 100644
--- a/src/tests/containerizer/provisioner_docker_tests.cpp
+++ b/src/tests/containerizer/provisioner_docker_tests.cpp
@@ -14,984 +14,50 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-#include <utility>
-
#include <gmock/gmock.h>
-#include <gtest/gtest.h>
-
-#include <stout/duration.hpp>
-
#include <stout/gtest.hpp>
#include <stout/json.hpp>
#include <stout/os.hpp>
#include <stout/path.hpp>
#include <stout/stringify.hpp>
-#include <process/address.hpp>
-#include <process/clock.hpp>
#include <process/future.hpp>
#include <process/gmock.hpp>
-#include <process/io.hpp>
#include <process/owned.hpp>
-#include <process/socket.hpp>
-#include <process/subprocess.hpp>
-
-#include <process/ssl/gtest.hpp>
#include <mesos/docker/spec.hpp>
#include "slave/containerizer/mesos/provisioner/docker/metadata_manager.hpp"
#include "slave/containerizer/mesos/provisioner/docker/paths.hpp"
#include "slave/containerizer/mesos/provisioner/docker/puller.hpp"
-#include "slave/containerizer/mesos/provisioner/docker/registry_client.hpp"
#include "slave/containerizer/mesos/provisioner/docker/registry_puller.hpp"
#include "slave/containerizer/mesos/provisioner/docker/store.hpp"
-#include "slave/containerizer/mesos/provisioner/docker/token_manager.hpp"
#include "tests/mesos.hpp"
#include "tests/utils.hpp"
-namespace io = process::io;
+namespace paths = mesos::internal::slave::docker::paths;
namespace slave = mesos::internal::slave;
namespace spec = ::docker::spec;
-using std::list;
-using std::map;
-using std::pair;
using std::string;
using std::vector;
-using process::Clock;
using process::Future;
using process::Owned;
using process::Promise;
-using process::Subprocess;
-using process::network::Socket;
+using slave::ImageInfo;
using slave::docker::Puller;
using slave::docker::RegistryPuller;
-
-using slave::docker::paths::getImageLayerRootfsPath;
-
-using slave::docker::registry::RegistryClient;
-using slave::docker::registry::Token;
-using slave::docker::registry::TokenManager;
+using slave::docker::Store;
namespace mesos {
namespace internal {
namespace tests {
-/**
- * Provides token operations and defaults.
- */
-class TokenHelper {
-protected:
- const string hdrBase64 = base64::encode(
- "{ \
- \"alg\":\"ES256\", \
- \"typ\":\"JWT\", \
- \"x5c\":[\"test\"] \
- }");
-
- string getClaimsBase64() const
- {
- return base64::encode(claimsJsonString);
- }
-
- string getTokenString() const
- {
- return hdrBase64 + "." + getClaimsBase64() + "." + signBase64;
- }
-
- string getDefaultTokenString()
- {
- // Construct response and send(server side).
- const double expirySecs = Clock::now().secs() + Days(365).secs();
-
- claimsJsonString =
- "{\"access\" \
- :[ \
- { \
- \"type\":\"repository\", \
- \"name\":\"library/busybox\", \
- \"actions\":[\"pull\"]}], \
- \"aud\":\"registry.docker.io\", \
- \"exp\":" + stringify(expirySecs) + ", \
- \"iat\":1438887168, \
- \"iss\":\"auth.docker.io\", \
- \"jti\":\"l2PJDFkzwvoL7-TajJF7\", \
- \"nbf\":1438887166, \
- \"sub\":\"\" \
- }";
-
- return getTokenString();
- }
-
- const string signBase64 = base64::encode("{\"\"}");
- string claimsJsonString;
-};
-
-
-/**
- * Fixture for testing TokenManager component.
- */
-class RegistryTokenTest : public TokenHelper, public ::testing::Test {};
-
-
-// Tests JSON Web Token parsing for a valid token string.
-TEST_F(RegistryTokenTest, ValidToken)
-{
- const double expirySecs = Clock::now().secs() + Days(365).secs();
-
- claimsJsonString =
- "{\"access\" \
- :[ \
- { \
- \"type\":\"repository\", \
- \"name\":\"library/busybox\", \
- \"actions\":[\"pull\"]}], \
- \"aud\":\"registry.docker.io\", \
- \"exp\":" + stringify(expirySecs) + ", \
- \"iat\":1438887168, \
- \"iss\":\"auth.docker.io\", \
- \"jti\":\"l2PJDFkzwvoL7-TajJF7\", \
- \"nbf\":1438887166, \
- \"sub\":\"\" \
- }";
-
- Try<Token> token = Token::create(getTokenString());
-
- ASSERT_SOME(token);
-}
-
-
-// Tests JSON Web Token parsing for a token string with expiration
-// date in the past.
-TEST_F(RegistryTokenTest, ExpiredToken)
-{
- // Use an arbitrary fixed date that is far in the past (12 weeks
- // after the Unix epoch).
- const double expirySecs = Weeks(12).secs();
-
- claimsJsonString =
- "{\"access\" \
- :[ \
- { \
- \"type\":\"repository\", \
- \"name\":\"library/busybox\", \
- \"actions\":[\"pull\"]}], \
- \"aud\":\"registry.docker.io\", \
- \"exp\":" + stringify(expirySecs) + ", \
- \"iat\":1438887166, \
- \"iss\":\"auth.docker.io\", \
- \"jti\":\"l2PJDFkzwvoL7-TajJF7\", \
- \"nbf\":1438887166, \
- \"sub\":\"\" \
- }";
-
- Try<Token> token = Token::create(getTokenString());
-
- EXPECT_ERROR(token);
-}
-
-
-// Tests JSON Web Token parsing for a token string with no expiration date.
-TEST_F(RegistryTokenTest, NoExpiration)
-{
- claimsJsonString =
- "{\"access\" \
- :[ \
- { \
- \"type\":\"repository\", \
- \"name\":\"library/busybox\", \
- \"actions\":[\"pull\"]}], \
- \"aud\":\"registry.docker.io\", \
- \"iat\":1438887166, \
- \"iss\":\"auth.docker.io\", \
- \"jti\":\"l2PJDFkzwvoL7-TajJF7\", \
- \"nbf\":1438887166, \
- \"sub\":\"\" \
- }";
-
- const Try<Token> token = Token::create(getTokenString());
-
- ASSERT_SOME(token);
-}
-
-
-// Tests JSON Web Token parsing for a token string with not-before date in the
-// future.
-TEST_F(RegistryTokenTest, NotBeforeInFuture)
-{
- const double expirySecs = Clock::now().secs() + Days(365).secs();
- const double nbfSecs = Clock::now().secs() + Days(7).secs();
-
- claimsJsonString =
- "{\"access\" \
- :[ \
- { \
- \"type\":\"repository\", \
- \"name\":\"library/busybox\", \
- \"actions\":[\"pull\"]}], \
- \"aud\":\"registry.docker.io\", \
- \"exp\":" + stringify(expirySecs) + ", \
- \"iat\":1438887166, \
- \"iss\":\"auth.docker.io\", \
- \"jti\":\"l2PJDFkzwvoL7-TajJF7\", \
- \"nbf\":" + stringify(nbfSecs) + ", \
- \"sub\":\"\" \
- }";
-
- const Try<Token> token = Token::create(getTokenString());
-
- ASSERT_SOME(token);
- ASSERT_EQ(token.get().isValid(), false);
-}
-
-
-#ifdef USE_SSL_SOCKET
-
-// Test suite for docker registry tests.
-class RegistryClientTest : public virtual SSLTest, public TokenHelper
-{
-protected:
- RegistryClientTest() {}
-
- Try<Socket> getServer() {
- return setup_server({
- {"SSL_ENABLED", "true"},
- {"SSL_KEY_FILE", key_path().value},
- {"SSL_CERT_FILE", certificate_path().value}});
- }
-};
-
-
-// Tests TokenManager for a simple token request.
-TEST_F(RegistryClientTest, SimpleGetToken)
-{
- Try<Socket> server = getServer();
-
- ASSERT_SOME(server);
- ASSERT_SOME(server.get().address());
- ASSERT_SOME(server.get().address().get().hostname());
-
- Future<Socket> socket = server.get().accept();
-
- // Create URL from server hostname and port.
- const process::http::URL url(
- "https",
- server.get().address().get().hostname().get(),
- server.get().address().get().port);
-
- Try<Owned<TokenManager>> tokenMgr = TokenManager::create(url);
- ASSERT_SOME(tokenMgr);
-
- Future<Token> token =
- tokenMgr.get()->getToken(
- "registry.docker.io",
- "repository:library/busybox:pull",
- None());
-
- AWAIT_ASSERT_READY(socket);
-
- // Construct response and send(server side).
- const double expirySecs = Clock::now().secs() + Days(365).secs();
-
- claimsJsonString =
- "{\"access\" \
- :[ \
- { \
- \"type\":\"repository\", \
- \"name\":\"library/busybox\", \
- \"actions\":[\"pull\"]}], \
- \"aud\":\"registry.docker.io\", \
- \"exp\":" + stringify(expirySecs) + ", \
- \"iat\":1438887168, \
- \"iss\":\"auth.docker.io\", \
- \"jti\":\"l2PJDFkzwvoL7-TajJF7\", \
- \"nbf\":1438887166, \
- \"sub\":\"\" \
- }";
-
- const string tokenString(getTokenString());
- const string tokenResponse = "{\"token\":\"" + tokenString + "\"}";
-
- const string buffer =
- string("HTTP/1.1 200 OK\r\n") +
- "Content-Length : " +
- stringify(tokenResponse.length()) + "\r\n" +
- "\r\n" +
- tokenResponse;
-
- AWAIT_ASSERT_READY(Socket(socket.get()).send(buffer));
-
- AWAIT_ASSERT_READY(token);
- ASSERT_EQ(token.get().raw, tokenString);
-}
-
-
-// Tests TokenManager for bad token response from server.
-TEST_F(RegistryClientTest, BadTokenResponse)
-{
- Try<Socket> server = getServer();
-
- ASSERT_SOME(server);
- ASSERT_SOME(server.get().address());
- ASSERT_SOME(server.get().address().get().hostname());
-
- Future<Socket> socket = server.get().accept();
-
- // Create URL from server hostname and port.
- const process::http::URL url(
- "https",
- server.get().address().get().hostname().get(),
- server.get().address().get().port);
-
- Try<Owned<TokenManager>> tokenMgr = TokenManager::create(url);
- ASSERT_SOME(tokenMgr);
-
- Future<Token> token =
- tokenMgr.get()->getToken(
- "registry.docker.io",
- "repository:library/busybox:pull",
- None());
-
- AWAIT_ASSERT_READY(socket);
-
- const string tokenString("bad token");
- const string tokenResponse = "{\"token\":\"" + tokenString + "\"}";
-
- const string buffer =
- string("HTTP/1.1 200 OK\r\n") +
- "Content-Length : " +
- stringify(tokenResponse.length()) + "\r\n" +
- "\r\n" +
- tokenResponse;
-
- AWAIT_ASSERT_READY(Socket(socket.get()).send(buffer));
-
- AWAIT_FAILED(token);
-}
-
-
-// Tests TokenManager for request to invalid server.
-TEST_F(RegistryClientTest, DISABLED_BadTokenServerAddress)
-{
- // Create an invalid URL with current time.
- const process::http::URL url("https", stringify(Clock::now().secs()), 0);
-
- Try<Owned<TokenManager>> tokenMgr = TokenManager::create(url);
- ASSERT_SOME(tokenMgr);
-
- Future<Token> token =
- tokenMgr.get()->getToken(
- "registry.docker.io",
- "repository:library/busybox:pull",
- None());
-
- AWAIT_FAILED(token);
-}
-
-
-// Tests docker registry's getManifest API.
-TEST_F(RegistryClientTest, SimpleGetManifest)
-{
- Try<Socket> server = getServer();
-
- ASSERT_SOME(server);
- ASSERT_SOME(server.get().address());
- ASSERT_SOME(server.get().address().get().hostname());
-
- Future<Socket> socket = server.get().accept();
-
- const process::http::URL url(
- "https",
- server.get().address().get().hostname().get(),
- server.get().address().get().port);
-
- Try<Owned<RegistryClient>> registryClient =
- RegistryClient::create(url, url, None());
-
- ASSERT_SOME(registryClient);
-
- Try<spec::ImageReference> reference =
- spec::parseImageReference("library/busybox");
- ASSERT_SOME(reference);
-
- Future<spec::v2::ImageManifest> manifestResponse =
- registryClient.get()->getManifest(reference.get());
-
- const string unauthResponseHeaders = "Www-Authenticate: Bearer"
- " realm=\"https://auth.docker.io/token\","
- "service=" + stringify(server.get().address().get()) + ","
- "scope=\"repository:library/busybox:pull\"";
-
- const string unauthHttpResponse =
- string("HTTP/1.1 401 Unauthorized\r\n") +
- unauthResponseHeaders + "\r\n" +
- "\r\n";
-
- AWAIT_ASSERT_READY(socket);
-
- // Send 401 Unauthorized response for a manifest request.
- Future<string> manifestHttpRequest = Socket(socket.get()).recv();
- AWAIT_ASSERT_READY(manifestHttpRequest);
- AWAIT_ASSERT_READY(Socket(socket.get()).send(unauthHttpResponse));
-
- // Token response.
- socket = server.get().accept();
- AWAIT_ASSERT_READY(socket);
-
- Future<string> tokenRequest = Socket(socket.get()).recv();
- AWAIT_ASSERT_READY(tokenRequest);
-
- const string tokenResponse =
- "{\"token\":\"" + getDefaultTokenString() + "\"}";
-
- const string tokenHttpResponse =
- string("HTTP/1.1 200 OK\r\n") +
- "Content-Length : " +
- stringify(tokenResponse.length()) + "\r\n" +
- "\r\n" +
- tokenResponse;
-
- AWAIT_ASSERT_READY(Socket(socket.get()).send(tokenHttpResponse));
-
- // Manifest response.
- socket = server.get().accept();
- AWAIT_ASSERT_READY(socket);
-
- manifestHttpRequest = Socket(socket.get()).recv();
- AWAIT_ASSERT_READY(manifestHttpRequest);
-
- const string manifestJSON =
- "{"
- " \"schemaVersion\": 1,"
- " \"name\": \"library/busybox\","
- " \"tag\": \"latest\","
- " \"architecture\": \"amd64\","
- " \"fsLayers\": ["
- " {"
- " \"blobSum\": "
- "\"sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4\""
- " },"
- " {"
- " \"blobSum\": "
- "\"sha256:1db09adb5ddd7f1a07b6d585a7db747a51c7bd17418d47e91f901bdf420abd66\""
- " },"
- " {"
- " \"blobSum\": "
- "\"sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4\""
- " }"
- " ],"
- " \"history\": ["
- " {"
- " \"v1Compatibility\": "
- " \"{"
- " \\\"id\\\": "
- "\\\"1ce2e90b0bc7224de3db1f0d646fe8e2c4dd37f1793928287f6074bc451a57ea\\\","
- " \\\"parent\\\": "
- "\\\"cf2616975b4a3cba083ca99bc3f0bf25f5f528c3c52be1596b30f60b0b1c37ff\\\""
- " }\""
- " },"
- " {"
- " \"v1Compatibility\": "
- " \"{"
- " \\\"id\\\": "
- "\\\"2ce2e90b0bc7224de3db1f0d646fe8e2c4dd37f1793928287f6074bc451a57ea\\\","
- " \\\"parent\\\": "
- "\\\"cf2616975b4a3cba083ca99bc3f0bf25f5f528c3c52be1596b30f60b0b1c37ff\\\""
- " }\""
- " },"
- " {"
- " \"v1Compatibility\": "
- " \"{"
- " \\\"id\\\": "
- "\\\"3ce2e90b0bc7224de3db1f0d646fe8e2c4dd37f1793928287f6074bc451a57ea\\\","
- " \\\"parent\\\": "
- "\\\"cf2616975b4a3cba083ca99bc3f0bf25f5f528c3c52be1596b30f60b0b1c37ff\\\""
- " }\""
- " }"
- " ],"
- " \"signatures\": ["
- " {"
- " \"header\": {"
- " \"jwk\": {"
- " \"crv\": \"P-256\","
- " \"kid\": "
- "\"OOI5:SI3T:LC7D:O7DX:FY6S:IAYW:WDRN:VQEM:BCFL:OIST:Q3LO:GTQQ\","
- " \"kty\": \"EC\","
- " \"x\": \"J2N5ePGhlblMI2cdsR6NrAG_xbNC_X7s1HRtk5GXvzM\","
- " \"y\": \"Idr-tEBjnNnfq6_71aeXBi3Z9ah_rrE209l4wiaohk0\""
- " },"
- " \"alg\": \"ES256\""
- " },"
- " \"signature\": \"65vq57TakC_yperuhfefF4uvTbKO2L45gYGDs5bIEgO"
- "EarAs7_4dbEV5u-W7uR8gF6EDKfowUCmTq3a5vEOJ3w\","
- " \"protected\": \"eyJmb3JtYXRMZW5ndGgiOjYwNjMsImZvcm1hdFRhaWwiOiJ"
- "DbjAiLCJ0aW1lIjoiMjAxNC0wOS0xMVQxNzoxNDozMFoifQ\""
- " }"
- " ]"
- "}";
-
- const string manifestHttpResponse =
- string("HTTP/1.1 200 OK\r\n") +
- "Content-Length : " +
- stringify(manifestJSON.length()) + "\r\n" +
- "Docker-Content-Digest: "
- "sha256:df9e13f36d2d5b30c16bfbf2a6110c45ebed0bfa1ea42d357651bc6c736d5322"
- + "\r\n" +
- "\r\n" +
- manifestJSON;
-
- AWAIT_ASSERT_READY(Socket(socket.get()).send(manifestHttpResponse));
-
- AWAIT_ASSERT_READY(manifestResponse);
-
- EXPECT_EQ(
- manifestResponse.get().history(2).v1().id(),
- "1ce2e90b0bc7224de3db1f0d646fe8e2c4dd37f1793928287f6074bc451a57ea");
-
- EXPECT_EQ(
- manifestResponse.get().history(1).v1().id(),
- "2ce2e90b0bc7224de3db1f0d646fe8e2c4dd37f1793928287f6074bc451a57ea");
-
- EXPECT_EQ(
- manifestResponse.get().history(0).v1().id(),
- "3ce2e90b0bc7224de3db1f0d646fe8e2c4dd37f1793928287f6074bc451a57ea");
-}
-
-
-// Tests docker registry's getBlob API.
-TEST_F(RegistryClientTest, SimpleGetBlob)
-{
- Try<Socket> server = getServer();
-
- ASSERT_SOME(server);
- ASSERT_SOME(server.get().address());
- ASSERT_SOME(server.get().address().get().hostname());
-
- Future<Socket> socket = server.get().accept();
-
- Try<Socket> blobServer = getServer();
-
- ASSERT_SOME(blobServer);
- ASSERT_SOME(blobServer.get().address());
- ASSERT_SOME(blobServer.get().address().get().hostname());
-
- Future<Socket> blobServerAcceptSocket = blobServer.get().accept();
-
- const process::http::URL url(
- "https",
- server.get().address().get().hostname().get(),
- server.get().address().get().port);
-
- Try<Owned<RegistryClient>> registryClient =
- RegistryClient::create(url, url, None());
-
- ASSERT_SOME(registryClient);
-
- const Path blobPath(path::join(os::getcwd(), "blob"));
-
- Try<spec::ImageReference> reference = spec::parseImageReference("blob");
- ASSERT_SOME(reference);
-
- Future<size_t> result =
- registryClient.get()->getBlob(
- reference.get(),
- "digest",
- blobPath);
-
- const string unauthResponseHeaders = "WWW-Authenticate: Bearer"
- " realm=\"https://auth.docker.io/token\","
- "service=" + stringify(server.get().address().get()) + ","
- "scope=\"repository:library/busybox:pull\"";
-
- const string unauthHttpResponse =
- string("HTTP/1.1 401 Unauthorized\r\n") +
- unauthResponseHeaders + "\r\n" +
- "\r\n";
-
- AWAIT_ASSERT_READY(socket);
-
- // Send 401 Unauthorized response.
- Future<string> blobHttpRequest = Socket(socket.get()).recv();
- AWAIT_ASSERT_READY(blobHttpRequest);
- AWAIT_ASSERT_READY(Socket(socket.get()).send(unauthHttpResponse));
-
- // Send token response.
- socket = server.get().accept();
- AWAIT_ASSERT_READY(socket);
-
- Future<string> tokenRequest = Socket(socket.get()).recv();
- AWAIT_ASSERT_READY(tokenRequest);
-
- const string tokenResponse =
- "{\"token\":\"" + getDefaultTokenString() + "\"}";
-
- const string tokenHttpResponse =
- string("HTTP/1.1 200 OK\r\n") +
- "Content-Length : " +
- stringify(tokenResponse.length()) + "\r\n" +
- "\r\n" +
- tokenResponse;
-
- AWAIT_ASSERT_READY(Socket(socket.get()).send(tokenHttpResponse));
-
- // Send redirect.
- socket = server.get().accept();
- AWAIT_ASSERT_READY(socket);
-
- blobHttpRequest = Socket(socket.get()).recv();
- AWAIT_ASSERT_READY(blobHttpRequest);
-
- const string redirectHttpResponse =
- string("HTTP/1.1 307 Temporary Redirect\r\n") +
- "Location: https://" +
- blobServer.get().address().get().hostname().get() + ":" +
- stringify(blobServer.get().address().get().port) + "/blob \r\n" +
- "\r\n";
-
- AWAIT_ASSERT_READY(Socket(socket.get()).send(redirectHttpResponse));
-
- // Finally send blob response.
- AWAIT_ASSERT_READY(blobServerAcceptSocket);
-
- blobHttpRequest = Socket(blobServerAcceptSocket.get()).recv();
- AWAIT_ASSERT_READY(blobHttpRequest);
-
- const string blobResponse = stringify(Clock::now());
-
- const string blobHttpResponse =
- string("HTTP/1.1 200 OK\r\n") +
- "Content-Length : " +
- stringify(blobResponse.length()) + "\r\n" +
- "\r\n" +
- blobResponse;
-
- AWAIT_ASSERT_READY(Socket(blobServerAcceptSocket.get()).send(
- blobHttpResponse));
-
- AWAIT_ASSERT_READY(result);
-
- Try<string> blob = os::read(blobPath);
- ASSERT_SOME(blob);
- ASSERT_EQ(blob.get(), blobResponse);
-}
-
-
-TEST_F(RegistryClientTest, BadRequest)
-{
- Try<Socket> server = getServer();
-
- ASSERT_SOME(server);
- ASSERT_SOME(server.get().address());
- ASSERT_SOME(server.get().address().get().hostname());
-
- Future<Socket> socket = server.get().accept();
-
- const process::http::URL url(
- "https",
- server.get().address().get().hostname().get(),
- server.get().address().get().port);
-
- Try<Owned<RegistryClient>> registryClient =
- RegistryClient::create(url, url, None());
-
- ASSERT_SOME(registryClient);
-
- const Path blobPath(path::join(os::getcwd(), "blob"));
-
- Try<spec::ImageReference> reference = spec::parseImageReference("blob");
- ASSERT_SOME(reference);
-
- Future<size_t> result =
- registryClient.get()->getBlob(
- reference.get(),
- "digest",
- blobPath);
-
- const string badRequestResponse =
- "{\"errors\": [{\"message\": \"Error1\" }, {\"message\": \"Error2\"}]}";
-
- const string badRequestHttpResponse =
- string("HTTP/1.1 400 Bad Request\r\n") +
- "Content-Length : " + stringify(badRequestResponse.length()) + "\r\n" +
- "\r\n" +
- badRequestResponse;
-
- AWAIT_ASSERT_READY(socket);
-
- // Send 400 Bad Request.
- Future<string> blobHttpRequest = Socket(socket.get()).recv();
- AWAIT_ASSERT_READY(blobHttpRequest);
- AWAIT_ASSERT_READY(Socket(socket.get()).send(badRequestHttpResponse));
-
- AWAIT_FAILED(result);
-
- ASSERT_TRUE(strings::contains(result.failure(), "Error1"));
- ASSERT_TRUE(strings::contains(result.failure(), "Error2"));
-}
-
-
-// Tests docker RegistryPuller component. It simulates pulling an image layer
-// from remote registry and then verifies the content saved on disk.
-TEST_F(RegistryClientTest, SimpleRegistryPuller)
-{
- Try<Socket> server = getServer();
-
- ASSERT_SOME(server);
- ASSERT_SOME(server.get().address());
- ASSERT_SOME(server.get().address().get().hostname());
-
- Future<Socket> socket = server.get().accept();
-
- Try<Socket> blobServer = getServer();
-
- ASSERT_SOME(blobServer);
- ASSERT_SOME(blobServer.get().address());
- ASSERT_SOME(blobServer.get().address().get().hostname());
-
- Future<Socket> blobServerAcceptSock = blobServer.get().accept();
-
- slave::Flags flags;
- process::network::Address address = server.get().address().get();
- const string url = "https://" + address.hostname().get() + ":" +
- stringify(address.port);
- flags.docker_registry = url;
- flags.docker_auth_server = url;
-
- Try<Owned<Puller>> registryPuller = RegistryPuller::create(flags);
- ASSERT_SOME(registryPuller);
-
- const string registryPullerPath = os::getcwd();
-
- Try<spec::ImageReference> reference = spec::parseImageReference("busybox");
- ASSERT_SOME(reference);
-
- Future<vector<string>> registryPullerFuture =
- registryPuller.get()->pull(reference.get(), registryPullerPath);
-
- const string unauthResponseHeaders = "WWW-Authenticate: Bearer"
- " realm=\"https://auth.docker.io/token\","
- "service=" + stringify(server.get().address().get()) + ","
- "scope=\"repository:library/busybox:pull\"";
-
- const string unauthHttpResponse =
- string("HTTP/1.1 401 Unauthorized\r\n") +
- unauthResponseHeaders + "\r\n" +
- "\r\n";
-
- AWAIT_ASSERT_READY(socket);
-
- // Send 401 Unauthorized response for a manifest request.
- Future<string> registryPullerHttpRequestFuture = Socket(socket.get()).recv();
- AWAIT_ASSERT_READY(registryPullerHttpRequestFuture);
- AWAIT_ASSERT_READY(Socket(socket.get()).send(unauthHttpResponse));
-
- // Token response.
- socket = server.get().accept();
- AWAIT_ASSERT_READY(socket);
-
- Future<string> tokenRequestFuture = Socket(socket.get()).recv();
- AWAIT_ASSERT_READY(tokenRequestFuture);
-
- const string tokenResponse =
- "{\"token\":\"" + getDefaultTokenString() + "\"}";
-
- const string tokenHttpResponse =
- string("HTTP/1.1 200 OK\r\n") +
- "Content-Length : " +
- stringify(tokenResponse.length()) + "\r\n" +
- "\r\n" +
- tokenResponse;
-
- AWAIT_ASSERT_READY(Socket(socket.get()).send(tokenHttpResponse));
-
- // Manifest response.
- socket = server.get().accept();
- AWAIT_ASSERT_READY(socket);
-
- registryPullerHttpRequestFuture = Socket(socket.get()).recv();
- AWAIT_ASSERT_READY(registryPullerHttpRequestFuture);
-
- const string manifestResponse =
- "{"
- " \"schemaVersion\": 1,"
- " \"name\": \"library/busybox\","
- " \"tag\": \"latest\","
- " \"architecture\": \"amd64\","
- " \"fsLayers\": ["
- " {"
- " \"blobSum\": "
- "\"sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4\""
- " }"
- " ],"
- " \"history\": ["
- " {"
- " \"v1Compatibility\": "
- " \"{"
- " \\\"id\\\": "
- "\\\"1ce2e90b0bc7224de3db1f0d646fe8e2c4dd37f1793928287f6074bc451a57ea\\\","
- " \\\"parent\\\": "
- "\\\"cf2616975b4a3cba083ca99bc3f0bf25f5f528c3c52be1596b30f60b0b1c37ff\\\""
- " }\""
- " }"
- " ],"
- " \"signatures\": ["
- " {"
- " \"header\": {"
- " \"jwk\": {"
- " \"crv\": \"P-256\","
- " \"kid\": "
- "\"OOI5:SI3T:LC7D:O7DX:FY6S:IAYW:WDRN:VQEM:BCFL:OIST:Q3LO:GTQQ\","
- " \"kty\": \"EC\","
- " \"x\": \"J2N5ePGhlblMI2cdsR6NrAG_xbNC_X7s1HRtk5GXvzM\","
- " \"y\": \"Idr-tEBjnNnfq6_71aeXBi3Z9ah_rrE209l4wiaohk0\""
- " },"
- " \"alg\": \"ES256\""
- " },"
- " \"signature\": \"65vq57TakC_yperuhfefF4uvTbKO2L45gYGDs5bIEgO"
- "EarAs7_4dbEV5u-W7uR8gF6EDKfowUCmTq3a5vEOJ3w\","
- " \"protected\": \"eyJmb3JtYXRMZW5ndGgiOjYwNjMsImZvcm1hdFRhaWwiOiJ"
- "DbjAiLCJ0aW1lIjoiMjAxNC0wOS0xMVQxNzoxNDozMFoifQ\""
- " }"
- " ]"
- "}";
-
- const string manifestHttpResponse =
- string("HTTP/1.1 200 OK\r\n") +
- "Content-Length : " +
- stringify(manifestResponse.length()) + "\r\n" +
- "Docker-Content-Digest: "
- "sha256:df9e13f36d2d5b30c16bfbf2a6110c45ebed0bfa1ea42d357651bc6c736d5322"
- + "\r\n" +
- "\r\n" +
- manifestResponse;
-
- AWAIT_ASSERT_READY(Socket(socket.get()).send(manifestHttpResponse));
-
- // Redirect response.
- socket = server.get().accept();
- AWAIT_ASSERT_READY(socket);
-
- registryPullerHttpRequestFuture = Socket(socket.get()).recv();
- AWAIT_ASSERT_READY(registryPullerHttpRequestFuture);
-
- const string redirectHttpResponse =
- string("HTTP/1.1 307 Temporary Redirect\r\n") +
- "Content-Length : 0\r\n" +
- "Location: https://" +
- blobServer.get().address().get().hostname().get() + ":" +
- stringify(blobServer.get().address().get().port) + "/blob \r\n" +
- "\r\n";
-
- AWAIT_ASSERT_READY(Socket(socket.get()).send(redirectHttpResponse));
-
- AWAIT_ASSERT_READY(blobServerAcceptSock);
-
- registryPullerHttpRequestFuture = Socket(blobServerAcceptSock.get()).recv();
- AWAIT_ASSERT_READY(registryPullerHttpRequestFuture);
-
- // Prepare the blob response from the server. The blob response buffer is a
- // tarball. So we create a tarball of our test response and send that.
- const string blobFile = "blob";
- const string blobResponse = "hello docker";
-
- Path blobPath(path::join(registryPullerPath, blobFile));
- ASSERT_SOME(os::write(blobPath, blobResponse));
-
- Path blobTarPath(path::join(registryPullerPath, blobFile + ".tar"));
-
- vector<string> argv = {
- "tar",
- "-C",
- registryPullerPath,
- "-c",
- "-f",
- blobTarPath,
- blobFile
- };
-
- Try<Subprocess> s = subprocess(
- "tar",
- argv,
- Subprocess::PATH("/dev/null"),
- Subprocess::PATH("/dev/null"),
- Subprocess::PATH("/dev/null"));
-
- ASSERT_SOME(s);
- AWAIT_ASSERT_READY(s.get().status());
-
- Try<Bytes> tarSize = os::stat::size(blobTarPath);
- ASSERT_SOME(tarSize);
-
- ASSERT_SOME(os::rm(blobPath));
-
- std::unique_ptr<char[]> tarBuffer(new char[tarSize.get().bytes()]);
- ASSERT_NE(tarBuffer.get(), nullptr);
-
- Try<int> fd = os::open(
- blobTarPath,
- O_RDONLY,
- S_IRUSR | S_IRGRP | S_IROTH);
- ASSERT_SOME(fd);
-
- ASSERT_SOME(os::nonblock(fd.get()));
-
- AWAIT_ASSERT_READY(
- io::read(fd.get(), tarBuffer.get(), tarSize.get().bytes()));
-
- ASSERT_SOME(os::close(fd.get()));
-
- const string blobHttpResponse =
- string("HTTP/1.1 200 OK\r\n") +
- "Content-type : application/octet-stream\r\n" +
- "Content-Length : " +
- stringify(tarSize.get().bytes()) + "\r\n" +
- "\r\n";
-
- const size_t blobResponseSize =
- blobHttpResponse.length() + tarSize.get().bytes();
-
- std::unique_ptr<char[]> responseBuffer(new char[blobResponseSize]);
- ASSERT_NE(responseBuffer.get(), nullptr);
-
- memcpy(
- responseBuffer.get(),
- blobHttpResponse.c_str(),
- blobHttpResponse.length());
-
- memcpy(
- responseBuffer.get() + blobHttpResponse.length(),
- tarBuffer.get(),
- tarSize.get().bytes());
-
- AWAIT_ASSERT_READY(Socket(blobServerAcceptSock.get()).send(
- responseBuffer.get(),
- blobResponseSize));
-
- AWAIT_ASSERT_READY(registryPullerFuture);
- vector<string> layers = registryPullerFuture.get();
- ASSERT_EQ(1u, layers.size());
- ASSERT_EQ(layers.front(),
- "1ce2e90b0bc7224de3db1f0d646fe8e2c4dd37f1793928287f6074bc451a57ea");
-
- Try<string> blob = os::read(path::join(
- registryPullerPath,
- layers.front(),
- "rootfs",
- blobFile));
-
- ASSERT_SOME(blob);
- ASSERT_EQ(blob.get(), blobResponse);
-}
-
-#endif // USE_SSL_SOCKET
-
-
class ProvisionerDockerLocalStoreTest : public TemporaryDirectoryTest
{
public:
@@ -999,20 +65,22 @@ public:
const slave::Flags& flags,
const vector<string>& layers)
{
- const string layersPath = path::join(flags.docker_store_dir, "layers");
-
// Verify contents of the image in store directory.
- const string layerPath1 =
- getImageLayerRootfsPath(flags.docker_store_dir, "123");
+ const string layerPath1 = paths::getImageLayerRootfsPath(
+ flags.docker_store_dir,
+ "123");
- const string layerPath2 =
- getImageLayerRootfsPath(flags.docker_store_dir, "456");
+ const string layerPath2 = paths::getImageLayerRootfsPath(
+ flags.docker_store_dir,
+ "456");
EXPECT_TRUE(os::exists(layerPath1));
EXPECT_TRUE(os::exists(layerPath2));
+
EXPECT_SOME_EQ(
"foo 123",
os::read(path::join(layerPath1 , "temp")));
+
EXPECT_SOME_EQ(
"bar 456",
os::read(path::join(layerPath2, "temp")));
@@ -1238,6 +306,30 @@ TEST_F(ProvisionerDockerLocalStoreTest, PullingSameImageSimutanuously)
EXPECT_EQ(imageInfo1.get().layers, imageInfo2.get().layers);
}
+
+class ProvisionerDockerRegistryPullerTest : public TemporaryDirectoryTest {};
+
+
+TEST_F(ProvisionerDockerRegistryPullerTest, INTERNET_CURL_Pull)
+{
+ slave::Flags flags;
+ flags.docker_registry = "https://registry-1.docker.io";
+ flags.docker_store_dir = os::getcwd();
+
+ Try<Owned<slave::Store>> store = Store::create(flags);
+ ASSERT_SOME(store);
+
+ Image image;
+ image.set_type(Image::DOCKER);
+ image.mutable_docker()->set_name("library/alpine");
+
+ Future<ImageInfo> imageInfo = store.get()->get(image);
+ AWAIT_READY_FOR(imageInfo, Seconds(60));
+
+ EXPECT_LE(1u, imageInfo->layers.size());
+ EXPECT_SOME(imageInfo->dockerManifest);
+}
+
} // namespace tests {
} // namespace internal {
} // namespace mesos {