You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by ji...@apache.org on 2017/01/17 14:23:45 UTC
mesos git commit: Added support for pulling Docker images by digest.
Repository: mesos
Updated Branches:
refs/heads/master 58eeec839 -> 92595f4f1
Added support for pulling Docker images by digest.
For now we can only use digests to pull images that were pushed with
Docker 1.9 and older or from Registry 2.2.1 and older. Newer versions
use Schema 2 manifests that are not converted by the registry when
pulling by digest.
Review: https://reviews.apache.org/r/53848/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/92595f4f
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/92595f4f
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/92595f4f
Branch: refs/heads/master
Commit: 92595f4f120e48c98b48add4a58548cba7745312
Parents: 58eeec8
Author: Ilya Pronin <ip...@twopensource.com>
Authored: Tue Jan 17 14:33:00 2017 +0100
Committer: Jie Yu <yu...@gmail.com>
Committed: Tue Jan 17 15:23:33 2017 +0100
----------------------------------------------------------------------
src/docker/spec.cpp | 4 +-
.../provisioner/docker/registry_puller.cpp | 8 +-
.../containerizer/provisioner_docker_tests.cpp | 82 ++++++++++++++++++++
src/uri/fetchers/docker.cpp | 13 ++++
src/uri/schemes/docker.hpp | 28 +++++--
5 files changed, 127 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/92595f4f/src/docker/spec.cpp
----------------------------------------------------------------------
diff --git a/src/docker/spec.cpp b/src/docker/spec.cpp
index 2f2c32e..88029c2 100644
--- a/src/docker/spec.cpp
+++ b/src/docker/spec.cpp
@@ -90,7 +90,9 @@ ostream& operator<<(ostream& stream, const ImageReference& reference)
stream << reference.repository();
}
- if (reference.has_tag()) {
+ if (reference.has_digest()) {
+ stream << "@" << reference.digest();
+ } else if (reference.has_tag()) {
stream << ":" << reference.tag();
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/92595f4f/src/slave/containerizer/mesos/provisioner/docker/registry_puller.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/docker/registry_puller.cpp b/src/slave/containerizer/mesos/provisioner/docker/registry_puller.cpp
index b06ddff..cecf34a 100644
--- a/src/slave/containerizer/mesos/provisioner/docker/registry_puller.cpp
+++ b/src/slave/containerizer/mesos/provisioner/docker/registry_puller.cpp
@@ -217,7 +217,9 @@ Future<vector<string>> RegistryPullerProcess::pull(
manifestUri = uri::docker::manifest(
reference.repository(),
- (reference.has_tag() ? reference.tag() : "latest"),
+ (reference.has_digest()
+ ? reference.digest()
+ : (reference.has_tag() ? reference.tag() : "latest")),
spec::getRegistryHost(reference.registry()),
scheme.get(),
port.isSome() ? port.get() : Option<int>());
@@ -232,7 +234,9 @@ Future<vector<string>> RegistryPullerProcess::pull(
manifestUri = uri::docker::manifest(
reference.repository(),
- (reference.has_tag() ? reference.tag() : "latest"),
+ (reference.has_digest()
+ ? reference.digest()
+ : (reference.has_tag() ? reference.tag() : "latest")),
registry,
defaultRegistryUrl.scheme,
port);
http://git-wip-us.apache.org/repos/asf/mesos/blob/92595f4f/src/tests/containerizer/provisioner_docker_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/containerizer/provisioner_docker_tests.cpp b/src/tests/containerizer/provisioner_docker_tests.cpp
index 9682169..d9472bb 100644
--- a/src/tests/containerizer/provisioner_docker_tests.cpp
+++ b/src/tests/containerizer/provisioner_docker_tests.cpp
@@ -749,6 +749,88 @@ TEST_P(ProvisionerDockerWhiteoutTest, ROOT_INTERNET_CURL_Whiteout)
driver.join();
}
+
+// This test verifies that Docker image can be pulled from the
+// repository by digest.
+TEST_F(ProvisionerDockerPullerTest, ROOT_INTERNET_CURL_ImageDigest)
+{
+ Try<Owned<cluster::Master>> master = StartMaster();
+ ASSERT_SOME(master);
+
+ slave::Flags flags = CreateSlaveFlags();
+ flags.isolation = "docker/runtime,filesystem/linux";
+ flags.image_providers = "docker";
+
+ Owned<MasterDetector> detector = master.get()->createDetector();
+ Try<Owned<cluster::Slave>> slave = StartSlave(detector.get(), flags);
+ ASSERT_SOME(slave);
+
+ MockScheduler sched;
+ MesosSchedulerDriver driver(
+ &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL);
+
+ EXPECT_CALL(sched, registered(&driver, _, _));
+
+ Future<vector<Offer>> offers;
+ EXPECT_CALL(sched, resourceOffers(&driver, _))
+ .WillOnce(FutureArg<1>(&offers))
+ .WillRepeatedly(Return()); // Ignore subsequent offers.
+
+ driver.start();
+
+ AWAIT_READY(offers);
+ ASSERT_EQ(1u, offers->size());
+
+ const Offer& offer = offers.get()[0];
+
+ // NOTE: We use a non-shell command here because 'sh' might not be
+ // in the PATH. 'alpine' does not specify env PATH in the image. On
+ // some linux distribution, '/bin' is not in the PATH by default.
+ CommandInfo command;
+ command.set_shell(false);
+ command.set_value("/bin/ls");
+ command.add_arguments("ls");
+ command.add_arguments("-al");
+ command.add_arguments("/");
+
+ TaskInfo task = createTask(
+ offer.slave_id(),
+ Resources::parse("cpus:1;mem:128").get(),
+ command);
+
+ // NOTE: We use the digest of the 'alpine:2.7' image because it has a
+ // Schema 1 manifest (the only manifest schema that we currently support).
+ const string digest =
+ "sha256:9f08005dff552038f0ad2f46b8e65ff3d25641747d3912e3ea8da6785046561a";
+
+ Image image;
+ image.set_type(Image::DOCKER);
+ image.mutable_docker()->set_name("library/alpine@" + digest);
+
+ ContainerInfo* container = task.mutable_container();
+ container->set_type(ContainerInfo::MESOS);
+ container->mutable_mesos()->mutable_image()->CopyFrom(image);
+
+ Future<TaskStatus> statusRunning;
+ Future<TaskStatus> statusFinished;
+ EXPECT_CALL(sched, statusUpdate(&driver, _))
+ .WillOnce(FutureArg<1>(&statusRunning))
+ .WillOnce(FutureArg<1>(&statusFinished));
+
+ driver.launchTasks(offer.id(), {task});
+
+ AWAIT_READY_FOR(statusRunning, Seconds(60));
+ EXPECT_EQ(task.task_id(), statusRunning->task_id());
+ EXPECT_EQ(TASK_RUNNING, statusRunning->state());
+
+ AWAIT_READY(statusFinished);
+ EXPECT_EQ(task.task_id(), statusFinished->task_id());
+ EXPECT_EQ(TASK_FINISHED, statusFinished->state());
+
+ driver.stop();
+ driver.join();
+}
+
#endif
} // namespace tests {
http://git-wip-us.apache.org/repos/asf/mesos/blob/92595f4f/src/uri/fetchers/docker.cpp
----------------------------------------------------------------------
diff --git a/src/uri/fetchers/docker.cpp b/src/uri/fetchers/docker.cpp
index 3f38ddd..027e748 100644
--- a/src/uri/fetchers/docker.cpp
+++ b/src/uri/fetchers/docker.cpp
@@ -473,6 +473,19 @@ Future<Nothing> DockerFetcherPluginProcess::__fetch(
CHECK_EQ(response.type, http::Response::BODY);
+ // Check if we got a V2 Schema 1 manifest.
+ // TODO(ipronin): We have to support Schema 2 manifests to be able to use
+ // digests for pulling images that were pushed with Docker 1.10+ to
+ // Registry 2.3+.
+ Option<string> contentType = response.headers.get("Content-Type");
+ if (contentType.isSome() &&
+ !strings::startsWith(
+ contentType.get(),
+ "application/vnd.docker.distribution.manifest.v1")) {
+ return Failure(
+ "Unsupported manifest MIME type: " + contentType.get());
+ }
+
Try<spec::v2::ImageManifest> manifest = spec::v2::parse(response.body);
if (manifest.isError()) {
return Failure("Failed to parse the image manifest: " + manifest.error());
http://git-wip-us.apache.org/repos/asf/mesos/blob/92595f4f/src/uri/schemes/docker.hpp
----------------------------------------------------------------------
diff --git a/src/uri/schemes/docker.hpp b/src/uri/schemes/docker.hpp
index 68dbf3f..527c7e4 100644
--- a/src/uri/schemes/docker.hpp
+++ b/src/uri/schemes/docker.hpp
@@ -29,23 +29,35 @@ namespace docker {
inline URI image(
const std::string& repository,
- const std::string& tag,
+ const std::string& reference, // Either tag or digest.
const std::string& registry,
const Option<std::string>& scheme = None(),
const Option<int>& port = None())
{
- return construct("docker", repository, registry, port, tag, scheme);
+ return construct(
+ "docker",
+ repository,
+ registry,
+ port,
+ reference,
+ scheme);
}
inline URI manifest(
const std::string& repository,
- const std::string& tag,
+ const std::string& reference, // Either tag or digest.
const std::string& registry,
const Option<std::string>& scheme = None(),
const Option<int>& port = None())
{
- return construct("docker-manifest", repository, registry, port, tag, scheme);
+ return construct(
+ "docker-manifest",
+ repository,
+ registry,
+ port,
+ reference,
+ scheme);
}
@@ -56,7 +68,13 @@ inline URI blob(
const Option<std::string>& scheme = None(),
const Option<int>& port = None())
{
- return construct("docker-blob", repository, registry, port, digest, scheme);
+ return construct(
+ "docker-blob",
+ repository,
+ registry,
+ port,
+ digest,
+ scheme);
}
} // namespace docker {