You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by ji...@apache.org on 2016/01/19 01:12:35 UTC
[4/9] mesos git commit: Added protobuf for docker ImageReference and
the parsing function.
Added protobuf for docker ImageReference and the parsing function.
The goal here is to replace Image::Name with ImageReference and the
corresponding parsing method, which will be done in a subsequent patch.
Review: https://reviews.apache.org/r/42235
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/49c0fee9
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/49c0fee9
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/49c0fee9
Branch: refs/heads/master
Commit: 49c0fee9787d5bd41dec85ee1cdd0356ae6c78ae
Parents: 81ee91e
Author: Jie Yu <yu...@gmail.com>
Authored: Sat Jan 9 23:04:09 2016 -0800
Committer: Jie Yu <yu...@gmail.com>
Committed: Mon Jan 18 16:09:14 2016 -0800
----------------------------------------------------------------------
include/mesos/docker/spec.hpp | 88 ++++++++++++++++++++
include/mesos/docker/spec.proto | 38 +++++++++
src/CMakeLists.txt | 2 +
src/Makefile.am | 8 +-
src/docker/spec.cpp | 57 ++++++++++++-
src/docker/spec.hpp | 64 --------------
.../mesos/provisioner/docker/message.hpp | 2 +
.../mesos/provisioner/docker/message.proto | 1 +
.../provisioner/docker/registry_client.hpp | 2 +-
src/tests/containerizer/docker_spec_tests.cpp | 71 +++++++++++++++-
.../containerizer/provisioner_docker_tests.cpp | 4 +-
11 files changed, 268 insertions(+), 69 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/49c0fee9/include/mesos/docker/spec.hpp
----------------------------------------------------------------------
diff --git a/include/mesos/docker/spec.hpp b/include/mesos/docker/spec.hpp
new file mode 100644
index 0000000..61b55b4
--- /dev/null
+++ b/include/mesos/docker/spec.hpp
@@ -0,0 +1,88 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef __MESOS_DOCKER_SPEC_HPP__
+#define __MESOS_DOCKER_SPEC_HPP__
+
+#include <string>
+
+#include <stout/error.hpp>
+#include <stout/json.hpp>
+#include <stout/option.hpp>
+#include <stout/try.hpp>
+
+// ONLY USEFUL AFTER RUNNING PROTOC.
+#include <mesos/docker/spec.pb.h>
+
+#include <mesos/docker/v1.hpp>
+#include <mesos/docker/v2.hpp>
+
+namespace docker {
+namespace spec {
+
+/**
+ * Parse the docker image reference. Docker expects the image
+ * reference to be in the following format:
+ * [REGISTRY_HOST[:REGISTRY_PORT]/]REPOSITORY[:TAG|@TYPE:DIGEST]
+ *
+ * This format is inherently ambiguous when dealing with repository
+ * names that include forward slashes. To disambiguate, the docker
+ * code looks for '.', or ':', or 'localhost' to decide if the first
+ * component is a registry or a respository name. For more detail,
+ * drill into the implementation of docker pull.
+ *
+ * See docker implementation:
+ * https://github.com/docker/distribution/blob/master/reference/reference.go
+ */
+Try<ImageReference> parseImageReference(const std::string& s);
+
+
+namespace v1 {
+
+/**
+ * Validates if the specified docker v1 image manifest conforms to the
+ * Docker v1 spec. Returns the error if the validation fails.
+ */
+Option<Error> validate(const ImageManifest& manifest);
+
+
+/**
+ * Returns the docker v1 image manifest from the given JSON object.
+ */
+Try<ImageManifest> parse(const JSON::Object& json);
+
+} // namespace v1 {
+
+
+namespace v2 {
+
+/**
+ * Validates if the specified v2 image manifest conforms to the Docker
+ * v2 spec. Returns the error if the validation fails.
+ */
+Option<Error> validate(const ImageManifest& manifest);
+
+
+/**
+ * Returns the docker v2 image manifest from the given JSON object.
+ */
+Try<ImageManifest> parse(const JSON::Object& json);
+
+} // namespace v2 {
+} // namespace spec {
+} // namespace docker {
+
+#endif // __MESOS_DOCKER_SPEC_HPP__
http://git-wip-us.apache.org/repos/asf/mesos/blob/49c0fee9/include/mesos/docker/spec.proto
----------------------------------------------------------------------
diff --git a/include/mesos/docker/spec.proto b/include/mesos/docker/spec.proto
new file mode 100644
index 0000000..4f2ff2d
--- /dev/null
+++ b/include/mesos/docker/spec.proto
@@ -0,0 +1,38 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package docker.spec;
+
+/**
+ * The docker image reference:
+ * [REGISTRY_HOST[:REGISTRY_PORT]/]REPOSITORY[:TAG|@DIGEST]
+ *
+ * See more details in:
+ * https://github.com/docker/distribution/blob/master/reference/reference.go
+ */
+message ImageReference {
+ // The registry in the following format: 'HOST:PORT'.
+ optional string registry = 1;
+
+ // The repository name (e.g., library/busybox, ubuntu).
+ required string repository = 2;
+
+ // For example: latest, 2.0.2, etc.
+ optional string tag = 3;
+
+ // In the format of 'ALGORITHM:HEX'.
+ optional string digest = 4;
+}
http://git-wip-us.apache.org/repos/asf/mesos/blob/49c0fee9/src/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 39a23df..a52203a 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -24,6 +24,7 @@ PROTOC_TO_INCLUDE_DIR(V1_MESOS mesos/v1/mesos)
PROTOC_TO_INCLUDE_DIR(AUTHENTICATION mesos/authentication/authentication)
PROTOC_TO_INCLUDE_DIR(AUTHORIZATION mesos/authorizer/authorizer)
PROTOC_TO_INCLUDE_DIR(CONTAINERIZER mesos/containerizer/containerizer)
+PROTOC_TO_INCLUDE_DIR(DOCKER_SPEC mesos/docker/spec)
PROTOC_TO_INCLUDE_DIR(DOCKER_V1 mesos/docker/v1)
PROTOC_TO_INCLUDE_DIR(DOCKER_V2 mesos/docker/v2)
PROTOC_TO_INCLUDE_DIR(EXECUTOR mesos/executor/executor)
@@ -53,6 +54,7 @@ set(MESOS_PROTOBUF_SRC
${AUTHENTICATION_PROTO_CC}
${AUTHORIZATION_PROTO_CC}
${CONTAINERIZER_PROTO_CC}
+ ${DOCKER_SPEC_PROTO_CC}
${DOCKER_V1_PROTO_CC}
${DOCKER_V2_PROTO_CC}
${EXECUTOR_PROTO_CC}
http://git-wip-us.apache.org/repos/asf/mesos/blob/49c0fee9/src/Makefile.am
----------------------------------------------------------------------
diff --git a/src/Makefile.am b/src/Makefile.am
index d23e350..69383b6 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -150,6 +150,7 @@ ALLOCATOR_PROTO = $(top_srcdir)/include/mesos/master/allocator.proto
AUTHENTICATION_PROTO = $(top_srcdir)/include/mesos/authentication/authentication.proto
AUTHORIZATION_PROTO = $(top_srcdir)/include/mesos/authorizer/authorizer.proto
CONTAINERIZER_PROTO = $(top_srcdir)/include/mesos/containerizer/containerizer.proto
+DOCKER_SPEC_PROTO = $(top_srcdir)/include/mesos/docker/spec.proto
DOCKER_V1_PROTO = $(top_srcdir)/include/mesos/docker/v1.proto
DOCKER_V2_PROTO = $(top_srcdir)/include/mesos/docker/v2.proto
EXECUTOR_PROTO = $(top_srcdir)/include/mesos/executor/executor.proto
@@ -176,6 +177,8 @@ CXX_PROTOS = \
../include/mesos/authorizer/authorizer.pb.h \
../include/mesos/containerizer/containerizer.pb.cc \
../include/mesos/containerizer/containerizer.pb.h \
+ ../include/mesos/docker/spec.pb.cc \
+ ../include/mesos/docker/spec.pb.h \
../include/mesos/docker/v1.pb.cc \
../include/mesos/docker/v1.pb.h \
../include/mesos/docker/v2.pb.cc \
@@ -400,12 +403,15 @@ nodist_containerizer_HEADERS = \
dockerdir = $(pkgincludedir)/docker
docker_HEADERS = \
+ $(top_srcdir)/include/mesos/docker/spec.hpp \
+ $(top_srcdir)/include/mesos/docker/spec.proto \
$(top_srcdir)/include/mesos/docker/v1.hpp \
$(top_srcdir)/include/mesos/docker/v1.proto \
$(top_srcdir)/include/mesos/docker/v2.hpp \
$(top_srcdir)/include/mesos/docker/v2.proto
nodist_docker_HEADERS = \
+ ../include/mesos/docker/spec.pb.h \
../include/mesos/docker/v1.pb.h \
../include/mesos/docker/v2.pb.h
@@ -684,7 +690,6 @@ libmesos_no_3rdparty_la_SOURCES += \
credentials/credentials.hpp \
docker/docker.hpp \
docker/executor.hpp \
- docker/spec.hpp \
examples/test_anonymous_module.hpp \
examples/test_module.hpp \
examples/utils.hpp \
@@ -995,6 +1000,7 @@ libmesos_la_SOURCES = \
$(AUTHORIZATION_PROTO) \
$(CONTAINERIZER_PROTO) \
$(EXECUTOR_PROTO) \
+ $(DOCKER_SPEC_PROTO) \
$(DOCKER_V1_PROTO) \
$(DOCKER_V2_PROTO) \
$(FETCHER_PROTO) \
http://git-wip-us.apache.org/repos/asf/mesos/blob/49c0fee9/src/docker/spec.cpp
----------------------------------------------------------------------
diff --git a/src/docker/spec.cpp b/src/docker/spec.cpp
index 0188078..29d2f11 100644
--- a/src/docker/spec.cpp
+++ b/src/docker/spec.cpp
@@ -20,12 +20,67 @@
#include <stout/protobuf.hpp>
#include <stout/strings.hpp>
-#include "docker/spec.hpp"
+#include <mesos/docker/spec.hpp>
using std::string;
+using std::vector;
namespace docker {
namespace spec {
+
+// TODO(jieyu): Use regex to parse and verify the reference.
+Try<ImageReference> parseImageReference(const string& _s)
+{
+ ImageReference reference;
+ string s(_s);
+
+ // Extract the digest.
+ if (strings::contains(s, "@")) {
+ vector<string> split = strings::split(s, "@");
+ if (split.size() != 2) {
+ return Error("Multiple '@' symbols found");
+ }
+
+ s = split[0];
+ reference.set_digest(split[1]);
+ }
+
+ // Remove the tag. We need to watch out for a
+ // host:port registry, which also contains ':'.
+ if (strings::contains(s, ":")) {
+ vector<string> split = strings::split(s, ":");
+
+ // The tag must be the last component. If a slash is
+ // present there is a registry port and no tag.
+ if (!strings::contains(split.back(), "/")) {
+ reference.set_tag(split.back());
+ split.pop_back();
+
+ s = strings::join(":", split);
+ }
+ }
+
+ // Extract the registry and repository. The first component can
+ // either be the registry, or the first part of the repository!
+ // We resolve this ambiguity using the same hacks used in the
+ // docker code ('.', ':', 'localhost' indicate a registry).
+ vector<string> split = strings::split(s, "/", 2);
+
+ if (split.size() == 1) {
+ reference.set_repository(s);
+ } else if (strings::contains(split[0], ".") ||
+ strings::contains(split[0], ":") ||
+ split[0] == "localhost") {
+ reference.set_registry(split[0]);
+ reference.set_repository(split[1]);
+ } else {
+ reference.set_repository(s);
+ }
+
+ return reference;
+}
+
+
namespace v1 {
Option<Error> validate(const ImageManifest& manifest)
http://git-wip-us.apache.org/repos/asf/mesos/blob/49c0fee9/src/docker/spec.hpp
----------------------------------------------------------------------
diff --git a/src/docker/spec.hpp b/src/docker/spec.hpp
deleted file mode 100644
index 822b238..0000000
--- a/src/docker/spec.hpp
+++ /dev/null
@@ -1,64 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#ifndef __DOCKER_SPEC_HPP__
-#define __DOCKER_SPEC_HPP__
-
-#include <stout/error.hpp>
-#include <stout/json.hpp>
-#include <stout/option.hpp>
-
-#include <mesos/docker/v1.hpp>
-#include <mesos/docker/v2.hpp>
-
-namespace docker {
-namespace spec {
-namespace v1 {
-
-/**
- * Validates if the specified docker v1 image manifest conforms to the
- * Docker v1 spec. Returns the error if the validation fails.
- */
-Option<Error> validate(const ImageManifest& manifest);
-
-
-/**
- * Returns the docker v1 image manifest from the given JSON object.
- */
-Try<ImageManifest> parse(const JSON::Object& json);
-
-} // namespace v1 {
-
-
-namespace v2 {
-
-/**
- * Validates if the specified v2 image manifest conforms to the Docker
- * v2 spec. Returns the error if the validation fails.
- */
-Option<Error> validate(const ImageManifest& manifest);
-
-
-/**
- * Returns the docker v1 image manifest from the given JSON object.
- */
-Try<ImageManifest> parse(const JSON::Object& json);
-
-} // namespace v2 {
-} // namespace spec {
-} // namespace docker {
-
-#endif // __DOCKER_SPEC_HPP__
http://git-wip-us.apache.org/repos/asf/mesos/blob/49c0fee9/src/slave/containerizer/mesos/provisioner/docker/message.hpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/docker/message.hpp b/src/slave/containerizer/mesos/provisioner/docker/message.hpp
index 162e4c6..43c3608 100644
--- a/src/slave/containerizer/mesos/provisioner/docker/message.hpp
+++ b/src/slave/containerizer/mesos/provisioner/docker/message.hpp
@@ -41,6 +41,8 @@ namespace docker {
//
// TODO(bmahler): Validate based on docker's validation logic
// and return a Try here.
+//
+// TODO(jieyu): Remove this in favor of using 'spec::parseImageName'.
inline Image::Name parseImageName(std::string s)
{
Image::Name name;
http://git-wip-us.apache.org/repos/asf/mesos/blob/49c0fee9/src/slave/containerizer/mesos/provisioner/docker/message.proto
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/docker/message.proto b/src/slave/containerizer/mesos/provisioner/docker/message.proto
index 2b2ed05..003c666 100644
--- a/src/slave/containerizer/mesos/provisioner/docker/message.proto
+++ b/src/slave/containerizer/mesos/provisioner/docker/message.proto
@@ -24,6 +24,7 @@ package mesos.internal.slave.docker;
* and the leaf layer id last.
*/
message Image {
+ // TODO(jieyu): Remove this in favor of using spec::ImageReference.
message Name {
optional string registry = 1;
required string repository = 2;
http://git-wip-us.apache.org/repos/asf/mesos/blob/49c0fee9/src/slave/containerizer/mesos/provisioner/docker/registry_client.hpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/docker/registry_client.hpp b/src/slave/containerizer/mesos/provisioner/docker/registry_client.hpp
index bd6dace..bfe3086 100644
--- a/src/slave/containerizer/mesos/provisioner/docker/registry_client.hpp
+++ b/src/slave/containerizer/mesos/provisioner/docker/registry_client.hpp
@@ -29,7 +29,7 @@
#include <process/http.hpp>
#include <process/process.hpp>
-#include "docker/spec.hpp"
+#include <mesos/docker/spec.hpp>
#include "slave/containerizer/mesos/provisioner/docker/message.hpp"
http://git-wip-us.apache.org/repos/asf/mesos/blob/49c0fee9/src/tests/containerizer/docker_spec_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/containerizer/docker_spec_tests.cpp b/src/tests/containerizer/docker_spec_tests.cpp
index aa4faf9..5799dc9 100644
--- a/src/tests/containerizer/docker_spec_tests.cpp
+++ b/src/tests/containerizer/docker_spec_tests.cpp
@@ -19,7 +19,7 @@
#include <stout/gtest.hpp>
#include <stout/json.hpp>
-#include "docker/spec.hpp"
+#include <mesos/docker/spec.hpp>
namespace spec = docker::spec;
@@ -30,6 +30,75 @@ namespace tests {
class DockerSpecTest : public ::testing::Test {};
+TEST_F(DockerSpecTest, ParseImageReference)
+{
+ Try<spec::ImageReference> reference =
+ spec::parseImageReference("library/busybox");
+
+ ASSERT_SOME(reference);
+ EXPECT_FALSE(reference->has_registry());
+ EXPECT_EQ("library/busybox", reference->repository());
+ EXPECT_FALSE(reference->has_tag());
+ EXPECT_FALSE(reference->has_digest());
+
+ reference = spec::parseImageReference("busybox");
+
+ ASSERT_SOME(reference);
+ EXPECT_FALSE(reference->has_registry());
+ EXPECT_EQ("busybox", reference->repository());
+ EXPECT_FALSE(reference->has_tag());
+ EXPECT_FALSE(reference->has_digest());
+
+ reference = spec::parseImageReference("library/busybox:tag");
+
+ ASSERT_SOME(reference);
+ EXPECT_FALSE(reference->has_registry());
+ EXPECT_EQ("library/busybox", reference->repository());
+ EXPECT_EQ("tag", reference->tag());
+ EXPECT_FALSE(reference->has_digest());
+
+ reference = spec::parseImageReference("library/busybox@sha256:bc8813ea7b3603864987522f02a76101c17ad122e1c46d790efc0fca78ca7bfb"); // NOLINT(whitespace/line_length)
+
+ ASSERT_SOME(reference);
+ EXPECT_FALSE(reference->has_registry());
+ EXPECT_EQ("library/busybox", reference->repository());
+ EXPECT_FALSE(reference->has_tag());
+ EXPECT_EQ("sha256:bc8813ea7b3603864987522f02a76101c17ad122e1c46d790efc0fca78ca7bfb", reference->digest()); // NOLINT(whitespace/line_length)
+
+ reference = spec::parseImageReference("registry.io/library/busybox");
+
+ ASSERT_SOME(reference);
+ EXPECT_EQ("registry.io", reference->registry());
+ EXPECT_EQ("library/busybox", reference->repository());
+ EXPECT_FALSE(reference->has_tag());
+ EXPECT_FALSE(reference->has_digest());
+
+ reference = spec::parseImageReference("registry.io/library/busybox:tag");
+
+ ASSERT_SOME(reference);
+ EXPECT_EQ("registry.io", reference->registry());
+ EXPECT_EQ("library/busybox", reference->repository());
+ EXPECT_EQ("tag", reference->tag());
+ EXPECT_FALSE(reference->has_digest());
+
+ reference = spec::parseImageReference("registry.io:80/library/busybox:tag");
+
+ ASSERT_SOME(reference);
+ EXPECT_EQ("registry.io:80", reference->registry());
+ EXPECT_EQ("library/busybox", reference->repository());
+ EXPECT_EQ("tag", reference->tag());
+ EXPECT_FALSE(reference->has_digest());
+
+ reference = spec::parseImageReference("registry.io:80/library/busybox@sha256:bc8813ea7b3603864987522f02a76101c17ad122e1c46d790efc0fca78ca7bfb"); // NOLINT(whitespace/line_length)
+
+ ASSERT_SOME(reference);
+ EXPECT_EQ("registry.io:80", reference->registry());
+ EXPECT_EQ("library/busybox", reference->repository());
+ EXPECT_FALSE(reference->has_tag());
+ EXPECT_EQ("sha256:bc8813ea7b3603864987522f02a76101c17ad122e1c46d790efc0fca78ca7bfb", reference->digest()); // NOLINT(whitespace/line_length)
+}
+
+
TEST_F(DockerSpecTest, ParseV1ImageManifest)
{
Try<JSON::Object> json = JSON::parse<JSON::Object>(
http://git-wip-us.apache.org/repos/asf/mesos/blob/49c0fee9/src/tests/containerizer/provisioner_docker_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/containerizer/provisioner_docker_tests.cpp b/src/tests/containerizer/provisioner_docker_tests.cpp
index f81f003..33df600 100644
--- a/src/tests/containerizer/provisioner_docker_tests.cpp
+++ b/src/tests/containerizer/provisioner_docker_tests.cpp
@@ -39,7 +39,7 @@
#include <process/ssl/gtest.hpp>
-#include "docker/spec.hpp"
+#include <mesos/docker/spec.hpp>
#include "slave/containerizer/mesos/provisioner/docker/metadata_manager.hpp"
#include "slave/containerizer/mesos/provisioner/docker/paths.hpp"
@@ -84,6 +84,8 @@ namespace mesos {
namespace internal {
namespace tests {
+// TODO(jieyu): Remove this test in favor of using
+// DockerSpecTest.ParseImageReference.
TEST(DockerUtilsTest, ParseImageName)
{
slave::docker::Image::Name name;