You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by tn...@apache.org on 2015/10/16 14:16:10 UTC
[2/2] mesos git commit: Added serialization of Docker Image Spec as
protobuf.
Added serialization of Docker Image Spec as protobuf.
Review: https://reviews.apache.org/r/38901
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/cb9e40dc
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/cb9e40dc
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/cb9e40dc
Branch: refs/heads/master
Commit: cb9e40dca394e49714cb64de7574de0c937dd64f
Parents: 7b53bb1
Author: Gilbert Song <gi...@mesoshere.io>
Authored: Fri Oct 16 04:40:06 2015 +0000
Committer: Timothy Chen <tn...@gmail.com>
Committed: Fri Oct 16 12:15:54 2015 +0000
----------------------------------------------------------------------
src/CMakeLists.txt | 1 +
src/Makefile.am | 2 +
.../provisioner/docker/message.proto | 56 +++++++-
.../containerizer/provisioner/docker/spec.cpp | 49 +++++++
.../containerizer/provisioner/docker/spec.hpp | 47 +++++++
.../containerizer/provisioner_docker_tests.cpp | 136 +++++++++++++++++++
6 files changed, 290 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/cb9e40dc/src/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 98e76ce..35e5cdc 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -195,6 +195,7 @@ set(AGENT_SRC
slave/containerizer/provisioner/backend.cpp
slave/containerizer/provisioner/backends/copy.cpp
slave/containerizer/provisioner/docker/registry_client.cpp
+ slave/containerizer/provisioner/docker/spec.cpp
slave/containerizer/provisioner/docker/token_manager.cpp
slave/resource_estimators/noop.cpp
)
http://git-wip-us.apache.org/repos/asf/mesos/blob/cb9e40dc/src/Makefile.am
----------------------------------------------------------------------
diff --git a/src/Makefile.am b/src/Makefile.am
index 96ce73b..8523832 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -560,6 +560,7 @@ libmesos_no_3rdparty_la_SOURCES = \
slave/containerizer/provisioner/docker/paths.cpp \
slave/containerizer/provisioner/docker/puller.cpp \
slave/containerizer/provisioner/docker/registry_client.cpp \
+ slave/containerizer/provisioner/docker/spec.cpp \
slave/containerizer/provisioner/docker/store.cpp \
slave/containerizer/provisioner/docker/token_manager.cpp \
slave/resource_estimators/noop.cpp \
@@ -866,6 +867,7 @@ libmesos_no_3rdparty_la_SOURCES += \
slave/containerizer/provisioner/docker/paths.hpp \
slave/containerizer/provisioner/docker/puller.hpp \
slave/containerizer/provisioner/docker/registry_client.hpp \
+ slave/containerizer/provisioner/docker/spec.hpp \
slave/containerizer/provisioner/docker/store.hpp \
slave/containerizer/provisioner/docker/token_manager.hpp \
slave/containerizer/isolators/posix.hpp \
http://git-wip-us.apache.org/repos/asf/mesos/blob/cb9e40dc/src/slave/containerizer/provisioner/docker/message.proto
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/provisioner/docker/message.proto b/src/slave/containerizer/provisioner/docker/message.proto
index bbac2e6..edbb1a8 100644
--- a/src/slave/containerizer/provisioner/docker/message.proto
+++ b/src/slave/containerizer/provisioner/docker/message.proto
@@ -41,4 +41,58 @@ message Image {
message Images {
repeated Image images = 1;
-}
\ No newline at end of file
+}
+
+
+/**
+* Protobuf for the Docker image manifest JSON schema:
+* https://github.com/docker/distribution/blob/master/docs/spec/manifest-v2-1.md
+*/
+message DockerImageManifest {
+ required string name = 1;
+ required string tag = 2;
+ required string architecture = 3;
+
+ message FsLayers {
+ required string blobSum = 1;
+ }
+
+ repeated FsLayers fsLayers = 4;
+
+ message History {
+ message V1Compatibility {
+ required string id = 1;
+ required string parent = 2;
+ }
+
+ required V1Compatibility v1Compatibility = 1;
+ }
+
+ repeated History history = 5;
+ required uint32 schemaVersion = 6;
+
+ message Signatures {
+
+ //JOSE (A JSON Web Signature).
+ message Header {
+
+ //JSON Web Key.
+ message Jwk {
+ required string crv = 1;
+ required string kid = 2;
+ required string kty = 3;
+ required string x = 4;
+ required string y = 5;
+ }
+
+ optional Jwk jwk = 1;
+ required string alg = 2;
+ }
+
+ required Header header = 1;
+ required string signature = 2;
+ required string protected = 3;
+ }
+
+ repeated Signatures signatures = 7;
+}
http://git-wip-us.apache.org/repos/asf/mesos/blob/cb9e40dc/src/slave/containerizer/provisioner/docker/spec.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/provisioner/docker/spec.cpp b/src/slave/containerizer/provisioner/docker/spec.cpp
new file mode 100644
index 0000000..4de8c5a
--- /dev/null
+++ b/src/slave/containerizer/provisioner/docker/spec.cpp
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <stout/json.hpp>
+#include <stout/protobuf.hpp>
+#include <stout/strings.hpp>
+
+#include "slave/containerizer/provisioner/docker/spec.hpp"
+
+using std::string;
+
+namespace mesos {
+namespace internal {
+namespace slave {
+namespace docker {
+namespace spec {
+
+Try<docker::DockerImageManifest> parse(const JSON::Object& json)
+{
+ Try<docker::DockerImageManifest> manifest =
+ protobuf::parse<docker::DockerImageManifest>(json);
+
+ if (manifest.isError()) {
+ return Error("Protobuf parse failed: " + manifest.error());
+ }
+
+ return manifest.get();
+}
+
+} // namespace spec {
+} // namespace docker {
+} // namespace slave {
+} // namespace internal {
+} // namespace mesos {
http://git-wip-us.apache.org/repos/asf/mesos/blob/cb9e40dc/src/slave/containerizer/provisioner/docker/spec.hpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/provisioner/docker/spec.hpp b/src/slave/containerizer/provisioner/docker/spec.hpp
new file mode 100644
index 0000000..1aa7a03
--- /dev/null
+++ b/src/slave/containerizer/provisioner/docker/spec.hpp
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __PROVISIONER_DOCKER_SPEC_HPP__
+#define __PROVISIONER_DOCKER_SPEC_HPP__
+
+#include <stout/error.hpp>
+#include <stout/json.hpp>
+#include <stout/option.hpp>
+
+#include <mesos/mesos.hpp>
+
+#include "slave/containerizer/provisioner/docker/message.hpp"
+
+namespace mesos {
+namespace internal {
+namespace slave {
+namespace docker {
+namespace spec {
+
+// TODO(Gilbert): add validations here, e.g., Manifest, Blob, Layout, ImageID.
+
+// Parse the DockerImageManifest from the specified JSON object.
+Try<docker::DockerImageManifest> parse(const JSON::Object& json);
+
+} // namespace spec {
+} // namespace docker {
+} // namespace slave {
+} // namespace internal {
+} // namespace mesos {
+
+#endif // __PROVISIONER_DOCKER_SPEC_HPP__
http://git-wip-us.apache.org/repos/asf/mesos/blob/cb9e40dc/src/tests/containerizer/provisioner_docker_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/containerizer/provisioner_docker_tests.cpp b/src/tests/containerizer/provisioner_docker_tests.cpp
index 822aa77..11d7457 100644
--- a/src/tests/containerizer/provisioner_docker_tests.cpp
+++ b/src/tests/containerizer/provisioner_docker_tests.cpp
@@ -40,6 +40,7 @@
#include "slave/containerizer/provisioner/docker/metadata_manager.hpp"
#include "slave/containerizer/provisioner/docker/paths.hpp"
#include "slave/containerizer/provisioner/docker/registry_client.hpp"
+#include "slave/containerizer/provisioner/docker/spec.hpp"
#include "slave/containerizer/provisioner/docker/store.hpp"
#include "slave/containerizer/provisioner/docker/token_manager.hpp"
@@ -236,6 +237,141 @@ TEST_F(RegistryTokenTest, NotBeforeInFuture)
}
+class DockerSpecTest : public ::testing::Test {};
+
+TEST_F(DockerSpecTest, SerializeDockerManifest)
+{
+ JSON::Value manifest = JSON::parse(
+ "{"
+ " \"name\": \"dmcgowan/test-image\","
+ " \"tag\": \"latest\","
+ " \"architecture\": \"amd64\","
+ " \"fsLayers\": ["
+ " {"
+ " \"blobSum\": "
+ "\"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\""
+ " },"
+ " {"
+ " \"blobSum\": "
+ "\"sha256:cea0d2071b01b0a79aa4a05ea56ab6fdf3fafa03369d9f4eea8d46ea33c43e5f\""
+ " },"
+ " {"
+ " \"blobSum\": "
+ "\"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\""
+ " },"
+ " {"
+ " \"blobSum\": "
+ "\"sha256:2a7812e636235448785062100bb9103096aa6655a8f6bb9ac9b13fe8290f66df\""
+ " }"
+ " ],"
+ " \"history\": ["
+ " {"
+ " \"v1Compatibility\": "
+ " {"
+ " \"id\": "
+ "\"2ce2e90b0bc7224de3db1f0d646fe8e2c4dd37f1793928287f6074bc451a57ea\","
+ " \"parent\": "
+ "\"cf2616975b4a3cba083ca99bc3f0bf25f5f528c3c52be1596b30f60b0b1c37ff\""
+ " }"
+ " },"
+ " {"
+ " \"v1Compatibility\": "
+ " {"
+ " \"id\": "
+ "\"2ce2e90b0bc7224de3db1f0d646fe8e2c4dd37f1793928287f6074bc451a57ea\","
+ " \"parent\": "
+ "\"cf2616975b4a3cba083ca99bc3f0bf25f5f528c3c52be1596b30f60b0b1c37ff\""
+ " }"
+ " },"
+ " {"
+ " \"v1Compatibility\": "
+ " {"
+ " \"id\": "
+ "\"2ce2e90b0bc7224de3db1f0d646fe8e2c4dd37f1793928287f6074bc451a57ea\","
+ " \"parent\": "
+ "\"cf2616975b4a3cba083ca99bc3f0bf25f5f528c3c52be1596b30f60b0b1c37ff\""
+ " }"
+ " }"
+ " ],"
+ " \"schemaVersion\": 1,"
+ " \"signatures\": ["
+ " {"
+ " \"header\": {"
+ " \"jwk\": {"
+ " \"crv\": \"P-256\","
+ " \"kid\": "
+ "\"LYRA:YAG2:QQKS:376F:QQXY:3UNK:SXH7:K6ES:Y5AU:XUN5:ZLVY:KBYL\","
+ " \"kty\": \"EC\","
+ " \"x\": \"Cu_UyxwLgHzE9rvlYSmvVdqYCXY42E9eNhBb0xNv0SQ\","
+ " \"y\": \"zUsjWJkeKQ5tv7S-hl1Tg71cd-CqnrtiiLxSi6N_yc8\""
+ " },"
+ " \"alg\": \"ES256\""
+ " },"
+ " \"signature\": \"m3bgdBXZYRQ4ssAbrgj8Kjl7GNgrKQvmCSY-00yzQosKi-8"
+ "UBrIRrn3Iu5alj82B6u_jNrkGCjEx3TxrfT1rig\","
+ " \"protected\": \"eyJmb3JtYXRMZW5ndGgiOjYwNjMsImZvcm1hdFRhaWwiOiJ"
+ "DbjAiLCJ0aW1lIjoiMjAxNC0wOS0xMVQxNzoxNDozMFoifQ\""
+ " }"
+ " ]"
+ "}").get();
+
+ Try<JSON::Object> json = JSON::parse<JSON::Object>(stringify(manifest));
+ ASSERT_SOME(json);
+
+ Try<slave::docker::DockerImageManifest> dockerImageManifest =
+ spec::parse(json.get());
+
+ ASSERT_SOME(dockerImageManifest);
+
+ EXPECT_EQ(dockerImageManifest.get().name(), "dmcgowan/test-image");
+ EXPECT_EQ(dockerImageManifest.get().tag(), "latest");
+ EXPECT_EQ(dockerImageManifest.get().architecture(), "amd64");
+
+ EXPECT_EQ(dockerImageManifest.get().fslayers(0).blobsum(),
+ "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855");
+ EXPECT_EQ(dockerImageManifest.get().fslayers(1).blobsum(),
+ "sha256:cea0d2071b01b0a79aa4a05ea56ab6fdf3fafa03369d9f4eea8d46ea33c43e5f");
+ EXPECT_EQ(dockerImageManifest.get().fslayers(2).blobsum(),
+ "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855");
+ EXPECT_EQ(dockerImageManifest.get().fslayers(3).blobsum(),
+ "sha256:2a7812e636235448785062100bb9103096aa6655a8f6bb9ac9b13fe8290f66df");
+
+ EXPECT_EQ(dockerImageManifest.get().history(1).v1compatibility().id(),
+ "2ce2e90b0bc7224de3db1f0d646fe8e2c4dd37f1793928287f6074bc451a57ea");
+ EXPECT_EQ(dockerImageManifest.get().history(2).v1compatibility().parent(),
+ "cf2616975b4a3cba083ca99bc3f0bf25f5f528c3c52be1596b30f60b0b1c37ff");
+
+ EXPECT_EQ(dockerImageManifest.get().schemaversion(), 1);
+
+ EXPECT_EQ(dockerImageManifest.get().signatures(0).header().jwk().kid(),
+ "LYRA:YAG2:QQKS:376F:QQXY:3UNK:SXH7:K6ES:Y5AU:XUN5:ZLVY:KBYL");
+ EXPECT_EQ(dockerImageManifest.get().signatures(0).signature(),
+ "m3bgdBXZYRQ4ssAbrgj8Kjl7GNgrKQvmCSY-00yzQosKi-8"
+ "UBrIRrn3Iu5alj82B6u_jNrkGCjEx3TxrfT1rig");
+}
+
+// Test invalid JSON object, expecting an error.
+TEST_F(DockerSpecTest, SerializeDockerInvalidManifest)
+{
+ // This is an invalid manifest. The repeated fields 'history' and 'fsLayers'
+ // must be >= 1. The 'signatures' and 'schemaVersion' are not set.
+ JSON::Value manifest = JSON::parse(
+ "{"
+ " \"name\": \"dmcgowan/test-image\","
+ " \"tag\": \"latest\","
+ " \"architecture\": \"amd64\""
+ "}").get();
+
+ Try<JSON::Object> json = JSON::parse<JSON::Object>(stringify(manifest));
+ ASSERT_SOME(json);
+
+ Try<slave::docker::DockerImageManifest> dockerImageManifest =
+ spec::parse(json.get());
+
+ EXPECT_ERROR(dockerImageManifest);
+}
+
+
#ifdef USE_SSL_SOCKET
// Test suite for docker registry tests.