You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by qi...@apache.org on 2017/02/07 08:56:12 UTC
[1/3] mesos git commit: Add protobuf messages for OCI image spec.
Repository: mesos
Updated Branches:
refs/heads/master 974581bdb -> 25f4feae4
Add protobuf messages for OCI image spec.
Review: https://reviews.apache.org/r/52349/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/6726cd19
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/6726cd19
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/6726cd19
Branch: refs/heads/master
Commit: 6726cd19cad23480d68f92fe3b6019b9452ab1e7
Parents: 974581b
Author: Qian Zhang <zh...@gmail.com>
Authored: Tue Feb 7 16:54:24 2017 +0800
Committer: Qian Zhang <zh...@gmail.com>
Committed: Tue Feb 7 16:54:24 2017 +0800
----------------------------------------------------------------------
include/mesos/oci/spec.hpp | 34 +++++++++
include/mesos/oci/spec.proto | 156 ++++++++++++++++++++++++++++++++++++++
src/Makefile.am | 13 ++++
3 files changed, 203 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/6726cd19/include/mesos/oci/spec.hpp
----------------------------------------------------------------------
diff --git a/include/mesos/oci/spec.hpp b/include/mesos/oci/spec.hpp
new file mode 100644
index 0000000..ac0b063
--- /dev/null
+++ b/include/mesos/oci/spec.hpp
@@ -0,0 +1,34 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef __MESOS_OCI_SPEC_HPP__
+#define __MESOS_OCI_SPEC_HPP__
+
+#include <mesos/oci/spec.pb.h>
+
+namespace oci {
+namespace spec {
+namespace image {
+namespace v1 {
+
+// TODO(qianzhang): Add methods to parse OCI image spec
+
+} // namespace v1 {
+} // namespace image {
+} // namespace spec {
+} // namespace oci {
+
+#endif // __MESOS_OCI_SPEC_HPP__
http://git-wip-us.apache.org/repos/asf/mesos/blob/6726cd19/include/mesos/oci/spec.proto
----------------------------------------------------------------------
diff --git a/include/mesos/oci/spec.proto b/include/mesos/oci/spec.proto
new file mode 100644
index 0000000..f3083dd
--- /dev/null
+++ b/include/mesos/oci/spec.proto
@@ -0,0 +1,156 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package oci.spec.image.v1;
+
+/**
+ * Protobuf for the OCI content descriptor JSON schema:
+ * https://github.com/opencontainers/image-spec/blob/master/descriptor.md
+ */
+message Descriptor {
+ // Media types identify their referenced resources, see more details in:
+ // https://github.com/opencontainers/image-spec/blob/master/media-types.md
+ required string mediaType = 1;
+ required string digest = 2;
+ required int64 size = 3;
+ repeated string urls = 4;
+}
+
+
+message Platform {
+ required string architecture = 1;
+ required string os = 2;
+
+ // NOTE: The actual field names in OCI spec are 'os.version' and 'os.features',
+ // but we can not use them in protobuf message since "." is not allowed in
+ // field name, so these two fields will be handled manually during parsing.
+ optional string os_version = 3;
+ repeated string os_features = 4;
+ optional string variant = 5;
+ repeated string features = 6;
+}
+
+
+message ManifestDescriptor {
+ required string mediaType = 1;
+ required string digest = 2;
+ required int64 size = 3;
+ required Platform platform = 4;
+ repeated string urls = 5;
+}
+
+
+/**
+ * Protobuf for the string-string map field entry.
+ */
+message Label {
+ required string key = 1;
+ required string value = 2;
+}
+
+
+/**
+ * Protobuf for the OCI image manifest list JSON schema:
+ * https://github.com/opencontainers/image-spec/blob/master/manifest-list.md
+ *
+ * The OCI MIME type of this message is:
+ * application/vnd.oci.image.manifest.list.v1+json
+ */
+message ManifestList {
+ required int64 schemaVersion = 1;
+ repeated ManifestDescriptor manifests = 2;
+
+ // NOTE: We cannot use 'annotations' here because otherwise,
+ // json->protobuf parsing will fail. 'Annotations' is manually
+ // set during parsing.
+ repeated Label Annotations = 3;
+}
+
+
+/**
+ * Protobuf for the OCI image manifest JSON schema:
+ * https://github.com/opencontainers/image-spec/blob/master/manifest.md
+ *
+ * The OCI MIME type of this message is:
+ * application/vnd.oci.image.manifest.v1+json
+ */
+message Manifest {
+ required int64 schemaVersion = 1;
+ required Descriptor config = 2;
+ repeated Descriptor layers = 3;
+
+ // NOTE: We cannot use 'annotations' here because otherwise,
+ // json->protobuf parsing will fail. 'Annotations' is manually
+ // set during parsing.
+ repeated Label Annotations = 4;
+}
+
+
+/**
+ * Protobuf for the OCI image configuration JSON schema:
+ * https://github.com/opencontainers/image-spec/blob/master/config.md
+ *
+ * The OCI MIME type of this message is:
+ * application/vnd.oci.image.config.v1+json
+ */
+message Configuration {
+ required string architecture = 1;
+ required string os = 2;
+
+ message Rootfs {
+ required string type = 1;
+ repeated string diff_ids = 2;
+ }
+
+ required Rootfs rootfs = 3;
+ optional string created = 4;
+ optional string author = 5;
+
+ message Config {
+ optional string User = 1;
+
+ // NOTE: We cannot use 'ExposedPorts' here because otherwise,
+ // json->protobuf parsing will fail. 'exposedPorts' is manually
+ // set during parsing.
+ repeated string exposedPorts = 2;
+ repeated string Env = 3;
+ repeated string Entrypoint = 4;
+ repeated string Cmd = 5;
+
+ // NOTE: We cannot use 'Volumes' here because otherwise,
+ // json->protobuf parsing will fail. 'volumes' is manually
+ // set during parsing.
+ repeated string volumes = 6;
+ optional string WorkingDir = 7;
+
+ // NOTE: We cannot use 'Labels' here because otherwise,
+ // json->protobuf parsing will fail. 'labels' is manually
+ // set during parsing.
+ repeated Label labels = 8;
+ }
+
+ optional Config config = 6;
+
+ message History {
+ optional string created = 1;
+ optional string author = 2;
+ optional string created_by = 3;
+ optional string comment = 4;
+ optional bool empty_layer = 5;
+ }
+
+ repeated History history = 7;
+}
http://git-wip-us.apache.org/repos/asf/mesos/blob/6726cd19/src/Makefile.am
----------------------------------------------------------------------
diff --git a/src/Makefile.am b/src/Makefile.am
index 78d55d8..f432571 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -240,6 +240,7 @@ MAINTENANCE_PROTO = $(top_srcdir)/include/mesos/maintenance/maintenance.proto
MASTER_PROTO = $(top_srcdir)/include/mesos/master/master.proto
MESOS_PROTO = $(top_srcdir)/include/mesos/mesos.proto
MODULE_PROTO = $(top_srcdir)/include/mesos/module/module.proto
+OCI_SPEC_PROTO = $(top_srcdir)/include/mesos/oci/spec.proto
QUOTA_PROTO = $(top_srcdir)/include/mesos/quota/quota.proto
SCHEDULER_PROTO = $(top_srcdir)/include/mesos/scheduler/scheduler.proto
STATE_PROTO = $(top_srcdir)/include/mesos/state/state.proto
@@ -288,6 +289,8 @@ CXX_PROTOS = \
../include/mesos/module/hook.pb.h \
../include/mesos/module/module.pb.cc \
../include/mesos/module/module.pb.h \
+ ../include/mesos/oci/spec.pb.cc \
+ ../include/mesos/oci/spec.pb.h \
../include/mesos/quota/quota.pb.cc \
../include/mesos/quota/quota.pb.h \
../include/mesos/scheduler/scheduler.pb.cc \
@@ -656,6 +659,15 @@ nodist_module_HEADERS = \
../include/mesos/module/hook.pb.h \
../include/mesos/module/module.pb.h
+ocidir = $(pkgincludedir)/oci
+
+oci_HEADERS = \
+ $(top_srcdir)/include/mesos/oci/spec.hpp \
+ $(top_srcdir)/include/mesos/oci/spec.proto
+
+nodist_oci_HEADERS = \
+ ../include/mesos/oci/spec.pb.h
+
quotadir = $(pkgincludedir)/quota
quota_HEADERS = \
@@ -1386,6 +1398,7 @@ libmesos_la_SOURCES = \
$(MASTER_PROTO) \
$(MESOS_PROTO) \
$(MODULE_PROTO) \
+ $(OCI_SPEC_PROTO) \
$(OVERSUBSCRIPTION_PROTO) \
$(QUOTA_PROTO) \
$(SCHEDULER_PROTO) \
[3/3] mesos git commit: Added tests for parsing OCI image spec.
Posted by qi...@apache.org.
Added tests for parsing OCI image spec.
Review: https://reviews.apache.org/r/55140/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/25f4feae
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/25f4feae
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/25f4feae
Branch: refs/heads/master
Commit: 25f4feae487d53a701adb787fd8a2e5f6166b789
Parents: 3681d61
Author: Qian Zhang <zh...@gmail.com>
Authored: Tue Feb 7 16:54:56 2017 +0800
Committer: Qian Zhang <zh...@gmail.com>
Committed: Tue Feb 7 16:54:56 2017 +0800
----------------------------------------------------------------------
src/Makefile.am | 1 +
src/tests/CMakeLists.txt | 1 +
src/tests/containerizer/oci_spec_tests.cpp | 401 ++++++++++++++++++++++++
3 files changed, 403 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/25f4feae/src/Makefile.am
----------------------------------------------------------------------
diff --git a/src/Makefile.am b/src/Makefile.am
index cb8e604..c21a073 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -2309,6 +2309,7 @@ mesos_tests_SOURCES = \
tests/containerizer/memory_test_helper.cpp \
tests/containerizer/mesos_containerizer_tests.cpp \
tests/containerizer/mesos_containerizer_paths_tests.cpp \
+ tests/containerizer/oci_spec_tests.cpp \
tests/containerizer/posix_rlimits_isolator_tests.cpp \
tests/containerizer/provisioner_appc_tests.cpp \
tests/containerizer/provisioner_backend_tests.cpp \
http://git-wip-us.apache.org/repos/asf/mesos/blob/25f4feae/src/tests/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt
index c12a9f7..7898833 100644
--- a/src/tests/CMakeLists.txt
+++ b/src/tests/CMakeLists.txt
@@ -190,6 +190,7 @@ if (NOT WIN32)
containerizer/memory_isolator_tests.cpp
containerizer/mesos_containerizer_paths_tests.cpp
containerizer/mesos_containerizer_tests.cpp
+ containerizer/oci_spec_tests.cpp
containerizer/posix_rlimits_isolator_tests.cpp
containerizer/provisioner_appc_tests.cpp
containerizer/provisioner_backend_tests.cpp
http://git-wip-us.apache.org/repos/asf/mesos/blob/25f4feae/src/tests/containerizer/oci_spec_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/containerizer/oci_spec_tests.cpp b/src/tests/containerizer/oci_spec_tests.cpp
new file mode 100644
index 0000000..29a8f23
--- /dev/null
+++ b/src/tests/containerizer/oci_spec_tests.cpp
@@ -0,0 +1,401 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <string>
+
+#include <stout/gtest.hpp>
+#include <stout/json.hpp>
+
+#include <mesos/oci/spec.hpp>
+
+#include "tests/mesos.hpp"
+
+namespace image = ::oci::spec::image;
+
+using std::string;
+
+namespace mesos {
+namespace internal {
+namespace tests {
+
+class OCISpecTest : public ::testing::Test {};
+
+
+TEST_F(OCISpecTest, ParseDescriptor)
+{
+ const string json =
+ R"~(
+ {
+ "mediaType": "application/vnd.oci.image.manifest.v1+json",
+ "digest": "sha256:5b0bcabd1ed22e9fb1310cf6c2dec7cdef19f0ad69efa1f392e94a4333501270",
+ "size": 7682,
+ "urls": [
+ "https://example.com/example-manifest"
+ ]
+ })~";
+
+ Try<image::v1::Descriptor> descriptor =
+ image::v1::parse<image::v1::Descriptor>(json);
+
+ ASSERT_SOME(descriptor);
+
+ EXPECT_EQ(
+ "application/vnd.oci.image.manifest.v1+json",
+ descriptor->mediatype());
+
+ EXPECT_EQ(
+ "sha256:5b0bcabd1ed22e9fb1310cf6c2dec7cdef19f0ad69efa1f392e94a4333501270",
+ descriptor->digest());
+
+ EXPECT_EQ(7682u, descriptor->size());
+
+ EXPECT_EQ(
+ "https://example.com/example-manifest",
+ descriptor->urls(0));
+}
+
+
+TEST_F(OCISpecTest, ParseManifestList)
+{
+ const string json =
+ R"~(
+ {
+ "schemaVersion": 2,
+ "manifests": [
+ {
+ "mediaType": "application/vnd.oci.image.manifest.v1+json",
+ "size": 7143,
+ "digest": "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f",
+ "platform": {
+ "architecture": "ppc64le",
+ "os": "linux",
+ "os.version": "16.04"
+ }
+ },
+ {
+ "mediaType": "application/vnd.oci.image.manifest.v1+json",
+ "size": 7682,
+ "digest": "sha256:5b0bcabd1ed22e9fb1310cf6c2dec7cdef19f0ad69efa1f392e94a4333501270",
+ "platform": {
+ "architecture": "amd64",
+ "os": "linux",
+ "os.features": [
+ "sse4"
+ ]
+ }
+ }
+ ],
+ "annotations": {
+ "com.example.key1": "value1",
+ "com.example.key2": "value2"
+ }
+ })~";
+
+ Try<image::v1::ManifestList> manifestList =
+ image::v1::parse<image::v1::ManifestList>(json);
+
+ ASSERT_SOME(manifestList);
+
+ EXPECT_EQ(2u, manifestList->schemaversion());
+
+ EXPECT_EQ(
+ "application/vnd.oci.image.manifest.v1+json",
+ manifestList->manifests(0).mediatype());
+
+ EXPECT_EQ(7143u, manifestList->manifests(0).size());
+
+ EXPECT_EQ(
+ "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f",
+ manifestList->manifests(0).digest());
+
+ EXPECT_EQ(
+ "ppc64le",
+ manifestList->manifests(0).platform().architecture());
+
+ EXPECT_EQ(
+ "linux",
+ manifestList->manifests(0).platform().os());
+
+ EXPECT_EQ(
+ "16.04",
+ manifestList->manifests(0).platform().os_version());
+
+ EXPECT_EQ(
+ "sse4",
+ manifestList->manifests(1).platform().os_features(0));
+
+ EXPECT_EQ(
+ "com.example.key1",
+ manifestList->annotations(0).key());
+
+ EXPECT_EQ(
+ "value1",
+ manifestList->annotations(0).value());
+
+ EXPECT_EQ(
+ "com.example.key2",
+ manifestList->annotations(1).key());
+
+ EXPECT_EQ(
+ "value2",
+ manifestList->annotations(1).value());
+}
+
+
+TEST_F(OCISpecTest, ParseManifest)
+{
+ const string json =
+ R"~(
+ {
+ "schemaVersion": 2,
+ "config": {
+ "mediaType": "application/vnd.oci.image.config.v1+json",
+ "size": 7023,
+ "digest": "sha256:b5b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7"
+ },
+ "layers": [
+ {
+ "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip",
+ "size": 32654,
+ "digest": "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f"
+ },
+ {
+ "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip",
+ "size": 16724,
+ "digest": "sha256:3c3a4604a545cdc127456d94e421cd355bca5b528f4a9c1905b15da2eb4a4c6b"
+ }
+ ],
+ "annotations": {
+ "com.example.key1": "value1",
+ "com.example.key2": "value2"
+ }
+ })~";
+
+ Try<image::v1::Manifest> manifest=
+ image::v1::parse<image::v1::Manifest>(json);
+
+ ASSERT_SOME(manifest);
+
+ EXPECT_EQ(2u, manifest->schemaversion());
+
+ EXPECT_EQ(
+ "application/vnd.oci.image.config.v1+json",
+ manifest->config().mediatype());
+
+ EXPECT_EQ(7023u, manifest->config().size());
+
+ EXPECT_EQ(
+ "sha256:b5b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7",
+ manifest->config().digest());
+
+ EXPECT_EQ(
+ "application/vnd.oci.image.layer.v1.tar+gzip",
+ manifest->layers(0).mediatype());
+
+ EXPECT_EQ(32654u, manifest->layers(0).size());
+
+ EXPECT_EQ(
+ "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f",
+ manifest->layers(0).digest());
+
+ EXPECT_EQ(
+ "application/vnd.oci.image.layer.v1.tar+gzip",
+ manifest->layers(1).mediatype());
+
+ EXPECT_EQ(16724u, manifest->layers(1).size());
+
+ EXPECT_EQ(
+ "sha256:3c3a4604a545cdc127456d94e421cd355bca5b528f4a9c1905b15da2eb4a4c6b",
+ manifest->layers(1).digest());
+
+ EXPECT_EQ(
+ "com.example.key1",
+ manifest->annotations(0).key());
+
+ EXPECT_EQ(
+ "value1",
+ manifest->annotations(0).value());
+
+ EXPECT_EQ(
+ "com.example.key2",
+ manifest->annotations(1).key());
+
+ EXPECT_EQ(
+ "value2",
+ manifest->annotations(1).value());
+}
+
+
+TEST_F(OCISpecTest, ParseConfiguration)
+{
+ const string json =
+ R"~(
+ {
+ "created": "2015-10-31T22:22:56.015925234Z",
+ "author": "Alyssa P. Hacker <al...@example.com>",
+ "architecture": "amd64",
+ "os": "linux",
+ "config": {
+ "User": "alice",
+ "ExposedPorts": {
+ "8080/tcp": {}
+ },
+ "Env": [
+ "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
+ "FOO=oci_is_a"
+ ],
+ "Entrypoint": [
+ "/bin/my-app-binary"
+ ],
+ "Cmd": [
+ "--config",
+ "/etc/my-app.d/default.cfg"
+ ],
+ "Volumes": {
+ "/var/job-result-data": {},
+ "/var/log/my-app-logs": {}
+ },
+ "WorkingDir": "/home/alice",
+ "Labels": {
+ "com.example.project.git.commit": "45a939b2999782a3f005621a8d0f29aa387e1d6b",
+ "com.example.project.git.url": "https://example.com/project.git"
+ }
+ },
+ "rootfs": {
+ "diff_ids": [
+ "sha256:c6f988f4874bb0add23a778f753c65efe992244e148a1d2ec2a8b664fb66bbd1",
+ "sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef"
+ ],
+ "type": "layers"
+ },
+ "history": [
+ {
+ "created": "2015-10-31T22:22:54.690851953Z",
+ "created_by": "/bin/sh -c #(nop) ADD file:a3bc1e842b69636f9df5256c49c5 in /"
+ },
+ {
+ "created": "2015-10-31T22:22:55.613815829Z",
+ "created_by": "/bin/sh -c #(nop) CMD [\"sh\"]",
+ "empty_layer": true
+ }
+ ]
+ })~";
+
+ Try<image::v1::Configuration> configuration =
+ image::v1::parse<image::v1::Configuration>(json);
+
+ ASSERT_SOME(configuration);
+
+ EXPECT_EQ(
+ "2015-10-31T22:22:56.015925234Z",
+ configuration->created());
+
+ EXPECT_EQ(
+ "Alyssa P. Hacker <al...@example.com>",
+ configuration->author());
+
+ EXPECT_EQ(
+ "amd64",
+ configuration->architecture());
+
+ EXPECT_EQ(
+ "linux",
+ configuration->os());
+
+ EXPECT_EQ(
+ "alice",
+ configuration->config().user());
+
+ EXPECT_EQ(
+ "8080/tcp",
+ configuration->config().exposedports(0));
+
+ EXPECT_EQ(
+ "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
+ configuration->config().env(0));
+
+ EXPECT_EQ(
+ "FOO=oci_is_a",
+ configuration->config().env(1));
+
+ EXPECT_EQ(
+ "/bin/my-app-binary",
+ configuration->config().entrypoint(0));
+
+ EXPECT_EQ(
+ "--config",
+ configuration->config().cmd(0));
+
+ EXPECT_EQ(
+ "/etc/my-app.d/default.cfg",
+ configuration->config().cmd(1));
+
+ EXPECT_EQ(
+ "/var/job-result-data",
+ configuration->config().volumes(0));
+
+ EXPECT_EQ(
+ "/var/log/my-app-logs",
+ configuration->config().volumes(1));
+
+ EXPECT_EQ(
+ "/home/alice",
+ configuration->config().workingdir());
+
+ EXPECT_EQ(
+ "com.example.project.git.commit",
+ configuration->config().labels(0).key());
+
+ EXPECT_EQ(
+ "45a939b2999782a3f005621a8d0f29aa387e1d6b",
+ configuration->config().labels(0).value());
+
+ EXPECT_EQ(
+ "com.example.project.git.url",
+ configuration->config().labels(1).key());
+
+ EXPECT_EQ(
+ "https://example.com/project.git",
+ configuration->config().labels(1).value());
+
+ EXPECT_EQ(
+ "sha256:c6f988f4874bb0add23a778f753c65efe992244e148a1d2ec2a8b664fb66bbd1",
+ configuration->rootfs().diff_ids(0));
+
+ EXPECT_EQ(
+ "sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef",
+ configuration->rootfs().diff_ids(1));
+
+ EXPECT_EQ(
+ "layers",
+ configuration->rootfs().type());
+
+ EXPECT_EQ(
+ "2015-10-31T22:22:54.690851953Z",
+ configuration->history(0).created());
+
+ EXPECT_EQ(
+ "/bin/sh -c #(nop) ADD file:a3bc1e842b69636f9df5256c49c5 in /",
+ configuration->history(0).created_by());
+
+ EXPECT_EQ(
+ true,
+ configuration->history(1).empty_layer());
+}
+
+} // namespace tests {
+} // namespace internal {
+} // namespace mesos {
[2/3] mesos git commit: Implemented parse methods for OCI image spec.
Posted by qi...@apache.org.
Implemented parse methods for OCI image spec.
Review: https://reviews.apache.org/r/55139/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/3681d61a
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/3681d61a
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/3681d61a
Branch: refs/heads/master
Commit: 3681d61aa6469bdb0ea7955dd7c9680c0fb5496b
Parents: 6726cd1
Author: Qian Zhang <zh...@gmail.com>
Authored: Tue Feb 7 16:54:35 2017 +0800
Committer: Qian Zhang <zh...@gmail.com>
Committed: Tue Feb 7 16:54:35 2017 +0800
----------------------------------------------------------------------
include/mesos/oci/spec.hpp | 24 ++-
src/CMakeLists.txt | 7 +
src/Makefile.am | 1 +
src/oci/spec.cpp | 373 ++++++++++++++++++++++++++++++++++++++++
4 files changed, 404 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/3681d61a/include/mesos/oci/spec.hpp
----------------------------------------------------------------------
diff --git a/include/mesos/oci/spec.hpp b/include/mesos/oci/spec.hpp
index ac0b063..ea4f29e 100644
--- a/include/mesos/oci/spec.hpp
+++ b/include/mesos/oci/spec.hpp
@@ -24,7 +24,29 @@ namespace spec {
namespace image {
namespace v1 {
-// TODO(qianzhang): Add methods to parse OCI image spec
+// Constant strings for OCI image media types:
+// https://github.com/opencontainers/image-spec/blob/master/media-types.md
+constexpr char MEDIA_TYPE_MANIFEST_LIST[] =
+ "application/vnd.oci.image.manifest.list.v1+json";
+
+constexpr char MEDIA_TYPE_MANIFEST[] =
+ "application/vnd.oci.image.manifest.v1+json";
+
+constexpr char MEDIA_TYPE_CONFIG[] =
+ "application/vnd.oci.image.config.v1+json";
+
+constexpr char MEDIA_TYPE_LAYER[] =
+ "application/vnd.oci.image.layer.v1.tar+gzip";
+
+constexpr char MEDIA_TYPE_NONDIST_LAYER[] =
+ "application/vnd.oci.image.layer.nondistributable.v1.tar+gzip";
+
+/**
+ * Returns the OCI v1 descriptor, image manifest list, image manifest
+ * and image configuration from the given string.
+ */
+template <typename T>
+Try<T> parse(const std::string& s);
} // namespace v1 {
} // namespace image {
http://git-wip-us.apache.org/repos/asf/mesos/blob/3681d61a/src/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index c09bcde..3a4ace9 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -37,6 +37,7 @@ PROTOC_TO_INCLUDE_DIR(MASTER mesos/master/master)
PROTOC_TO_INCLUDE_DIR(MESOS mesos/mesos)
PROTOC_TO_INCLUDE_DIR(MODULE mesos/module/module)
PROTOC_TO_INCLUDE_DIR(OVERSUBSCRIPTION mesos/slave/oversubscription)
+PROTOC_TO_INCLUDE_DIR(OCI_SPEC mesos/oci/spec)
PROTOC_TO_INCLUDE_DIR(QUOTA mesos/quota/quota)
PROTOC_TO_INCLUDE_DIR(SCHEDULER mesos/scheduler/scheduler)
PROTOC_TO_INCLUDE_DIR(STATE mesos/state/state)
@@ -77,6 +78,7 @@ set(PUBLIC_PROTOBUF_SRC
${MASTER_PROTO_CC}
${MESOS_PROTO_CC}
${MODULE_PROTO_CC}
+ ${OCI_SPEC_PROTO_CC}
${OVERSUBSCRIPTION_PROTO_CC}
${QUOTA_PROTO_CC}
${SCHEDULER_PROTO_CC}
@@ -365,6 +367,10 @@ set(MODULE_SRC
module/manager.cpp
)
+set(OCI_SRC
+ oci/spec.cpp
+ )
+
set(POSIX_SRC
posix/rlimits.cpp
)
@@ -452,6 +458,7 @@ set(MESOS_SRC
${MASTER_SRC}
${MESSAGES_SRC}
${MODULE_SRC}
+ ${OCI_SRC}
${SCHEDULER_SRC}
${STATE_SRC}
${URI_SRC}
http://git-wip-us.apache.org/repos/asf/mesos/blob/3681d61a/src/Makefile.am
----------------------------------------------------------------------
diff --git a/src/Makefile.am b/src/Makefile.am
index f432571..cb8e604 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -893,6 +893,7 @@ libmesos_no_3rdparty_la_SOURCES += \
master/detector/zookeeper.cpp \
messages/messages.cpp \
module/manager.cpp \
+ oci/spec.cpp \
posix/rlimits.cpp \
sched/sched.cpp \
scheduler/scheduler.cpp \
http://git-wip-us.apache.org/repos/asf/mesos/blob/3681d61a/src/oci/spec.cpp
----------------------------------------------------------------------
diff --git a/src/oci/spec.cpp b/src/oci/spec.cpp
new file mode 100644
index 0000000..f1713a2
--- /dev/null
+++ b/src/oci/spec.cpp
@@ -0,0 +1,373 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <vector>
+
+#include <stout/protobuf.hpp>
+
+#include <mesos/oci/spec.hpp>
+
+using std::string;
+using std::vector;
+
+namespace oci {
+namespace spec {
+namespace image {
+namespace v1 {
+
+namespace internal {
+
+Option<Error> validateDigest(const string& digest)
+{
+ vector<string> split = strings::split(digest, ":");
+ if (split.size() != 2) {
+ return Error("Incorrect 'digest' format: " + digest);
+ }
+
+ // TODO(qianzhang): Validate algorithm (split[0]) and hex (split[1]).
+
+ return None();
+}
+
+
+Option<Error> validate(const ManifestList& manifestList)
+{
+ if (manifestList.schemaversion() != 2) {
+ return Error("Incorrect 'schemaVersion': " + manifestList.schemaversion());
+ }
+
+ foreach (const ManifestDescriptor& manifest, manifestList.manifests()) {
+ Option<Error> error = validateDigest(manifest.digest());
+ if (error.isSome()) {
+ return Error(
+ "Failed to validate 'digest' of the 'manifest': " + error->message);
+ }
+
+ if (manifest.mediatype() != MEDIA_TYPE_MANIFEST) {
+ return Error(
+ "Incorrect 'mediaType' of the 'manifest': " + manifest.mediatype());
+ }
+ }
+
+ return None();
+}
+
+
+Option<Error> validate(const Manifest& manifest)
+{
+ if (manifest.schemaversion() != 2) {
+ return Error("Incorrect 'schemaVersion': " + manifest.schemaversion());
+ }
+
+ const Descriptor& config = manifest.config();
+
+ Option<Error> error = validateDigest(config.digest());
+ if (error.isSome()) {
+ return Error(
+ "Failed to validate 'digest' of the 'config': " + error->message);
+ }
+
+ if (config.mediatype() != MEDIA_TYPE_CONFIG) {
+ return Error(
+ "Incorrect 'mediaType' of the 'config': " + config.mediatype());
+ }
+
+ if (manifest.layers_size() <= 0) {
+ return Error("'layers' field size must be at least one");
+ }
+
+ foreach (const Descriptor& layer, manifest.layers()) {
+ Option<Error> error = validateDigest(layer.digest());
+ if (error.isSome()) {
+ return Error(
+ "Failed to validate 'digest' of the 'layer': " + error->message);
+ }
+
+ if (layer.mediatype() != MEDIA_TYPE_LAYER &&
+ layer.mediatype() != MEDIA_TYPE_NONDIST_LAYER) {
+ return Error(
+ "Incorrect 'mediaType' of the 'layer': " + layer.mediatype());
+ }
+ }
+
+ return None();
+}
+
+} // namespace internal {
+
+
+template <>
+Try<Descriptor> parse(const string& s)
+{
+ Try<JSON::Object> json = JSON::parse<JSON::Object>(s);
+ if (json.isError()) {
+ return Error("JSON parse failed: " + json.error());
+ }
+
+ Try<Descriptor> descriptor = protobuf::parse<Descriptor>(json.get());
+ if (descriptor.isError()) {
+ return Error("Protobuf parse failed: " + descriptor.error());
+ }
+
+ Option<Error> error = internal::validateDigest(descriptor->digest());
+ if (error.isSome()) {
+ return Error(
+ "OCI v1 image descriptor validation failed: " + error->message);
+ }
+
+ return descriptor.get();
+}
+
+
+template <>
+Try<ManifestList> parse(const string& s)
+{
+ Try<JSON::Object> json = JSON::parse<JSON::Object>(s);
+ if (json.isError()) {
+ return Error("JSON parse failed: " + json.error());
+ }
+
+ Try<ManifestList> manifestList = protobuf::parse<ManifestList>(json.get());
+ if (manifestList.isError()) {
+ return Error("Protobuf parse failed: " + manifestList.error());
+ }
+
+ // Manually parse 'manifest.platform.os.version' and
+ // 'manifest.platform.os.features'.
+ Result<JSON::Array> manifests = json->at<JSON::Array>("manifests");
+ if (manifests.isError()) {
+ return Error("Failed to find 'manifests': " + manifests.error());
+ } else if (manifests.isNone()) {
+ return Error("Unable to find 'manifests'");
+ }
+
+ foreach (const JSON::Value& value, manifests->values) {
+ if (!value.is<JSON::Object>()) {
+ return Error("Expecting 'manifest' to be JSON object type");
+ }
+
+ const JSON::Object manifest = value.as<JSON::Object>();
+ Result<JSON::String> digest = manifest.at<JSON::String>("digest");
+ if (digest.isError()) {
+ return Error("Failed to find 'digest': " + digest.error());
+ } else if (digest.isNone()) {
+ return Error("Unable to find 'digest'");
+ }
+
+ ManifestDescriptor* _manifest = nullptr;
+ for (int i = 0; i < manifestList->manifests_size(); i++) {
+ if (manifestList->manifests(i).digest() == digest.get()) {
+ _manifest = manifestList->mutable_manifests(i);
+ break;
+ }
+ }
+
+ if (_manifest == nullptr) {
+ return Error(
+ "Unable to find the manifest whose digest is '" +
+ digest->value + "'");
+ }
+
+ Result<JSON::Object> platform = manifest.at<JSON::Object>("platform");
+ if (platform.isError()) {
+ return Error("Failed to find 'platform': " + platform.error());
+ } else if (platform.isNone()) {
+ return Error("Unable to find 'platform'");
+ }
+
+ Result<JSON::String> osVersion = platform->at<JSON::String>("os.version");
+ if (osVersion.isError()) {
+ return Error(
+ "Failed to find 'platform.os.version': " + osVersion.error());
+ }
+
+ if (osVersion.isSome()) {
+ Platform* platform = _manifest->mutable_platform();
+ platform->set_os_version(osVersion->value);
+ }
+
+ Result<JSON::Array> osFeatures = platform->at<JSON::Array>("os.features");
+ if (osFeatures.isError()) {
+ return Error(
+ "Failed to find 'platform.os.features': " + osFeatures.error());
+ }
+
+ if (osFeatures.isSome()) {
+ const vector<JSON::Value>& values = osFeatures->values;
+ if (values.size() != 0) {
+ Platform* platform = _manifest->mutable_platform();
+ foreach (const JSON::Value& value, values) {
+ if (!value.is<JSON::String>()) {
+ return Error("Expecting OS feature to be string type");
+ }
+
+ platform->add_os_features(value.as<JSON::String>().value);
+ }
+ }
+ }
+ }
+
+ // Manually parse 'annotations' field.
+ Result<JSON::Value> annotations = json->find<JSON::Value>("annotations");
+ if (annotations.isError()) {
+ return Error(
+ "Failed to find 'annotations': " + annotations.error());
+ }
+
+ if (annotations.isSome() && !annotations->is<JSON::Null>()) {
+ foreachpair (const string& key,
+ const JSON::Value& value,
+ annotations->as<JSON::Object>().values) {
+ if (!value.is<JSON::String>()) {
+ return Error(
+ "The value of annotation key '" + key + "' is not a JSON string");
+ }
+
+ Label* annotation = manifestList->add_annotations();
+ annotation->set_key(key);
+ annotation->set_value(value.as<JSON::String>().value);
+ }
+ }
+
+ Option<Error> error = internal::validate(manifestList.get());
+ if (error.isSome()) {
+ return Error(
+ "OCI v1 image manifest list validation failed: " + error->message);
+ }
+
+ return manifestList.get();
+}
+
+
+template <>
+Try<Manifest> parse(const string& s)
+{
+ Try<JSON::Object> json = JSON::parse<JSON::Object>(s);
+ if (json.isError()) {
+ return Error("JSON parse failed: " + json.error());
+ }
+
+ Try<Manifest> manifest = protobuf::parse<Manifest>(json.get());
+ if (manifest.isError()) {
+ return Error("Protobuf parse failed: " + manifest.error());
+ }
+
+ // Manually parse 'annotations' field.
+ Result<JSON::Value> annotations = json->find<JSON::Value>("annotations");
+ if (annotations.isError()) {
+ return Error(
+ "Failed to find 'annotations': " + annotations.error());
+ }
+
+ if (annotations.isSome() && !annotations->is<JSON::Null>()) {
+ foreachpair (const string& key,
+ const JSON::Value& value,
+ annotations->as<JSON::Object>().values) {
+ if (!value.is<JSON::String>()) {
+ return Error(
+ "The value of annotation key '" + key + "' is not a JSON string");
+ }
+
+ Label* annotation = manifest->add_annotations();
+ annotation->set_key(key);
+ annotation->set_value(value.as<JSON::String>().value);
+ }
+ }
+
+ Option<Error> error = internal::validate(manifest.get());
+ if (error.isSome()) {
+ return Error(
+ "OCI v1 image manifest validation failed: " + error->message);
+ }
+
+ return manifest.get();
+}
+
+
+template <>
+Try<Configuration> parse(const string& s)
+{
+ Try<JSON::Object> json = JSON::parse<JSON::Object>(s);
+ if (json.isError()) {
+ return Error("JSON parse failed: " + json.error());
+ }
+
+ Try<Configuration> configuration =
+ protobuf::parse<Configuration>(json.get());
+
+ if (configuration.isError()) {
+ return Error("Protobuf parse failed: " + configuration.error());
+ }
+
+ Result<JSON::Object> config = json->find<JSON::Object>("config");
+ if (config.isError()) {
+ return Error("Failed to find 'config': " + config.error());
+ }
+
+ if (config.isSome()) {
+ // Manually parse 'ExposedPorts' field.
+ Result<JSON::Value> value = config->find<JSON::Value>("ExposedPorts");
+ if (value.isError()) {
+ return Error("Failed to find 'ExposedPorts': " + value.error());
+ }
+
+ if (value.isSome() && !value->is<JSON::Null>()) {
+ foreachkey (const string& key, value->as<JSON::Object>().values) {
+ configuration->mutable_config()->add_exposedports(key);
+ }
+ }
+
+ // Manually parse 'Volumes' field.
+ value = config->find<JSON::Value>("Volumes");
+ if (value.isError()) {
+ return Error("Failed to find 'Volumes': " + value.error());
+ }
+
+ if (value.isSome() && !value->is<JSON::Null>()) {
+ foreachkey (const string& key, value->as<JSON::Object>().values) {
+ configuration->mutable_config()->add_volumes(key);
+ }
+ }
+
+ // Manually parse 'Labels' field.
+ value = config->find<JSON::Value>("Labels");
+ if (value.isError()) {
+ return Error("Failed to find 'Labels': " + value.error());
+ }
+
+ if (value.isSome() && !value->is<JSON::Null>()) {
+ foreachpair (const string& key,
+ const JSON::Value& value,
+ value->as<JSON::Object>().values) {
+ if (!value.is<JSON::String>()) {
+ return Error(
+ "The value of label key '" + key + "' is not a JSON string");
+ }
+
+ Label* label = configuration->mutable_config()->add_labels();
+ label->set_key(key);
+ label->set_value(value.as<JSON::String>().value);
+ }
+ }
+ }
+
+ return configuration.get();
+}
+
+} // namespace v1 {
+} // namespace image {
+} // namespace spec {
+} // namespace oci {