You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by qi...@apache.org on 2017/02/07 08:56:13 UTC
[2/3] mesos git commit: Implemented parse methods for OCI image spec.
Implemented parse methods for OCI image spec.
Review: https://reviews.apache.org/r/55139/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/3681d61a
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/3681d61a
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/3681d61a
Branch: refs/heads/master
Commit: 3681d61aa6469bdb0ea7955dd7c9680c0fb5496b
Parents: 6726cd1
Author: Qian Zhang <zh...@gmail.com>
Authored: Tue Feb 7 16:54:35 2017 +0800
Committer: Qian Zhang <zh...@gmail.com>
Committed: Tue Feb 7 16:54:35 2017 +0800
----------------------------------------------------------------------
include/mesos/oci/spec.hpp | 24 ++-
src/CMakeLists.txt | 7 +
src/Makefile.am | 1 +
src/oci/spec.cpp | 373 ++++++++++++++++++++++++++++++++++++++++
4 files changed, 404 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/3681d61a/include/mesos/oci/spec.hpp
----------------------------------------------------------------------
diff --git a/include/mesos/oci/spec.hpp b/include/mesos/oci/spec.hpp
index ac0b063..ea4f29e 100644
--- a/include/mesos/oci/spec.hpp
+++ b/include/mesos/oci/spec.hpp
@@ -24,7 +24,29 @@ namespace spec {
namespace image {
namespace v1 {
-// TODO(qianzhang): Add methods to parse OCI image spec
+// Constant strings for OCI image media types:
+// https://github.com/opencontainers/image-spec/blob/master/media-types.md
+constexpr char MEDIA_TYPE_MANIFEST_LIST[] =
+ "application/vnd.oci.image.manifest.list.v1+json";
+
+constexpr char MEDIA_TYPE_MANIFEST[] =
+ "application/vnd.oci.image.manifest.v1+json";
+
+constexpr char MEDIA_TYPE_CONFIG[] =
+ "application/vnd.oci.image.config.v1+json";
+
+constexpr char MEDIA_TYPE_LAYER[] =
+ "application/vnd.oci.image.layer.v1.tar+gzip";
+
+constexpr char MEDIA_TYPE_NONDIST_LAYER[] =
+ "application/vnd.oci.image.layer.nondistributable.v1.tar+gzip";
+
+/**
+ * Returns the OCI v1 descriptor, image manifest list, image manifest
+ * and image configuration from the given string.
+ */
+template <typename T>
+Try<T> parse(const std::string& s);
} // namespace v1 {
} // namespace image {
http://git-wip-us.apache.org/repos/asf/mesos/blob/3681d61a/src/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index c09bcde..3a4ace9 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -37,6 +37,7 @@ PROTOC_TO_INCLUDE_DIR(MASTER mesos/master/master)
PROTOC_TO_INCLUDE_DIR(MESOS mesos/mesos)
PROTOC_TO_INCLUDE_DIR(MODULE mesos/module/module)
PROTOC_TO_INCLUDE_DIR(OVERSUBSCRIPTION mesos/slave/oversubscription)
+PROTOC_TO_INCLUDE_DIR(OCI_SPEC mesos/oci/spec)
PROTOC_TO_INCLUDE_DIR(QUOTA mesos/quota/quota)
PROTOC_TO_INCLUDE_DIR(SCHEDULER mesos/scheduler/scheduler)
PROTOC_TO_INCLUDE_DIR(STATE mesos/state/state)
@@ -77,6 +78,7 @@ set(PUBLIC_PROTOBUF_SRC
${MASTER_PROTO_CC}
${MESOS_PROTO_CC}
${MODULE_PROTO_CC}
+ ${OCI_SPEC_PROTO_CC}
${OVERSUBSCRIPTION_PROTO_CC}
${QUOTA_PROTO_CC}
${SCHEDULER_PROTO_CC}
@@ -365,6 +367,10 @@ set(MODULE_SRC
module/manager.cpp
)
+set(OCI_SRC
+ oci/spec.cpp
+ )
+
set(POSIX_SRC
posix/rlimits.cpp
)
@@ -452,6 +458,7 @@ set(MESOS_SRC
${MASTER_SRC}
${MESSAGES_SRC}
${MODULE_SRC}
+ ${OCI_SRC}
${SCHEDULER_SRC}
${STATE_SRC}
${URI_SRC}
http://git-wip-us.apache.org/repos/asf/mesos/blob/3681d61a/src/Makefile.am
----------------------------------------------------------------------
diff --git a/src/Makefile.am b/src/Makefile.am
index f432571..cb8e604 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -893,6 +893,7 @@ libmesos_no_3rdparty_la_SOURCES += \
master/detector/zookeeper.cpp \
messages/messages.cpp \
module/manager.cpp \
+ oci/spec.cpp \
posix/rlimits.cpp \
sched/sched.cpp \
scheduler/scheduler.cpp \
http://git-wip-us.apache.org/repos/asf/mesos/blob/3681d61a/src/oci/spec.cpp
----------------------------------------------------------------------
diff --git a/src/oci/spec.cpp b/src/oci/spec.cpp
new file mode 100644
index 0000000..f1713a2
--- /dev/null
+++ b/src/oci/spec.cpp
@@ -0,0 +1,373 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <vector>
+
+#include <stout/protobuf.hpp>
+
+#include <mesos/oci/spec.hpp>
+
+using std::string;
+using std::vector;
+
+namespace oci {
+namespace spec {
+namespace image {
+namespace v1 {
+
+namespace internal {
+
+Option<Error> validateDigest(const string& digest)
+{
+ vector<string> split = strings::split(digest, ":");
+ if (split.size() != 2) {
+ return Error("Incorrect 'digest' format: " + digest);
+ }
+
+ // TODO(qianzhang): Validate algorithm (split[0]) and hex (split[1]).
+
+ return None();
+}
+
+
+Option<Error> validate(const ManifestList& manifestList)
+{
+ if (manifestList.schemaversion() != 2) {
+ return Error("Incorrect 'schemaVersion': " + manifestList.schemaversion());
+ }
+
+ foreach (const ManifestDescriptor& manifest, manifestList.manifests()) {
+ Option<Error> error = validateDigest(manifest.digest());
+ if (error.isSome()) {
+ return Error(
+ "Failed to validate 'digest' of the 'manifest': " + error->message);
+ }
+
+ if (manifest.mediatype() != MEDIA_TYPE_MANIFEST) {
+ return Error(
+ "Incorrect 'mediaType' of the 'manifest': " + manifest.mediatype());
+ }
+ }
+
+ return None();
+}
+
+
+Option<Error> validate(const Manifest& manifest)
+{
+ if (manifest.schemaversion() != 2) {
+ return Error("Incorrect 'schemaVersion': " + manifest.schemaversion());
+ }
+
+ const Descriptor& config = manifest.config();
+
+ Option<Error> error = validateDigest(config.digest());
+ if (error.isSome()) {
+ return Error(
+ "Failed to validate 'digest' of the 'config': " + error->message);
+ }
+
+ if (config.mediatype() != MEDIA_TYPE_CONFIG) {
+ return Error(
+ "Incorrect 'mediaType' of the 'config': " + config.mediatype());
+ }
+
+ if (manifest.layers_size() <= 0) {
+ return Error("'layers' field size must be at least one");
+ }
+
+ foreach (const Descriptor& layer, manifest.layers()) {
+ Option<Error> error = validateDigest(layer.digest());
+ if (error.isSome()) {
+ return Error(
+ "Failed to validate 'digest' of the 'layer': " + error->message);
+ }
+
+ if (layer.mediatype() != MEDIA_TYPE_LAYER &&
+ layer.mediatype() != MEDIA_TYPE_NONDIST_LAYER) {
+ return Error(
+ "Incorrect 'mediaType' of the 'layer': " + layer.mediatype());
+ }
+ }
+
+ return None();
+}
+
+} // namespace internal {
+
+
+template <>
+Try<Descriptor> parse(const string& s)
+{
+ Try<JSON::Object> json = JSON::parse<JSON::Object>(s);
+ if (json.isError()) {
+ return Error("JSON parse failed: " + json.error());
+ }
+
+ Try<Descriptor> descriptor = protobuf::parse<Descriptor>(json.get());
+ if (descriptor.isError()) {
+ return Error("Protobuf parse failed: " + descriptor.error());
+ }
+
+ Option<Error> error = internal::validateDigest(descriptor->digest());
+ if (error.isSome()) {
+ return Error(
+ "OCI v1 image descriptor validation failed: " + error->message);
+ }
+
+ return descriptor.get();
+}
+
+
+template <>
+Try<ManifestList> parse(const string& s)
+{
+ Try<JSON::Object> json = JSON::parse<JSON::Object>(s);
+ if (json.isError()) {
+ return Error("JSON parse failed: " + json.error());
+ }
+
+ Try<ManifestList> manifestList = protobuf::parse<ManifestList>(json.get());
+ if (manifestList.isError()) {
+ return Error("Protobuf parse failed: " + manifestList.error());
+ }
+
+ // Manually parse 'manifest.platform.os.version' and
+ // 'manifest.platform.os.features'.
+ Result<JSON::Array> manifests = json->at<JSON::Array>("manifests");
+ if (manifests.isError()) {
+ return Error("Failed to find 'manifests': " + manifests.error());
+ } else if (manifests.isNone()) {
+ return Error("Unable to find 'manifests'");
+ }
+
+ foreach (const JSON::Value& value, manifests->values) {
+ if (!value.is<JSON::Object>()) {
+ return Error("Expecting 'manifest' to be JSON object type");
+ }
+
+ const JSON::Object manifest = value.as<JSON::Object>();
+ Result<JSON::String> digest = manifest.at<JSON::String>("digest");
+ if (digest.isError()) {
+ return Error("Failed to find 'digest': " + digest.error());
+ } else if (digest.isNone()) {
+ return Error("Unable to find 'digest'");
+ }
+
+ ManifestDescriptor* _manifest = nullptr;
+ for (int i = 0; i < manifestList->manifests_size(); i++) {
+ if (manifestList->manifests(i).digest() == digest.get()) {
+ _manifest = manifestList->mutable_manifests(i);
+ break;
+ }
+ }
+
+ if (_manifest == nullptr) {
+ return Error(
+ "Unable to find the manifest whose digest is '" +
+ digest->value + "'");
+ }
+
+ Result<JSON::Object> platform = manifest.at<JSON::Object>("platform");
+ if (platform.isError()) {
+ return Error("Failed to find 'platform': " + platform.error());
+ } else if (platform.isNone()) {
+ return Error("Unable to find 'platform'");
+ }
+
+ Result<JSON::String> osVersion = platform->at<JSON::String>("os.version");
+ if (osVersion.isError()) {
+ return Error(
+ "Failed to find 'platform.os.version': " + osVersion.error());
+ }
+
+ if (osVersion.isSome()) {
+ Platform* platform = _manifest->mutable_platform();
+ platform->set_os_version(osVersion->value);
+ }
+
+ Result<JSON::Array> osFeatures = platform->at<JSON::Array>("os.features");
+ if (osFeatures.isError()) {
+ return Error(
+ "Failed to find 'platform.os.features': " + osFeatures.error());
+ }
+
+ if (osFeatures.isSome()) {
+ const vector<JSON::Value>& values = osFeatures->values;
+ if (values.size() != 0) {
+ Platform* platform = _manifest->mutable_platform();
+ foreach (const JSON::Value& value, values) {
+ if (!value.is<JSON::String>()) {
+ return Error("Expecting OS feature to be string type");
+ }
+
+ platform->add_os_features(value.as<JSON::String>().value);
+ }
+ }
+ }
+ }
+
+ // Manually parse 'annotations' field.
+ Result<JSON::Value> annotations = json->find<JSON::Value>("annotations");
+ if (annotations.isError()) {
+ return Error(
+ "Failed to find 'annotations': " + annotations.error());
+ }
+
+ if (annotations.isSome() && !annotations->is<JSON::Null>()) {
+ foreachpair (const string& key,
+ const JSON::Value& value,
+ annotations->as<JSON::Object>().values) {
+ if (!value.is<JSON::String>()) {
+ return Error(
+ "The value of annotation key '" + key + "' is not a JSON string");
+ }
+
+ Label* annotation = manifestList->add_annotations();
+ annotation->set_key(key);
+ annotation->set_value(value.as<JSON::String>().value);
+ }
+ }
+
+ Option<Error> error = internal::validate(manifestList.get());
+ if (error.isSome()) {
+ return Error(
+ "OCI v1 image manifest list validation failed: " + error->message);
+ }
+
+ return manifestList.get();
+}
+
+
+template <>
+Try<Manifest> parse(const string& s)
+{
+ Try<JSON::Object> json = JSON::parse<JSON::Object>(s);
+ if (json.isError()) {
+ return Error("JSON parse failed: " + json.error());
+ }
+
+ Try<Manifest> manifest = protobuf::parse<Manifest>(json.get());
+ if (manifest.isError()) {
+ return Error("Protobuf parse failed: " + manifest.error());
+ }
+
+ // Manually parse 'annotations' field.
+ Result<JSON::Value> annotations = json->find<JSON::Value>("annotations");
+ if (annotations.isError()) {
+ return Error(
+ "Failed to find 'annotations': " + annotations.error());
+ }
+
+ if (annotations.isSome() && !annotations->is<JSON::Null>()) {
+ foreachpair (const string& key,
+ const JSON::Value& value,
+ annotations->as<JSON::Object>().values) {
+ if (!value.is<JSON::String>()) {
+ return Error(
+ "The value of annotation key '" + key + "' is not a JSON string");
+ }
+
+ Label* annotation = manifest->add_annotations();
+ annotation->set_key(key);
+ annotation->set_value(value.as<JSON::String>().value);
+ }
+ }
+
+ Option<Error> error = internal::validate(manifest.get());
+ if (error.isSome()) {
+ return Error(
+ "OCI v1 image manifest validation failed: " + error->message);
+ }
+
+ return manifest.get();
+}
+
+
+template <>
+Try<Configuration> parse(const string& s)
+{
+ Try<JSON::Object> json = JSON::parse<JSON::Object>(s);
+ if (json.isError()) {
+ return Error("JSON parse failed: " + json.error());
+ }
+
+ Try<Configuration> configuration =
+ protobuf::parse<Configuration>(json.get());
+
+ if (configuration.isError()) {
+ return Error("Protobuf parse failed: " + configuration.error());
+ }
+
+ Result<JSON::Object> config = json->find<JSON::Object>("config");
+ if (config.isError()) {
+ return Error("Failed to find 'config': " + config.error());
+ }
+
+ if (config.isSome()) {
+ // Manually parse 'ExposedPorts' field.
+ Result<JSON::Value> value = config->find<JSON::Value>("ExposedPorts");
+ if (value.isError()) {
+ return Error("Failed to find 'ExposedPorts': " + value.error());
+ }
+
+ if (value.isSome() && !value->is<JSON::Null>()) {
+ foreachkey (const string& key, value->as<JSON::Object>().values) {
+ configuration->mutable_config()->add_exposedports(key);
+ }
+ }
+
+ // Manually parse 'Volumes' field.
+ value = config->find<JSON::Value>("Volumes");
+ if (value.isError()) {
+ return Error("Failed to find 'Volumes': " + value.error());
+ }
+
+ if (value.isSome() && !value->is<JSON::Null>()) {
+ foreachkey (const string& key, value->as<JSON::Object>().values) {
+ configuration->mutable_config()->add_volumes(key);
+ }
+ }
+
+ // Manually parse 'Labels' field.
+ value = config->find<JSON::Value>("Labels");
+ if (value.isError()) {
+ return Error("Failed to find 'Labels': " + value.error());
+ }
+
+ if (value.isSome() && !value->is<JSON::Null>()) {
+ foreachpair (const string& key,
+ const JSON::Value& value,
+ value->as<JSON::Object>().values) {
+ if (!value.is<JSON::String>()) {
+ return Error(
+ "The value of label key '" + key + "' is not a JSON string");
+ }
+
+ Label* label = configuration->mutable_config()->add_labels();
+ label->set_key(key);
+ label->set_value(value.as<JSON::String>().value);
+ }
+ }
+ }
+
+ return configuration.get();
+}
+
+} // namespace v1 {
+} // namespace image {
+} // namespace spec {
+} // namespace oci {