You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by ji...@apache.org on 2018/08/28 20:56:33 UTC
[mesos] branch master updated (0a58ecd -> bf17c98)
This is an automated email from the ASF dual-hosted git repository.
jieyu pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git.
from 0a58ecd Added a CNI test for networking statistics.
new dc14362 Windows: Update curl version to 7.61.0.
new bf17c98 Windows: Parse version 2 schema 2 Docker image manifest.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
3rdparty/CMakeLists.txt | 4 +-
3rdparty/cmake/Versions.cmake | 4 +-
include/mesos/docker/spec.hpp | 18 +++
.../mesos/{fetcher/fetcher.hpp => docker/v2_2.hpp} | 8 +-
.../mesos/docker/v2_2.proto | 35 +++---
src/CMakeLists.txt | 1 +
src/Makefile.am | 11 +-
src/docker/spec.cpp | 66 +++++++++++
src/tests/CMakeLists.txt | 2 +-
src/tests/containerizer/docker_spec_tests.cpp | 121 +++++++++++++++++++++
10 files changed, 245 insertions(+), 25 deletions(-)
copy include/mesos/{fetcher/fetcher.hpp => docker/v2_2.hpp} (85%)
copy src/slave/containerizer/mesos/provisioner/docker/message.proto => include/mesos/docker/v2_2.proto (57%)
[mesos] 01/02: Windows: Update curl version to 7.61.0.
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jieyu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit dc1436289129e5339a7e9d6d9350d64a352ece6b
Author: Liangyu Zhao <t-...@microsoft.com>
AuthorDate: Tue Aug 28 13:30:42 2018 -0700
Windows: Update curl version to 7.61.0.
A bug is encountered in version 7.57.0, and is fixed in 7.61.0.
Review: https://reviews.apache.org/r/68450/
---
3rdparty/CMakeLists.txt | 4 ++--
3rdparty/cmake/Versions.cmake | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/3rdparty/CMakeLists.txt b/3rdparty/CMakeLists.txt
index 8a69685..d083aa5 100644
--- a/3rdparty/CMakeLists.txt
+++ b/3rdparty/CMakeLists.txt
@@ -740,7 +740,7 @@ if (WIN32)
set_target_properties(
curl PROPERTIES
- IMPORTED_LOCATION_DEBUG ${CURL_ROOT}-build/src/Debug/curl-d.exe
+ IMPORTED_LOCATION_DEBUG ${CURL_ROOT}-build/src/Debug/curl.exe
IMPORTED_LOCATION_RELEASE ${CURL_ROOT}-build/src/Release/curl.exe)
else ()
# This is for single-configuration generators such as Ninja.
@@ -755,7 +755,7 @@ if (WIN32)
set_target_properties(
curl PROPERTIES
- IMPORTED_LOCATION ${CURL_ROOT}-build/src/curl${CURL_SUFFIX}.exe)
+ IMPORTED_LOCATION ${CURL_ROOT}-build/src/curl.exe)
endif ()
MAKE_INCLUDE_DIR(libcurl)
diff --git a/3rdparty/cmake/Versions.cmake b/3rdparty/cmake/Versions.cmake
index c65d7b3..2b3a1ac 100644
--- a/3rdparty/cmake/Versions.cmake
+++ b/3rdparty/cmake/Versions.cmake
@@ -4,8 +4,8 @@ set(CONCURRENTQUEUE_VERSION "7b69a8f")
set(CONCURRENTQUEUE_HASH "SHA256=B2741A1FB2172C2A829503A85D5EE7548BE7ED04236A3FD1EFD2B6088E065CB7")
set(CSI_VERSION "0.2.0")
set(CSI_HASH "SHA256=CB75FC99B03F3C7C30E407AE86BA63EB069AE4A167A26C94FE97F09CB7FF8771")
-set(CURL_VERSION "7.57.0")
-set(CURL_HASH "SHA256=7CE35F207562674E71DBADA6891B37E3F043C1E7A82915CB9C2A17AD3A9D659B")
+set(CURL_VERSION "7.61.0")
+set(CURL_HASH "SHA256=64141F0DB4945268A21B490D58806B97C615D3D0C75BF8C335BBE0EFD13B45B5")
set(ELFIO_VERSION "3.2")
set(ELFIO_HASH "SHA256=964BE1D401F98FA7A1242BCF048DF32B7D56DBAAAE5D02834900499073AC2E95")
set(GOOGLETEST_VERSION "1.8.0")
[mesos] 02/02: Windows: Parse version 2 schema 2 Docker image
manifest.
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jieyu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit bf17c98f440df0998281f8619642d1d9ebcf49eb
Author: Liangyu Zhao <t-...@microsoft.com>
AuthorDate: Tue Aug 28 13:32:17 2018 -0700
Windows: Parse version 2 schema 2 Docker image manifest.
Added support to parse V2S2 Docker image manifest
(https://docs.docker.com/registry/spec/manifest-v2-2/). Adopted the
validation code from patch 53850.
Review: https://reviews.apache.org/r/68451/
---
include/mesos/docker/spec.hpp | 18 ++++
include/mesos/docker/v2_2.hpp | 23 +++++
include/mesos/docker/v2_2.proto | 45 ++++++++++
src/CMakeLists.txt | 1 +
src/Makefile.am | 11 ++-
src/docker/spec.cpp | 66 ++++++++++++++
src/tests/CMakeLists.txt | 2 +-
src/tests/containerizer/docker_spec_tests.cpp | 121 ++++++++++++++++++++++++++
8 files changed, 284 insertions(+), 3 deletions(-)
diff --git a/include/mesos/docker/spec.hpp b/include/mesos/docker/spec.hpp
index 2879414..56827fe 100644
--- a/include/mesos/docker/spec.hpp
+++ b/include/mesos/docker/spec.hpp
@@ -30,6 +30,7 @@
#include <mesos/docker/v1.hpp>
#include <mesos/docker/v2.hpp>
+#include <mesos/docker/v2_2.hpp>
namespace docker {
namespace spec {
@@ -119,6 +120,23 @@ Try<ImageManifest> parse(const JSON::Object& json);
Try<ImageManifest> parse(const std::string& s);
} // namespace v2 {
+
+
+namespace v2_2 {
+
+// Validates if the specified docker v2 s2 image manifest conforms to the
+// Docker v2 s2 spec. Returns the error if the validation fails.
+Option<Error> validate(const ImageManifest& manifest);
+
+
+// Returns the docker v2 s2 image manifest from the given JSON object.
+Try<ImageManifest> parse(const JSON::Object& json);
+
+
+// Returns the docker v2 s2 image manifest from the given string.
+Try<ImageManifest> parse(const std::string& s);
+
+} // namespace v2_2 {
} // namespace spec {
} // namespace docker {
diff --git a/include/mesos/docker/v2_2.hpp b/include/mesos/docker/v2_2.hpp
new file mode 100644
index 0000000..19f710f
--- /dev/null
+++ b/include/mesos/docker/v2_2.hpp
@@ -0,0 +1,23 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef __MESOS_DOCKER_V2_2_HPP__
+#define __MESOS_DOCKER_V2_2_HPP__
+
+// ONLY USEFUL AFTER RUNNING PROTOC.
+#include <mesos/docker/v2_2.pb.h>
+
+#endif // __MESOS_DOCKER_V2_2_HPP__
diff --git a/include/mesos/docker/v2_2.proto b/include/mesos/docker/v2_2.proto
new file mode 100644
index 0000000..17bb11a
--- /dev/null
+++ b/include/mesos/docker/v2_2.proto
@@ -0,0 +1,45 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package docker.spec.v2_2;
+
+/**
+ * Protobuf for the Docker version 2 schema 2 image manifest JSON schema:
+ * https://github.com/docker/distribution/blob/master/docs/spec/manifest-v2-2.md
+ */
+message ImageManifest {
+ required uint32 schemaVersion = 1;
+ required string mediaType = 2;
+
+ message Config {
+ required string mediaType = 1;
+ required uint32 size = 2;
+ required string digest = 3;
+ }
+
+ required Config config = 3;
+
+ message Layer {
+ required string mediaType = 1;
+ required uint32 size = 2;
+ required string digest = 3;
+ repeated string urls = 4;
+ }
+
+ repeated Layer layers = 4;
+}
\ No newline at end of file
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 7b4a421..56081c0 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -44,6 +44,7 @@ PROTOC_GENERATE(TARGET mesos/authorizer/authorizer)
PROTOC_GENERATE(TARGET mesos/docker/spec)
PROTOC_GENERATE(TARGET mesos/docker/v1)
PROTOC_GENERATE(TARGET mesos/docker/v2)
+PROTOC_GENERATE(TARGET mesos/docker/v2_2)
PROTOC_GENERATE(TARGET mesos/maintenance/maintenance)
PROTOC_GENERATE(TARGET mesos/master/master)
PROTOC_GENERATE(TARGET mesos/module/hook)
diff --git a/src/Makefile.am b/src/Makefile.am
index db661c3..3023061 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -287,6 +287,7 @@ CONTAINERIZER_PROTO = $(top_srcdir)/include/mesos/slave/containerizer.proto
DOCKER_SPEC_PROTO = $(top_srcdir)/include/mesos/docker/spec.proto
DOCKER_V1_PROTO = $(top_srcdir)/include/mesos/docker/v1.proto
DOCKER_V2_PROTO = $(top_srcdir)/include/mesos/docker/v2.proto
+DOCKER_V2_2_PROTO = $(top_srcdir)/include/mesos/docker/v2_2.proto
EXECUTOR_PROTO = $(top_srcdir)/include/mesos/executor/executor.proto
FETCHER_PROTO = $(top_srcdir)/include/mesos/fetcher/fetcher.proto
HOOK_PROTO = $(top_srcdir)/include/mesos/module/hook.proto
@@ -333,6 +334,8 @@ CXX_PROTOS = \
../include/mesos/docker/v1.pb.h \
../include/mesos/docker/v2.pb.cc \
../include/mesos/docker/v2.pb.h \
+ ../include/mesos/docker/v2_2.pb.cc \
+ ../include/mesos/docker/v2_2.pb.h \
../include/mesos/executor/executor.pb.cc \
../include/mesos/executor/executor.pb.h \
../include/mesos/fetcher/fetcher.pb.cc \
@@ -688,12 +691,15 @@ docker_HEADERS = \
$(top_srcdir)/include/mesos/docker/v1.hpp \
$(top_srcdir)/include/mesos/docker/v1.proto \
$(top_srcdir)/include/mesos/docker/v2.hpp \
- $(top_srcdir)/include/mesos/docker/v2.proto
+ $(top_srcdir)/include/mesos/docker/v2.proto \
+ $(top_srcdir)/include/mesos/docker/v2_2.hpp \
+ $(top_srcdir)/include/mesos/docker/v2_2.proto
nodist_docker_HEADERS = \
../include/mesos/docker/spec.pb.h \
../include/mesos/docker/v1.pb.h \
- ../include/mesos/docker/v2.pb.h
+ ../include/mesos/docker/v2.pb.h \
+ ../include/mesos/docker/v2_2.pb.h
executordir = $(pkgincludedir)/executor
@@ -1608,6 +1614,7 @@ libmesos_la_SOURCES = \
$(DOCKER_SPEC_PROTO) \
$(DOCKER_V1_PROTO) \
$(DOCKER_V2_PROTO) \
+ $(DOCKER_V2_2_PROTO) \
$(FETCHER_PROTO) \
$(HOOK_PROTO) \
$(MAINTENANCE_PROTO) \
diff --git a/src/docker/spec.cpp b/src/docker/spec.cpp
index 96fbf1f..79ddcdb 100644
--- a/src/docker/spec.cpp
+++ b/src/docker/spec.cpp
@@ -341,5 +341,71 @@ Try<ImageManifest> parse(const string& s)
}
} // namespace v2 {
+
+namespace v2_2 {
+
+Option<Error> validate(const ImageManifest& manifest)
+{
+ // Validate required fields are present,
+ // e.g., repeated fields that has to be >= 1.
+ if (manifest.layers_size() <= 0) {
+ return Error("'layers' field size must be at least one");
+ }
+
+ // Verify 'config' field.
+ if (!strings::contains(manifest.config().digest(), ":")) {
+ return Error("Incorrect 'digest' format: " + manifest.config().digest());
+ }
+
+ // Verify 'layers' field.
+ for (int i = 0; i < manifest.layers_size(); ++i) {
+ if (!strings::contains(manifest.layers(i).digest(), ":")) {
+ return Error("Incorrect 'digest' format: " + manifest.layers(i).digest());
+ }
+ }
+
+ if (manifest.schemaversion() != 2) {
+ return Error("'schemaVersion' field must be 2");
+ }
+
+ if (manifest.mediatype() !=
+ "application/vnd.docker.distribution.manifest.v2+json") {
+ return Error(
+ "'mediaType' field must be "
+ "'application/vnd.docker.distribution.manifest.v2+json'");
+ }
+
+ return None();
+}
+
+
+Try<ImageManifest> parse(const JSON::Object& json)
+{
+ Try<ImageManifest> manifest = protobuf::parse<ImageManifest>(json);
+ if (manifest.isError()) {
+ return Error("Protobuf parse failed: " + manifest.error());
+ }
+
+ Option<Error> error = validate(manifest.get());
+ if (error.isSome()) {
+ return Error(
+ "Docker v2 s2 image manifest validation failed: " + error->message);
+ }
+
+ return manifest.get();
+}
+
+
+Try<ImageManifest> parse(const string& s)
+{
+ Try<JSON::Object> json = JSON::parse<JSON::Object>(s);
+ if (json.isError()) {
+ return Error("JSON parse failed: " + json.error());
+ }
+
+ return parse(json.get());
+}
+
+} // namespace v2_2 {
} // namespace spec {
} // namespace docker {
diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt
index fd01531..00dbdee 100644
--- a/src/tests/CMakeLists.txt
+++ b/src/tests/CMakeLists.txt
@@ -154,6 +154,7 @@ list(APPEND MESOS_TESTS_SRC
containerizer/containerizer_tests.cpp
containerizer/cpu_isolator_tests.cpp
containerizer/docker_containerizer_tests.cpp
+ containerizer/docker_spec_tests.cpp
containerizer/docker_tests.cpp
containerizer/memory_isolator_tests.cpp)
@@ -194,7 +195,6 @@ if (NOT WIN32)
list(APPEND MESOS_TESTS_SRC
containerizer/appc_spec_tests.cpp
containerizer/composing_containerizer_tests.cpp
- containerizer/docker_spec_tests.cpp
containerizer/environment_secret_isolator_tests.cpp
containerizer/io_switchboard_tests.cpp
containerizer/isolator_tests.cpp
diff --git a/src/tests/containerizer/docker_spec_tests.cpp b/src/tests/containerizer/docker_spec_tests.cpp
index 8f2fa4e..4be0716 100644
--- a/src/tests/containerizer/docker_spec_tests.cpp
+++ b/src/tests/containerizer/docker_spec_tests.cpp
@@ -603,6 +603,127 @@ TEST_F(DockerSpecTest, ValidateV2ImageManifestSignaturesNonEmpty)
EXPECT_ERROR(manifest);
}
+
+TEST_F(DockerSpecTest, ParseV2_2ImageManifest)
+{
+ Try<JSON::Object> json = JSON::parse<JSON::Object>(
+ R"~(
+ {
+ "schemaVersion": 2,
+ "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
+ "config": {
+ "mediaType": "application/vnd.docker.container.image.v1+json",
+ "size": 7023,
+ "digest": "sha256:b5b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7"
+ },
+ "layers": [
+ {
+ "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
+ "size": 32654,
+ "digest": "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f"
+ },
+ {
+ "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
+ "size": 16724,
+ "digest": "sha256:3c3a4604a545cdc127456d94e421cd355bca5b528f4a9c1905b15da2eb4a4c6b"
+ },
+ {
+ "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
+ "size": 73109,
+ "digest": "sha256:ec4b8955958665577945c89419d1af06b5f7636b4ac3da7f12184802ad867736"
+ }
+ ]
+ })~");
+
+ ASSERT_SOME(json);
+
+ Try<spec::v2_2::ImageManifest> manifest = spec::v2_2::parse(json.get());
+ ASSERT_SOME(manifest);
+
+ EXPECT_EQ(2u, manifest->schemaversion());
+ EXPECT_EQ(
+ "application/vnd.docker.distribution.manifest.v2+json",
+ manifest->mediatype());
+
+ EXPECT_EQ(
+ "application/vnd.docker.container.image.v1+json",
+ manifest->config().mediatype());
+ EXPECT_EQ(7023u, manifest->config().size());
+ EXPECT_EQ(
+ "sha256:b5b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7",
+ manifest->config().digest());
+
+ EXPECT_EQ(3, manifest->layers_size());
+
+ EXPECT_EQ(
+ "application/vnd.docker.image.rootfs.diff.tar.gzip",
+ manifest->layers(0).mediatype());
+ EXPECT_EQ(32654u, manifest->layers(0).size());
+ EXPECT_EQ(
+ "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f",
+ manifest->layers(0).digest());
+
+ EXPECT_EQ(
+ "application/vnd.docker.image.rootfs.diff.tar.gzip",
+ manifest->layers(1).mediatype());
+ EXPECT_EQ(16724u, manifest->layers(1).size());
+ EXPECT_EQ(
+ "sha256:3c3a4604a545cdc127456d94e421cd355bca5b528f4a9c1905b15da2eb4a4c6b",
+ manifest->layers(1).digest());
+
+ EXPECT_EQ(
+ "application/vnd.docker.image.rootfs.diff.tar.gzip",
+ manifest->layers(2).mediatype());
+ EXPECT_EQ(73109u, manifest->layers(2).size());
+ EXPECT_EQ(
+ "sha256:ec4b8955958665577945c89419d1af06b5f7636b4ac3da7f12184802ad867736",
+ manifest->layers(2).digest());
+}
+
+
+TEST_F(DockerSpecTest, ParseInvalidV2_2ImageManifest)
+{
+ // This is an invalid manifest. The size of the repeated fields
+ // 'layers' must be >= 1. The 'signatures' and
+ // 'schemaVersion' fields are not set.
+ Try<JSON::Object> json = JSON::parse<JSON::Object>(
+ R"~(
+ {
+ "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
+ "config": {
+ "mediaType": "application/vnd.docker.container.image.v1+json",
+ "size": 7023,
+ "digest": "sha256:b5b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7"
+ }
+ })~");
+
+ ASSERT_SOME(json);
+
+ Try<spec::v2_2::ImageManifest> manifest = spec::v2_2::parse(json.get());
+ EXPECT_ERROR(manifest);
+}
+
+
+TEST_F(DockerSpecTest, ValidateV2_2ImageManifestLayersNonEmpty)
+{
+ Try<JSON::Object> json = JSON::parse<JSON::Object>(
+ R"~(
+ {
+ "schemaVersion": 2,
+ "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
+ "config": {
+ "mediaType": "application/vnd.docker.container.image.v1+json",
+ "size": 7023,
+ "digest": "sha256:b5b2b2c507a0944348e0303114d8d93aaaa081732b86451d9bce1f432a537bc7"
+ }
+ })~");
+
+ ASSERT_SOME(json);
+
+ Try<spec::v2_2::ImageManifest> manifest = spec::v2_2::parse(json.get());
+ EXPECT_ERROR(manifest);
+}
+
} // namespace tests {
} // namespace internal {
} // namespace mesos {