You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by ji...@apache.org on 2016/01/19 01:12:32 UTC
[1/9] mesos git commit: Made Docker::create return an Owned object
instead of a raw pointer.
Repository: mesos
Updated Branches:
refs/heads/master 81ee91e57 -> 684b551b0
Made Docker::create return an Owned object instead of a raw pointer.
Review: https://reviews.apache.org/r/42442
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/97ed41c2
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/97ed41c2
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/97ed41c2
Branch: refs/heads/master
Commit: 97ed41c2deeda0f352addba49a869b438a1741f2
Parents: 6119eb0
Author: Jie Yu <yu...@gmail.com>
Authored: Sat Jan 16 23:24:30 2016 -0800
Committer: Jie Yu <yu...@gmail.com>
Committed: Mon Jan 18 16:09:14 2016 -0800
----------------------------------------------------------------------
src/docker/docker.cpp | 7 +-
src/docker/docker.hpp | 3 +-
src/docker/executor.cpp | 14 ++--
src/slave/containerizer/docker.cpp | 7 +-
.../docker_containerizer_tests.cpp | 9 ++-
src/tests/containerizer/docker_tests.cpp | 77 ++++++++++++--------
src/tests/environment.cpp | 7 +-
7 files changed, 72 insertions(+), 52 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/97ed41c2/src/docker/docker.cpp
----------------------------------------------------------------------
diff --git a/src/docker/docker.cpp b/src/docker/docker.cpp
index 037cd40..4d2f1fa 100755
--- a/src/docker/docker.cpp
+++ b/src/docker/docker.cpp
@@ -93,7 +93,8 @@ static Future<Nothing> checkError(const string& cmd, const Subprocess& s)
.then(lambda::bind(_checkError, cmd, s));
}
-Try<Docker*> Docker::create(
+
+Try<Owned<Docker>> Docker::create(
const string& path,
const string& socket,
bool validate)
@@ -102,7 +103,7 @@ Try<Docker*> Docker::create(
return Error("Invalid Docker socket path: " + socket);
}
- Docker* docker = new Docker(path, socket);
+ Owned<Docker> docker(new Docker(path, socket));
if (!validate) {
return docker;
}
@@ -113,7 +114,6 @@ Try<Docker*> Docker::create(
Result<string> hierarchy = cgroups::hierarchy("cpu");
if (hierarchy.isNone()) {
- delete docker;
return Error("Failed to find a mounted cgroups hierarchy "
"for the 'cpu' subsystem; you probably need "
"to mount cgroups manually");
@@ -122,7 +122,6 @@ Try<Docker*> Docker::create(
Try<Nothing> validateVersion = docker->validateVersion(Version(1, 0, 0));
if (validateVersion.isError()) {
- delete docker;
return Error(validateVersion.error());
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/97ed41c2/src/docker/docker.hpp
----------------------------------------------------------------------
diff --git a/src/docker/docker.hpp b/src/docker/docker.hpp
index dde2b29..7802f23 100644
--- a/src/docker/docker.hpp
+++ b/src/docker/docker.hpp
@@ -22,6 +22,7 @@
#include <string>
#include <process/future.hpp>
+#include <process/owned.hpp>
#include <process/subprocess.hpp>
#include <stout/duration.hpp>
@@ -43,7 +44,7 @@ class Docker
{
public:
// Create Docker abstraction and optionally validate docker.
- static Try<Docker*> create(
+ static Try<process::Owned<Docker>> create(
const std::string& path,
const std::string& socket,
bool validate = true);
http://git-wip-us.apache.org/repos/asf/mesos/blob/97ed41c2/src/docker/executor.cpp
----------------------------------------------------------------------
diff --git a/src/docker/executor.cpp b/src/docker/executor.cpp
index 7512d07..654a41d 100644
--- a/src/docker/executor.cpp
+++ b/src/docker/executor.cpp
@@ -41,6 +41,9 @@
#include "messages/messages.hpp"
+using namespace mesos;
+using namespace process;
+
using std::cerr;
using std::cout;
using std::endl;
@@ -51,9 +54,6 @@ namespace mesos {
namespace internal {
namespace docker {
-using namespace mesos;
-using namespace process;
-
const Duration DOCKER_INSPECT_DELAY = Milliseconds(500);
const Duration DOCKER_INSPECT_TIMEOUT = Seconds(5);
@@ -571,8 +571,10 @@ int main(int argc, char** argv)
// The 2nd argument for docker create is set to false so we skip
// validation when creating a docker abstraction, as the slave
// should have already validated docker.
- Try<Docker*> docker =
- Docker::create(flags.docker.get(), flags.docker_socket.get(), false);
+ Try<Owned<Docker>> docker = Docker::create(
+ flags.docker.get(),
+ flags.docker_socket.get(),
+ false);
if (docker.isError()) {
cerr << "Unable to create docker abstraction: " << docker.error() << endl;
@@ -580,7 +582,7 @@ int main(int argc, char** argv)
}
mesos::internal::docker::DockerExecutor executor(
- process::Owned<Docker>(docker.get()),
+ docker.get(),
flags.container.get(),
flags.sandbox_directory.get(),
flags.mapped_directory.get(),
http://git-wip-us.apache.org/repos/asf/mesos/blob/97ed41c2/src/slave/containerizer/docker.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/docker.cpp b/src/slave/containerizer/docker.cpp
index da19975..40f6f0b 100644
--- a/src/slave/containerizer/docker.cpp
+++ b/src/slave/containerizer/docker.cpp
@@ -134,13 +134,16 @@ Try<DockerContainerizer*> DockerContainerizer::create(
return Error("Failed to create container logger: " + logger.error());
}
- Try<Docker*> create = Docker::create(flags.docker, flags.docker_socket, true);
+ Try<Owned<Docker>> create = Docker::create(
+ flags.docker,
+ flags.docker_socket,
+ true);
if (create.isError()) {
return Error("Failed to create docker: " + create.error());
}
- Shared<Docker> docker(create.get());
+ Shared<Docker> docker = create->share();
if (flags.docker_mesos_image.isSome()) {
Try<Nothing> validateResult = docker->validateVersion(Version(1, 5, 0));
http://git-wip-us.apache.org/repos/asf/mesos/blob/97ed41c2/src/tests/containerizer/docker_containerizer_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/containerizer/docker_containerizer_tests.cpp b/src/tests/containerizer/docker_containerizer_tests.cpp
index a1a44da..81ab362 100644
--- a/src/tests/containerizer/docker_containerizer_tests.cpp
+++ b/src/tests/containerizer/docker_containerizer_tests.cpp
@@ -144,10 +144,13 @@ public:
virtual void TearDown()
{
- Try<Docker*> docker =
- Docker::create(tests::flags.docker, tests::flags.docker_socket, false);
+ Try<Owned<Docker>> docker = Docker::create(
+ tests::flags.docker,
+ tests::flags.docker_socket,
+ false);
ASSERT_SOME(docker);
+
Future<list<Docker::Container>> containers =
docker.get()->ps(true, slave::DOCKER_NAME_PREFIX);
@@ -157,8 +160,6 @@ public:
foreach (const Docker::Container& container, containers.get()) {
AWAIT_READY_FOR(docker.get()->rm(container.id, true), Seconds(30));
}
-
- delete docker.get();
}
};
http://git-wip-us.apache.org/repos/asf/mesos/blob/97ed41c2/src/tests/containerizer/docker_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/containerizer/docker_tests.cpp b/src/tests/containerizer/docker_tests.cpp
index a65d552..9583070 100644
--- a/src/tests/containerizer/docker_tests.cpp
+++ b/src/tests/containerizer/docker_tests.cpp
@@ -50,9 +50,10 @@ class DockerTest : public MesosTest
{
virtual void TearDown()
{
- Try<Docker*> docker =
- Docker::create(tests::flags.docker, tests::flags.docker_socket,
- false);
+ Try<Owned<Docker>> docker = Docker::create(
+ tests::flags.docker,
+ tests::flags.docker_socket,
+ false);
ASSERT_SOME(docker);
@@ -65,8 +66,6 @@ class DockerTest : public MesosTest
foreach (const Docker::Container& container, containers.get()) {
AWAIT_READY_FOR(docker.get()->rm(container.id, true), Seconds(30));
}
-
- delete docker.get();
}
};
@@ -77,9 +76,10 @@ TEST_F(DockerTest, ROOT_DOCKER_interface)
const string containerName = NAME_PREFIX + "-test";
Resources resources = Resources::parse("cpus:1;mem:512").get();
- Owned<Docker> docker(Docker::create(tests::flags.docker,
- tests::flags.docker_socket,
- false).get());
+ Owned<Docker> docker = Docker::create(
+ tests::flags.docker,
+ tests::flags.docker_socket,
+ false).get();
// Verify that we do not see the container.
Future<list<Docker::Container> > containers = docker->ps(true, containerName);
@@ -112,6 +112,7 @@ TEST_F(DockerTest, ROOT_DOCKER_interface)
Future<Docker::Container> inspect =
docker->inspect(containerName, Seconds(1));
+
AWAIT_READY(inspect);
// Should be able to see the container now.
@@ -227,23 +228,29 @@ TEST_F(DockerTest, ROOT_DOCKER_interface)
// This test tests parsing docker version output.
TEST_F(DockerTest, ROOT_DOCKER_parsing_version)
{
- Owned<Docker> docker1(Docker::create("echo Docker version 1.7.1, build",
- tests::flags.docker_socket,
- false).get());
+ Owned<Docker> docker1 = Docker::create(
+ "echo Docker version 1.7.1, build",
+ tests::flags.docker_socket,
+ false).get();
+
Future<Version> version1 = docker1->version();
AWAIT_READY(version1);
EXPECT_EQ(version1.get(), Version::parse("1.7.1").get());
- Owned<Docker> docker2(Docker::create("echo Docker version 1.7.1.fc22, build",
- tests::flags.docker_socket,
- false).get());
+ Owned<Docker> docker2 = Docker::create(
+ "echo Docker version 1.7.1.fc22, build",
+ tests::flags.docker_socket,
+ false).get();
+
Future<Version> version2 = docker2->version();
AWAIT_READY(version2);
EXPECT_EQ(version2.get(), Version::parse("1.7.1").get());
- Owned<Docker> docker3(Docker::create("echo Docker version 1.7.1-fc22, build",
- tests::flags.docker_socket,
- false).get());
+ Owned<Docker> docker3 = Docker::create(
+ "echo Docker version 1.7.1-fc22, build",
+ tests::flags.docker_socket,
+ false).get();
+
Future<Version> version3 = docker3->version();
AWAIT_READY(version3);
EXPECT_EQ(version3.get(), Version::parse("1.7.1").get());
@@ -252,9 +259,10 @@ TEST_F(DockerTest, ROOT_DOCKER_parsing_version)
TEST_F(DockerTest, ROOT_DOCKER_CheckCommandWithShell)
{
- Owned<Docker> docker(Docker::create(tests::flags.docker,
- tests::flags.docker_socket,
- false).get());
+ Owned<Docker> docker = Docker::create(
+ tests::flags.docker,
+ tests::flags.docker_socket,
+ false).get();
ContainerInfo containerInfo;
containerInfo.set_type(ContainerInfo::DOCKER);
@@ -280,9 +288,11 @@ TEST_F(DockerTest, ROOT_DOCKER_CheckCommandWithShell)
TEST_F(DockerTest, ROOT_DOCKER_CheckPortResource)
{
const string containerName = NAME_PREFIX + "-port-resource-test";
- Owned<Docker> docker(Docker::create(tests::flags.docker,
- tests::flags.docker_socket,
- false).get());
+
+ Owned<Docker> docker = Docker::create(
+ tests::flags.docker,
+ tests::flags.docker_socket,
+ false).get();
// Make sure the container is removed.
Future<Nothing> remove = docker->rm(containerName, true);
@@ -352,9 +362,10 @@ TEST_F(DockerTest, ROOT_DOCKER_CancelPull)
AWAIT_READY_FOR(s.get().status(), Seconds(30));
- Owned<Docker> docker(Docker::create(tests::flags.docker,
- tests::flags.docker_socket,
- false).get());
+ Owned<Docker> docker = Docker::create(
+ tests::flags.docker,
+ tests::flags.docker_socket,
+ false).get();
Try<string> directory = environment->mkdtemp();
@@ -376,9 +387,10 @@ TEST_F(DockerTest, ROOT_DOCKER_CancelPull)
// docker container works.
TEST_F(DockerTest, ROOT_DOCKER_MountRelative)
{
- Owned<Docker> docker(Docker::create(tests::flags.docker,
- tests::flags.docker_socket,
- false).get());
+ Owned<Docker> docker = Docker::create(
+ tests::flags.docker,
+ tests::flags.docker_socket,
+ false).get();
ContainerInfo containerInfo;
containerInfo.set_type(ContainerInfo::DOCKER);
@@ -418,9 +430,10 @@ TEST_F(DockerTest, ROOT_DOCKER_MountRelative)
// docker container works.
TEST_F(DockerTest, ROOT_DOCKER_MountAbsolute)
{
- Owned<Docker> docker(Docker::create(tests::flags.docker,
- tests::flags.docker_socket,
- false).get());
+ Owned<Docker> docker = Docker::create(
+ tests::flags.docker,
+ tests::flags.docker_socket,
+ false).get();
ContainerInfo containerInfo;
containerInfo.set_type(ContainerInfo::DOCKER);
http://git-wip-us.apache.org/repos/asf/mesos/blob/97ed41c2/src/tests/environment.cpp
----------------------------------------------------------------------
diff --git a/src/tests/environment.cpp b/src/tests/environment.cpp
index ebb7660..51e759b 100644
--- a/src/tests/environment.cpp
+++ b/src/tests/environment.cpp
@@ -242,11 +242,12 @@ public:
DockerFilter()
{
#ifdef __linux__
- Try<Docker*> docker = Docker::create(flags.docker, flags.docker_socket);
+ Try<Owned<Docker>> docker = Docker::create(
+ flags.docker,
+ flags.docker_socket);
+
if (docker.isError()) {
dockerError = docker.error();
- } else {
- delete docker.get();
}
#else
dockerError = Error("Docker tests not supported on non-Linux systems");
[8/9] mesos git commit: Reordered filters in test environment
according to lexical order.
Posted by ji...@apache.org.
Reordered filters in test environment according to lexical order.
Review: https://reviews.apache.org/r/42443
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/0e3091bc
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/0e3091bc
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/0e3091bc
Branch: refs/heads/master
Commit: 0e3091bc48f07578129b068c7457f2af7f8451e2
Parents: 97ed41c
Author: Jie Yu <yu...@gmail.com>
Authored: Sun Jan 17 11:23:43 2016 -0800
Committer: Jie Yu <yu...@gmail.com>
Committed: Mon Jan 18 16:10:41 2016 -0800
----------------------------------------------------------------------
src/tests/environment.cpp | 208 ++++++++++++++++++++---------------------
1 file changed, 104 insertions(+), 104 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/0e3091bc/src/tests/environment.cpp
----------------------------------------------------------------------
diff --git a/src/tests/environment.cpp b/src/tests/environment.cpp
index 51e759b..0b994a3 100644
--- a/src/tests/environment.cpp
+++ b/src/tests/environment.cpp
@@ -104,23 +104,12 @@ public:
};
-class RootFilter : public TestFilter
+class BenchmarkFilter : public TestFilter
{
public:
bool disable(const ::testing::TestInfo* test) const
{
- Result<string> user = os::user();
- CHECK_SOME(user);
-
-#ifdef __linux__
- // On Linux non-privileged users are limited to 64k of locked
- // memory so we cannot run the MemIsolatorTest.Usage.
- if (matches(test, "MemIsolatorTest")) {
- return user.get() != "root";
- }
-#endif // __linux__
-
- return matches(test, "ROOT_") && user.get() != "root";
+ return matches(test, "BENCHMARK_") && !flags.benchmark;
}
};
@@ -236,6 +225,31 @@ private:
};
+class CurlFilter : public TestFilter
+{
+public:
+ CurlFilter()
+ {
+ curlError = os::system("which curl") != 0;
+ if (curlError) {
+ std::cerr
+ << "-------------------------------------------------------------\n"
+ << "No 'curl' command found so no 'curl' tests will be run\n"
+ << "-------------------------------------------------------------"
+ << std::endl;
+ }
+ }
+
+ bool disable(const ::testing::TestInfo* test) const
+ {
+ return matches(test, "CURL_") && curlError;
+ }
+
+private:
+ bool curlError;
+};
+
+
class DockerFilter : public TestFilter
{
public:
@@ -273,38 +287,66 @@ private:
};
-class PerfFilter : public TestFilter
+class NetcatFilter : public TestFilter
{
public:
- PerfFilter()
+ NetcatFilter()
{
-#ifdef __linux__
- perfError = os::system("perf help >&-") != 0;
- if (perfError) {
+ netcatError = os::system("which nc") != 0;
+ if (netcatError) {
std::cerr
<< "-------------------------------------------------------------\n"
- << "No 'perf' command found so no 'perf' tests will be run\n"
+ << "No 'nc' command found so no tests depending on 'nc' will run\n"
<< "-------------------------------------------------------------"
<< std::endl;
}
-#else
- perfError = true;
-#endif // __linux__
}
bool disable(const ::testing::TestInfo* test) const
{
- // Currently all tests that require 'perf' are part of the
- // 'PerfTest' test fixture, hence we check for 'Perf' here.
- //
- // TODO(ijimenez): Replace all tests which require 'perf' with
- // the prefix 'PERF_' to be more consistent with the filter
- // naming we've done (i.e., ROOT_, CGROUPS_, etc).
- return matches(test, "Perf") && perfError;
+ return matches(test, "NC_") && netcatError;
}
private:
- bool perfError;
+ bool netcatError;
+};
+
+
+class NetworkIsolatorTestFilter : public TestFilter
+{
+public:
+ NetworkIsolatorTestFilter()
+ {
+#ifdef WITH_NETWORK_ISOLATOR
+ Try<Nothing> check = routing::check();
+ if (check.isError()) {
+ std::cerr
+ << "-------------------------------------------------------------\n"
+ << "We cannot run any PortMapping tests because:\n"
+ << check.error() << "\n"
+ << "-------------------------------------------------------------\n";
+
+ portMappingError = Error(check.error());
+ }
+#endif
+ }
+
+ bool disable(const ::testing::TestInfo* test) const
+ {
+ if (matches(test, "PortMappingIsolatorTest") ||
+ matches(test, "PortMappingMesosTest")) {
+#ifdef WITH_NETWORK_ISOLATOR
+ return !portMappingError.isNone();
+#else
+ return true;
+#endif
+ }
+
+ return false;
+ }
+
+private:
+ Option<Error> portMappingError;
};
@@ -358,101 +400,59 @@ private:
};
-class NetcatFilter : public TestFilter
-{
-public:
- NetcatFilter()
- {
- netcatError = os::system("which nc") != 0;
- if (netcatError) {
- std::cerr
- << "-------------------------------------------------------------\n"
- << "No 'nc' command found so no tests depending on 'nc' will run\n"
- << "-------------------------------------------------------------"
- << std::endl;
- }
- }
-
- bool disable(const ::testing::TestInfo* test) const
- {
- return matches(test, "NC_") && netcatError;
- }
-
-private:
- bool netcatError;
-};
-
-
-class CurlFilter : public TestFilter
+class PerfFilter : public TestFilter
{
public:
- CurlFilter()
+ PerfFilter()
{
- curlError = os::system("which curl") != 0;
- if (curlError) {
+#ifdef __linux__
+ perfError = os::system("perf help >&-") != 0;
+ if (perfError) {
std::cerr
<< "-------------------------------------------------------------\n"
- << "No 'curl' command found so no 'curl' tests will be run\n"
+ << "No 'perf' command found so no 'perf' tests will be run\n"
<< "-------------------------------------------------------------"
<< std::endl;
}
+#else
+ perfError = true;
+#endif // __linux__
}
bool disable(const ::testing::TestInfo* test) const
{
- return matches(test, "CURL_") && curlError;
+ // Currently all tests that require 'perf' are part of the
+ // 'PerfTest' test fixture, hence we check for 'Perf' here.
+ //
+ // TODO(ijimenez): Replace all tests which require 'perf' with
+ // the prefix 'PERF_' to be more consistent with the filter
+ // naming we've done (i.e., ROOT_, CGROUPS_, etc).
+ return matches(test, "Perf") && perfError;
}
private:
- bool curlError;
+ bool perfError;
};
-class BenchmarkFilter : public TestFilter
+class RootFilter : public TestFilter
{
public:
bool disable(const ::testing::TestInfo* test) const
{
- return matches(test, "BENCHMARK_") && !flags.benchmark;
- }
-};
-
-
-class NetworkIsolatorTestFilter : public TestFilter
-{
-public:
- NetworkIsolatorTestFilter()
- {
-#ifdef WITH_NETWORK_ISOLATOR
- Try<Nothing> check = routing::check();
- if (check.isError()) {
- std::cerr
- << "-------------------------------------------------------------\n"
- << "We cannot run any PortMapping tests because:\n"
- << check.error() << "\n"
- << "-------------------------------------------------------------\n";
-
- portMappingError = Error(check.error());
- }
-#endif
- }
+ Result<string> user = os::user();
+ CHECK_SOME(user);
- bool disable(const ::testing::TestInfo* test) const
- {
- if (matches(test, "PortMappingIsolatorTest") ||
- matches(test, "PortMappingMesosTest")) {
-#ifdef WITH_NETWORK_ISOLATOR
- return !portMappingError.isNone();
-#else
- return true;
-#endif
+#ifdef __linux__
+ // On Linux non-privileged users are limited to 64k of locked
+ // memory so we cannot run the MemIsolatorTest.Usage.
+ if (matches(test, "MemIsolatorTest")) {
+ return user.get() != "root";
}
+#endif // __linux__
- return false;
+ return matches(test, "ROOT_") && user.get() != "root";
}
-
-private:
- Option<Error> portMappingError;
};
@@ -522,16 +522,16 @@ Environment::Environment(const Flags& _flags) : flags(_flags)
vector<Owned<TestFilter> > filters;
- filters.push_back(Owned<TestFilter>(new RootFilter()));
+ filters.push_back(Owned<TestFilter>(new BenchmarkFilter()));
filters.push_back(Owned<TestFilter>(new CfsFilter()));
filters.push_back(Owned<TestFilter>(new CgroupsFilter()));
+ filters.push_back(Owned<TestFilter>(new CurlFilter()));
filters.push_back(Owned<TestFilter>(new DockerFilter()));
- filters.push_back(Owned<TestFilter>(new BenchmarkFilter()));
+ filters.push_back(Owned<TestFilter>(new NetcatFilter()));
filters.push_back(Owned<TestFilter>(new NetworkIsolatorTestFilter()));
- filters.push_back(Owned<TestFilter>(new PerfFilter()));
filters.push_back(Owned<TestFilter>(new PerfCPUCyclesFilter()));
- filters.push_back(Owned<TestFilter>(new NetcatFilter()));
- filters.push_back(Owned<TestFilter>(new CurlFilter()));
+ filters.push_back(Owned<TestFilter>(new PerfFilter()));
+ filters.push_back(Owned<TestFilter>(new RootFilter()));
// Construct the filter string to handle system or platform specific tests.
::testing::UnitTest* unitTest = ::testing::UnitTest::GetInstance();
[6/9] mesos git commit: Added utility functions to create docker URI.
Posted by ji...@apache.org.
Added utility functions to create docker URI.
Review: https://reviews.apache.org/r/42275
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/b589a3ac
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/b589a3ac
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/b589a3ac
Branch: refs/heads/master
Commit: b589a3ac67c740ac84c54f2db2cc83b6a043d5ca
Parents: f070799
Author: Jie Yu <yu...@gmail.com>
Authored: Wed Jan 13 14:42:31 2016 -0800
Committer: Jie Yu <yu...@gmail.com>
Committed: Mon Jan 18 16:09:14 2016 -0800
----------------------------------------------------------------------
src/Makefile.am | 1 +
src/uri/schemes/docker.hpp | 66 +++++++++++++++++++++++++++++++++++++++++
2 files changed, 67 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/b589a3ac/src/Makefile.am
----------------------------------------------------------------------
diff --git a/src/Makefile.am b/src/Makefile.am
index 69383b6..4fabe60 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -790,6 +790,7 @@ libmesos_no_3rdparty_la_SOURCES += \
uri/fetchers/curl.hpp \
uri/fetchers/docker.hpp \
uri/fetchers/hadoop.hpp \
+ uri/schemes/docker.hpp \
uri/schemes/file.hpp \
uri/schemes/hdfs.hpp \
uri/schemes/http.hpp \
http://git-wip-us.apache.org/repos/asf/mesos/blob/b589a3ac/src/uri/schemes/docker.hpp
----------------------------------------------------------------------
diff --git a/src/uri/schemes/docker.hpp b/src/uri/schemes/docker.hpp
new file mode 100644
index 0000000..68dbf3f
--- /dev/null
+++ b/src/uri/schemes/docker.hpp
@@ -0,0 +1,66 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef __URI_SCHEMES_DOCKER_HPP__
+#define __URI_SCHEMES_DOCKER_HPP__
+
+#include <string>
+
+#include <mesos/uri/uri.hpp>
+
+#include "uri/utils.hpp"
+
+namespace mesos {
+namespace uri {
+namespace docker {
+
+inline URI image(
+ const std::string& repository,
+ const std::string& tag,
+ const std::string& registry,
+ const Option<std::string>& scheme = None(),
+ const Option<int>& port = None())
+{
+ return construct("docker", repository, registry, port, tag, scheme);
+}
+
+
+inline URI manifest(
+ const std::string& repository,
+ const std::string& tag,
+ const std::string& registry,
+ const Option<std::string>& scheme = None(),
+ const Option<int>& port = None())
+{
+ return construct("docker-manifest", repository, registry, port, tag, scheme);
+}
+
+
+inline URI blob(
+ const std::string& repository,
+ const std::string& digest,
+ const std::string& registry,
+ const Option<std::string>& scheme = None(),
+ const Option<int>& port = None())
+{
+ return construct("docker-blob", repository, registry, port, digest, scheme);
+}
+
+} // namespace docker {
+} // namespace uri {
+} // namespace mesos {
+
+#endif // __URI_SCHEMES_DOCKER_HPP__
[9/9] mesos git commit: Added an internet access test filter.
Posted by ji...@apache.org.
Added an internet access test filter.
Review: https://reviews.apache.org/r/42444
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/684b551b
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/684b551b
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/684b551b
Branch: refs/heads/master
Commit: 684b551b02da55b62f401dc05936e3f067f55a70
Parents: 0e3091b
Author: Jie Yu <yu...@gmail.com>
Authored: Sun Jan 17 11:38:01 2016 -0800
Committer: Jie Yu <yu...@gmail.com>
Committed: Mon Jan 18 16:10:41 2016 -0800
----------------------------------------------------------------------
src/tests/environment.cpp | 26 ++++++++++++++++++++++++++
1 file changed, 26 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/684b551b/src/tests/environment.cpp
----------------------------------------------------------------------
diff --git a/src/tests/environment.cpp b/src/tests/environment.cpp
index 0b994a3..a6322f2 100644
--- a/src/tests/environment.cpp
+++ b/src/tests/environment.cpp
@@ -287,6 +287,31 @@ private:
};
+class InternetFilter : public TestFilter
+{
+public:
+ InternetFilter()
+ {
+ error = os::system("ping -c 1 -W 1 google.com") != 0;
+ if (error) {
+ std::cerr
+ << "-------------------------------------------------------------\n"
+ << "We cannot run any INTERNET tests because no internet access\n"
+ << "-------------------------------------------------------------"
+ << std::endl;
+ }
+ }
+
+ bool disable(const ::testing::TestInfo* test) const
+ {
+ return matches(test, "INTERNET_") && error;
+ }
+
+private:
+ bool error;
+};
+
+
class NetcatFilter : public TestFilter
{
public:
@@ -527,6 +552,7 @@ Environment::Environment(const Flags& _flags) : flags(_flags)
filters.push_back(Owned<TestFilter>(new CgroupsFilter()));
filters.push_back(Owned<TestFilter>(new CurlFilter()));
filters.push_back(Owned<TestFilter>(new DockerFilter()));
+ filters.push_back(Owned<TestFilter>(new InternetFilter()));
filters.push_back(Owned<TestFilter>(new NetcatFilter()));
filters.push_back(Owned<TestFilter>(new NetworkIsolatorTestFilter()));
filters.push_back(Owned<TestFilter>(new PerfCPUCyclesFilter()));
[2/9] mesos git commit: Added docker image manifest parsing functions
for strings.
Posted by ji...@apache.org.
Added docker image manifest parsing functions for strings.
Review: https://reviews.apache.org/r/42237
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/f0707990
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/f0707990
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/f0707990
Branch: refs/heads/master
Commit: f07079904270eb4685eba57c537d59092d52543a
Parents: 6815d3f
Author: Jie Yu <yu...@gmail.com>
Authored: Tue Jan 12 16:53:51 2016 -0800
Committer: Jie Yu <yu...@gmail.com>
Committed: Mon Jan 18 16:09:14 2016 -0800
----------------------------------------------------------------------
include/mesos/docker/spec.hpp | 12 ++++++++++++
src/docker/spec.cpp | 22 ++++++++++++++++++++++
2 files changed, 34 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/f0707990/include/mesos/docker/spec.hpp
----------------------------------------------------------------------
diff --git a/include/mesos/docker/spec.hpp b/include/mesos/docker/spec.hpp
index 61b55b4..b42f422 100644
--- a/include/mesos/docker/spec.hpp
+++ b/include/mesos/docker/spec.hpp
@@ -64,6 +64,12 @@ Option<Error> validate(const ImageManifest& manifest);
*/
Try<ImageManifest> parse(const JSON::Object& json);
+
+/**
+ * Returns the docker v1 image manifest from the given string.
+ */
+Try<ImageManifest> parse(const std::string& s);
+
} // namespace v1 {
@@ -81,6 +87,12 @@ Option<Error> validate(const ImageManifest& manifest);
*/
Try<ImageManifest> parse(const JSON::Object& json);
+
+/**
+ * Returns the docker v2 image manifest from the given string.
+ */
+Try<ImageManifest> parse(const std::string& s);
+
} // namespace v2 {
} // namespace spec {
} // namespace docker {
http://git-wip-us.apache.org/repos/asf/mesos/blob/f0707990/src/docker/spec.cpp
----------------------------------------------------------------------
diff --git a/src/docker/spec.cpp b/src/docker/spec.cpp
index 29d2f11..1f7adf7 100644
--- a/src/docker/spec.cpp
+++ b/src/docker/spec.cpp
@@ -106,6 +106,17 @@ Try<ImageManifest> parse(const JSON::Object& json)
return manifest.get();
}
+
+Try<ImageManifest> parse(const string& s)
+{
+ Try<JSON::Object> json = JSON::parse<JSON::Object>(s);
+ if (json.isError()) {
+ return Error("JSON parse failed: " + json.error());
+ }
+
+ return parse(json.get());
+}
+
} // namespace v1 {
@@ -180,6 +191,17 @@ Try<ImageManifest> parse(const JSON::Object& json)
return manifest.get();
}
+
+Try<ImageManifest> parse(const string& s)
+{
+ Try<JSON::Object> json = JSON::parse<JSON::Object>(s);
+ if (json.isError()) {
+ return Error("JSON parse failed: " + json.error());
+ }
+
+ return parse(json.get());
+}
+
} // namespace v2 {
} // namespace spec {
} // namespace docker {
[3/9] mesos git commit: Added a utility function to create https URI.
Posted by ji...@apache.org.
Added a utility function to create https URI.
Review: https://reviews.apache.org/r/42236
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/6815d3f5
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/6815d3f5
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/6815d3f5
Branch: refs/heads/master
Commit: 6815d3f529a60520e4f2db256db76f34155834f8
Parents: c6a2f8a
Author: Jie Yu <yu...@gmail.com>
Authored: Sun Jan 10 22:10:18 2016 -0800
Committer: Jie Yu <yu...@gmail.com>
Committed: Mon Jan 18 16:09:14 2016 -0800
----------------------------------------------------------------------
src/uri/schemes/http.hpp | 16 ++++++++++++++++
1 file changed, 16 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/6815d3f5/src/uri/schemes/http.hpp
----------------------------------------------------------------------
diff --git a/src/uri/schemes/http.hpp b/src/uri/schemes/http.hpp
index 84a80b9..a35f175 100644
--- a/src/uri/schemes/http.hpp
+++ b/src/uri/schemes/http.hpp
@@ -44,6 +44,22 @@ inline URI http(
return construct("http", path, host, port, query, fragment, user, password);
}
+
+/**
+ * Creates an https URI with the given parameters.
+ */
+inline URI https(
+ const std::string& host,
+ const std::string& path = "/",
+ const Option<int>& port = None(),
+ const Option<std::string>& query = None(),
+ const Option<std::string>& fragment = None(),
+ const Option<std::string>& user = None(),
+ const Option<std::string>& password = None())
+{
+ return construct("https", path, host, port, query, fragment, user, password);
+}
+
} // namespace uri {
} // namespace mesos {
[5/9] mesos git commit: Implemented the Docker URI fetcher plugin
based on curl.
Posted by ji...@apache.org.
Implemented the Docker URI fetcher plugin based on curl.
Review: https://reviews.apache.org/r/42238
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/6119eb0a
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/6119eb0a
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/6119eb0a
Branch: refs/heads/master
Commit: 6119eb0a396b4ba9ddaa2c8f86c9437997df406b
Parents: b589a3a
Author: Jie Yu <yu...@gmail.com>
Authored: Thu Dec 31 23:14:02 2015 -0800
Committer: Jie Yu <yu...@gmail.com>
Committed: Mon Jan 18 16:09:14 2016 -0800
----------------------------------------------------------------------
include/mesos/uri/fetcher.hpp | 1 +
src/uri/fetchers/docker.cpp | 585 ++++++++++++++++++++++++++++++++++++-
2 files changed, 584 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/6119eb0a/include/mesos/uri/fetcher.hpp
----------------------------------------------------------------------
diff --git a/include/mesos/uri/fetcher.hpp b/include/mesos/uri/fetcher.hpp
index af2cf00..baa32d3 100644
--- a/include/mesos/uri/fetcher.hpp
+++ b/include/mesos/uri/fetcher.hpp
@@ -83,6 +83,7 @@ public:
* @param uri the URI to fetch
* @param directory the directory the URI will be downloaded to
*/
+ // TODO(jieyu): Consider using 'Path' for 'directory' here.
process::Future<Nothing> fetch(
const URI& uri,
const std::string& directory);
http://git-wip-us.apache.org/repos/asf/mesos/blob/6119eb0a/src/uri/fetchers/docker.cpp
----------------------------------------------------------------------
diff --git a/src/uri/fetchers/docker.cpp b/src/uri/fetchers/docker.cpp
index 3595f91..2ceee19 100644
--- a/src/uri/fetchers/docker.cpp
+++ b/src/uri/fetchers/docker.cpp
@@ -14,13 +14,45 @@
// See the License for the specific language governing permissions and
// limitations under the License.
+#include <list>
+#include <sstream>
+#include <string>
+#include <tuple>
+#include <vector>
+
+#include <process/collect.hpp>
#include <process/dispatch.hpp>
+#include <process/http.hpp>
+#include <process/io.hpp>
#include <process/process.hpp>
+#include <process/subprocess.hpp>
+
+#include <stout/none.hpp>
+#include <stout/option.hpp>
+#include <stout/strings.hpp>
+
+#include <stout/os/mkdir.hpp>
+#include <stout/os/write.hpp>
+
+#include <mesos/docker/spec.hpp>
+
+#include "uri/utils.hpp"
#include "uri/fetchers/docker.hpp"
+#include "uri/schemes/docker.hpp"
+#include "uri/schemes/http.hpp"
+
+namespace http = process::http;
+namespace io = process::io;
+namespace spec = docker::spec;
+
+using std::list;
+using std::ostringstream;
using std::set;
using std::string;
+using std::tuple;
+using std::vector;
using process::dispatch;
using process::spawn;
@@ -31,21 +63,257 @@ using process::Failure;
using process::Future;
using process::Owned;
using process::Process;
+using process::Subprocess;
namespace mesos {
namespace uri {
+//-------------------------------------------------------------------
+// Helper and utility functions.
+//-------------------------------------------------------------------
+
+// Returns the set of schemes supported by this URI fetcher plugin.
+static set<string> schemes()
+{
+ return {
+ "docker", // Fetch image manifest and blobs.
+ "docker-manifest", // Fetch image manifest only.
+ "docker-blob" // Fetch a single image blob.
+ };
+}
+
+// TODO(jieyu): Move the following curl based utility functions to a
+// command utils common directory.
+
+// Uses the curl command to send an HTTP request to the given URL and
+// returns the HTTP response it received. The location redirection and
+// HTTPS connections will be handled automatically by the curl
+// command. The returned HTTP response will have the type 'BODY' (no
+// streaming).
+static Future<http::Response> curl(
+ const string& uri,
+ const http::Headers& headers = http::Headers())
+{
+ vector<string> argv = {
+ "curl",
+ "-s", // Don’t show progress meter or error messages.
+ "-S", // Make curl show an error message if it fails.
+ "-L", // Follow HTTP 3xx redirects.
+ "-D", "-", // Write the protocol headers to stdout.
+ };
+
+ // Add additional headers.
+ foreachpair (const string& key, const string& value, headers) {
+ argv.push_back("-H");
+ argv.push_back(key + ": " + value);
+ }
+
+ argv.push_back(strings::trim(uri));
+
+ // TODO(jieyu): Kill the process if discard is called.
+ Try<Subprocess> s = subprocess(
+ "curl",
+ argv,
+ Subprocess::PATH("/dev/null"),
+ Subprocess::PIPE(),
+ Subprocess::PIPE());
+
+ if (s.isError()) {
+ return Failure("Failed to exec the curl subprocess: " + s.error());
+ }
+
+ return await(
+ s.get().status(),
+ io::read(s.get().out().get()),
+ io::read(s.get().err().get()))
+ .then([](const tuple<
+ Future<Option<int>>,
+ Future<string>,
+ Future<string>>& t) -> Future<http::Response> {
+ Future<Option<int>> status = std::get<0>(t);
+ if (!status.isReady()) {
+ return Failure(
+ "Failed to get the exit status of the curl subprocess: " +
+ (status.isFailed() ? status.failure() : "discarded"));
+ }
+
+ if (status->isNone()) {
+ return Failure("Failed to reap the curl subprocess");
+ }
+
+ if (status->get() != 0) {
+ Future<string> error = std::get<2>(t);
+ if (!error.isReady()) {
+ return Failure(
+ "Failed to perform 'curl'. Reading stderr failed: " +
+ (error.isFailed() ? error.failure() : "discarded"));
+ }
+
+ return Failure("Failed to perform 'curl': " + error.get());
+ }
+
+ Future<string> output = std::get<1>(t);
+ if (!output.isReady()) {
+ return Failure(
+ "Failed to read stdout from 'curl': " +
+ (output.isFailed() ? output.failure() : "discarded"));
+ }
+
+ // Decode HTTP responses.
+ Try<vector<http::Response>> responses =
+ http::decodeResponses(output.get());
+
+ if (responses.isError()) {
+ return Failure(
+ "Failed to decode HTTP responses: " + responses.error() +
+ "\n" + output.get());
+ }
+
+ // NOTE: We always return the last response because there might
+ // be a '307 Temporary Redirect' response before that.
+ return responses->back();
+ });
+}
+
+
+static Future<http::Response> curl(
+ const URI& uri,
+ const http::Headers& headers = http::Headers())
+{
+ return curl(stringify(uri), headers);
+}
+
+
+// TODO(jieyu): Add a comment here.
+static Future<int> download(
+ const URI& uri,
+ const string& directory,
+ const http::Headers& headers = http::Headers())
+{
+ const string output = path::join(directory, Path(uri.path()).basename());
+
+ vector<string> argv = {
+ "curl",
+ "-s", // Don’t show progress meter or error messages.
+ "-S", // Make curl show an error message if it fails.
+ "-L", // Follow HTTP 3xx redirects.
+ "-w", "%{http_code}", // Display HTTP response code on stdout.
+ "-o", output // Write output to the file.
+ };
+
+ // Add additional headers.
+ foreachpair (const string& key, const string& value, headers) {
+ argv.push_back("-H");
+ argv.push_back(key + ": " + value);
+ }
+
+ argv.push_back(strings::trim(stringify(uri)));
+
+ // TODO(jieyu): Kill the process if discard is called.
+ Try<Subprocess> s = subprocess(
+ "curl",
+ argv,
+ Subprocess::PATH("/dev/null"),
+ Subprocess::PIPE(),
+ Subprocess::PIPE());
+
+ if (s.isError()) {
+ return Failure("Failed to exec the curl subprocess: " + s.error());
+ }
+
+ return await(
+ s.get().status(),
+ io::read(s.get().out().get()),
+ io::read(s.get().err().get()))
+ .then([](const tuple<
+ Future<Option<int>>,
+ Future<string>,
+ Future<string>>& t) -> Future<int> {
+ Future<Option<int>> status = std::get<0>(t);
+ if (!status.isReady()) {
+ return Failure(
+ "Failed to get the exit status of the curl subprocess: " +
+ (status.isFailed() ? status.failure() : "discarded"));
+ }
+
+ if (status->isNone()) {
+ return Failure("Failed to reap the curl subprocess");
+ }
+
+ if (status->get() != 0) {
+ Future<string> error = std::get<2>(t);
+ if (!error.isReady()) {
+ return Failure(
+ "Failed to perform 'curl'. Reading stderr failed: " +
+ (error.isFailed() ? error.failure() : "discarded"));
+ }
+
+ return Failure("Failed to perform 'curl': " + error.get());
+ }
+
+ Future<string> output = std::get<1>(t);
+ if (!output.isReady()) {
+ return Failure(
+ "Failed to read stdout from 'curl': " +
+ (output.isFailed() ? output.failure() : "discarded"));
+ }
+
+ // Parse the output and get the HTTP response code.
+ Try<int> code = numify<int>(output.get());
+ if (code.isError()) {
+ return Failure("Unexpected output from 'curl': " + output.get());
+ }
+
+ return code.get();
+ });
+}
+
+//-------------------------------------------------------------------
+// DockerFetcherPlugin implementation.
+//-------------------------------------------------------------------
+
class DockerFetcherPluginProcess : public Process<DockerFetcherPluginProcess>
{
public:
DockerFetcherPluginProcess() {}
Future<Nothing> fetch(const URI& uri, const string& directory);
+
+private:
+ Future<Nothing> _fetch(
+ const URI& uri,
+ const string& directory,
+ const URI& manifestUri,
+ const http::Response& response);
+
+ Future<Nothing> __fetch(
+ const URI& uri,
+ const string& directory,
+ const Option<string>& authToken,
+ const http::Response& response);
+
+ Future<Nothing> fetchBlob(
+ const URI& uri,
+ const string& directory,
+ const Option<string>& authToken);
+
+ Future<Nothing> _fetchBlob(
+ const URI& uri,
+ const string& directory,
+ const URI& blobUri);
+
+ Future<string> getAuthToken(const http::Response& response);
+ http::Headers getAuthHeader(const Option<string>& authToken);
+
+ URI getManifestUri(const URI& uri);
+ URI getBlobUri(const URI& uri);
};
Try<Owned<Fetcher::Plugin>> DockerFetcherPlugin::create(const Flags& flags)
{
+ // TODO(jieyu): Make sure curl is available.
+
Owned<DockerFetcherPluginProcess> process(
new DockerFetcherPluginProcess());
@@ -70,7 +338,8 @@ DockerFetcherPlugin::~DockerFetcherPlugin()
set<string> DockerFetcherPlugin::schemes()
{
- return {"docker"};
+ // Use uri:: prefix to disambiguate.
+ return uri::schemes();
}
@@ -90,7 +359,319 @@ Future<Nothing> DockerFetcherPluginProcess::fetch(
const URI& uri,
const string& directory)
{
- return Failure("Unimplemented");
+ if (schemes().count(uri.scheme()) == 0) {
+ return Failure(
+ "Docker fetcher plugin does not support "
+ "'" + uri.scheme() + "' URI scheme");
+ }
+
+ if (!uri.has_host()) {
+ return Failure("Registry host (uri.host) is not specified");
+ }
+
+ if (!uri.has_query()) {
+ return Failure("Image tag/digest (uri.query) is not specified");
+ }
+
+ Try<Nothing> mkdir = os::mkdir(directory);
+ if (mkdir.isError()) {
+ return Failure(
+ "Failed to create directory '" +
+ directory + "': " + mkdir.error());
+ }
+
+ if (uri.scheme() == "docker-blob") {
+ return fetchBlob(uri, directory, None());
+ }
+
+ URI manifestUri = getManifestUri(uri);
+
+ return curl(manifestUri)
+ .then(defer(self(),
+ &Self::_fetch,
+ uri,
+ directory,
+ manifestUri,
+ lambda::_1));
+}
+
+
+Future<Nothing> DockerFetcherPluginProcess::_fetch(
+ const URI& uri,
+ const string& directory,
+ const URI& manifestUri,
+ const http::Response& response)
+{
+ if (response.code == http::Status::UNAUTHORIZED) {
+ return getAuthToken(response)
+ .then(defer(self(), [=](const string& authToken) -> Future<Nothing> {
+ return curl(manifestUri, getAuthHeader(authToken))
+ .then(defer(self(),
+ &Self::__fetch,
+ uri,
+ directory,
+ authToken,
+ lambda::_1));
+ }));
+ }
+
+ return __fetch(uri, directory, None(), response);
+}
+
+
+Future<Nothing> DockerFetcherPluginProcess::__fetch(
+ const URI& uri,
+ const string& directory,
+ const Option<string>& authToken,
+ const http::Response& response)
+{
+ if (response.code != http::Status::OK) {
+ return Failure(
+ "Unexpected HTTP response '" + response.status + "' "
+ "when trying to get the manifest");
+ }
+
+ CHECK_EQ(response.type, http::Response::BODY);
+
+ Try<spec::v2::ImageManifest> manifest = spec::v2::parse(response.body);
+ if (manifest.isError()) {
+ return Failure("Failed to parse the image manifest: " + manifest.error());
+ }
+
+ // Save manifest to 'directory'.
+ Try<Nothing> write = os::write(
+ path::join(directory, "manifest"),
+ response.body);
+
+ if (write.isError()) {
+ return Failure(
+ "Failed to write the image manifest to "
+ "'" + directory + "': " + write.error());
+ }
+
+ // No need to proceed if we only want manifest.
+ if (uri.scheme() == "docker-manifest") {
+ return Nothing();
+ }
+
+ // Download all the filesystem layers.
+ list<Future<Nothing>> futures;
+ for (int i = 0; i < manifest->fslayers_size(); i++) {
+ URI blob = uri::docker::blob(
+ uri.path(), // The 'repository'.
+ manifest->fslayers(i).blobsum(), // The 'digest'.
+ uri.host(), // The 'registry'.
+ (uri.has_fragment() // The 'scheme'.
+ ? Option<string>(uri.fragment())
+ : None()),
+ (uri.has_port() // The 'port'.
+ ? Option<int>(uri.port())
+ : None()));
+
+ futures.push_back(fetchBlob(
+ blob,
+ directory,
+ authToken));
+ }
+
+ return collect(futures)
+ .then([]() -> Future<Nothing> { return Nothing(); });
+}
+
+
+Future<Nothing> DockerFetcherPluginProcess::fetchBlob(
+ const URI& uri,
+ const string& directory,
+ const Option<string>& authToken)
+{
+ URI blobUri = getBlobUri(uri);
+
+ return download(blobUri, directory, getAuthHeader(authToken))
+ .then(defer(self(), [=](int code) -> Future<Nothing> {
+ if (code == http::Status::OK) {
+ return Nothing();
+ }
+
+ // Note that if 'authToken' is specified, but we still get a
+ // '401 Unauthorized' response, we return a Failure. This can
+ // prevent us from entering an infinite loop.
+ if (code == http::Status::UNAUTHORIZED && authToken.isNone()) {
+ return _fetchBlob(uri, directory, blobUri);
+ }
+
+ return Failure(
+ "Unexpected HTTP response '" + http::Status::string(code) + "' "
+ "when trying to download the blob");
+ }));
+}
+
+
+Future<Nothing> DockerFetcherPluginProcess::_fetchBlob(
+ const URI& uri,
+ const string& directory,
+ const URI& blobUri)
+{
+ // TODO(jieyu): This extra 'curl' call can be avoided if we can get
+ // HTTP headers from 'download'. Currently, 'download' only returns
+ // the HTTP response code because we don't support parsing HTTP
+ // headers alone. Revisit this once that's supported.
+ return curl(blobUri)
+ .then(defer(self(), [=](const http::Response& response) -> Future<Nothing> {
+ // We expect a '401 Unauthorized' response here since the
+ // 'download' with the same URI returns a '401 Unauthorized'.
+ if (response.code != http::Status::UNAUTHORIZED) {
+ return Failure(
+ "Expecting a '401 Unauthorized' response when fetching a blob, "
+ "but get '" + response.status + "' instead");
+ }
+
+ return getAuthToken(response)
+ .then(defer(self(),
+ &Self::fetchBlob,
+ uri,
+ directory,
+ lambda::_1));
+ }));
+}
+
+
+// If a '401 Unauthorized' response is received, we expect a header
+// 'Www-Authenticate' containing the auth server information. This
+// function takes the '401 Unauthorized' response, extracts the auth
+// server information, and then contacts the auth server to get the
+// token. The token will then be placed in the subsequent HTTP
+// requests as a header.
+//
+// See details here:
+// https://docs.docker.com/registry/spec/auth/token/
+Future<string> DockerFetcherPluginProcess::getAuthToken(
+ const http::Response& response)
+{
+ // The expected HTTP response here is:
+ //
+ // HTTP/1.1 401 Unauthorized
+ // Www-Authenticate: Bearer realm="xxx",service="yyy",scope="zzz"
+ CHECK_EQ(response.code, http::Status::UNAUTHORIZED);
+
+ if (!response.headers.contains("WWW-Authenticate")) {
+ return Failure("WWW-Authorization header is not found");
+ }
+
+ const vector<string> tokens = strings::tokenize(
+ response.headers.at("WWW-Authenticate"), " ");
+
+ if (tokens.size() != 2) {
+ return Failure(
+ "Unexpected WWW-Authenticate header format: "
+ "'" + response.headers.at("WWW-Authenticate") + "'");
+ }
+
+ if (tokens[0] != "Bearer") {
+ return Failure("Not a Bearer authentication challenge");
+ }
+
+ // Map containing the 'realm', 'service' and 'scope' information.
+ hashmap<string, string> attributes;
+
+ foreach (const string& token, strings::tokenize(tokens[1], ",")) {
+ const vector<string> split = strings::split(token, "=");
+ if (split.size() != 2) {
+ return Failure("Unexpected attribute format: '" + token + "'");
+ }
+
+ attributes[split[0]] = strings::trim(split[1], strings::ANY, "\"");
+ }
+
+ if (!attributes.contains("realm")) {
+ return Failure("Missing 'realm' in WWW-Authenticate header");
+ }
+
+ if (!attributes.contains("service")) {
+ return Failure("Missing 'service' in WWW-Authenticate header");
+ }
+
+ if (!attributes.contains("scope")) {
+ return Failure("Missing 'scope' in WWW-Authenticate header");
+ }
+
+ ostringstream stream;
+
+ // TODO(jieyu): Currently, we don't expect the auth server to return
+ // a service or a scope that needs encoding.
+ // TODO(jieyu): Getting auth token here might require HTTP
+ // authentication. We need to support that later.
+ string uri =
+ attributes.at("realm") + "?" +
+ "service=" + attributes.at("service") + "&" +
+ "scope=" + attributes.at("scope");
+
+ return curl(uri)
+ .then([uri](const http::Response& response) -> Future<string> {
+ if (response.code != http::Status::OK) {
+ return Failure(
+ "Unexpected HTTP response '" + response.status + "' "
+ "when trying to GET '" + uri + "'");
+ }
+
+ CHECK_EQ(response.type, http::Response::BODY);
+
+ Try<JSON::Object> object = JSON::parse<JSON::Object>(response.body);
+ if (object.isError()) {
+ return Failure("Parsing the JSON object failed: " + object.error());
+ }
+
+ Result<JSON::String> token = object->find<JSON::String>("token");
+ if (token.isError()) {
+ return Failure("Finding token in JSON object failed: " + token.error());
+ } else if (token.isNone()) {
+ return Failure("Failed to find token in JSON object");
+ }
+
+ return token->value;
+ });
+}
+
+
+http::Headers DockerFetcherPluginProcess::getAuthHeader(
+ const Option<string>& authToken)
+{
+ http::Headers headers;
+
+ if (authToken.isSome()) {
+ headers["Authorization"] = "Bearer " + authToken.get();
+ }
+
+ return headers;
+}
+
+
+URI DockerFetcherPluginProcess::getManifestUri(const URI& uri)
+{
+ string scheme = "https";
+ if (uri.has_fragment()) {
+ scheme = uri.fragment();
+ }
+
+ return uri::construct(
+ scheme,
+ path::join("/v2", uri.path(), "manifests", uri.query()),
+ uri.host(),
+ (uri.has_port() ? Option<int>(uri.port()) : None()));
+}
+
+
+URI DockerFetcherPluginProcess::getBlobUri(const URI& uri)
+{
+ string scheme = "https";
+ if (uri.has_fragment()) {
+ scheme = uri.fragment();
+ }
+
+ return uri::construct(
+ scheme,
+ path::join("/v2", uri.path(), "blobs", uri.query()),
+ uri.host(),
+ (uri.has_port() ? Option<int>(uri.port()) : None()));
}
} // namespace uri {
[4/9] mesos git commit: Added protobuf for docker ImageReference and
the parsing function.
Posted by ji...@apache.org.
Added protobuf for docker ImageReference and the parsing function.
The goal here is to replace Image::Name with ImageReference and the
corresponding parsing method, which will be done in a subsequent patch.
Review: https://reviews.apache.org/r/42235
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/49c0fee9
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/49c0fee9
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/49c0fee9
Branch: refs/heads/master
Commit: 49c0fee9787d5bd41dec85ee1cdd0356ae6c78ae
Parents: 81ee91e
Author: Jie Yu <yu...@gmail.com>
Authored: Sat Jan 9 23:04:09 2016 -0800
Committer: Jie Yu <yu...@gmail.com>
Committed: Mon Jan 18 16:09:14 2016 -0800
----------------------------------------------------------------------
include/mesos/docker/spec.hpp | 88 ++++++++++++++++++++
include/mesos/docker/spec.proto | 38 +++++++++
src/CMakeLists.txt | 2 +
src/Makefile.am | 8 +-
src/docker/spec.cpp | 57 ++++++++++++-
src/docker/spec.hpp | 64 --------------
.../mesos/provisioner/docker/message.hpp | 2 +
.../mesos/provisioner/docker/message.proto | 1 +
.../provisioner/docker/registry_client.hpp | 2 +-
src/tests/containerizer/docker_spec_tests.cpp | 71 +++++++++++++++-
.../containerizer/provisioner_docker_tests.cpp | 4 +-
11 files changed, 268 insertions(+), 69 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/49c0fee9/include/mesos/docker/spec.hpp
----------------------------------------------------------------------
diff --git a/include/mesos/docker/spec.hpp b/include/mesos/docker/spec.hpp
new file mode 100644
index 0000000..61b55b4
--- /dev/null
+++ b/include/mesos/docker/spec.hpp
@@ -0,0 +1,88 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef __MESOS_DOCKER_SPEC_HPP__
+#define __MESOS_DOCKER_SPEC_HPP__
+
+#include <string>
+
+#include <stout/error.hpp>
+#include <stout/json.hpp>
+#include <stout/option.hpp>
+#include <stout/try.hpp>
+
+// ONLY USEFUL AFTER RUNNING PROTOC.
+#include <mesos/docker/spec.pb.h>
+
+#include <mesos/docker/v1.hpp>
+#include <mesos/docker/v2.hpp>
+
+namespace docker {
+namespace spec {
+
+/**
+ * Parse the docker image reference. Docker expects the image
+ * reference to be in the following format:
+ * [REGISTRY_HOST[:REGISTRY_PORT]/]REPOSITORY[:TAG|@TYPE:DIGEST]
+ *
+ * This format is inherently ambiguous when dealing with repository
+ * names that include forward slashes. To disambiguate, the docker
+ * code looks for '.', or ':', or 'localhost' to decide if the first
+ * component is a registry or a respository name. For more detail,
+ * drill into the implementation of docker pull.
+ *
+ * See docker implementation:
+ * https://github.com/docker/distribution/blob/master/reference/reference.go
+ */
+Try<ImageReference> parseImageReference(const std::string& s);
+
+
+namespace v1 {
+
+/**
+ * Validates if the specified docker v1 image manifest conforms to the
+ * Docker v1 spec. Returns the error if the validation fails.
+ */
+Option<Error> validate(const ImageManifest& manifest);
+
+
+/**
+ * Returns the docker v1 image manifest from the given JSON object.
+ */
+Try<ImageManifest> parse(const JSON::Object& json);
+
+} // namespace v1 {
+
+
+namespace v2 {
+
+/**
+ * Validates if the specified v2 image manifest conforms to the Docker
+ * v2 spec. Returns the error if the validation fails.
+ */
+Option<Error> validate(const ImageManifest& manifest);
+
+
+/**
+ * Returns the docker v2 image manifest from the given JSON object.
+ */
+Try<ImageManifest> parse(const JSON::Object& json);
+
+} // namespace v2 {
+} // namespace spec {
+} // namespace docker {
+
+#endif // __MESOS_DOCKER_SPEC_HPP__
http://git-wip-us.apache.org/repos/asf/mesos/blob/49c0fee9/include/mesos/docker/spec.proto
----------------------------------------------------------------------
diff --git a/include/mesos/docker/spec.proto b/include/mesos/docker/spec.proto
new file mode 100644
index 0000000..4f2ff2d
--- /dev/null
+++ b/include/mesos/docker/spec.proto
@@ -0,0 +1,38 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package docker.spec;
+
+/**
+ * The docker image reference:
+ * [REGISTRY_HOST[:REGISTRY_PORT]/]REPOSITORY[:TAG|@DIGEST]
+ *
+ * See more details in:
+ * https://github.com/docker/distribution/blob/master/reference/reference.go
+ */
+message ImageReference {
+ // The registry in the following format: 'HOST:PORT'.
+ optional string registry = 1;
+
+ // The repository name (e.g., library/busybox, ubuntu).
+ required string repository = 2;
+
+ // For example: latest, 2.0.2, etc.
+ optional string tag = 3;
+
+ // In the format of 'ALGORITHM:HEX'.
+ optional string digest = 4;
+}
http://git-wip-us.apache.org/repos/asf/mesos/blob/49c0fee9/src/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 39a23df..a52203a 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -24,6 +24,7 @@ PROTOC_TO_INCLUDE_DIR(V1_MESOS mesos/v1/mesos)
PROTOC_TO_INCLUDE_DIR(AUTHENTICATION mesos/authentication/authentication)
PROTOC_TO_INCLUDE_DIR(AUTHORIZATION mesos/authorizer/authorizer)
PROTOC_TO_INCLUDE_DIR(CONTAINERIZER mesos/containerizer/containerizer)
+PROTOC_TO_INCLUDE_DIR(DOCKER_SPEC mesos/docker/spec)
PROTOC_TO_INCLUDE_DIR(DOCKER_V1 mesos/docker/v1)
PROTOC_TO_INCLUDE_DIR(DOCKER_V2 mesos/docker/v2)
PROTOC_TO_INCLUDE_DIR(EXECUTOR mesos/executor/executor)
@@ -53,6 +54,7 @@ set(MESOS_PROTOBUF_SRC
${AUTHENTICATION_PROTO_CC}
${AUTHORIZATION_PROTO_CC}
${CONTAINERIZER_PROTO_CC}
+ ${DOCKER_SPEC_PROTO_CC}
${DOCKER_V1_PROTO_CC}
${DOCKER_V2_PROTO_CC}
${EXECUTOR_PROTO_CC}
http://git-wip-us.apache.org/repos/asf/mesos/blob/49c0fee9/src/Makefile.am
----------------------------------------------------------------------
diff --git a/src/Makefile.am b/src/Makefile.am
index d23e350..69383b6 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -150,6 +150,7 @@ ALLOCATOR_PROTO = $(top_srcdir)/include/mesos/master/allocator.proto
AUTHENTICATION_PROTO = $(top_srcdir)/include/mesos/authentication/authentication.proto
AUTHORIZATION_PROTO = $(top_srcdir)/include/mesos/authorizer/authorizer.proto
CONTAINERIZER_PROTO = $(top_srcdir)/include/mesos/containerizer/containerizer.proto
+DOCKER_SPEC_PROTO = $(top_srcdir)/include/mesos/docker/spec.proto
DOCKER_V1_PROTO = $(top_srcdir)/include/mesos/docker/v1.proto
DOCKER_V2_PROTO = $(top_srcdir)/include/mesos/docker/v2.proto
EXECUTOR_PROTO = $(top_srcdir)/include/mesos/executor/executor.proto
@@ -176,6 +177,8 @@ CXX_PROTOS = \
../include/mesos/authorizer/authorizer.pb.h \
../include/mesos/containerizer/containerizer.pb.cc \
../include/mesos/containerizer/containerizer.pb.h \
+ ../include/mesos/docker/spec.pb.cc \
+ ../include/mesos/docker/spec.pb.h \
../include/mesos/docker/v1.pb.cc \
../include/mesos/docker/v1.pb.h \
../include/mesos/docker/v2.pb.cc \
@@ -400,12 +403,15 @@ nodist_containerizer_HEADERS = \
dockerdir = $(pkgincludedir)/docker
docker_HEADERS = \
+ $(top_srcdir)/include/mesos/docker/spec.hpp \
+ $(top_srcdir)/include/mesos/docker/spec.proto \
$(top_srcdir)/include/mesos/docker/v1.hpp \
$(top_srcdir)/include/mesos/docker/v1.proto \
$(top_srcdir)/include/mesos/docker/v2.hpp \
$(top_srcdir)/include/mesos/docker/v2.proto
nodist_docker_HEADERS = \
+ ../include/mesos/docker/spec.pb.h \
../include/mesos/docker/v1.pb.h \
../include/mesos/docker/v2.pb.h
@@ -684,7 +690,6 @@ libmesos_no_3rdparty_la_SOURCES += \
credentials/credentials.hpp \
docker/docker.hpp \
docker/executor.hpp \
- docker/spec.hpp \
examples/test_anonymous_module.hpp \
examples/test_module.hpp \
examples/utils.hpp \
@@ -995,6 +1000,7 @@ libmesos_la_SOURCES = \
$(AUTHORIZATION_PROTO) \
$(CONTAINERIZER_PROTO) \
$(EXECUTOR_PROTO) \
+ $(DOCKER_SPEC_PROTO) \
$(DOCKER_V1_PROTO) \
$(DOCKER_V2_PROTO) \
$(FETCHER_PROTO) \
http://git-wip-us.apache.org/repos/asf/mesos/blob/49c0fee9/src/docker/spec.cpp
----------------------------------------------------------------------
diff --git a/src/docker/spec.cpp b/src/docker/spec.cpp
index 0188078..29d2f11 100644
--- a/src/docker/spec.cpp
+++ b/src/docker/spec.cpp
@@ -20,12 +20,67 @@
#include <stout/protobuf.hpp>
#include <stout/strings.hpp>
-#include "docker/spec.hpp"
+#include <mesos/docker/spec.hpp>
using std::string;
+using std::vector;
namespace docker {
namespace spec {
+
+// TODO(jieyu): Use regex to parse and verify the reference.
+Try<ImageReference> parseImageReference(const string& _s)
+{
+ ImageReference reference;
+ string s(_s);
+
+ // Extract the digest.
+ if (strings::contains(s, "@")) {
+ vector<string> split = strings::split(s, "@");
+ if (split.size() != 2) {
+ return Error("Multiple '@' symbols found");
+ }
+
+ s = split[0];
+ reference.set_digest(split[1]);
+ }
+
+ // Remove the tag. We need to watch out for a
+ // host:port registry, which also contains ':'.
+ if (strings::contains(s, ":")) {
+ vector<string> split = strings::split(s, ":");
+
+ // The tag must be the last component. If a slash is
+ // present there is a registry port and no tag.
+ if (!strings::contains(split.back(), "/")) {
+ reference.set_tag(split.back());
+ split.pop_back();
+
+ s = strings::join(":", split);
+ }
+ }
+
+ // Extract the registry and repository. The first component can
+ // either be the registry, or the first part of the repository!
+ // We resolve this ambiguity using the same hacks used in the
+ // docker code ('.', ':', 'localhost' indicate a registry).
+ vector<string> split = strings::split(s, "/", 2);
+
+ if (split.size() == 1) {
+ reference.set_repository(s);
+ } else if (strings::contains(split[0], ".") ||
+ strings::contains(split[0], ":") ||
+ split[0] == "localhost") {
+ reference.set_registry(split[0]);
+ reference.set_repository(split[1]);
+ } else {
+ reference.set_repository(s);
+ }
+
+ return reference;
+}
+
+
namespace v1 {
Option<Error> validate(const ImageManifest& manifest)
http://git-wip-us.apache.org/repos/asf/mesos/blob/49c0fee9/src/docker/spec.hpp
----------------------------------------------------------------------
diff --git a/src/docker/spec.hpp b/src/docker/spec.hpp
deleted file mode 100644
index 822b238..0000000
--- a/src/docker/spec.hpp
+++ /dev/null
@@ -1,64 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#ifndef __DOCKER_SPEC_HPP__
-#define __DOCKER_SPEC_HPP__
-
-#include <stout/error.hpp>
-#include <stout/json.hpp>
-#include <stout/option.hpp>
-
-#include <mesos/docker/v1.hpp>
-#include <mesos/docker/v2.hpp>
-
-namespace docker {
-namespace spec {
-namespace v1 {
-
-/**
- * Validates if the specified docker v1 image manifest conforms to the
- * Docker v1 spec. Returns the error if the validation fails.
- */
-Option<Error> validate(const ImageManifest& manifest);
-
-
-/**
- * Returns the docker v1 image manifest from the given JSON object.
- */
-Try<ImageManifest> parse(const JSON::Object& json);
-
-} // namespace v1 {
-
-
-namespace v2 {
-
-/**
- * Validates if the specified v2 image manifest conforms to the Docker
- * v2 spec. Returns the error if the validation fails.
- */
-Option<Error> validate(const ImageManifest& manifest);
-
-
-/**
- * Returns the docker v1 image manifest from the given JSON object.
- */
-Try<ImageManifest> parse(const JSON::Object& json);
-
-} // namespace v2 {
-} // namespace spec {
-} // namespace docker {
-
-#endif // __DOCKER_SPEC_HPP__
http://git-wip-us.apache.org/repos/asf/mesos/blob/49c0fee9/src/slave/containerizer/mesos/provisioner/docker/message.hpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/docker/message.hpp b/src/slave/containerizer/mesos/provisioner/docker/message.hpp
index 162e4c6..43c3608 100644
--- a/src/slave/containerizer/mesos/provisioner/docker/message.hpp
+++ b/src/slave/containerizer/mesos/provisioner/docker/message.hpp
@@ -41,6 +41,8 @@ namespace docker {
//
// TODO(bmahler): Validate based on docker's validation logic
// and return a Try here.
+//
+// TODO(jieyu): Remove this in favor of using 'spec::parseImageName'.
inline Image::Name parseImageName(std::string s)
{
Image::Name name;
http://git-wip-us.apache.org/repos/asf/mesos/blob/49c0fee9/src/slave/containerizer/mesos/provisioner/docker/message.proto
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/docker/message.proto b/src/slave/containerizer/mesos/provisioner/docker/message.proto
index 2b2ed05..003c666 100644
--- a/src/slave/containerizer/mesos/provisioner/docker/message.proto
+++ b/src/slave/containerizer/mesos/provisioner/docker/message.proto
@@ -24,6 +24,7 @@ package mesos.internal.slave.docker;
* and the leaf layer id last.
*/
message Image {
+ // TODO(jieyu): Remove this in favor of using spec::ImageReference.
message Name {
optional string registry = 1;
required string repository = 2;
http://git-wip-us.apache.org/repos/asf/mesos/blob/49c0fee9/src/slave/containerizer/mesos/provisioner/docker/registry_client.hpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/docker/registry_client.hpp b/src/slave/containerizer/mesos/provisioner/docker/registry_client.hpp
index bd6dace..bfe3086 100644
--- a/src/slave/containerizer/mesos/provisioner/docker/registry_client.hpp
+++ b/src/slave/containerizer/mesos/provisioner/docker/registry_client.hpp
@@ -29,7 +29,7 @@
#include <process/http.hpp>
#include <process/process.hpp>
-#include "docker/spec.hpp"
+#include <mesos/docker/spec.hpp>
#include "slave/containerizer/mesos/provisioner/docker/message.hpp"
http://git-wip-us.apache.org/repos/asf/mesos/blob/49c0fee9/src/tests/containerizer/docker_spec_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/containerizer/docker_spec_tests.cpp b/src/tests/containerizer/docker_spec_tests.cpp
index aa4faf9..5799dc9 100644
--- a/src/tests/containerizer/docker_spec_tests.cpp
+++ b/src/tests/containerizer/docker_spec_tests.cpp
@@ -19,7 +19,7 @@
#include <stout/gtest.hpp>
#include <stout/json.hpp>
-#include "docker/spec.hpp"
+#include <mesos/docker/spec.hpp>
namespace spec = docker::spec;
@@ -30,6 +30,75 @@ namespace tests {
class DockerSpecTest : public ::testing::Test {};
+TEST_F(DockerSpecTest, ParseImageReference)
+{
+ Try<spec::ImageReference> reference =
+ spec::parseImageReference("library/busybox");
+
+ ASSERT_SOME(reference);
+ EXPECT_FALSE(reference->has_registry());
+ EXPECT_EQ("library/busybox", reference->repository());
+ EXPECT_FALSE(reference->has_tag());
+ EXPECT_FALSE(reference->has_digest());
+
+ reference = spec::parseImageReference("busybox");
+
+ ASSERT_SOME(reference);
+ EXPECT_FALSE(reference->has_registry());
+ EXPECT_EQ("busybox", reference->repository());
+ EXPECT_FALSE(reference->has_tag());
+ EXPECT_FALSE(reference->has_digest());
+
+ reference = spec::parseImageReference("library/busybox:tag");
+
+ ASSERT_SOME(reference);
+ EXPECT_FALSE(reference->has_registry());
+ EXPECT_EQ("library/busybox", reference->repository());
+ EXPECT_EQ("tag", reference->tag());
+ EXPECT_FALSE(reference->has_digest());
+
+ reference = spec::parseImageReference("library/busybox@sha256:bc8813ea7b3603864987522f02a76101c17ad122e1c46d790efc0fca78ca7bfb"); // NOLINT(whitespace/line_length)
+
+ ASSERT_SOME(reference);
+ EXPECT_FALSE(reference->has_registry());
+ EXPECT_EQ("library/busybox", reference->repository());
+ EXPECT_FALSE(reference->has_tag());
+ EXPECT_EQ("sha256:bc8813ea7b3603864987522f02a76101c17ad122e1c46d790efc0fca78ca7bfb", reference->digest()); // NOLINT(whitespace/line_length)
+
+ reference = spec::parseImageReference("registry.io/library/busybox");
+
+ ASSERT_SOME(reference);
+ EXPECT_EQ("registry.io", reference->registry());
+ EXPECT_EQ("library/busybox", reference->repository());
+ EXPECT_FALSE(reference->has_tag());
+ EXPECT_FALSE(reference->has_digest());
+
+ reference = spec::parseImageReference("registry.io/library/busybox:tag");
+
+ ASSERT_SOME(reference);
+ EXPECT_EQ("registry.io", reference->registry());
+ EXPECT_EQ("library/busybox", reference->repository());
+ EXPECT_EQ("tag", reference->tag());
+ EXPECT_FALSE(reference->has_digest());
+
+ reference = spec::parseImageReference("registry.io:80/library/busybox:tag");
+
+ ASSERT_SOME(reference);
+ EXPECT_EQ("registry.io:80", reference->registry());
+ EXPECT_EQ("library/busybox", reference->repository());
+ EXPECT_EQ("tag", reference->tag());
+ EXPECT_FALSE(reference->has_digest());
+
+ reference = spec::parseImageReference("registry.io:80/library/busybox@sha256:bc8813ea7b3603864987522f02a76101c17ad122e1c46d790efc0fca78ca7bfb"); // NOLINT(whitespace/line_length)
+
+ ASSERT_SOME(reference);
+ EXPECT_EQ("registry.io:80", reference->registry());
+ EXPECT_EQ("library/busybox", reference->repository());
+ EXPECT_FALSE(reference->has_tag());
+ EXPECT_EQ("sha256:bc8813ea7b3603864987522f02a76101c17ad122e1c46d790efc0fca78ca7bfb", reference->digest()); // NOLINT(whitespace/line_length)
+}
+
+
TEST_F(DockerSpecTest, ParseV1ImageManifest)
{
Try<JSON::Object> json = JSON::parse<JSON::Object>(
http://git-wip-us.apache.org/repos/asf/mesos/blob/49c0fee9/src/tests/containerizer/provisioner_docker_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/containerizer/provisioner_docker_tests.cpp b/src/tests/containerizer/provisioner_docker_tests.cpp
index f81f003..33df600 100644
--- a/src/tests/containerizer/provisioner_docker_tests.cpp
+++ b/src/tests/containerizer/provisioner_docker_tests.cpp
@@ -39,7 +39,7 @@
#include <process/ssl/gtest.hpp>
-#include "docker/spec.hpp"
+#include <mesos/docker/spec.hpp>
#include "slave/containerizer/mesos/provisioner/docker/metadata_manager.hpp"
#include "slave/containerizer/mesos/provisioner/docker/paths.hpp"
@@ -84,6 +84,8 @@ namespace mesos {
namespace internal {
namespace tests {
+// TODO(jieyu): Remove this test in favor of using
+// DockerSpecTest.ParseImageReference.
TEST(DockerUtilsTest, ParseImageName)
{
slave::docker::Image::Name name;
[7/9] mesos git commit: Added an HTTP decode response method.
Posted by ji...@apache.org.
Added an HTTP decode response method.
Review: https://reviews.apache.org/r/41961
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/c6a2f8af
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/c6a2f8af
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/c6a2f8af
Branch: refs/heads/master
Commit: c6a2f8af369fb07eb4b604a86738efbccd105c01
Parents: 49c0fee
Author: Jie Yu <yu...@gmail.com>
Authored: Sun Jan 3 22:14:22 2016 -0800
Committer: Jie Yu <yu...@gmail.com>
Committed: Mon Jan 18 16:09:14 2016 -0800
----------------------------------------------------------------------
3rdparty/libprocess/include/process/http.hpp | 8 +++++++
3rdparty/libprocess/src/http.cpp | 29 +++++++++++++++++++++++
2 files changed, 37 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/c6a2f8af/3rdparty/libprocess/include/process/http.hpp
----------------------------------------------------------------------
diff --git a/3rdparty/libprocess/include/process/http.hpp b/3rdparty/libprocess/include/process/http.hpp
index 1fe549e..1d9a944 100644
--- a/3rdparty/libprocess/include/process/http.hpp
+++ b/3rdparty/libprocess/include/process/http.hpp
@@ -718,6 +718,14 @@ std::string encode(const std::string& s);
Try<std::string> decode(const std::string& s);
+/**
+ * Decode HTTP responses from the given string.
+ *
+ * @param s the given string.
+ */
+Try<std::vector<Response>> decodeResponses(const std::string& s);
+
+
namespace query {
// Decodes an HTTP query string into a map. For example:
http://git-wip-us.apache.org/repos/asf/mesos/blob/c6a2f8af/3rdparty/libprocess/src/http.cpp
----------------------------------------------------------------------
diff --git a/3rdparty/libprocess/src/http.cpp b/3rdparty/libprocess/src/http.cpp
index 40fd87c..762da9a 100644
--- a/3rdparty/libprocess/src/http.cpp
+++ b/3rdparty/libprocess/src/http.cpp
@@ -718,6 +718,35 @@ Try<string> decode(const string& s)
}
+Try<vector<Response>> decodeResponses(const string& s)
+{
+ ResponseDecoder decoder;
+
+ deque<http::Response*> responses = decoder.decode(s.data(), s.length());
+
+ if (decoder.failed()) {
+ foreach (Response* response, responses) {
+ delete response;
+ }
+
+ return Error("Decoding failed");
+ }
+
+ if (responses.empty()) {
+ return Error("No response decoded");
+ }
+
+ vector<Response> result;
+
+ foreach (Response* response, responses) {
+ result.push_back(*response);
+ delete response;
+ }
+
+ return result;
+}
+
+
namespace query {
Try<hashmap<std::string, std::string>> decode(const std::string& query)