You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by ji...@apache.org on 2016/02/17 21:46:33 UTC
[2/2] mesos git commit: Introduced Appc image fetcher.
Introduced Appc image fetcher.
Added implementation for simple image discovery for Appc images.
Review: https://reviews.apache.org/r/43336/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/4107f14e
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/4107f14e
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/4107f14e
Branch: refs/heads/master
Commit: 4107f14e3f2864e0840c6712207f5c49bb8cb6b4
Parents: 906566d
Author: Jojy Varghese <jo...@mesosphere.io>
Authored: Wed Feb 17 11:33:38 2016 -0800
Committer: Jie Yu <yu...@gmail.com>
Committed: Wed Feb 17 12:01:53 2016 -0800
----------------------------------------------------------------------
src/CMakeLists.txt | 1 +
src/Makefile.am | 2 +
.../mesos/provisioner/appc/fetcher.cpp | 228 +++++++++++++++++++
.../mesos/provisioner/appc/fetcher.hpp | 80 +++++++
src/slave/flags.cpp | 6 +
src/slave/flags.hpp | 2 +
6 files changed, 319 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/4107f14e/src/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 9ab84c0..5cf0ec8 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -244,6 +244,7 @@ if (NOT WIN32)
slave/containerizer/mesos/provisioner/provisioner.cpp
slave/containerizer/mesos/provisioner/store.cpp
slave/containerizer/mesos/provisioner/appc/cache.cpp
+ slave/containerizer/mesos/provisioner/appc/fetcher.cpp
slave/containerizer/mesos/provisioner/appc/paths.cpp
slave/containerizer/mesos/provisioner/appc/store.cpp
slave/containerizer/mesos/provisioner/backend.cpp
http://git-wip-us.apache.org/repos/asf/mesos/blob/4107f14e/src/Makefile.am
----------------------------------------------------------------------
diff --git a/src/Makefile.am b/src/Makefile.am
index 5813ab2..54ebe91 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -668,6 +668,7 @@ libmesos_no_3rdparty_la_SOURCES += \
slave/containerizer/mesos/provisioner/provisioner.cpp \
slave/containerizer/mesos/provisioner/store.cpp \
slave/containerizer/mesos/provisioner/appc/cache.cpp \
+ slave/containerizer/mesos/provisioner/appc/fetcher.cpp \
slave/containerizer/mesos/provisioner/appc/paths.cpp \
slave/containerizer/mesos/provisioner/appc/store.cpp \
slave/containerizer/mesos/provisioner/backends/copy.cpp \
@@ -778,6 +779,7 @@ libmesos_no_3rdparty_la_SOURCES += \
slave/containerizer/mesos/provisioner/provisioner.hpp \
slave/containerizer/mesos/provisioner/store.hpp \
slave/containerizer/mesos/provisioner/appc/cache.hpp \
+ slave/containerizer/mesos/provisioner/appc/fetcher.hpp \
slave/containerizer/mesos/provisioner/appc/paths.hpp \
slave/containerizer/mesos/provisioner/appc/store.hpp \
slave/containerizer/mesos/provisioner/backends/copy.hpp \
http://git-wip-us.apache.org/repos/asf/mesos/blob/4107f14e/src/slave/containerizer/mesos/provisioner/appc/fetcher.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/appc/fetcher.cpp b/src/slave/containerizer/mesos/provisioner/appc/fetcher.cpp
new file mode 100644
index 0000000..e12a6f2
--- /dev/null
+++ b/src/slave/containerizer/mesos/provisioner/appc/fetcher.cpp
@@ -0,0 +1,228 @@
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stout/os.hpp>
+#include <stout/strings.hpp>
+
+#include <mesos/mesos.hpp>
+
+#include "common/command_utils.hpp"
+
+#include "slave/containerizer/mesos/provisioner/appc/fetcher.hpp"
+#include "slave/containerizer/mesos/provisioner/appc/paths.hpp"
+
+#include "uri/schemes/http.hpp"
+
+namespace http = process::http;
+
+using std::string;
+
+using process::Failure;
+using process::Future;
+using process::Owned;
+using process::Shared;
+
+namespace mesos {
+namespace internal {
+namespace slave {
+namespace appc {
+
+static const char EXT[] = "aci";
+static const char LABEL_VERSION[] = "version";
+static const char LABEL_OS[] = "os";
+static const char LABEL_ARCH[] = "arch";
+
+
+static Try<string> getSimpleDiscoveryImagePath(const Image::Appc& appc)
+{
+ CHECK(!appc.name().empty());
+
+ hashmap<string, string> labels;
+ foreach (const mesos::Label& label, appc.labels().labels()) {
+ labels[label.key()] = label.value();
+ }
+
+ if (!labels.contains(LABEL_VERSION)) {
+ labels.insert({LABEL_VERSION, "latest"});
+ }
+
+ if (!labels.contains(LABEL_OS)) {
+ return Error(
+ "Failed to form simple discovery url: label '" +
+ string(LABEL_OS) + "' is missing");
+ }
+
+ if (!labels.contains(LABEL_ARCH)) {
+ return Error(
+ "Failed to form simple discovery url: label '" +
+ string(LABEL_ARCH) + "' is missing");
+ }
+
+ return strings::format(
+ "%s-%s-%s-%s.%s",
+ appc.name(), // Image name.
+ labels.at(LABEL_VERSION), // Version label.
+ labels.at(LABEL_OS), // OS label.
+ labels.at(LABEL_ARCH), // ARCH label.
+ EXT); // Extension.
+}
+
+
+static Try<URI> getUri(const string& prefix, const string& path)
+{
+ const string rawUrl = prefix + path;
+
+ // TODO(jojy): Add parse URI function in URI namespace.
+ Try<http::URL> _url = http::URL::parse(rawUrl);
+ if (_url.isError()) {
+ return Error(
+ "Failed to parse '" + rawUrl + "' as a valid URL: " + _url.error());
+ }
+
+ const http::URL& url = _url.get();
+
+ if (url.domain.isNone() && url.ip.isNone()) {
+ return Error(
+ "Failed to parse host name from image url '" + rawUrl + "'");
+ }
+
+ // Get image server port.
+ if (url.port.isNone()) {
+ return Error(
+ "Failed to parse port for image url '" + rawUrl + "'");
+ }
+
+ // Get image server host.
+ const string host = url.domain.isSome()
+ ? url.domain.get()
+ : stringify(url.ip.get());
+
+ int port = static_cast<int>(url.port.get());
+
+ if (url.scheme.get() == "http") {
+ return uri::http( host, url.path, port);
+ }
+
+ if (url.scheme.get() == "https") {
+ return uri::https(host, url.path, port);
+ }
+
+ // TODO(jojy): Add support for hdfs.
+
+ return Error("Unsupported scheme '" + url.scheme.get() + "'");
+}
+
+
+Try<Owned<Fetcher>> Fetcher::create(
+ const Flags& flags,
+ const Shared<uri::Fetcher>& fetcher)
+{
+ const string prefix = flags.appc_simple_discovery_uri_prefix;
+
+ // TODO(jojy): Add support for hdfs.
+ if (!strings::startsWith(prefix, "http") &&
+ !strings::startsWith(prefix, "https")) {
+ return Error("Invalid simple discovery uri prefix: " + prefix);
+ }
+
+ return new Fetcher(prefix, fetcher);
+}
+
+
+Fetcher::Fetcher(
+ const string& _uriPrefix,
+ const Shared<uri::Fetcher>& _fetcher)
+ : uriPrefix(_uriPrefix),
+ fetcher(_fetcher) {}
+
+
+Future<Nothing> Fetcher::fetch(const Image::Appc& appc, const Path& directory)
+{
+ // TODO(jojy): Add more discovery implementations. We use simple
+ // discovery now but in the future, this will be extended to use
+ // multiple discoveries and try sequentially based on priority.
+
+ if (appc.name().empty()) {
+ return Failure("Image name cannot be empty");
+ }
+
+ Try<string> path = getSimpleDiscoveryImagePath(appc);
+ if (path.isError()) {
+ return Failure(
+ "Failed to get discovery path for image '" +
+ appc.name() + "': " + path.error());
+ }
+
+ // First construct a URI based on the scheme.
+ Try<URI> uri = getUri(uriPrefix, path.get());
+ if (uri.isError()) {
+ return Failure(
+ "Failed to get URI for image discovery path '" +
+ path.get() + "': " + uri.error());
+ }
+
+ VLOG(1) << "Fetching image from URI '" << uri.get() << "'";
+
+ // NOTE: URI fetcher will fetch the image into 'directory' with file
+ // name as URI's basename.
+ const Path aciBundle(path::join(
+ directory,
+ Path(uri->path()).basename()));
+
+ return fetcher->fetch(uri.get(), directory)
+ .then([=]() -> Future<Nothing> {
+ // Change the extension to ".gz" as gzip utility expects it.
+ const Path _aciBundle(aciBundle.value + ".gz");
+
+ Try<Nothing> rename = os::rename(aciBundle, _aciBundle);
+ if (rename.isError()) {
+ return Failure(
+ "Failed to change extension to 'gz' for bundle '" +
+ stringify(aciBundle) + "': " + rename.error());
+ }
+
+ return command::decompress(_aciBundle);
+ })
+ .then([=]() -> Future<string> {
+ return command::sha512(aciBundle);
+ })
+ .then([=](const string& shasum) -> Future<Nothing> {
+ const string imagePath(path::join(directory, "sha512-" + shasum));
+
+ Try<Nothing> mkdir = os::mkdir(imagePath);
+ if (mkdir.isError()) {
+ return Failure(
+ "Failed to create directory for untarring image '" +
+ appc.name() + "': " + mkdir.error());
+ }
+
+ return command::untar(aciBundle, imagePath);
+ })
+ .then([=]() -> Future<Nothing> {
+ // Remove the bundle file if everything goes well.
+ Try<Nothing> remove = os::rm(aciBundle);
+ if (remove.isError()) {
+ return Failure(
+ "Failed to remove aci bundle file '" + stringify(aciBundle) +
+ "': " + remove.error());
+ }
+
+ return Nothing();
+ });
+}
+
+} // namespace appc {
+} // namespace slave {
+} // namespace internal {
+} // namespace mesos {
http://git-wip-us.apache.org/repos/asf/mesos/blob/4107f14e/src/slave/containerizer/mesos/provisioner/appc/fetcher.hpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/mesos/provisioner/appc/fetcher.hpp b/src/slave/containerizer/mesos/provisioner/appc/fetcher.hpp
new file mode 100644
index 0000000..373c7d4
--- /dev/null
+++ b/src/slave/containerizer/mesos/provisioner/appc/fetcher.hpp
@@ -0,0 +1,80 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef __PROVISIONER_APPC_FETCHER_HPP__
+#define __PROVISIONER_APPC_FETCHER_HPP__
+
+#include <string>
+
+#include <process/process.hpp>
+#include <process/shared.hpp>
+
+#include <stout/path.hpp>
+
+#include "slave/flags.hpp"
+
+#include "uri/fetcher.hpp"
+
+namespace mesos {
+namespace internal {
+namespace slave {
+namespace appc {
+
+class Fetcher
+{
+public:
+ /**
+ * Factory method for creating the fetcher component.
+ *
+ * @param flags Slave flags.
+ * @param fetcher Shared pointer to the uri fetcher.
+ */
+ static Try<process::Owned<Fetcher>> create(
+ const Flags& flags,
+ const process::Shared<uri::Fetcher>& fetcher);
+
+ /*
+ * Fetches Appc image to the given directory.
+ *
+ * Reference: https://github.com/appc/spec/blob/master/spec/discovery.md
+ *
+ * @param image Encapsulated information about the appc image.
+ * @param directory Path of directory where the image has to be saved.
+ * @returns Nothing on success.
+ * Failure in case of any error.
+ */
+ process::Future<Nothing> fetch(
+ const Image::Appc& appc,
+ const Path& directory);
+
+private:
+ Fetcher(
+ const std::string& uriPrefix,
+ const process::Shared<uri::Fetcher>& fetcher);
+
+ Fetcher(const Fetcher&) = delete;
+ Fetcher& operator=(const Fetcher&) = delete;
+
+ const std::string uriPrefix;
+ process::Shared<uri::Fetcher> fetcher;
+};
+
+} // namespace appc {
+} // namespace slave {
+} // namespace internal {
+} // namespace mesos {
+
+#endif // __PROVISIONER_APPC_FETCHER_HPP__
http://git-wip-us.apache.org/repos/asf/mesos/blob/4107f14e/src/slave/flags.cpp
----------------------------------------------------------------------
diff --git a/src/slave/flags.cpp b/src/slave/flags.cpp
index 46ccf7f..d4b4e52 100644
--- a/src/slave/flags.cpp
+++ b/src/slave/flags.cpp
@@ -113,6 +113,12 @@ mesos::internal::slave::Flags::Flags()
"e.g., `bind`, `copy`.",
"copy");
+ add(&Flags::appc_simple_discovery_uri_prefix,
+ "appc_simple_discovery_uri_prefix",
+ "URI prefix to be used for simple discovery of appc images,\n"
+ "e.g., 'http://', 'https://', 'hdfs://<hostname>:9000/user/abc/cde'.",
+ "http://");
+
add(&Flags::appc_store_dir,
"appc_store_dir",
"Directory the appc provisioner will store images in.\n",
http://git-wip-us.apache.org/repos/asf/mesos/blob/4107f14e/src/slave/flags.hpp
----------------------------------------------------------------------
diff --git a/src/slave/flags.hpp b/src/slave/flags.hpp
index 3704376..bd52b4f 100644
--- a/src/slave/flags.hpp
+++ b/src/slave/flags.hpp
@@ -49,6 +49,8 @@ public:
Option<std::string> image_providers;
std::string image_provisioner_backend;
+
+ std::string appc_simple_discovery_uri_prefix;
std::string appc_store_dir;
std::string docker_auth_server;