You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by ji...@apache.org on 2014/06/06 18:19:40 UTC
git commit: Added an abstraction for launching operations in a
subprocess.
Repository: mesos
Updated Branches:
refs/heads/master 9a19ea7f3 -> 1920efdd8
Added an abstraction for launching operations in a subprocess.
Review: https://reviews.apache.org/r/22224
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/1920efdd
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/1920efdd
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/1920efdd
Branch: refs/heads/master
Commit: 1920efdd828fdd0cbcf7d08ff81449214699d902
Parents: 9a19ea7
Author: Jie Yu <yu...@gmail.com>
Authored: Mon Jun 2 10:01:08 2014 -0700
Committer: Jie Yu <yu...@gmail.com>
Committed: Fri Jun 6 09:19:10 2014 -0700
----------------------------------------------------------------------
src/Makefile.am | 10 +-
src/launcher/launcher.cpp | 222 ++++++++++++++++++++++++++++++++++++++
src/launcher/launcher.hpp | 133 +++++++++++++++++++++++
src/launcher/main.cpp | 29 +++++
src/tests/environment.cpp | 5 +
src/tests/launcher_tests.cpp | 79 ++++++++++++++
6 files changed, 477 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/1920efdd/src/Makefile.am
----------------------------------------------------------------------
diff --git a/src/Makefile.am b/src/Makefile.am
index c3ecb94..4a3f2e1 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -218,6 +218,7 @@ libmesos_no_3rdparty_la_SOURCES = \
sched/sched.cpp \
scheduler/scheduler.cpp \
local/local.cpp \
+ launcher/launcher.cpp \
master/contender.cpp \
master/constants.cpp \
master/detector.cpp \
@@ -330,9 +331,10 @@ libmesos_no_3rdparty_la_SOURCES += common/attributes.hpp \
common/http.hpp \
common/lock.hpp \
common/type_utils.hpp common/thread.hpp \
- credentials/credentials.hpp \
+ credentials/credentials.hpp \
examples/utils.hpp files/files.hpp \
hdfs/hdfs.hpp \
+ launcher/launcher.hpp \
linux/cgroups.hpp \
linux/fs.hpp local/flags.hpp local/local.hpp \
logging/flags.hpp logging/logging.hpp \
@@ -547,6 +549,11 @@ mesos_usage_SOURCES = usage/main.cpp
mesos_usage_CPPFLAGS = $(MESOS_CPPFLAGS)
mesos_usage_LDADD = libmesos.la
+pkglibexec_PROGRAMS += mesos-launcher
+mesos_launcher_SOURCES = launcher/main.cpp
+mesos_launcher_CPPFLAGS = $(MESOS_CPPFLAGS)
+mesos_launcher_LDADD = libmesos.la
+
bin_PROGRAMS += mesos-log
mesos_log_SOURCES = log/main.cpp
mesos_log_CPPFLAGS = $(MESOS_CPPFLAGS)
@@ -962,6 +969,7 @@ mesos_tests_SOURCES = \
tests/gc_tests.cpp \
tests/isolator_tests.cpp \
tests/external_containerizer_test.cpp \
+ tests/launcher_tests.cpp \
tests/log_tests.cpp \
tests/logging_tests.cpp \
tests/main.cpp \
http://git-wip-us.apache.org/repos/asf/mesos/blob/1920efdd/src/launcher/launcher.cpp
----------------------------------------------------------------------
diff --git a/src/launcher/launcher.cpp b/src/launcher/launcher.cpp
new file mode 100644
index 0000000..1d352b6
--- /dev/null
+++ b/src/launcher/launcher.cpp
@@ -0,0 +1,222 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <process/internal.hpp>
+#include <process/io.hpp>
+#include <process/subprocess.hpp>
+
+#include <stout/foreach.hpp>
+#include <stout/hashmap.hpp>
+#include <stout/os.hpp>
+#include <stout/path.hpp>
+#include <stout/strings.hpp>
+
+#include "launcher/launcher.hpp"
+
+using namespace process;
+
+using std::cerr;
+using std::endl;
+using std::map;
+using std::string;
+
+namespace mesos {
+namespace internal {
+namespace launcher {
+
+// The default executable.
+const string DEFAULT_EXECUTABLE = "mesos-launcher";
+
+
+// The prefix of the environment variables that launcher uses.
+static const string LAUNCHER_PREFIX = "MESOS_LAUNCHER_";
+
+
+// The default directory to search for the executable.
+static Option<string> defaultPath;
+static int defaultPathLock = 0;
+
+
+// Stores all the registered operations.
+static hashmap<string, Owned<Operation> > operations;
+
+
+static void usage(const char* argv0)
+{
+ cerr << "Usage: " << argv0 << " <operation> [OPTIONS]" << endl
+ << endl
+ << "Available operations:" << endl
+ << " help" << endl;
+
+ // Get a list of available operations.
+ foreachkey (const string& name, operations) {
+ cerr << " " << name << endl;
+ }
+}
+
+
+void setDefaultPath(const string& path)
+{
+ process::internal::acquire(&defaultPathLock);
+ {
+ defaultPath = path;
+ }
+ process::internal::release(&defaultPathLock);
+}
+
+
+static Option<string> getDefaultPath()
+{
+ Option<string> path;
+
+ process::internal::acquire(&defaultPathLock);
+ {
+ path = defaultPath;
+ }
+ process::internal::release(&defaultPathLock);
+
+ return path;
+}
+
+
+void add(const Owned<Operation>& operation)
+{
+ operations[operation->name()] = operation;
+}
+
+
+int main(int argc, char** argv)
+{
+ if (argc < 2) {
+ usage(argv[0]);
+ return 1;
+ }
+
+ if (!strcmp(argv[1], "help")) {
+ if (argc == 2) {
+ usage(argv[0]);
+ return 1;
+ }
+
+ // 'argv[0] help operation' => 'argv[0] operation --help'
+ argv[1] = argv[2];
+ argv[2] = (char*) "--help";
+ }
+
+ const string operation = argv[1];
+
+ if (!operations.contains(operation)) {
+ cerr << "Operation '" << operation << "' is not available" << endl;
+ usage(argv[0]);
+ return 1;
+ }
+
+ // Create the operation specific flags.
+ flags::FlagsBase* flags = operations[operation]->getFlags();
+
+ // Parse the flags from the environment and the command line.
+ Try<Nothing> load = flags->load(LAUNCHER_PREFIX, argc, argv);
+ if (load.isError()) {
+ cerr << "Failed to parse the flags: " << load.error() << endl;
+ return 1;
+ }
+
+ // Execute the operation.
+ return operations[operation]->execute();
+}
+
+
+ShellOperation::Flags::Flags()
+{
+ add(&command,
+ "command",
+ "The shell command to be executed");
+}
+
+
+int ShellOperation::execute()
+{
+ if (flags.command.isNone()) {
+ cerr << "The command is not specified" << endl;
+ return 1;
+ }
+
+ int status = os::system(flags.command.get());
+ if (!WIFEXITED(status)) {
+ return 1;
+ }
+
+ return WEXITSTATUS(status);
+}
+
+
+process::Future<Option<int> > Operation::launch(
+ const Option<int>& stdout,
+ const Option<int>& stderr,
+ const string& executable,
+ const Option<string>& _path)
+{
+ // Determine the path to search for the executable. If the path is
+ // specified by the user, use it. Otherwise, use the default path.
+ // If both are not specified, return failure.
+ string path;
+ if (_path.isSome()) {
+ path = _path.get();
+ } else {
+ Option<string> _defaultPath = getDefaultPath();
+ if (_defaultPath.isNone()) {
+ return Failure("Path is not specified and no default path is found");
+ }
+ path = _defaultPath.get();
+ }
+
+ Result<string> realpath = os::realpath(path::join(path, executable));
+ if (!realpath.isSome()) {
+ return Failure(
+ "Failed to determine the canonical path for '" + executable + "': " +
+ (realpath.isError() ? realpath.error() : "No such file or directory"));
+ }
+
+ // Prepare the environment variables.
+ map<string, string> environment;
+ foreachpair (const string& name, const flags::Flag& flag, *getFlags()) {
+ Option<string> value = flag.stringify(*getFlags());
+ if (value.isSome()) {
+ string key = LAUNCHER_PREFIX + name;
+ environment[key] = value.get();
+ VLOG(1) << "Setting launcher environment " << key << "=" << value.get();
+ }
+ }
+
+ // Prepare the command: 'mesos-launcher <operation_name> ...'.
+ string command = strings::join(" ", realpath.get(), name());
+
+ Try<Subprocess> s = subprocess(command, environment);
+ if (s.isError()) {
+ return Failure("Launch subprocess failed: " + s.error());
+ }
+
+ io::redirect(s.get().out(), stdout);
+ io::redirect(s.get().err(), stderr);
+
+ return s.get().status();
+}
+
+} // namespace launcher {
+} // namespace internal {
+} // namespace mesos {
http://git-wip-us.apache.org/repos/asf/mesos/blob/1920efdd/src/launcher/launcher.hpp
----------------------------------------------------------------------
diff --git a/src/launcher/launcher.hpp b/src/launcher/launcher.hpp
new file mode 100644
index 0000000..551b445
--- /dev/null
+++ b/src/launcher/launcher.hpp
@@ -0,0 +1,133 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __LAUNCHER_LAUNCHER_HPP__
+#define __LAUNCHER_LAUNCHER_HPP__
+
+#include <string>
+
+#include <process/future.hpp>
+#include <process/owned.hpp>
+
+#include <stout/flags.hpp>
+#include <stout/none.hpp>
+#include <stout/nothing.hpp>
+#include <stout/option.hpp>
+#include <stout/try.hpp>
+
+namespace mesos {
+namespace internal {
+namespace launcher {
+
+// The default executable used by the launcher.
+extern const std::string DEFAULT_EXECUTABLE;
+
+
+// Represents an operation to be executed by a subprocess.
+class Operation
+{
+public:
+ // Launches this operation in a subprocess. The user may choose to
+ // specify the executable and the path in which to search for the
+ // executable. If not specified, the default executable and the
+ // default path will be used.
+ process::Future<Option<int> > launch(
+ const Option<int>& stdout = None(),
+ const Option<int>& stderr = None(),
+ const std::string& executable = DEFAULT_EXECUTABLE,
+ const Option<std::string>& path = None());
+
+protected:
+ // Returns the name of this operation.
+ virtual std::string name() const = 0;
+
+ // Defines the operation that will be executed by a subprocess. The
+ // return value will be the exit code of the subprocess.
+ virtual int execute() = 0;
+
+ // Returns the pointer to the flags that will be used for this
+ // operation. By default, the flags is empty.
+ virtual flags::FlagsBase* getFlags() { return &flags; }
+
+private:
+ friend void add(const process::Owned<Operation>& operation);
+ friend int main(int argc, char** argv);
+
+ // The default flags which is empty.
+ flags::FlagsBase flags;
+};
+
+
+// Tell the launcher which directory to search for the executable by
+// default if it is not specified by the user. When launching an
+// operation, if the user does not specify the 'path' and no default
+// 'path' is set, the 'launch' will fail.
+void setDefaultPath(const std::string& path);
+
+
+// Register an operation. This is supposed to be called in the main
+// function of the subprocess.
+void add(const process::Owned<Operation>& operation);
+
+
+// Syntactic sugar for registering an operation. For example, the
+// following code shows a typical main function of the subprocess.
+//
+// int main(int argc, char** argv)
+// {
+// launcher::add<Operation1>();
+// launcher::add<OPeration2>();
+//
+// return launcher::main(argc, argv);
+// }
+template <typename T>
+void add()
+{
+ add(process::Owned<Operation>(new T()));
+}
+
+
+// The main entry of the subprocess.
+int main(int argc, char** argv);
+
+
+// An operation which takes a shell command and executes it. This is
+// mainly used for testing.
+class ShellOperation : public Operation
+{
+public:
+ struct Flags : public flags::FlagsBase
+ {
+ Flags();
+
+ Option<std::string> command;
+ };
+
+ Flags flags;
+
+protected:
+ virtual std::string name() const { return "shell"; }
+ virtual int execute();
+ virtual flags::FlagsBase* getFlags() { return &flags; }
+};
+
+} // namespace launcher {
+} // namespace internal {
+} // namespace mesos {
+
+#endif // __LAUNCHER_LAUNCHER_HPP__
http://git-wip-us.apache.org/repos/asf/mesos/blob/1920efdd/src/launcher/main.cpp
----------------------------------------------------------------------
diff --git a/src/launcher/main.cpp b/src/launcher/main.cpp
new file mode 100644
index 0000000..b497e98
--- /dev/null
+++ b/src/launcher/main.cpp
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "launcher/launcher.hpp"
+
+using namespace mesos::internal;
+
+
+int main(int argc, char** argv)
+{
+ launcher::add<launcher::ShellOperation>();
+
+ return launcher::main(argc, argv);
+}
http://git-wip-us.apache.org/repos/asf/mesos/blob/1920efdd/src/tests/environment.cpp
----------------------------------------------------------------------
diff --git a/src/tests/environment.cpp b/src/tests/environment.cpp
index 3e10508..005fc54 100644
--- a/src/tests/environment.cpp
+++ b/src/tests/environment.cpp
@@ -38,6 +38,8 @@
#include "linux/cgroups.hpp"
#endif
+#include "launcher/launcher.hpp"
+
#include "logging/logging.hpp"
#include "tests/environment.hpp"
@@ -241,6 +243,9 @@ void Environment::SetUp()
os::setenv("MESOS_NATIVE_JAVA_LIBRARY", path);
}
+ // Set the default path for the launcher.
+ launcher::setDefaultPath(path::join(tests::flags.build_dir, "src"));
+
if (!GTEST_IS_THREADSAFE) {
EXIT(1) << "Testing environment is not thread safe, bailing!";
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/1920efdd/src/tests/launcher_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/launcher_tests.cpp b/src/tests/launcher_tests.cpp
new file mode 100644
index 0000000..e293cc5
--- /dev/null
+++ b/src/tests/launcher_tests.cpp
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <stdio.h>
+
+#include <gtest/gtest.h>
+
+#include <process/gtest.hpp>
+
+#include <stout/gtest.hpp>
+#include <stout/os.hpp>
+#include <stout/path.hpp>
+
+#include "launcher/launcher.hpp"
+
+#include "tests/flags.hpp"
+#include "tests/utils.hpp"
+
+using namespace process;
+
+using namespace mesos::internal;
+using namespace mesos::internal::launcher;
+
+using std::string;
+
+
+class LauncherTest: public tests::TemporaryDirectoryTest {};
+
+
+TEST_F(LauncherTest, Launch)
+{
+ Option<int> stdout = None();
+ Option<int> stderr = None();
+
+ // Redirect output if running the tests verbosely.
+ if (tests::flags.verbose) {
+ stdout = STDOUT_FILENO;
+ stderr = STDERR_FILENO;
+ }
+
+ string temp1 = path::join(os::getcwd(), "temp1");
+ string temp2 = path::join(os::getcwd(), "temp2");
+
+ ASSERT_SOME(os::write(temp1, "hello world"));
+
+ ShellOperation operation;
+ operation.flags.command = "cp " + temp1 + " " + temp2;
+
+ Future<Option<int> > launch = operation.launch(stdout, stderr);
+ AWAIT_READY(launch);
+ EXPECT_SOME_EQ(0, launch.get());
+ ASSERT_SOME_EQ("hello world", os::read(temp2));
+
+ AWAIT_FAILED(operation.launch(
+ stdout,
+ stderr,
+ "non-exist"));
+
+ AWAIT_FAILED(operation.launch(
+ stdout,
+ stderr,
+ launcher::DEFAULT_EXECUTABLE,
+ "non-exist"));
+}