You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by ji...@apache.org on 2016/05/28 00:19:54 UTC
[1/6] mesos git commit: Split DockerContainerizerProcess::launch into
two functions.
Repository: mesos
Updated Branches:
refs/heads/master eb805f0e6 -> f5b3f2a6c
Split DockerContainerizerProcess::launch into two functions.
This prepares the `::launch` method for an asynchronous hook.
Review: https://reviews.apache.org/r/47149/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/21a3ab6f
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/21a3ab6f
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/21a3ab6f
Branch: refs/heads/master
Commit: 21a3ab6fbb89945fbd7b2ea773fff67894bf24bb
Parents: eb805f0
Author: Joseph Wu <jo...@mesosphere.io>
Authored: Fri May 27 17:19:10 2016 -0700
Committer: Jie Yu <yu...@gmail.com>
Committed: Fri May 27 17:19:10 2016 -0700
----------------------------------------------------------------------
src/slave/containerizer/docker.cpp | 50 +++++++++++++++++++++++++--------
src/slave/containerizer/docker.hpp | 7 +++++
2 files changed, 46 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/21a3ab6f/src/slave/containerizer/docker.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/docker.cpp b/src/slave/containerizer/docker.cpp
index 7af14f4..0b866c1 100644
--- a/src/slave/containerizer/docker.cpp
+++ b/src/slave/containerizer/docker.cpp
@@ -1018,17 +1018,45 @@ Future<bool> DockerContainerizerProcess::launch(
<< "' and framework '" << executorInfo.framework_id() << "'";
}
+ Future<Nothing> f = Nothing();
+
+ // TODO(josephw): Add a hook here.
+
+ return f.then(defer(
+ self(),
+ &Self::_launch,
+ containerId,
+ taskInfo,
+ executorInfo,
+ directory,
+ slaveId));
+}
+
+
+Future<bool> DockerContainerizerProcess::_launch(
+ const ContainerID& containerId,
+ const Option<TaskInfo>& taskInfo,
+ const ExecutorInfo& executorInfo,
+ const string& directory,
+ const SlaveID& slaveId)
+{
+ if (!containers_.contains(containerId)) {
+ return Failure("Container is already destroyed");
+ }
+
+ Container* container = containers_[containerId];
+
if (HookManager::hooksAvailable()) {
HookManager::slavePreLaunchDockerHook(
- container.get()->container,
- container.get()->command,
+ container->container,
+ container->command,
taskInfo,
executorInfo,
- container.get()->name(),
- container.get()->directory,
+ container->name(),
+ container->directory,
flags.sandbox_directory,
- container.get()->resources,
- container.get()->environment);
+ container->resources,
+ container->environment);
}
if (taskInfo.isSome() && flags.docker_mesos_image.isNone()) {
@@ -1038,7 +1066,7 @@ Future<bool> DockerContainerizerProcess::launch(
// where 'update' can be called before mesos-docker-executor
// creates the Docker container for the task. See more details in
// the comments of r33174.
- return container.get()->launch = fetch(containerId, slaveId)
+ return container->launch = fetch(containerId, slaveId)
.then(defer(self(), [=]() {
return pull(containerId);
}))
@@ -1057,12 +1085,12 @@ Future<bool> DockerContainerizerProcess::launch(
}));
}
- string containerName = container.get()->name();
+ string containerName = container->name();
- if (container.get()->executorName().isSome()) {
+ if (container->executorName().isSome()) {
// Launch the container with the executor name as we expect the
// executor will launch the docker container.
- containerName = container.get()->executorName().get();
+ containerName = container->executorName().get();
}
// Launching task or executor by launching a separate docker
@@ -1070,7 +1098,7 @@ Future<bool> DockerContainerizerProcess::launch(
// We need to do so for launching a task because as the slave is
// running in a container (via docker_mesos_image flag) we want the
// executor to keep running when the slave container dies.
- return container.get()->launch = fetch(containerId, slaveId)
+ return container->launch = fetch(containerId, slaveId)
.then(defer(self(), [=]() {
return pull(containerId);
}))
http://git-wip-us.apache.org/repos/asf/mesos/blob/21a3ab6f/src/slave/containerizer/docker.hpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/docker.hpp b/src/slave/containerizer/docker.hpp
index fac26a2..fde0b20 100644
--- a/src/slave/containerizer/docker.hpp
+++ b/src/slave/containerizer/docker.hpp
@@ -177,6 +177,13 @@ private:
const ContainerID& containerId,
pid_t pid);
+ process::Future<bool> _launch(
+ const ContainerID& containerId,
+ const Option<TaskInfo>& taskInfo,
+ const ExecutorInfo& executorInfo,
+ const std::string& directory,
+ const SlaveID& slaveId);
+
process::Future<Nothing> _recover(
const Option<state::SlaveState>& state,
const std::list<Docker::Container>& containers);
[2/6] mesos git commit: Added optional environment variable argument
to mesos-docker-executor.
Posted by ji...@apache.org.
Added optional environment variable argument to mesos-docker-executor.
This flag opens up a way for hooks to specify environment variables for
docker tasks. Existing hooks can only affect the environment variables
of docker executors.
Review: https://reviews.apache.org/r/47205/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/82029372
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/82029372
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/82029372
Branch: refs/heads/master
Commit: 82029372c3eb0a12218fd9864cc0f5da38f5b108
Parents: 21a3ab6
Author: Joseph Wu <jo...@mesosphere.io>
Authored: Fri May 27 17:19:15 2016 -0700
Committer: Jie Yu <yu...@gmail.com>
Committed: Fri May 27 17:19:15 2016 -0700
----------------------------------------------------------------------
src/docker/executor.cpp | 54 +++++++++++++++++++++++++++++++++++++++-----
src/docker/executor.hpp | 14 ++++++++++--
2 files changed, 60 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/82029372/src/docker/executor.cpp
----------------------------------------------------------------------
diff --git a/src/docker/executor.cpp b/src/docker/executor.cpp
index 04f4404..f4796e4 100644
--- a/src/docker/executor.cpp
+++ b/src/docker/executor.cpp
@@ -16,6 +16,7 @@
#include <stdio.h>
+#include <map>
#include <string>
#include <mesos/mesos.hpp>
@@ -27,9 +28,12 @@
#include <process/reap.hpp>
#include <process/owned.hpp>
+#include <stout/error.hpp>
#include <stout/flags.hpp>
-#include <stout/protobuf.hpp>
+#include <stout/json.hpp>
#include <stout/os.hpp>
+#include <stout/protobuf.hpp>
+#include <stout/try.hpp>
#include "common/status_utils.hpp"
@@ -49,6 +53,7 @@ using namespace process;
using std::cerr;
using std::cout;
using std::endl;
+using std::map;
using std::string;
using std::vector;
@@ -73,7 +78,8 @@ public:
const string& sandboxDirectory,
const string& mappedDirectory,
const Duration& shutdownGracePeriod,
- const string& healthCheckDir)
+ const string& healthCheckDir,
+ const map<string, string>& taskEnvironment)
: killed(false),
killedByHealthCheck(false),
terminated(false),
@@ -84,6 +90,7 @@ public:
sandboxDirectory(sandboxDirectory),
mappedDirectory(mappedDirectory),
shutdownGracePeriod(shutdownGracePeriod),
+ taskEnvironment(taskEnvironment),
stop(Nothing()),
inspect(Nothing()) {}
@@ -153,7 +160,7 @@ public:
sandboxDirectory,
mappedDirectory,
task.resources() + task.executor().resources(),
- None(),
+ taskEnvironment,
Subprocess::FD(STDOUT_FILENO),
Subprocess::FD(STDERR_FILENO));
@@ -495,6 +502,8 @@ private:
string sandboxDirectory;
string mappedDirectory;
Duration shutdownGracePeriod;
+ map<string, string> taskEnvironment;
+
Option<KillPolicy> killPolicy;
Option<Future<Nothing>> run;
Future<Nothing> stop;
@@ -514,7 +523,8 @@ public:
const string& sandboxDirectory,
const string& mappedDirectory,
const Duration& shutdownGracePeriod,
- const string& healthCheckDir)
+ const string& healthCheckDir,
+ const map<string, string>& taskEnvironment)
{
process = Owned<DockerExecutorProcess>(new DockerExecutorProcess(
docker,
@@ -522,7 +532,8 @@ public:
sandboxDirectory,
mappedDirectory,
shutdownGracePeriod,
- healthCheckDir));
+ healthCheckDir,
+ taskEnvironment));
spawn(process.get());
}
@@ -650,6 +661,36 @@ int main(int argc, char** argv)
return EXIT_FAILURE;
}
+ map<string, string> taskEnvironment;
+ if (flags.task_environment.isSome()) {
+ // Parse the string as JSON.
+ Try<JSON::Object> json =
+ JSON::parse<JSON::Object>(flags.task_environment.get());
+
+ if (json.isError()) {
+ cerr << flags.usage("Failed to parse --task_environment: " + json.error())
+ << endl;
+ return EXIT_FAILURE;
+ }
+
+ // Convert from JSON to map.
+ foreachpair (
+ const std::string& key,
+ const JSON::Value& value,
+ json->values) {
+ if (!value.is<JSON::String>()) {
+ cerr << flags.usage(
+ "Value of key '" + key +
+ "' in --task_environment is not a string")
+ << endl;
+ return EXIT_FAILURE;
+ }
+
+ // Save the parsed and validated key/value.
+ taskEnvironment[key] = value.as<JSON::String>().value;
+ }
+ }
+
// Get executor shutdown grace period from the environment.
//
// NOTE: We avoided introducing a docker executor flag for this
@@ -702,7 +743,8 @@ int main(int argc, char** argv)
flags.sandbox_directory.get(),
flags.mapped_directory.get(),
shutdownGracePeriod,
- flags.launcher_dir.get());
+ flags.launcher_dir.get(),
+ taskEnvironment);
mesos::MesosExecutorDriver driver(&executor);
return driver.run() == mesos::DRIVER_STOPPED ? EXIT_SUCCESS : EXIT_FAILURE;
http://git-wip-us.apache.org/repos/asf/mesos/blob/82029372/src/docker/executor.hpp
----------------------------------------------------------------------
diff --git a/src/docker/executor.hpp b/src/docker/executor.hpp
index 798ca3d..d20a8fb 100644
--- a/src/docker/executor.hpp
+++ b/src/docker/executor.hpp
@@ -19,8 +19,13 @@
#include <stdio.h>
+#include <map>
+#include <string>
+
#include <process/process.hpp>
+#include <stout/option.hpp>
+
#include "logging/flags.hpp"
namespace mesos {
@@ -64,6 +69,11 @@ struct Flags : public mesos::internal::logging::Flags
"Directory path of Mesos binaries. Mesos would find health-check,\n"
"fetcher, containerizer and executor binary files under this\n"
"directory.");
+
+ add(&task_environment,
+ "task_environment",
+ "A JSON map of environment variables and values that should\n"
+ "be passed into the task launched by this executor.");
}
Option<std::string> container;
@@ -71,11 +81,11 @@ struct Flags : public mesos::internal::logging::Flags
Option<std::string> docker_socket;
Option<std::string> sandbox_directory;
Option<std::string> mapped_directory;
+ Option<std::string> launcher_dir;
+ Option<std::string> task_environment;
// TODO(alexr): Remove this after the deprecation cycle (started in 0.29).
Option<Duration> stop_timeout;
-
- Option<std::string> launcher_dir;
};
} // namespace docker {
[5/6] mesos git commit: Implemented new asynchronous docker
pre-launch hook.
Posted by ji...@apache.org.
Implemented new asynchronous docker pre-launch hook.
Introduces, but does not fully wire up a new hook.
The new hook, "slavePreLaunchDockerEnvironmentDecorator",
has divergent semantics compared with existing hooks:
* The hook is asynchronous,
* can prevent a task from launching if it errors,
* can overwrite environment variables.
The new hook is intended to be a strictly-superior
replacement for the existing hook "slavePreLaunchDockerHook".
Review: https://reviews.apache.org/r/47150/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/8486e982
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/8486e982
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/8486e982
Branch: refs/heads/master
Commit: 8486e9829435d9f09ad0c13de8a4c14257d8a988
Parents: cd46db8
Author: Joseph Wu <jo...@mesosphere.io>
Authored: Fri May 27 17:19:30 2016 -0700
Committer: Jie Yu <yu...@gmail.com>
Committed: Fri May 27 17:19:30 2016 -0700
----------------------------------------------------------------------
include/mesos/hook.hpp | 35 ++++++
src/examples/test_hook_module.cpp | 32 ++++++
src/hook/manager.cpp | 57 ++++++++++
src/hook/manager.hpp | 11 ++
src/tests/hook_tests.cpp | 197 +++++++++++++++++++++++++++++++++
5 files changed, 332 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/8486e982/include/mesos/hook.hpp
----------------------------------------------------------------------
diff --git a/include/mesos/hook.hpp b/include/mesos/hook.hpp
index 210ffba..c8a5864 100644
--- a/include/mesos/hook.hpp
+++ b/include/mesos/hook.hpp
@@ -24,6 +24,8 @@
#include <mesos/mesos.hpp>
#include <mesos/resources.hpp>
+#include <process/future.hpp>
+
#include <stout/none.hpp>
#include <stout/nothing.hpp>
#include <stout/result.hpp>
@@ -84,9 +86,42 @@ public:
return None();
}
+ // This environment decorator is called from within the slave after
+ // receiving a run task request from the master but before the docker
+ // containerizer launches the task. A module implementing the hook can
+ // inspect the arguments and return a `Failure` if the task should be
+ // rejected with a `TASK_FAILED`.
+ // The hook can return a set of environment variables. For command tasks
+ // the environment variables will become part of the task's environment.
+ // For custom executors, the environment variables will be part of the
+ // custom executor's environment.
+ //
+ // NOTE: The order of hooks matters for environment variables.
+ // If there is a conflict, the hook loaded last will take priority.
+ //
+ // NOTE: Unlike `slaveExecutorEnvironmentDecorator`, environment variables
+ // returned from this hook *will* overwrite environment variables inside
+ // the `ExecutorInfo`.
+ //
+ // NOTE: This hook is designed to be an asynchronous replacement for
+ // `slavePreLaunchDockerHook`. This hook is called first.
+ virtual process::Future<Option<Environment>>
+ slavePreLaunchDockerEnvironmentDecorator(
+ const Option<TaskInfo>& taskInfo,
+ const ExecutorInfo& executorInfo,
+ const std::string& name,
+ const std::string& sandboxDirectory,
+ const std::string& mappedDirectory,
+ const Option<std::map<std::string, std::string>>& env)
+ {
+ return None();
+ }
+
// This hook is called from within slave before docker is launched.
// A typical module implementing the hook will perform some settings
// as required.
+ //
+ // NOTE: Superceded by `slavePreLaunchDockerEnvironmentDecorator`.
virtual Try<Nothing> slavePreLaunchDockerHook(
const ContainerInfo& containerInfo,
const CommandInfo& commandInfo,
http://git-wip-us.apache.org/repos/asf/mesos/blob/8486e982/src/examples/test_hook_module.cpp
----------------------------------------------------------------------
diff --git a/src/examples/test_hook_module.cpp b/src/examples/test_hook_module.cpp
index 4b97f84..bc0bcb9 100644
--- a/src/examples/test_hook_module.cpp
+++ b/src/examples/test_hook_module.cpp
@@ -35,6 +35,7 @@ using namespace mesos;
using std::map;
using std::string;
+using process::Failure;
using process::Future;
// Must be kept in sync with variables of the same name in
@@ -42,6 +43,7 @@ using process::Future;
const char* testLabelKey = "MESOS_Test_Label";
const char* testLabelValue = "ApacheMesos";
const char* testRemoveLabelKey = "MESOS_Test_Remove_Label";
+const char* testErrorLabelKey = "MESOS_Test_Error_Label";
class HookProcess : public ProtobufProcess<HookProcess>
{
@@ -182,6 +184,36 @@ public:
}
+ // In this hook, look for the existence of a specific label.
+ // If found, return a `Failure`.
+ // Otherwise, add an environment variable to the task.
+ virtual Future<Option<Environment>> slavePreLaunchDockerEnvironmentDecorator(
+ const Option<TaskInfo>& taskInfo,
+ const ExecutorInfo& executorInfo,
+ const string& name,
+ const string& sandboxDirectory,
+ const string& mappedDirectory,
+ const Option<map<string, string>>& env)
+ {
+ LOG(INFO) << "Executing 'slavePreLaunchDockerEnvironmentDecorator' hook";
+
+ if (taskInfo.isSome()) {
+ foreach (const Label& label, taskInfo->labels().labels()) {
+ if (label.key() == testErrorLabelKey) {
+ return Failure("Spotted error label");
+ }
+ }
+ }
+
+ Environment environment;
+ Environment::Variable* variable = environment.add_variables();
+ variable->set_name("FOO_DOCKER");
+ variable->set_value("docker_bar");
+
+ return environment;
+ }
+
+
virtual Try<Nothing> slavePreLaunchDockerHook(
const ContainerInfo& containerInfo,
const CommandInfo& commandInfo,
http://git-wip-us.apache.org/repos/asf/mesos/blob/8486e982/src/hook/manager.cpp
----------------------------------------------------------------------
diff --git a/src/hook/manager.cpp b/src/hook/manager.cpp
index 381807d..2488522 100644
--- a/src/hook/manager.cpp
+++ b/src/hook/manager.cpp
@@ -14,6 +14,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
+#include <list>
#include <mutex>
#include <string>
#include <vector>
@@ -22,6 +23,9 @@
#include <mesos/module/hook.hpp>
+#include <process/collect.hpp>
+#include <process/future.hpp>
+
#include <stout/check.hpp>
#include <stout/foreach.hpp>
#include <stout/linkedhashmap.hpp>
@@ -32,10 +36,14 @@
#include "hook/manager.hpp"
#include "module/manager.hpp"
+using std::list;
using std::map;
using std::string;
using std::vector;
+using process::collect;
+using process::Future;
+
using mesos::modules::ModuleManager;
namespace mesos {
@@ -200,6 +208,55 @@ Environment HookManager::slaveExecutorEnvironmentDecorator(
}
+Future<map<string, string>>
+ HookManager::slavePreLaunchDockerEnvironmentDecorator(
+ const Option<TaskInfo>& taskInfo,
+ const ExecutorInfo& executorInfo,
+ const string& containerName,
+ const string& sandboxDirectory,
+ const string& mappedDirectory,
+ const Option<map<string, string>>& env)
+{
+ // We execute these hooks according to their ordering so any conflicting
+ // environment variables can be deterministically resolved
+ // (the last hook takes priority).
+ list<Future<Option<Environment>>> futures;
+
+ foreach (const string& name, availableHooks.keys()) {
+ Hook* hook = availableHooks[name];
+
+ // Chain together each hook.
+ futures.push_back(
+ hook->slavePreLaunchDockerEnvironmentDecorator(
+ taskInfo,
+ executorInfo,
+ containerName,
+ sandboxDirectory,
+ mappedDirectory,
+ env));
+ }
+
+ return collect(futures)
+ .then([](const list<Option<Environment>>& results)
+ -> Future<map<string, string>> {
+ // Combine all the Environments.
+ map<string, string> environment;
+
+ foreach (const Option<Environment>& result, results) {
+ if (result.isNone()) {
+ continue;
+ }
+
+ foreach (const Environment::Variable& variable, result->variables()) {
+ environment[variable.name()] = variable.value();
+ }
+ }
+
+ return environment;
+ });
+}
+
+
void HookManager::slavePreLaunchDockerHook(
const ContainerInfo& containerInfo,
const CommandInfo& commandInfo,
http://git-wip-us.apache.org/repos/asf/mesos/blob/8486e982/src/hook/manager.hpp
----------------------------------------------------------------------
diff --git a/src/hook/manager.hpp b/src/hook/manager.hpp
index 528674e..5ecfcab 100644
--- a/src/hook/manager.hpp
+++ b/src/hook/manager.hpp
@@ -22,6 +22,8 @@
#include <mesos/mesos.hpp>
#include <mesos/hook.hpp>
+#include <process/future.hpp>
+
#include <stout/try.hpp>
namespace mesos {
@@ -54,6 +56,15 @@ public:
static Environment slaveExecutorEnvironmentDecorator(
ExecutorInfo executorInfo);
+ static process::Future<std::map<std::string, std::string>>
+ slavePreLaunchDockerEnvironmentDecorator(
+ const Option<TaskInfo>& taskInfo,
+ const ExecutorInfo& executorInfo,
+ const std::string& containerName,
+ const std::string& sandboxDirectory,
+ const std::string& mappedDirectory,
+ const Option<std::map<std::string, std::string>>& env);
+
static void slavePreLaunchDockerHook(
const ContainerInfo& containerInfo,
const CommandInfo& commandInfo,
http://git-wip-us.apache.org/repos/asf/mesos/blob/8486e982/src/tests/hook_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/hook_tests.cpp b/src/tests/hook_tests.cpp
index c6b4e8a..36b64df 100644
--- a/src/tests/hook_tests.cpp
+++ b/src/tests/hook_tests.cpp
@@ -96,6 +96,7 @@ const char* testLabelKey = "MESOS_Test_Label";
const char* testLabelValue = "ApacheMesos";
const char* testRemoveLabelKey = "MESOS_Test_Remove_Label";
const char* testRemoveLabelValue = "FooBar";
+const char* testErrorLabelKey = "MESOS_Test_Error_Label";
const char* testEnvironmentVariableName = "MESOS_TEST_ENVIRONMENT_VARIABLE";
class HookTest : public MesosTest
@@ -614,6 +615,202 @@ TEST_F(HookTest, VerifySlaveTaskStatusDecorator)
}
+// This test verifies that the slave pre-launch docker environment
+// decorator can attach environment variables to a task.
+TEST_F(HookTest, ROOT_DOCKER_VerifySlavePreLaunchDockerEnvironmentDecorator)
+{
+ Try<Owned<cluster::Master>> master = StartMaster();
+ ASSERT_SOME(master);
+
+ MockDocker* mockDocker =
+ new MockDocker(tests::flags.docker, tests::flags.docker_socket);
+
+ Shared<Docker> docker(mockDocker);
+
+ slave::Flags flags = CreateSlaveFlags();
+
+ Fetcher fetcher;
+
+ Try<ContainerLogger*> logger =
+ ContainerLogger::create(flags.container_logger);
+
+ ASSERT_SOME(logger);
+
+ MockDockerContainerizer containerizer(
+ flags,
+ &fetcher,
+ Owned<ContainerLogger>(logger.get()),
+ docker);
+
+ Owned<MasterDetector> detector = master.get()->createDetector();
+
+ Try<Owned<cluster::Slave>> slave =
+ StartSlave(detector.get(), &containerizer, flags);
+ ASSERT_SOME(slave);
+
+ MockScheduler sched;
+ MesosSchedulerDriver driver(
+ &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL);
+
+ EXPECT_CALL(sched, registered(&driver, _, _));
+
+ Future<vector<Offer>> offers;
+ EXPECT_CALL(sched, resourceOffers(&driver, _))
+ .WillOnce(FutureArg<1>(&offers))
+ .WillRepeatedly(Return()); // Ignore subsequent offers.
+
+ driver.start();
+
+ AWAIT_READY(offers);
+ ASSERT_EQ(1u, offers.get().size());
+
+ TaskInfo task = createTask(
+ offers.get()[0],
+ "test \"$FOO_DOCKER\" = 'docker_bar'");
+
+ ContainerInfo containerInfo;
+ containerInfo.set_type(ContainerInfo::DOCKER);
+
+ // TODO(tnachen): Use local image to test if possible.
+ ContainerInfo::DockerInfo dockerInfo;
+ dockerInfo.set_image("alpine");
+ containerInfo.mutable_docker()->CopyFrom(dockerInfo);
+
+ task.mutable_container()->CopyFrom(containerInfo);
+
+ Future<ContainerID> containerId;
+ EXPECT_CALL(containerizer, launch(_, _, _, _, _, _, _, _))
+ .WillOnce(DoAll(FutureArg<0>(&containerId),
+ Invoke(&containerizer,
+ &MockDockerContainerizer::_launch)));
+
+ Future<TaskStatus> statusRunning;
+ Future<TaskStatus> statusFinished;
+ EXPECT_CALL(sched, statusUpdate(&driver, _))
+ .WillOnce(FutureArg<1>(&statusRunning))
+ .WillOnce(FutureArg<1>(&statusFinished))
+ .WillRepeatedly(DoDefault());
+
+ driver.launchTasks(offers.get()[0].id(), {task});
+
+ AWAIT_READY_FOR(containerId, Seconds(60));
+ AWAIT_READY_FOR(statusRunning, Seconds(60));
+ EXPECT_EQ(TASK_RUNNING, statusRunning.get().state());
+ AWAIT_READY_FOR(statusFinished, Seconds(60));
+ EXPECT_EQ(TASK_FINISHED, statusFinished.get().state());
+
+ Future<containerizer::Termination> termination =
+ containerizer.wait(containerId.get());
+
+ driver.stop();
+ driver.join();
+
+ AWAIT_READY(termination);
+
+ Future<list<Docker::Container>> containers =
+ docker.get()->ps(true, slave::DOCKER_NAME_PREFIX);
+
+ AWAIT_READY(containers);
+
+ // Cleanup all mesos launched containers.
+ foreach (const Docker::Container& container, containers.get()) {
+ AWAIT_READY_FOR(docker.get()->rm(container.id, true), Seconds(30));
+ }
+}
+
+
+// This test verifies that the slave pre-launch docker validator hook can check
+// labels on a task and subsequently prevent the task from being launched
+// if a specific label is present.
+TEST_F(HookTest, ROOT_DOCKER_VerifySlavePreLaunchDockerValidator)
+{
+ Try<Owned<cluster::Master>> master = StartMaster();
+ ASSERT_SOME(master);
+
+ MockDocker* mockDocker =
+ new MockDocker(tests::flags.docker, tests::flags.docker_socket);
+
+ Shared<Docker> docker(mockDocker);
+
+ slave::Flags flags = CreateSlaveFlags();
+
+ Fetcher fetcher;
+
+ Try<ContainerLogger*> logger =
+ ContainerLogger::create(flags.container_logger);
+
+ ASSERT_SOME(logger);
+
+ MockDockerContainerizer containerizer(
+ flags,
+ &fetcher,
+ Owned<ContainerLogger>(logger.get()),
+ docker);
+
+ Owned<MasterDetector> detector = master.get()->createDetector();
+
+ Try<Owned<cluster::Slave>> slave =
+ StartSlave(detector.get(), &containerizer, flags);
+ ASSERT_SOME(slave);
+
+ MockScheduler sched;
+ MesosSchedulerDriver driver(
+ &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL);
+
+ EXPECT_CALL(sched, registered(&driver, _, _));
+
+ Future<vector<Offer>> offers;
+ EXPECT_CALL(sched, resourceOffers(&driver, _))
+ .WillOnce(FutureArg<1>(&offers))
+ .WillRepeatedly(Return()); // Ignore subsequent offers.
+
+ driver.start();
+
+ AWAIT_READY(offers);
+ ASSERT_EQ(1u, offers.get().size());
+
+ TaskInfo task;
+ task.set_name("");
+ task.mutable_task_id()->set_value("1");
+ task.mutable_slave_id()->CopyFrom(offers.get()[0].slave_id());
+ task.mutable_resources()->CopyFrom(offers.get()[0].resources());
+
+ // Add a special label which the validator hook checks for.
+ // The existence of this label will cause the hook to reject the task.
+ Labels* labels = task.mutable_labels();
+ labels->add_labels()->CopyFrom(
+ createLabel(testErrorLabelKey, testLabelValue));
+
+ ContainerInfo containerInfo;
+ containerInfo.set_type(ContainerInfo::DOCKER);
+
+ CommandInfo command;
+ command.set_value("exit 0");
+
+ // TODO(tnachen): Use local image to test if possible.
+ ContainerInfo::DockerInfo dockerInfo;
+ dockerInfo.set_image("alpine");
+ containerInfo.mutable_docker()->CopyFrom(dockerInfo);
+
+ task.mutable_command()->CopyFrom(command);
+ task.mutable_container()->CopyFrom(containerInfo);
+
+ driver.launchTasks(offers.get()[0].id(), {task});
+
+ Future<TaskStatus> statusError;
+ EXPECT_CALL(sched, statusUpdate(&driver, _))
+ .WillOnce(FutureArg<1>(&statusError));
+
+ AWAIT_READY(statusError);
+
+ EXPECT_EQ(TASK_FAILED, statusError.get().state());
+ EXPECT_EQ(TaskStatus::REASON_CONTAINER_LAUNCH_FAILED, statusError->reason());
+
+ driver.stop();
+ driver.join();
+}
+
+
// Test that the prepare launch docker hook execute before launch
// a docker container. Test hook create a file "foo" in the sandbox
// directory. When the docker container launched, the sandbox directory
[6/6] mesos git commit: Wired up the new docker environment hook.
Posted by ji...@apache.org.
Wired up the new docker environment hook.
Modifies the code path for docker executors.
Docker command executors are now launched with an additional flag
that is filled in by a hook. The --task_environment flag tells the
command executor to pass some specified mapping of environment
variables to the task.
Custom executors are launched with the environment variables directly.
It is up to custom executors to pass these variables into tasks.
Review: https://reviews.apache.org/r/47216/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/f5b3f2a6
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/f5b3f2a6
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/f5b3f2a6
Branch: refs/heads/master
Commit: f5b3f2a6c4795d2c4e7effbb594749cdc9b8ea4e
Parents: 8486e98
Author: Joseph Wu <jo...@mesosphere.io>
Authored: Fri May 27 17:19:43 2016 -0700
Committer: Jie Yu <yu...@gmail.com>
Committed: Fri May 27 17:19:43 2016 -0700
----------------------------------------------------------------------
src/slave/containerizer/docker.cpp | 66 +++++++++++++++++++++++++++++++--
src/slave/containerizer/docker.hpp | 4 ++
2 files changed, 66 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/f5b3f2a6/src/slave/containerizer/docker.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/docker.cpp b/src/slave/containerizer/docker.cpp
index abc066b..52caf9f 100644
--- a/src/slave/containerizer/docker.cpp
+++ b/src/slave/containerizer/docker.cpp
@@ -34,6 +34,7 @@
#include <stout/fs.hpp>
#include <stout/hashmap.hpp>
#include <stout/hashset.hpp>
+#include <stout/jsonify.hpp>
#include <stout/os.hpp>
#include "common/status_utils.hpp"
@@ -200,10 +201,16 @@ DockerContainerizer::~DockerContainerizer()
}
+// Constructs the flags for the `mesos-docker-executor`.
+// Custom docker executors will also be invoked with these flags.
+//
+// NOTE: `taskEnvironment` is currently used to propagate environment variables
+// from a hook: `slavePreLaunchDockerEnvironmentDecorator`.
docker::Flags dockerFlags(
const Flags& flags,
const string& name,
- const string& directory)
+ const string& directory,
+ const Option<map<string, string>>& taskEnvironment)
{
docker::Flags dockerFlags;
dockerFlags.container = name;
@@ -213,6 +220,10 @@ docker::Flags dockerFlags(
dockerFlags.docker_socket = flags.docker_socket;
dockerFlags.launcher_dir = flags.launcher_dir;
+ if (taskEnvironment.isSome()) {
+ dockerFlags.task_environment = string(jsonify(taskEnvironment.get()));
+ }
+
// TODO(alexr): Remove this after the deprecation cycle (started in 0.29).
dockerFlags.stop_timeout = flags.docker_stop_timeout;
@@ -310,10 +321,14 @@ DockerContainerizerProcess::Container::create(
newContainerInfo.mutable_docker()->CopyFrom(dockerInfo);
+ // NOTE: We do not set the optional `taskEnvironment` here as
+ // this field is currently used to propagate environment variables
+ // from a hook. This hook is called after `Container::create`.
docker::Flags dockerExecutorFlags = dockerFlags(
flags,
Container::name(slaveId, stringify(id)),
- containerWorkdir);
+ containerWorkdir,
+ None());
// Override the command with the docker command executor.
CommandInfo newCommandInfo;
@@ -1029,7 +1044,46 @@ Future<bool> DockerContainerizerProcess::launch(
Future<Nothing> f = Nothing();
- // TODO(josephw): Add a hook here.
+ if (HookManager::hooksAvailable()) {
+ f = HookManager::slavePreLaunchDockerEnvironmentDecorator(
+ taskInfo,
+ executorInfo,
+ container.get()->name(),
+ container.get()->directory,
+ flags.sandbox_directory,
+ container.get()->environment)
+ .then([this, taskInfo, containerId](
+ const map<string, string>& environment) -> Future<Nothing> {
+ if (!containers_.contains(containerId)) {
+ return Failure("Container is already destroyed");
+ }
+
+ Container* container = containers_[containerId];
+
+ if (taskInfo.isSome()) {
+ // The built-in command executors explicitly support passing
+ // environment variables from a hook into a task.
+ container->taskEnvironment = environment;
+
+ // For dockerized command executors, the flags have already been
+ // serialized into the command, albeit without these environment
+ // variables. Append the last flag to the overridden command.
+ if (container->launchesExecutorContainer) {
+ container->command.add_arguments(
+ "--task_environment=" + string(jsonify(environment)));
+ }
+ } else {
+ // For custom executors, the environment variables from a hook
+ // are passed directly into the executor. It is up to the custom
+ // executor whether individual tasks should inherit these variables.
+ foreachpair (const string& key, const string& value, environment) {
+ container->environment[key] = value;
+ }
+ }
+
+ return Nothing();
+ });
+ }
return f.then(defer(
self(),
@@ -1269,7 +1323,11 @@ Future<pid_t> DockerContainerizerProcess::launchExecutorProcess(
subprocessInfo.out,
subprocessInfo.err,
SETSID,
- dockerFlags(flags, container->name(), container->directory),
+ dockerFlags(
+ flags,
+ container->name(),
+ container->directory,
+ container->taskEnvironment),
environment,
None(),
parentHooks,
http://git-wip-us.apache.org/repos/asf/mesos/blob/f5b3f2a6/src/slave/containerizer/docker.hpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/docker.hpp b/src/slave/containerizer/docker.hpp
index fde0b20..311dca2 100644
--- a/src/slave/containerizer/docker.hpp
+++ b/src/slave/containerizer/docker.hpp
@@ -443,6 +443,10 @@ private:
CommandInfo command;
std::map<std::string, std::string> environment;
+ // Environment variables that the command executor should pass
+ // onto a docker-ized task. This is set by a hook.
+ Option<std::map<std::string, std::string>> taskEnvironment;
+
// The sandbox directory for the container. This holds the
// symlinked path if symlinked boolean is true.
std::string directory;
[3/6] mesos git commit: Removed duplicate call to
containerizer::executorEnvironment.
Posted by ji...@apache.org.
Removed duplicate call to containerizer::executorEnvironment.
In this code path, where the task uses the default command executor,
and the agent is not dockerized
(i.e. `taskInfo.isSome() && flags.docker_mesos_image.isNone()`),
the `executorEnvironment` function is called twice.
The first call is inside the `Container*` constructor called by
`Container::create`. Since `Container::create` gives passes `None`
for the `environment` field, the constructor will call
`executorEnvironment` to populate the `environment` field.
The populated field is then accessible by `launchExecutorProcess`.
Review: https://reviews.apache.org/r/47212/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/9b054cc4
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/9b054cc4
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/9b054cc4
Branch: refs/heads/master
Commit: 9b054cc46d462ad5c8c5074b8b5c9e7eeac3dabf
Parents: 8202937
Author: Joseph Wu <jo...@mesosphere.io>
Authored: Fri May 27 17:19:22 2016 -0700
Committer: Jie Yu <yu...@gmail.com>
Committed: Fri May 27 17:19:22 2016 -0700
----------------------------------------------------------------------
src/slave/containerizer/docker.cpp | 9 +--------
1 file changed, 1 insertion(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/9b054cc4/src/slave/containerizer/docker.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/docker.cpp b/src/slave/containerizer/docker.cpp
index 0b866c1..47c4148 100644
--- a/src/slave/containerizer/docker.cpp
+++ b/src/slave/containerizer/docker.cpp
@@ -1197,14 +1197,7 @@ Future<pid_t> DockerContainerizerProcess::launchExecutorProcess(
container->state = Container::RUNNING;
// Prepare environment variables for the executor.
- map<string, string> environment = executorEnvironment(
- container->executor,
- container->directory,
- container->slaveId,
- container->slavePid,
- container->checkpoint,
- flags,
- false);
+ map<string, string> environment = container->environment;
// Include any enviroment variables from ExecutorInfo.
foreach (const Environment::Variable& variable,
[4/6] mesos git commit: Changed the dockerized docker command
executor CommandInfo usage.
Posted by ji...@apache.org.
Changed the dockerized docker command executor CommandInfo usage.
This changes how we override the `CommandInfo` when launching a
dockerized executor; from `shell == true` to `shell = false`.
This means that flags are now passed directly rather than as
a long string.
i.e.
From: 'mesos-docker-executor --foo="bar" --some="thing"'
To: [ 'mesos-docker-executor', '--foo=bar', '--some=thing' ]
Review: https://reviews.apache.org/r/47215/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/cd46db80
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/cd46db80
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/cd46db80
Branch: refs/heads/master
Commit: cd46db8073fd1ee3d8bd63d3bfedab2e7a522bd7
Parents: 9b054cc
Author: Joseph Wu <jo...@mesosphere.io>
Authored: Fri May 27 17:19:26 2016 -0700
Committer: Jie Yu <yu...@gmail.com>
Committed: Fri May 27 17:19:26 2016 -0700
----------------------------------------------------------------------
src/slave/containerizer/docker.cpp | 19 ++++++++++++++-----
1 file changed, 14 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/cd46db80/src/slave/containerizer/docker.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/docker.cpp b/src/slave/containerizer/docker.cpp
index 47c4148..abc066b 100644
--- a/src/slave/containerizer/docker.cpp
+++ b/src/slave/containerizer/docker.cpp
@@ -315,13 +315,22 @@ DockerContainerizerProcess::Container::create(
Container::name(slaveId, stringify(id)),
containerWorkdir);
+ // Override the command with the docker command executor.
CommandInfo newCommandInfo;
- // TODO(tnachen): Pass flags directly into docker run.
- newCommandInfo.set_value(
- path::join(flags.launcher_dir, "mesos-docker-executor") +
- " " + stringify(dockerExecutorFlags));
+ newCommandInfo.set_shell(false);
- newCommandInfo.set_shell(true);
+ newCommandInfo.set_value(
+ path::join(flags.launcher_dir, "mesos-docker-executor"));
+
+ // Stringify the flags as arguments.
+ // This minimizes the need for escaping flag values.
+ foreachvalue (const flags::Flag& flag, dockerExecutorFlags) {
+ Option<string> value = flag.stringify(dockerExecutorFlags);
+ if (value.isSome()) {
+ newCommandInfo.add_arguments(
+ "--" + flag.effective_name().value + "=" + value.get());
+ }
+ }
if (taskInfo->has_command()) {
newCommandInfo.mutable_uris()->CopyFrom(taskInfo->command().uris());