You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by vi...@apache.org on 2016/03/02 00:37:53 UTC
[1/5] mesos git commit: Removed `grace_period_seconds` field from
executor API protobufs.
Repository: mesos
Updated Branches:
refs/heads/master 18882493c -> f01677183
Removed `grace_period_seconds` field from executor API protobufs.
This change removes the \`grace_period_second\` field from the
executor API protobufs. Instead, we would be relying on the environment
variable \`MESOS_EXECUTOR_SHUTDOWN_GRACE_PERIOD\` set by the agent.
Review: https://reviews.apache.org/r/44222/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/c2f284df
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/c2f284df
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/c2f284df
Branch: refs/heads/master
Commit: c2f284df1a00fe8910a6df0fa5ce493af7a9c586
Parents: 1888249
Author: Anand Mazumdar <ma...@gmail.com>
Authored: Tue Mar 1 15:37:19 2016 -0800
Committer: Vinod Kone <vi...@gmail.com>
Committed: Tue Mar 1 15:37:19 2016 -0800
----------------------------------------------------------------------
include/mesos/executor/executor.proto | 26 ++++++++++++--------------
include/mesos/v1/executor/executor.proto | 26 ++++++++++++--------------
2 files changed, 24 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/c2f284df/include/mesos/executor/executor.proto
----------------------------------------------------------------------
diff --git a/include/mesos/executor/executor.proto b/include/mesos/executor/executor.proto
index e905a12..a67dbad 100644
--- a/include/mesos/executor/executor.proto
+++ b/include/mesos/executor/executor.proto
@@ -37,8 +37,17 @@ message Event {
KILL = 3; // See 'Kill' below.
ACKNOWLEDGED = 4; // See 'Acknowledged' below.
MESSAGE = 5; // See 'Message' below.
- SHUTDOWN = 6; // See 'Shutdown' below.
- ERROR = 7; // See 'Error' below.
+ ERROR = 6; // See 'Error' below.
+
+ // Received when the agent asks the executor to shutdown/kill itself. The
+ // executor is then required to kill all its active tasks, send
+ // 'TASK_KILLED' status updates and gracefully exit. If an
+ // executor does not terminate within a certain period
+ // `MESOS_EXECUTOR_SHUTDOWN_GRACE_PERIOD` (an environment variable set by
+ // the agent upon executor startup), the agent will forcefully destroy the
+ // container where the executor is running. The agent would then send
+ // 'TASK_LOST' updates for any remaining active tasks of this executor.
+ SHUTDOWN = 7;
}
// First event received when the executor subscribes.
@@ -84,16 +93,6 @@ message Event {
required bytes data = 1;
}
- // Received when the agent asks the executor to shutdown/kill itself. The
- // executor is then required to kill all its active tasks, send 'TASK_KILLED'
- // status updates and gracefully exit. If an executor does not terminate
- // within a certain period ('grace_period_seconds'), the agent will forcefully
- // destroy the container where the executor is running. The agent would then
- // send 'TASK_LOST' updates for any remaining active tasks of this executor.
- message Shutdown {
- optional double grace_period_seconds = 1 [default = 5.0];
- }
-
// Received in case the executor sends invalid calls (e.g.,
// required values not set).
// TODO(arojas): Remove this once the old executor driver is no
@@ -112,8 +111,7 @@ message Event {
optional Launch launch = 4;
optional Kill kill = 5;
optional Message message = 6;
- optional Shutdown shutdown = 7;
- optional Error error = 8;
+ optional Error error = 7;
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/c2f284df/include/mesos/v1/executor/executor.proto
----------------------------------------------------------------------
diff --git a/include/mesos/v1/executor/executor.proto b/include/mesos/v1/executor/executor.proto
index dbbc265..c4ad0b4 100644
--- a/include/mesos/v1/executor/executor.proto
+++ b/include/mesos/v1/executor/executor.proto
@@ -37,8 +37,17 @@ message Event {
KILL = 3; // See 'Kill' below.
ACKNOWLEDGED = 4; // See 'Acknowledged' below.
MESSAGE = 5; // See 'Message' below.
- SHUTDOWN = 6; // See 'Shutdown' below.
- ERROR = 7; // See 'Error' below.
+ ERROR = 6; // See 'Error' below.
+
+ // Received when the agent asks the executor to shutdown/kill itself. The
+ // executor is then required to kill all its active tasks, send
+ // 'TASK_KILLED' status updates and gracefully exit. If an
+ // executor does not terminate within a certain period
+ // `MESOS_EXECUTOR_SHUTDOWN_GRACE_PERIOD` (an environment variable set by
+ // the agent upon executor startup), the agent will forcefully destroy the
+ // container where the executor is running. The agent would then send
+ // 'TASK_LOST' updates for any remaining active tasks of this executor.
+ SHUTDOWN = 7;
}
// First event received when the executor subscribes.
@@ -84,16 +93,6 @@ message Event {
required bytes data = 1;
}
- // Received when the agent asks the executor to shutdown/kill itself. The
- // executor is then required to kill all its active tasks, send 'TASK_KILLED'
- // status updates and gracefully exit. If an executor does not terminate
- // within a certain period ('grace_period_seconds'), the agent will forcefully
- // destroy the container where the executor is running. The agent would then
- // send 'TASK_LOST' updates for any remaining active tasks of this executor.
- message Shutdown {
- optional double grace_period_seconds = 1 [default = 5.0];
- }
-
// Received in case the executor sends invalid calls (e.g.,
// required values not set).
// TODO(arojas): Remove this once the old executor driver is no
@@ -112,8 +111,7 @@ message Event {
optional Launch launch = 4;
optional Kill kill = 5;
optional Message message = 6;
- optional Shutdown shutdown = 7;
- optional Error error = 8;
+ optional Error error = 7;
}
[3/5] mesos git commit: Modified executor library to read grace
period from environment.
Posted by vi...@apache.org.
Modified executor library to read grace period from environment.
This change modifies the executor library to read shutdown grace
period from the environment instead of reading it from the `Shutdown`
event in executor protobuf.
Review: https://reviews.apache.org/r/44223/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/588ecefa
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/588ecefa
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/588ecefa
Branch: refs/heads/master
Commit: 588ecefacf532c4166331bb858f136c94bf845e5
Parents: 7f9a3b3
Author: Anand Mazumdar <ma...@gmail.com>
Authored: Tue Mar 1 15:37:28 2016 -0800
Committer: Vinod Kone <vi...@gmail.com>
Committed: Tue Mar 1 15:37:28 2016 -0800
----------------------------------------------------------------------
src/executor/executor.cpp | 40 ++++++++++++++++++++++------------------
1 file changed, 22 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/588ecefa/src/executor/executor.cpp
----------------------------------------------------------------------
diff --git a/src/executor/executor.cpp b/src/executor/executor.cpp
index c5f22e7..c3e95ea 100644
--- a/src/executor/executor.cpp
+++ b/src/executor/executor.cpp
@@ -87,19 +87,16 @@ namespace executor {
class ShutdownProcess : public process::Process<ShutdownProcess>
{
public:
- ShutdownProcess(double _gracePeriod)
+ explicit ShutdownProcess(const Duration& _gracePeriod)
: gracePeriod(_gracePeriod) {}
protected:
virtual void initialize()
{
- Try<Duration> gracePeriod_ = Duration::create(gracePeriod);
- CHECK_SOME(gracePeriod_);
-
VLOG(1) << "Scheduling shutdown of the executor with grace period: "
- << gracePeriod_.get();
+ << gracePeriod;
- delay(gracePeriod_.get(), self(), &Self::kill);
+ delay(gracePeriod, self(), &Self::kill);
}
void kill()
@@ -117,7 +114,7 @@ protected:
}
private:
- const double gracePeriod;
+ const Duration gracePeriod;
};
@@ -229,6 +226,21 @@ public:
<< "in the environment";
}
}
+
+ // Get shutdown grace period from environment.
+ value = os::getenv("MESOS_EXECUTOR_SHUTDOWN_GRACE_PERIOD");
+ if (value.isSome()) {
+ Try<Duration> _shutdownGracePeriod = Duration::parse(value.get());
+
+ CHECK_SOME(_shutdownGracePeriod)
+ << "Failed to parse MESOS_EXECUTOR_SHUTDOWN_GRACE_PERIOD '"
+ << value.get() << "': " << _shutdownGracePeriod.error();
+
+ shutdownGracePeriod = _shutdownGracePeriod.get();
+ } else {
+ EXIT(1) << "Expecting 'MESOS_EXECUTOR_SHUTDOWN_GRACE_PERIOD' to be set "
+ << "in the environment";
+ }
}
void send(const Call& call)
@@ -601,23 +613,14 @@ protected:
}
if (event.type() == Event::SHUTDOWN) {
- shutdown(event.shutdown().grace_period_seconds());
+ shutdown();
}
}
void shutdown()
{
- Event event;
- event.set_type(Event::SHUTDOWN);
- event.mutable_shutdown();
-
- receive(event, true);
- }
-
- void shutdown(double gracePeriod)
- {
if (!local) {
- spawn(new ShutdownProcess(gracePeriod), true);
+ spawn(new ShutdownProcess(shutdownGracePeriod), true);
} else {
// Process any pending received events from agent and then terminate.
terminate(this, false);
@@ -660,6 +663,7 @@ private:
bool checkpoint;
Option<Duration> recoveryTimeout;
Option<Duration> maxBackoff;
+ Duration shutdownGracePeriod;
};
[4/5] mesos git commit: Modified docs to add info on
`MESOS_EXECUTOR_SHUTDOWN_GRACE_PERIOD`.
Posted by vi...@apache.org.
Modified docs to add info on `MESOS_EXECUTOR_SHUTDOWN_GRACE_PERIOD`.
Review: https://reviews.apache.org/r/44225/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/85b5a693
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/85b5a693
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/85b5a693
Branch: refs/heads/master
Commit: 85b5a693afa7e9dd51692e9c8b3015eac6e42c88
Parents: 588ecef
Author: Anand Mazumdar <ma...@gmail.com>
Authored: Tue Mar 1 15:37:31 2016 -0800
Committer: Vinod Kone <vi...@gmail.com>
Committed: Tue Mar 1 15:37:31 2016 -0800
----------------------------------------------------------------------
docs/executor-http-api.md | 10 ++++------
1 file changed, 4 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/85b5a693/docs/executor-http-api.md
----------------------------------------------------------------------
diff --git a/docs/executor-http-api.md b/docs/executor-http-api.md
index e41c2d2..d0e415f 100644
--- a/docs/executor-http-api.md
+++ b/docs/executor-http-api.md
@@ -5,7 +5,7 @@ layout: documentation
# Executor HTTP API
-Mesos 0.27.0 added **experimental** support for V1 Executor HTTP API.
+Mesos 0.28.0 added **experimental** support for V1 Executor HTTP API.
## Overview
@@ -292,16 +292,13 @@ MESSAGE Event (JSON)
### SHUTDOWN
-Sent by the agent in order to shutdown the executor. Once an executor gets a `SHUTDOWN` event it is required to kill all its tasks, send `TASK_KILLED` updates and gracefully exit. If an executor doesn't terminate within a certain period after the event was emitted (`grace_period_seconds`), the agent will forcefully destroy the container where the executor is running. The agent would then send `TASK_LOST` updates for any remaining active tasks of this executor.
+Sent by the agent in order to shutdown the executor. Once an executor gets a `SHUTDOWN` event it is required to kill all its tasks, send `TASK_KILLED` updates and gracefully exit. If an executor doesn't terminate within a certain period `MESOS_EXECUTOR_SHUTDOWN_GRACE_PERIOD` (an environment variable set by the agent upon executor startup), the agent will forcefully destroy the container where the executor is running. The agent would then send `TASK_LOST` updates for any remaining active tasks of this executor.
```
SHUTDOWN Event (JSON)
<event-length>
{
- "type" : "SHUTDOWN",
- "shutdown" : {
- "grace_period_seconds" : 5
- }
+ "type" : "SHUTDOWN"
}
```
@@ -329,6 +326,7 @@ The following environment variables are set by the agent that can be used by the
* `MESOS_DIRECTORY`: Path to the working directory for the executor.
* `MESOS_AGENT_ENDPOINT`: Agent endpoint i.e. ip:port to be used by the executor to connect to the agent.
* `MESOS_CHECKPOINT`: If set to true, denotes that framework has checkpointing enabled.
+* `MESOS_EXECUTOR_SHUTDOWN_GRACE_PERIOD`: Amount of time the agent would wait for an executor to shut down (e.g., 60 secs, 3mins etc.) after sending a `SHUTDOWN` event.
If `MESOS_CHECKPOINT` is set i.e. when framework checkpointing is enabled, the following additional variables are also set that can be used by the executor for retrying upon a disconnection with the agent:
[5/5] mesos git commit: Modified tests to remove `Shutdown` event.
Posted by vi...@apache.org.
Modified tests to remove `Shutdown` event.
This change modifies the existing tests to not rely on the `Shutdown` event
in the executor protobufs.
Review: https://reviews.apache.org/r/44226/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/f0167718
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/f0167718
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/f0167718
Branch: refs/heads/master
Commit: f01677183f4d14ae33fe0c3de8d76d0f65c7157e
Parents: 85b5a69
Author: Anand Mazumdar <ma...@gmail.com>
Authored: Tue Mar 1 15:37:36 2016 -0800
Committer: Vinod Kone <vi...@gmail.com>
Committed: Tue Mar 1 15:37:36 2016 -0800
----------------------------------------------------------------------
src/tests/mesos.hpp | 4 ++--
src/tests/scheduler_tests.cpp | 12 ++++++------
2 files changed, 8 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/f0167718/src/tests/mesos.hpp
----------------------------------------------------------------------
diff --git a/src/tests/mesos.hpp b/src/tests/mesos.hpp
index 0c55d59..5f29085 100644
--- a/src/tests/mesos.hpp
+++ b/src/tests/mesos.hpp
@@ -1006,7 +1006,7 @@ public:
MOCK_METHOD2_T(launch, void(Mesos*, const typename Event::Launch&));
MOCK_METHOD2_T(kill, void(Mesos*, const typename Event::Kill&));
MOCK_METHOD2_T(message, void(Mesos*, const typename Event::Message&));
- MOCK_METHOD2_T(shutdown, void(Mesos*, const typename Event::Shutdown&));
+ MOCK_METHOD1_T(shutdown, void(Mesos*));
MOCK_METHOD2_T(error, void(Mesos*, const typename Event::Error&));
MOCK_METHOD2_T(acknowledged,
void(Mesos*, const typename Event::Acknowledged&));
@@ -1030,7 +1030,7 @@ public:
message(mesos, event.message());
break;
case Event::SHUTDOWN:
- shutdown(mesos, event.shutdown());
+ shutdown(mesos);
break;
case Event::ERROR:
error(mesos, event.error());
http://git-wip-us.apache.org/repos/asf/mesos/blob/f0167718/src/tests/scheduler_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/scheduler_tests.cpp b/src/tests/scheduler_tests.cpp
index 70c5b21..2b1693e 100644
--- a/src/tests/scheduler_tests.cpp
+++ b/src/tests/scheduler_tests.cpp
@@ -467,7 +467,7 @@ TEST_P(SchedulerTest, TaskRunning)
AWAIT_READY(update);
- EXPECT_CALL(*executor, shutdown(_, _))
+ EXPECT_CALL(*executor, shutdown(_))
.Times(AtMost(1));
EXPECT_CALL(*executor, disconnected(_))
@@ -597,7 +597,7 @@ TEST_P(SchedulerTest, ReconcileTask)
EXPECT_EQ(v1::TaskStatus::REASON_RECONCILIATION,
event.get().update().status().reason());
- EXPECT_CALL(*executor, shutdown(_, _))
+ EXPECT_CALL(*executor, shutdown(_))
.Times(AtMost(1));
EXPECT_CALL(*executor, disconnected(_))
@@ -744,7 +744,7 @@ TEST_P(SchedulerTest, KillTask)
EXPECT_EQ(Event::UPDATE, event.get().type());
EXPECT_EQ(v1::TASK_KILLED, event.get().update().status().state());
- EXPECT_CALL(*executor, shutdown(_, _))
+ EXPECT_CALL(*executor, shutdown(_))
.Times(AtMost(1));
EXPECT_CALL(*executor, disconnected(_))
@@ -856,7 +856,7 @@ TEST_P(SchedulerTest, ShutdownExecutor)
EXPECT_EQ(v1::TASK_FINISHED, event.get().update().status().state());
Future<Nothing> shutdown;
- EXPECT_CALL(*executor, shutdown(_, _))
+ EXPECT_CALL(*executor, shutdown(_))
.WillOnce(FutureSatisfy(&shutdown));
{
@@ -986,7 +986,7 @@ TEST_P(SchedulerTest, Teardown)
EXPECT_EQ(v1::TASK_RUNNING, event.get().update().status().state());
Future<Nothing> shutdown;
- EXPECT_CALL(*executor, shutdown(_, _))
+ EXPECT_CALL(*executor, shutdown(_))
.WillOnce(FutureSatisfy(&shutdown));
{
@@ -1427,7 +1427,7 @@ TEST_P(SchedulerTest, Message)
AWAIT_READY(message);
ASSERT_EQ("hello world", message->data());
- EXPECT_CALL(*executor, shutdown(_, _))
+ EXPECT_CALL(*executor, shutdown(_))
.Times(AtMost(1));
EXPECT_CALL(*executor, disconnected(_))
[2/5] mesos git commit: Set `MESOS_EXECUTOR_SHUTDOWN_GRACE_PERIOD` in
agent.
Posted by vi...@apache.org.
Set `MESOS_EXECUTOR_SHUTDOWN_GRACE_PERIOD` in agent.
Review: https://reviews.apache.org/r/44224/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/7f9a3b3b
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/7f9a3b3b
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/7f9a3b3b
Branch: refs/heads/master
Commit: 7f9a3b3b1030dfe83df7507efd327ef41b46f3e6
Parents: c2f284d
Author: Anand Mazumdar <ma...@gmail.com>
Authored: Tue Mar 1 15:37:24 2016 -0800
Committer: Vinod Kone <vi...@gmail.com>
Committed: Tue Mar 1 15:37:24 2016 -0800
----------------------------------------------------------------------
src/slave/containerizer/containerizer.cpp | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/7f9a3b3b/src/slave/containerizer/containerizer.cpp
----------------------------------------------------------------------
diff --git a/src/slave/containerizer/containerizer.cpp b/src/slave/containerizer/containerizer.cpp
index 5990468..f6fc786 100644
--- a/src/slave/containerizer/containerizer.cpp
+++ b/src/slave/containerizer/containerizer.cpp
@@ -329,6 +329,8 @@ map<string, string> executorEnvironment(
environment["MESOS_SLAVE_PID"] = stringify(slavePid);
environment["MESOS_AGENT_ENDPOINT"] = stringify(slavePid.address);
environment["MESOS_CHECKPOINT"] = checkpoint ? "1" : "0";
+ environment["MESOS_EXECUTOR_SHUTDOWN_GRACE_PERIOD"] =
+ stringify(flags.executor_shutdown_grace_period);
if (checkpoint) {
environment["MESOS_RECOVERY_TIMEOUT"] = stringify(flags.recovery_timeout);