You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by vi...@apache.org on 2015/11/24 01:12:50 UTC
[1/5] mesos git commit: Removed redundant fields from Update message
in Executor API protobuf.
Repository: mesos
Updated Branches:
refs/heads/master 0bb09121f -> 619a0aa08
Removed redundant fields from Update message in Executor API protobuf.
This change removes the redundant fields in `Call::Update` message.
These fields can be retrieved from the corresponding fields in the
`TaskStatus` message.
Review: https://reviews.apache.org/r/40560
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/7383af0c
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/7383af0c
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/7383af0c
Branch: refs/heads/master
Commit: 7383af0c1264175e817a00c0695188b3552038ae
Parents: 0bb0912
Author: Anand Mazumdar <ma...@gmail.com>
Authored: Mon Nov 23 16:09:14 2015 -0800
Committer: Vinod Kone <vi...@gmail.com>
Committed: Mon Nov 23 16:09:14 2015 -0800
----------------------------------------------------------------------
include/mesos/executor/executor.proto | 2 --
include/mesos/v1/executor/executor.proto | 2 --
src/tests/executor_http_api_tests.cpp | 9 ---------
3 files changed, 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/7383af0c/include/mesos/executor/executor.proto
----------------------------------------------------------------------
diff --git a/include/mesos/executor/executor.proto b/include/mesos/executor/executor.proto
index 7cfad9f..8f97ad8 100644
--- a/include/mesos/executor/executor.proto
+++ b/include/mesos/executor/executor.proto
@@ -143,8 +143,6 @@ message Event {
// 'Acknowledged' in the 'Events' section above for the semantics.
message Update {
required TaskStatus status = 1;
- required double timestamp = 2;
- required bytes uuid = 3;
}
// Sends arbitrary binary data to the scheduler. Note that Mesos
http://git-wip-us.apache.org/repos/asf/mesos/blob/7383af0c/include/mesos/v1/executor/executor.proto
----------------------------------------------------------------------
diff --git a/include/mesos/v1/executor/executor.proto b/include/mesos/v1/executor/executor.proto
index 6807839..7eaa40b 100644
--- a/include/mesos/v1/executor/executor.proto
+++ b/include/mesos/v1/executor/executor.proto
@@ -143,8 +143,6 @@ message Event {
// 'Acknowledged' in the 'Events' section above for the semantics.
message Update {
required TaskStatus status = 1;
- required double timestamp = 2;
- required bytes uuid = 3;
}
// Sends arbitrary binary data to the scheduler. Note that Mesos
http://git-wip-us.apache.org/repos/asf/mesos/blob/7383af0c/src/tests/executor_http_api_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/executor_http_api_tests.cpp b/src/tests/executor_http_api_tests.cpp
index 612890f..fe9df1f 100644
--- a/src/tests/executor_http_api_tests.cpp
+++ b/src/tests/executor_http_api_tests.cpp
@@ -616,9 +616,6 @@ TEST_P(ExecutorHttpApiTest, StatusUpdateCallFailedValidation)
call.mutable_framework_id()->set_value("dummy_framework_id");
call.mutable_executor_id()->set_value("call_level_executor_id");
- call.mutable_update()->set_uuid(UUID::random().toBytes());
- call.mutable_update()->set_timestamp(0);
-
v1::TaskStatus* status = call.mutable_update()->mutable_status();
status->mutable_executor_id()->set_value("update_level_executor_id");
@@ -646,9 +643,6 @@ TEST_P(ExecutorHttpApiTest, StatusUpdateCallFailedValidation)
call.mutable_framework_id()->set_value("dummy_framework_id");
call.mutable_executor_id()->set_value("call_level_executor_id");
- call.mutable_update()->set_uuid(UUID::random().toBytes());
- call.mutable_update()->set_timestamp(0);
-
v1::TaskStatus* status = call.mutable_update()->mutable_status();
status->mutable_executor_id()->set_value("call_level_executor_id");
@@ -676,9 +670,6 @@ TEST_P(ExecutorHttpApiTest, StatusUpdateCallFailedValidation)
call.mutable_framework_id()->set_value("dummy_framework_id");
call.mutable_executor_id()->set_value("call_level_executor_id");
- call.mutable_update()->set_uuid(UUID::random().toBytes());
- call.mutable_update()->set_timestamp(0);
-
v1::TaskStatus* status = call.mutable_update()->mutable_status();
status->mutable_executor_id()->set_value("call_level_executor_id");
[2/5] mesos git commit: Added an overload for createStatusUpdate(...)
function.
Posted by vi...@apache.org.
Added an overload for createStatusUpdate(...) function.
This change adds an overload for the existing `createStatusUpdate(...)`
function. This is later used by the Agent Executor HTTP API handler to
get an `StatusUpdateMessage` from an `Call::Update` message and then
invoke `Slave::statusUpdate` function.
Review: https://reviews.apache.org/r/40561
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/0861f8fc
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/0861f8fc
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/0861f8fc
Branch: refs/heads/master
Commit: 0861f8fcf55d4acae98d9e7f9fdfdf0b250d805c
Parents: 7383af0
Author: Anand Mazumdar <ma...@gmail.com>
Authored: Mon Nov 23 16:09:44 2015 -0800
Committer: Vinod Kone <vi...@gmail.com>
Committed: Mon Nov 23 16:09:45 2015 -0800
----------------------------------------------------------------------
src/common/protobuf_utils.cpp | 33 +++++++++++++++++++++++++++++++++
src/common/protobuf_utils.hpp | 6 ++++++
2 files changed, 39 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/0861f8fc/src/common/protobuf_utils.cpp
----------------------------------------------------------------------
diff --git a/src/common/protobuf_utils.cpp b/src/common/protobuf_utils.cpp
index 6fec566..6e1eb0b 100644
--- a/src/common/protobuf_utils.cpp
+++ b/src/common/protobuf_utils.cpp
@@ -116,6 +116,39 @@ StatusUpdate createStatusUpdate(
}
+StatusUpdate createStatusUpdate(
+ const FrameworkID& frameworkId,
+ const TaskStatus& status,
+ const Option<SlaveID>& slaveId)
+{
+ StatusUpdate update;
+
+ update.mutable_framework_id()->MergeFrom(frameworkId);
+
+ if (status.has_executor_id()) {
+ update.mutable_executor_id()->MergeFrom(status.executor_id());
+ }
+
+ if (slaveId.isSome()) {
+ update.mutable_slave_id()->MergeFrom(slaveId.get());
+ }
+
+ update.mutable_status()->MergeFrom(status);
+
+ if (!status.has_timestamp()) {
+ update.set_timestamp(process::Clock::now().secs());
+ } else {
+ update.set_timestamp(status.timestamp());
+ }
+
+ if (status.has_uuid()) {
+ update.set_uuid(status.uuid());
+ }
+
+ return update;
+}
+
+
Task createTask(
const TaskInfo& task,
const TaskState& state,
http://git-wip-us.apache.org/repos/asf/mesos/blob/0861f8fc/src/common/protobuf_utils.hpp
----------------------------------------------------------------------
diff --git a/src/common/protobuf_utils.hpp b/src/common/protobuf_utils.hpp
index 520490d..7280c9f 100644
--- a/src/common/protobuf_utils.hpp
+++ b/src/common/protobuf_utils.hpp
@@ -68,6 +68,12 @@ StatusUpdate createStatusUpdate(
const Option<ContainerStatus>& containerStatus = None());
+StatusUpdate createStatusUpdate(
+ const FrameworkID& frameworkId,
+ const TaskStatus& status,
+ const Option<SlaveID>& slaveId);
+
+
Task createTask(
const TaskInfo& task,
const TaskState& state,
[4/5] mesos git commit: Modified stateUpdate(...) function to get the
TaskStatus object by constant reference instead of copy.
Posted by vi...@apache.org.
Modified stateUpdate(...) function to get the TaskStatus object by
constant reference instead of copy.
Review: https://reviews.apache.org/r/40570
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/aa67fc56
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/aa67fc56
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/aa67fc56
Branch: refs/heads/master
Commit: aa67fc566adf7fb5a0e67e37382d8b15119d0d3f
Parents: 591a2cb
Author: Anand Mazumdar <ma...@gmail.com>
Authored: Mon Nov 23 16:10:50 2015 -0800
Committer: Vinod Kone <vi...@gmail.com>
Committed: Mon Nov 23 16:10:51 2015 -0800
----------------------------------------------------------------------
src/slave/slave.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/aa67fc56/src/slave/slave.cpp
----------------------------------------------------------------------
diff --git a/src/slave/slave.cpp b/src/slave/slave.cpp
index 7b151ea..4dc60c8 100644
--- a/src/slave/slave.cpp
+++ b/src/slave/slave.cpp
@@ -2850,7 +2850,7 @@ void Slave::statusUpdate(StatusUpdate update, const UPID& pid)
ipAddress->set_ip_address(stringify(self().address.ip));
}
- TaskStatus status = update.status();
+ const TaskStatus& status = update.status();
Executor* executor = framework->getExecutor(status.task_id());
if (executor == NULL) {
[5/5] mesos git commit: Added functionality for handling status
updates from HTTP based executors.
Posted by vi...@apache.org.
Added functionality for handling status updates from HTTP based
executors.
This change adds the ability for the agent to handle status updates
from HTTP based executors. Previously, the existing `statusUpdate`
method used to handle status updates sent from slave with
`pid == UPID()`, sent from other executors/on behalf of other
executors with `pid == Some()`. This change modifies the argument to be
`Option<PID>`. This ensures that `pid == None()` when set, the ACK is
correctly routed to the corresponding HTTP based executor.
Review: https://reviews.apache.org/r/40563
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/619a0aa0
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/619a0aa0
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/619a0aa0
Branch: refs/heads/master
Commit: 619a0aa081232f060d4e4941714c05fa8ed34303
Parents: aa67fc5
Author: Anand Mazumdar <ma...@gmail.com>
Authored: Mon Nov 23 16:11:28 2015 -0800
Committer: Vinod Kone <vi...@gmail.com>
Committed: Mon Nov 23 16:11:29 2015 -0800
----------------------------------------------------------------------
src/slave/http.cpp | 6 +++++
src/slave/slave.cpp | 61 ++++++++++++++++++++++++++++++++++--------------
src/slave/slave.hpp | 9 +++----
3 files changed, 55 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/619a0aa0/src/slave/http.cpp
----------------------------------------------------------------------
diff --git a/src/slave/http.cpp b/src/slave/http.cpp
index 35740e5..c3247f1 100644
--- a/src/slave/http.cpp
+++ b/src/slave/http.cpp
@@ -312,6 +312,12 @@ Future<Response> Slave::Http::executor(const Request& request) const
}
case executor::Call::UPDATE: {
+ slave->statusUpdate(protobuf::createStatusUpdate(
+ call.framework_id(),
+ call.update().status(),
+ slave->info.id()),
+ None());
+
return Accepted();
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/619a0aa0/src/slave/slave.cpp
----------------------------------------------------------------------
diff --git a/src/slave/slave.cpp b/src/slave/slave.cpp
index 4dc60c8..9055f2a 100644
--- a/src/slave/slave.cpp
+++ b/src/slave/slave.cpp
@@ -2758,9 +2758,10 @@ void Slave::reregisterExecutorTimeout()
// reliable delivery of status updates. Since executor driver caches
// unacked updates it is important that whoever sent the update gets
// acknowledgement for it.
-void Slave::statusUpdate(StatusUpdate update, const UPID& pid)
+void Slave::statusUpdate(StatusUpdate update, const Option<UPID>& pid)
{
- LOG(INFO) << "Handling status update " << update << " from " << pid;
+ LOG(INFO) << "Handling status update " << update
+ << (pid.isSome() ? " from " + stringify(pid.get()) : "");
CHECK(state == RECOVERING || state == DISCONNECTED ||
state == RUNNING || state == TERMINATING)
@@ -2787,7 +2788,8 @@ void Slave::statusUpdate(StatusUpdate update, const UPID& pid)
if (update.has_executor_id()) {
if (update.status().has_executor_id() &&
update.status().executor_id() != update.executor_id()) {
- LOG(WARNING) << "Executor ID mismatch in status update from " << pid
+ LOG(WARNING) << "Executor ID mismatch in status update"
+ << (pid.isSome() ? " from " + stringify(pid.get()) : "")
<< "; overwriting received '"
<< update.status().executor_id() << "' with expected'"
<< update.executor_id() << "'";
@@ -2899,10 +2901,11 @@ void Slave::statusUpdate(StatusUpdate update, const UPID& pid)
// TODO(vinod): Revisit these semantics when we disallow executors
// from sending updates for tasks that belong to other executors.
- if (pid != UPID() &&
+ if (pid.isSome() &&
+ pid != UPID() &&
executor->pid.isSome() &&
- executor->pid.get() != pid) {
- LOG(WARNING) << "Received status update " << update << " from " << pid
+ executor->pid != pid) {
+ LOG(WARNING) << "Received status update " << update << " from " << pid.get()
<< " on behalf of a different executor '" << executor->id
<< "' (" << executor->pid.get() << ")";
}
@@ -2955,7 +2958,7 @@ void Slave::statusUpdate(StatusUpdate update, const UPID& pid)
void Slave::_statusUpdate(
const Option<Future<Nothing>>& future,
const StatusUpdate& update,
- const UPID& pid,
+ const Option<UPID>& pid,
const ExecutorID& executorId,
const ContainerID& containerId,
bool checkpoint)
@@ -2999,25 +3002,49 @@ void Slave::_statusUpdate(
void Slave::__statusUpdate(
const Future<Nothing>& future,
const StatusUpdate& update,
- const UPID& pid)
+ const Option<UPID>& pid)
{
CHECK_READY(future) << "Failed to handle status update " << update;
VLOG(1) << "Status update manager successfully handled status update "
<< update;
+ if (pid == UPID()) {
+ return;
+ }
+
+ StatusUpdateAcknowledgementMessage message;
+ message.mutable_framework_id()->MergeFrom(update.framework_id());
+ message.mutable_slave_id()->MergeFrom(update.slave_id());
+ message.mutable_task_id()->MergeFrom(update.status().task_id());
+ message.set_uuid(update.uuid());
+
// Status update manager successfully handled the status update.
// Acknowledge the executor, if we have a valid pid.
- if (pid != UPID()) {
+ if (pid.isSome()) {
LOG(INFO) << "Sending acknowledgement for status update " << update
- << " to " << pid;
- StatusUpdateAcknowledgementMessage message;
- message.mutable_framework_id()->MergeFrom(update.framework_id());
- message.mutable_slave_id()->MergeFrom(update.slave_id());
- message.mutable_task_id()->MergeFrom(update.status().task_id());
- message.set_uuid(update.uuid());
-
- send(pid, message);
+ << " to " << pid.get();
+
+ send(pid.get(), message);
+ } else {
+ // Acknowledge the HTTP based executor.
+ Framework* framework = getFramework(update.framework_id());
+ if (framework == NULL) {
+ LOG(WARNING) << "Ignoring sending acknowledgement for status update "
+ << update << " of unknown framework";
+ return;
+ }
+
+ Executor* executor = framework->getExecutor(update.status().task_id());
+ if (executor == NULL) {
+ // Refer to the comments in 'statusUpdate()' on when this can
+ // happen.
+ LOG(WARNING) << "Ignoring sending acknowledgement for status update "
+ << update << " of unknown executor";
+ return;
+ }
+
+ executor->send(message);
}
}
http://git-wip-us.apache.org/repos/asf/mesos/blob/619a0aa0/src/slave/slave.hpp
----------------------------------------------------------------------
diff --git a/src/slave/slave.hpp b/src/slave/slave.hpp
index 2049b49..5ee133a 100644
--- a/src/slave/slave.hpp
+++ b/src/slave/slave.hpp
@@ -195,17 +195,18 @@ public:
// NOTE: If 'pid' is a valid UPID an ACK is sent to this pid
// after the update is successfully handled. If pid == UPID()
// no ACK is sent. The latter is used by the slave to send
- // status updates it generated (e.g., TASK_LOST).
+ // status updates it generated (e.g., TASK_LOST). If pid == None()
+ // an ACK is sent to the corresponding HTTP based executor.
// NOTE: StatusUpdate is passed by value because it is modified
// to ensure source field is set.
- void statusUpdate(StatusUpdate update, const process::UPID& pid);
+ void statusUpdate(StatusUpdate update, const Option<process::UPID>& pid);
// Continue handling the status update after optionally updating the
// container's resources.
void _statusUpdate(
const Option<Future<Nothing>>& future,
const StatusUpdate& update,
- const UPID& pid,
+ const Option<process::UPID>& pid,
const ExecutorID& executorId,
const ContainerID& containerId,
bool checkpoint);
@@ -216,7 +217,7 @@ public:
void __statusUpdate(
const process::Future<Nothing>& future,
const StatusUpdate& update,
- const process::UPID& pid);
+ const Option<process::UPID>& pid);
// This is called by status update manager to forward a status
// update to the master. Note that the latest state of the task is
[3/5] mesos git commit: Added validation for 'uuid' field being
present.
Posted by vi...@apache.org.
Added validation for 'uuid' field being present.
This change adds validation to ensure `uuid` field is always present
in `Call::Update::TaskStatus` message sent by the Executor.
Review: https://reviews.apache.org/r/40562
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/591a2cb6
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/591a2cb6
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/591a2cb6
Branch: refs/heads/master
Commit: 591a2cb6d111e0ae5dc82e11e7f3b0a81fda4389
Parents: 0861f8f
Author: Anand Mazumdar <ma...@gmail.com>
Authored: Mon Nov 23 16:10:15 2015 -0800
Committer: Vinod Kone <vi...@gmail.com>
Committed: Mon Nov 23 16:10:15 2015 -0800
----------------------------------------------------------------------
src/slave/validation.cpp | 4 ++++
1 file changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/591a2cb6/src/slave/validation.cpp
----------------------------------------------------------------------
diff --git a/src/slave/validation.cpp b/src/slave/validation.cpp
index 614a13f..bc8d671 100644
--- a/src/slave/validation.cpp
+++ b/src/slave/validation.cpp
@@ -54,6 +54,10 @@ Option<Error> validate(const mesos::executor::Call& call)
const TaskStatus& status = call.update().status();
+ if (!status.has_uuid()) {
+ return Error("Expecting 'uuid' to be present");
+ }
+
if (status.has_executor_id() &&
status.executor_id().value()
!= call.executor_id().value()) {