You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by bb...@apache.org on 2018/02/19 14:19:12 UTC
[1/4] mesos git commit: Added comparison operators for operations.
Repository: mesos
Updated Branches:
refs/heads/master 2c6952694 -> 8c9184a03
Added comparison operators for operations.
Review: https://reviews.apache.org/r/65671/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/ddecf8ee
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/ddecf8ee
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/ddecf8ee
Branch: refs/heads/master
Commit: ddecf8ee6f75ce631c5e91c49220eed12d600910
Parents: 2c69526
Author: Jan Schlicht <ja...@mesosphere.io>
Authored: Mon Feb 19 15:15:35 2018 +0100
Committer: Benjamin Bannier <bb...@apache.org>
Committed: Mon Feb 19 15:15:35 2018 +0100
----------------------------------------------------------------------
include/mesos/type_utils.hpp | 2 ++
include/mesos/v1/mesos.hpp | 2 ++
src/common/type_utils.cpp | 14 ++++++++++++++
src/v1/mesos.cpp | 14 ++++++++++++++
4 files changed, 32 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/ddecf8ee/include/mesos/type_utils.hpp
----------------------------------------------------------------------
diff --git a/include/mesos/type_utils.hpp b/include/mesos/type_utils.hpp
index af2b187..2fff71f 100644
--- a/include/mesos/type_utils.hpp
+++ b/include/mesos/type_utils.hpp
@@ -62,6 +62,7 @@ bool operator==(const ExecutorInfo& left, const ExecutorInfo& right);
bool operator==(const Label& left, const Label& right);
bool operator==(const Labels& left, const Labels& right);
bool operator==(const MasterInfo& left, const MasterInfo& right);
+bool operator==(const Offer::Operation& left, const Offer::Operation& right);
bool operator==(const OperationStatus& left, const OperationStatus& right);
bool operator==(
@@ -84,6 +85,7 @@ bool operator==(const Volume& left, const Volume& right);
bool operator!=(const CheckStatusInfo& left, const CheckStatusInfo& right);
bool operator!=(const ExecutorInfo& left, const ExecutorInfo& right);
bool operator!=(const Labels& left, const Labels& right);
+bool operator!=(const Offer::Operation& left, const Offer::Operation& right);
bool operator!=(const OperationStatus& left, const OperationStatus& right);
bool operator!=(const TaskStatus& left, const TaskStatus& right);
http://git-wip-us.apache.org/repos/asf/mesos/blob/ddecf8ee/include/mesos/v1/mesos.hpp
----------------------------------------------------------------------
diff --git a/include/mesos/v1/mesos.hpp b/include/mesos/v1/mesos.hpp
index f16568e..04b8fd5 100644
--- a/include/mesos/v1/mesos.hpp
+++ b/include/mesos/v1/mesos.hpp
@@ -60,6 +60,7 @@ bool operator==(const FileInfo& left, const FileInfo& right);
bool operator==(const Label& left, const Label& right);
bool operator==(const Labels& left, const Labels& right);
bool operator==(const MasterInfo& left, const MasterInfo& right);
+bool operator==(const Offer::Operation& left, const Offer::Operation& right);
bool operator==(
const ResourceProviderInfo& left,
@@ -75,6 +76,7 @@ bool operator==(const URL& left, const URL& right);
bool operator==(const Volume& left, const Volume& right);
bool operator!=(const Labels& left, const Labels& right);
+bool operator!=(const Offer::Operation& left, const Offer::Operation& right);
bool operator!=(const TaskStatus& left, const TaskStatus& right);
http://git-wip-us.apache.org/repos/asf/mesos/blob/ddecf8ee/src/common/type_utils.cpp
----------------------------------------------------------------------
diff --git a/src/common/type_utils.cpp b/src/common/type_utils.cpp
index a4d5dcb..3c2711b 100644
--- a/src/common/type_utils.cpp
+++ b/src/common/type_utils.cpp
@@ -16,6 +16,8 @@
#include <ostream>
+#include <google/protobuf/util/message_differencer.h>
+
#include <mesos/attributes.hpp>
#include <mesos/mesos.hpp>
#include <mesos/resources.hpp>
@@ -439,6 +441,12 @@ bool operator==(
}
+bool operator==(const Offer::Operation& left, const Offer::Operation& right)
+{
+ return google::protobuf::util::MessageDifferencer::Equals(left, right);
+}
+
+
bool operator==(const OperationStatus& left, const OperationStatus& right)
{
if (left.has_operation_id() != right.has_operation_id()) {
@@ -478,6 +486,12 @@ bool operator==(const OperationStatus& left, const OperationStatus& right)
}
+bool operator!=(const Offer::Operation& left, const Offer::Operation& right)
+{
+ return !(left == right);
+}
+
+
bool operator!=(const OperationStatus& left, const OperationStatus& right)
{
return !(left == right);
http://git-wip-us.apache.org/repos/asf/mesos/blob/ddecf8ee/src/v1/mesos.cpp
----------------------------------------------------------------------
diff --git a/src/v1/mesos.cpp b/src/v1/mesos.cpp
index 576f367..1eb53f3 100644
--- a/src/v1/mesos.cpp
+++ b/src/v1/mesos.cpp
@@ -16,6 +16,8 @@
#include <ostream>
+#include <google/protobuf/util/message_differencer.h>
+
#include <stout/protobuf.hpp>
#include <stout/uuid.hpp>
@@ -395,6 +397,18 @@ bool operator==(const MasterInfo& left, const MasterInfo& right)
}
+bool operator==(const Offer::Operation& left, const Offer::Operation& right)
+{
+ return google::protobuf::util::MessageDifferencer::Equals(left, right);
+}
+
+
+bool operator!=(const Offer::Operation& left, const Offer::Operation& right)
+{
+ return !(left == right);
+}
+
+
bool operator==(
const ResourceProviderInfo::Storage& left,
const ResourceProviderInfo::Storage& right)
[4/4] mesos git commit: Tested correct operation handling during
master failover.
Posted by bb...@apache.org.
Tested correct operation handling during master failover.
Review: https://reviews.apache.org/r/65045/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/8c9184a0
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/8c9184a0
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/8c9184a0
Branch: refs/heads/master
Commit: 8c9184a03fa6b6fe842eb3554220d3ed2c327cdc
Parents: c6de89e
Author: Jan Schlicht <ja...@mesosphere.io>
Authored: Mon Feb 19 15:15:55 2018 +0100
Committer: Benjamin Bannier <bb...@apache.org>
Committed: Mon Feb 19 15:15:55 2018 +0100
----------------------------------------------------------------------
src/tests/master_tests.cpp | 212 ++++++++++++++++++++++++++++++++++++++++
1 file changed, 212 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/8c9184a0/src/tests/master_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/master_tests.cpp b/src/tests/master_tests.cpp
index 28663c7..3705fa7 100644
--- a/src/tests/master_tests.cpp
+++ b/src/tests/master_tests.cpp
@@ -8743,6 +8743,218 @@ TEST_F(MasterTest, UpdateSlaveMessageWithPendingOffers)
}
+// Tests that the master correctly handles resource provider operations
+// that finished during a master failover.
+TEST_F(MasterTest, OperationUpdateDuringFailover)
+{
+ Clock::pause();
+
+ master::Flags masterFlags = CreateMasterFlags();
+
+ Try<Owned<cluster::Master>> master = StartMaster(masterFlags);
+ ASSERT_SOME(master);
+
+ slave::Flags slaveFlags = CreateSlaveFlags();
+
+ // TODO(nfnt): Remove this once 'MockResourceProvider' supports
+ // authentication.
+ slaveFlags.authenticate_http_readwrite = false;
+
+ // Set the resource provider capability.
+ vector<SlaveInfo::Capability> capabilities = slave::AGENT_CAPABILITIES();
+ SlaveInfo::Capability capability;
+ capability.set_type(SlaveInfo::Capability::RESOURCE_PROVIDER);
+ capabilities.push_back(capability);
+
+ slaveFlags.agent_features = SlaveCapabilities();
+ slaveFlags.agent_features->mutable_capabilities()->CopyFrom(
+ {capabilities.begin(), capabilities.end()});
+
+ Future<UpdateSlaveMessage> updateSlaveMessage =
+ FUTURE_PROTOBUF(UpdateSlaveMessage(), _, _);
+
+ StandaloneMasterDetector detector(master.get()->pid);
+ Try<Owned<cluster::Slave>> slave = StartSlave(&detector, slaveFlags);
+ ASSERT_SOME(slave);
+
+ Clock::advance(slaveFlags.registration_backoff_factor);
+
+ AWAIT_READY(updateSlaveMessage);
+
+ // Register a resource provider with the agent.
+ mesos::v1::ResourceProviderInfo resourceProviderInfo;
+ resourceProviderInfo.set_type("org.apache.mesos.resource_provider.test");
+ resourceProviderInfo.set_name("test");
+
+ v1::Resources resourceProviderResources = v1::createDiskResource(
+ "200",
+ "*",
+ None(),
+ None(),
+ v1::createDiskSourceRaw());
+
+ v1::MockResourceProvider resourceProvider(
+ resourceProviderInfo,
+ resourceProviderResources);
+
+ Owned<EndpointDetector> endpointDetector(
+ resource_provider::createEndpointDetector(slave.get()->pid));
+
+ updateSlaveMessage = FUTURE_PROTOBUF(UpdateSlaveMessage(), _, _);
+
+ resourceProvider.start(
+ endpointDetector,
+ ContentType::PROTOBUF,
+ v1::DEFAULT_CREDENTIAL);
+
+ AWAIT_READY(updateSlaveMessage);
+
+ // Start a framework to operate on offers.
+ MockScheduler sched;
+ TestingMesosSchedulerDriver driver(&sched, &detector);
+
+ // Expect a registration as well as a re-registration after master
+ // failover.
+ EXPECT_CALL(sched, registered(&driver, _, _))
+ .Times(2);
+
+ Future<vector<Offer>> offers;
+ EXPECT_CALL(sched, resourceOffers(&driver, _))
+ .WillOnce(FutureArg<1>(&offers))
+ .WillRepeatedly(Return()); // Ignore subsequent offers.
+
+ driver.start();
+
+ AWAIT_READY(offers);
+ ASSERT_FALSE(offers->empty());
+ const Offer& offer = offers->front();
+
+ Option<Resource> rawDisk;
+
+ foreach (const Resource& resource, offer.resources()) {
+ if (resource.has_provider_id() &&
+ resource.has_disk() &&
+ resource.disk().has_source() &&
+ resource.disk().source().type() == Resource::DiskInfo::Source::RAW) {
+ rawDisk = resource;
+ break;
+ }
+ }
+
+ ASSERT_SOME(rawDisk);
+
+ Future<mesos::v1::resource_provider::Event::ApplyOperation> operation;
+ EXPECT_CALL(resourceProvider, applyOperation(_))
+ .WillOnce(FutureArg<0>(&operation));
+
+ driver.acceptOffers(
+ {offer.id()},
+ {CREATE_VOLUME(rawDisk.get(), Resource::DiskInfo::Source::MOUNT)});
+
+ AWAIT_READY(operation);
+
+ Option<mesos::v1::UUID> operationUUID;
+
+ {
+ v1::master::Call call;
+ call.set_type(v1::master::Call::GET_OPERATIONS);
+
+ process::http::Headers headers = createBasicAuthHeaders(DEFAULT_CREDENTIAL);
+ headers["Accept"] = stringify(ContentType::PROTOBUF);
+
+ Future<Response> response = process::http::post(
+ master.get()->pid,
+ "api/v1",
+ headers,
+ serialize(ContentType::PROTOBUF, call),
+ stringify(ContentType::PROTOBUF));
+
+ AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
+
+ Try<v1::master::Response> response_ =
+ deserialize<v1::master::Response>(ContentType::PROTOBUF, response->body);
+
+ ASSERT_SOME(response_);
+ const v1::master::Response::GetOperations& operations =
+ response_->get_operations();
+
+ ASSERT_EQ(1, operations.operations_size());
+ EXPECT_EQ(
+ mesos::v1::OperationState::OPERATION_PENDING,
+ operations.operations(0).latest_status().state());
+
+ operationUUID = operations.operations(0).uuid();
+ }
+
+ CHECK_SOME(operationUUID);
+
+ EXPECT_CALL(sched, disconnected(&driver));
+
+ // Drop the operation update for the finished operation.
+ // As we fail over the master immediately afterwards, we expect
+ // that the operation update will be part of the agent's
+ // `UPDATE_STATE` message when re-registering with the master.
+ Future<UpdateOperationStatusMessage> updateOperationStatusMessage =
+ DROP_PROTOBUF(UpdateOperationStatusMessage(), _, _);
+
+ // Finish the pending operation.
+ resourceProvider.operationDefault(operation.get());
+
+ AWAIT_READY(updateOperationStatusMessage);
+
+ // Fail over the master.
+ master->reset();
+
+ updateSlaveMessage = FUTURE_PROTOBUF(UpdateSlaveMessage(), _, _);
+
+ EXPECT_CALL(sched, offerRescinded(&driver, _))
+ .WillRepeatedly(Return());
+
+ // Start a new master and have agent and framework reconnect.
+ // The reconnected agent should report the converted resources.
+ master = StartMaster(masterFlags);
+ detector.appoint(master.get()->pid);
+
+ Clock::advance(slaveFlags.registration_backoff_factor);
+ Clock::settle();
+
+ AWAIT_READY(updateSlaveMessage);
+
+ {
+ v1::master::Call call;
+ call.set_type(v1::master::Call::GET_OPERATIONS);
+
+ process::http::Headers headers = createBasicAuthHeaders(DEFAULT_CREDENTIAL);
+ headers["Accept"] = stringify(ContentType::PROTOBUF);
+
+ Future<Response> response = process::http::post(
+ master.get()->pid,
+ "api/v1",
+ headers,
+ serialize(ContentType::PROTOBUF, call),
+ stringify(ContentType::PROTOBUF));
+
+ AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
+
+ Try<v1::master::Response> response_ =
+ deserialize<v1::master::Response>(ContentType::PROTOBUF, response->body);
+
+ ASSERT_SOME(response_);
+ const v1::master::Response::GetOperations& operations =
+ response_->get_operations();
+
+ ASSERT_EQ(1, operations.operations_size());
+ EXPECT_EQ(
+ mesos::v1::OperationState::OPERATION_FINISHED,
+ operations.operations(0).latest_status().state());
+ EXPECT_EQ(operationUUID.get(), operations.operations(0).uuid());
+ }
+
+ driver.stop();
+ driver.join();
+}
+
+
class MasterTestPrePostReservationRefinement
: public MasterTest,
public WithParamInterface<bool> {
[2/4] mesos git commit: Added comparison operators for 'v1::UUID'.
Posted by bb...@apache.org.
Added comparison operators for 'v1::UUID'.
Review: https://reviews.apache.org/r/65672/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/2aed1d6c
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/2aed1d6c
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/2aed1d6c
Branch: refs/heads/master
Commit: 2aed1d6ca2c31546d700dd79484c193625fc8cc9
Parents: ddecf8e
Author: Jan Schlicht <ja...@mesosphere.io>
Authored: Mon Feb 19 15:15:40 2018 +0100
Committer: Benjamin Bannier <bb...@apache.org>
Committed: Mon Feb 19 15:15:40 2018 +0100
----------------------------------------------------------------------
include/mesos/v1/mesos.hpp | 12 ++++++++++++
1 file changed, 12 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/2aed1d6c/include/mesos/v1/mesos.hpp
----------------------------------------------------------------------
diff --git a/include/mesos/v1/mesos.hpp b/include/mesos/v1/mesos.hpp
index 04b8fd5..15723a2 100644
--- a/include/mesos/v1/mesos.hpp
+++ b/include/mesos/v1/mesos.hpp
@@ -124,6 +124,12 @@ inline bool operator==(
}
+inline bool operator==(const UUID& left, const UUID& right)
+{
+ return left.value() == right.value();
+}
+
+
inline bool operator==(const AgentID& left, const AgentID& right)
{
return left.value() == right.value();
@@ -277,6 +283,12 @@ inline bool operator!=(
}
+inline bool operator!=(const UUID& left, const UUID& right)
+{
+ return !(left == right);
+}
+
+
inline bool operator!=(const AgentID& left, const AgentID& right)
{
return left.value() != right.value();
[3/4] mesos git commit: Added the v1 API 'GET_OPERATIONS' call for
master and agent.
Posted by bb...@apache.org.
Added the v1 API 'GET_OPERATIONS' call for master and agent.
The 'GET_OPERATIONS' call lists all operations known to master or agent.
Review: https://reviews.apache.org/r/65044/
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/c6de89e1
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/c6de89e1
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/c6de89e1
Branch: refs/heads/master
Commit: c6de89e148bec735bb6fb975a1d6b17e3f6192c9
Parents: 2aed1d6
Author: Jan Schlicht <ja...@mesosphere.io>
Authored: Mon Feb 19 15:15:45 2018 +0100
Committer: Benjamin Bannier <bb...@apache.org>
Committed: Mon Feb 19 15:15:45 2018 +0100
----------------------------------------------------------------------
include/mesos/agent/agent.proto | 11 ++
include/mesos/master/master.proto | 9 +
include/mesos/v1/agent/agent.proto | 11 ++
include/mesos/v1/master/master.proto | 9 +
src/master/http.cpp | 28 +++
src/master/master.hpp | 5 +
src/master/validation.cpp | 3 +
src/slave/http.cpp | 28 +++
src/slave/http.hpp | 5 +
src/slave/validation.cpp | 3 +
src/tests/api_tests.cpp | 289 ++++++++++++++++++++++++++++++
11 files changed, 401 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/c6de89e1/include/mesos/agent/agent.proto
----------------------------------------------------------------------
diff --git a/include/mesos/agent/agent.proto b/include/mesos/agent/agent.proto
index 3158200..7d92cb8 100644
--- a/include/mesos/agent/agent.proto
+++ b/include/mesos/agent/agent.proto
@@ -60,6 +60,9 @@ message Call {
// Retrieves the information about known executors.
GET_EXECUTORS = 12;
+ // Retrieves the information about known operations.
+ GET_OPERATIONS = 31;
+
// Retrieves the information about known tasks.
GET_TASKS = 13;
@@ -413,6 +416,7 @@ message Response {
GET_CONTAINERS = 9;
GET_FRAMEWORKS = 10; // See 'GetFrameworks' below.
GET_EXECUTORS = 11; // See 'GetExecutors' below.
+ GET_OPERATIONS = 17; // See 'GetOperations' below.
GET_TASKS = 12; // See 'GetTasks' below.
GET_AGENT = 14; // See 'GetAgent' below.
GET_RESOURCE_PROVIDERS = 16; // See 'GetResourceProviders' below.
@@ -506,6 +510,12 @@ message Response {
repeated Executor completed_executors = 2;
}
+ // Lists information about all operations known to the agent at the
+ // current time.
+ message GetOperations {
+ repeated Operation operations = 1;
+ }
+
// Lists information about all the tasks known to the agent at the current
// time.
message GetTasks {
@@ -599,6 +609,7 @@ message Response {
optional GetContainers get_containers = 10;
optional GetFrameworks get_frameworks = 11;
optional GetExecutors get_executors = 12;
+ optional GetOperations get_operations = 18;
optional GetTasks get_tasks = 13;
optional GetAgent get_agent = 15;
optional GetResourceProviders get_resource_providers = 17;
http://git-wip-us.apache.org/repos/asf/mesos/blob/c6de89e1/include/mesos/master/master.proto
----------------------------------------------------------------------
diff --git a/include/mesos/master/master.proto b/include/mesos/master/master.proto
index 3e34634..f40caa2 100644
--- a/include/mesos/master/master.proto
+++ b/include/mesos/master/master.proto
@@ -62,6 +62,7 @@ message Call {
GET_AGENTS = 10;
GET_FRAMEWORKS = 11;
GET_EXECUTORS = 12; // Retrieves the information about all executors.
+ GET_OPERATIONS = 33; // Retrieves the information about known operations.
GET_TASKS = 13; // Retrieves the information about all known tasks.
GET_ROLES = 14; // Retrieves the information about roles.
@@ -260,6 +261,7 @@ message Response {
GET_AGENTS = 9;
GET_FRAMEWORKS = 10;
GET_EXECUTORS = 11; // See 'GetExecutors' below.
+ GET_OPERATIONS = 19; // See 'GetOperations' below.
GET_TASKS = 12; // See 'GetTasks' below.
GET_ROLES = 13; // See 'GetRoles' below.
@@ -416,6 +418,12 @@ message Response {
repeated Executor orphan_executors = 2 [deprecated=true];
}
+ // Lists information about all operations known to the master at the
+ // current time.
+ message GetOperations {
+ repeated Operation operations = 1;
+ }
+
// Lists information about all the tasks known to the master at the current
// time. Note that there might be tasks unknown to the master running on
// partitioned or unsubscribed agents.
@@ -498,6 +506,7 @@ message Response {
optional GetAgents get_agents = 10;
optional GetFrameworks get_frameworks = 11;
optional GetExecutors get_executors = 12;
+ optional GetOperations get_operations = 20;
optional GetTasks get_tasks = 13;
optional GetRoles get_roles = 14;
optional GetWeights get_weights = 15;
http://git-wip-us.apache.org/repos/asf/mesos/blob/c6de89e1/include/mesos/v1/agent/agent.proto
----------------------------------------------------------------------
diff --git a/include/mesos/v1/agent/agent.proto b/include/mesos/v1/agent/agent.proto
index 9e8b49d..59a9fd6 100644
--- a/include/mesos/v1/agent/agent.proto
+++ b/include/mesos/v1/agent/agent.proto
@@ -60,6 +60,9 @@ message Call {
// Retrieves the information about known executors.
GET_EXECUTORS = 12;
+ // Retrieves the information about known operations.
+ GET_OPERATIONS = 31;
+
// Retrieves the information about known tasks.
GET_TASKS = 13;
@@ -413,6 +416,7 @@ message Response {
GET_CONTAINERS = 9;
GET_FRAMEWORKS = 10; // See 'GetFrameworks' below.
GET_EXECUTORS = 11; // See 'GetExecutors' below.
+ GET_OPERATIONS = 17; // See 'GetOperations' below.
GET_TASKS = 12; // See 'GetTasks' below.
GET_AGENT = 14; // See 'GetAgent' below.
GET_RESOURCE_PROVIDERS = 16; // See 'GetResourceProviders' below.
@@ -506,6 +510,12 @@ message Response {
repeated Executor completed_executors = 2;
}
+ // Lists information about all operations known to the agent at the
+ // current time.
+ message GetOperations {
+ repeated Operation operations = 1;
+ }
+
// Lists information about all the tasks known to the agent at the current
// time.
message GetTasks {
@@ -599,6 +609,7 @@ message Response {
optional GetContainers get_containers = 10;
optional GetFrameworks get_frameworks = 11;
optional GetExecutors get_executors = 12;
+ optional GetOperations get_operations = 18;
optional GetTasks get_tasks = 13;
optional GetAgent get_agent = 15;
optional GetResourceProviders get_resource_providers = 17;
http://git-wip-us.apache.org/repos/asf/mesos/blob/c6de89e1/include/mesos/v1/master/master.proto
----------------------------------------------------------------------
diff --git a/include/mesos/v1/master/master.proto b/include/mesos/v1/master/master.proto
index 6759c30..67c9560 100644
--- a/include/mesos/v1/master/master.proto
+++ b/include/mesos/v1/master/master.proto
@@ -60,6 +60,7 @@ message Call {
GET_AGENTS = 10;
GET_FRAMEWORKS = 11;
GET_EXECUTORS = 12; // Retrieves the information about all executors.
+ GET_OPERATIONS = 33; // Retrieves the information about known operations.
GET_TASKS = 13; // Retrieves the information about all known tasks.
GET_ROLES = 14; // Retrieves the information about roles.
@@ -258,6 +259,7 @@ message Response {
GET_AGENTS = 9;
GET_FRAMEWORKS = 10;
GET_EXECUTORS = 11; // See 'GetExecutors' below.
+ GET_OPERATIONS = 19; // See 'GetOperations' below.
GET_TASKS = 12; // See 'GetTasks' below.
GET_ROLES = 13; // See 'GetRoles' below.
@@ -413,6 +415,12 @@ message Response {
repeated Executor orphan_executors = 2 [deprecated=true];
}
+ // Lists information about all operations known to the master at the
+ // current time.
+ message GetOperations {
+ repeated Operation operations = 1;
+ }
+
// Lists information about all the tasks known to the master at the current
// time. Note that there might be tasks unknown to the master running on
// partitioned or unsubscribed agents.
@@ -494,6 +502,7 @@ message Response {
optional GetAgents get_agents = 10;
optional GetFrameworks get_frameworks = 11;
optional GetExecutors get_executors = 12;
+ optional GetOperations get_operations = 20;
optional GetTasks get_tasks = 13;
optional GetRoles get_roles = 14;
optional GetWeights get_weights = 15;
http://git-wip-us.apache.org/repos/asf/mesos/blob/c6de89e1/src/master/http.cpp
----------------------------------------------------------------------
diff --git a/src/master/http.cpp b/src/master/http.cpp
index 46f2872..6f692e2 100644
--- a/src/master/http.cpp
+++ b/src/master/http.cpp
@@ -732,6 +732,9 @@ Future<Response> Master::Http::api(
case mesos::master::Call::GET_EXECUTORS:
return getExecutors(call, principal, acceptType);
+ case mesos::master::Call::GET_OPERATIONS:
+ return getOperations(call, principal, acceptType);
+
case mesos::master::Call::GET_TASKS:
return getTasks(call, principal, acceptType);
@@ -3887,6 +3890,31 @@ Future<Response> Master::Http::teardown(
}
+Future<Response> Master::Http::getOperations(
+ const mesos::master::Call& call,
+ const Option<Principal>& principal,
+ ContentType contentType) const
+{
+ CHECK_EQ(mesos::master::Call::GET_OPERATIONS, call.type());
+
+ // TODO(nfnt): Authorize this call (MESOS-8473).
+
+ mesos::master::Response response;
+ response.set_type(mesos::master::Response::GET_OPERATIONS);
+
+ mesos::master::Response::GetOperations* operations =
+ response.mutable_get_operations();
+
+ foreachvalue (const Slave* slave, master->slaves.registered) {
+ foreachvalue (Operation* operation, slave->operations) {
+ operations->add_operations()->CopyFrom(*operation);
+ }
+ }
+
+ return OK(serialize(contentType, evolve(response)), stringify(contentType));
+}
+
+
struct TaskComparator
{
static bool ascending(const Task* lhs, const Task* rhs)
http://git-wip-us.apache.org/repos/asf/mesos/blob/c6de89e1/src/master/master.hpp
----------------------------------------------------------------------
diff --git a/src/master/master.hpp b/src/master/master.hpp
index c4d3c80..92af852 100644
--- a/src/master/master.hpp
+++ b/src/master/master.hpp
@@ -1640,6 +1640,11 @@ private:
const Option<process::http::authentication::Principal>& principal,
ContentType contentType) const;
+ process::Future<process::http::Response> getOperations(
+ const mesos::master::Call& call,
+ const Option<process::http::authentication::Principal>& principal,
+ ContentType contentType) const;
+
process::Future<process::http::Response> getTasks(
const mesos::master::Call& call,
const Option<process::http::authentication::Principal>& principal,
http://git-wip-us.apache.org/repos/asf/mesos/blob/c6de89e1/src/master/validation.cpp
----------------------------------------------------------------------
diff --git a/src/master/validation.cpp b/src/master/validation.cpp
index 42f767e..b15b75c 100644
--- a/src/master/validation.cpp
+++ b/src/master/validation.cpp
@@ -124,6 +124,9 @@ Option<Error> validate(
case mesos::master::Call::GET_EXECUTORS:
return None();
+ case mesos::master::Call::GET_OPERATIONS:
+ return None();
+
case mesos::master::Call::GET_TASKS:
return None();
http://git-wip-us.apache.org/repos/asf/mesos/blob/c6de89e1/src/slave/http.cpp
----------------------------------------------------------------------
diff --git a/src/slave/http.cpp b/src/slave/http.cpp
index 59eef7a..7d7fa2b 100644
--- a/src/slave/http.cpp
+++ b/src/slave/http.cpp
@@ -575,6 +575,9 @@ Future<Response> Http::_api(
case mesos::agent::Call::GET_EXECUTORS:
return getExecutors(call, mediaTypes.accept, principal);
+ case mesos::agent::Call::GET_OPERATIONS:
+ return getOperations(call, mediaTypes.accept, principal);
+
case mesos::agent::Call::GET_TASKS:
return getTasks(call, mediaTypes.accept, principal);
@@ -1672,6 +1675,31 @@ mesos::agent::Response::GetExecutors Http::_getExecutors(
}
+Future<Response> Http::getOperations(
+ const mesos::agent::Call& call,
+ ContentType acceptType,
+ const Option<Principal>& principal) const
+{
+ CHECK_EQ(mesos::agent::Call::GET_OPERATIONS, call.type());
+
+ LOG(INFO) << "Processing GET_OPERATIONS call";
+
+ // TODO(nfnt): Authorize this call (MESOS-8473).
+
+ agent::Response response;
+ response.set_type(mesos::agent::Response::GET_OPERATIONS);
+
+ agent::Response::GetOperations* operations =
+ response.mutable_get_operations();
+
+ foreachvalue (Operation* operation, slave->operations) {
+ operations->add_operations()->CopyFrom(*operation);
+ }
+
+ return OK(serialize(acceptType, evolve(response)), stringify(acceptType));
+}
+
+
Future<Response> Http::getTasks(
const mesos::agent::Call& call,
ContentType acceptType,
http://git-wip-us.apache.org/repos/asf/mesos/blob/c6de89e1/src/slave/http.hpp
----------------------------------------------------------------------
diff --git a/src/slave/http.hpp b/src/slave/http.hpp
index 1619bb7..c33adeb 100644
--- a/src/slave/http.hpp
+++ b/src/slave/http.hpp
@@ -188,6 +188,11 @@ private:
const process::Owned<ObjectApprover>& frameworksApprover,
const process::Owned<ObjectApprover>& executorsApprover) const;
+ process::Future<process::http::Response> getOperations(
+ const mesos::agent::Call& call,
+ ContentType acceptType,
+ const Option<process::http::authentication::Principal>& principal) const;
+
process::Future<process::http::Response> getTasks(
const mesos::agent::Call& call,
ContentType acceptType,
http://git-wip-us.apache.org/repos/asf/mesos/blob/c6de89e1/src/slave/validation.cpp
----------------------------------------------------------------------
diff --git a/src/slave/validation.cpp b/src/slave/validation.cpp
index 0c2ccda..5d751d2 100644
--- a/src/slave/validation.cpp
+++ b/src/slave/validation.cpp
@@ -150,6 +150,9 @@ Option<Error> validate(
case mesos::agent::Call::GET_EXECUTORS:
return None();
+ case mesos::agent::Call::GET_OPERATIONS:
+ return None();
+
case mesos::agent::Call::GET_TASKS:
return None();
http://git-wip-us.apache.org/repos/asf/mesos/blob/c6de89e1/src/tests/api_tests.cpp
----------------------------------------------------------------------
diff --git a/src/tests/api_tests.cpp b/src/tests/api_tests.cpp
index b042201..c6383c4 100644
--- a/src/tests/api_tests.cpp
+++ b/src/tests/api_tests.cpp
@@ -1004,6 +1004,151 @@ TEST_P(MasterAPITest, GetRoles)
}
+TEST_P(MasterAPITest, GetOperations)
+{
+ Clock::pause();
+
+ master::Flags masterFlags = CreateMasterFlags();
+
+ Try<Owned<cluster::Master>> master = this->StartMaster(masterFlags);
+ ASSERT_SOME(master);
+
+ Owned<MasterDetector> detector = master.get()->createDetector();
+
+ // Start one agent.
+ Future<UpdateSlaveMessage> updateAgentMessage =
+ FUTURE_PROTOBUF(UpdateSlaveMessage(), _, _);
+
+ slave::Flags slaveFlags = CreateSlaveFlags();
+
+ // TODO(nfnt): Remove this once 'MockResourceProvider' supports
+ // authentication.
+ slaveFlags.authenticate_http_readwrite = false;
+
+ // Set the resource provider capability.
+ vector<SlaveInfo::Capability> capabilities = slave::AGENT_CAPABILITIES();
+ SlaveInfo::Capability capability;
+ capability.set_type(SlaveInfo::Capability::RESOURCE_PROVIDER);
+ capabilities.push_back(capability);
+
+ slaveFlags.agent_features = SlaveCapabilities();
+ slaveFlags.agent_features->mutable_capabilities()->CopyFrom(
+ {capabilities.begin(), capabilities.end()});
+
+ Try<Owned<cluster::Slave>> agent = StartSlave(detector.get(), slaveFlags);
+ ASSERT_SOME(agent);
+
+ Clock::advance(slaveFlags.registration_backoff_factor);
+ AWAIT_READY(updateAgentMessage);
+
+ mesos::v1::ResourceProviderInfo info;
+ info.set_type("org.apache.mesos.rp.test");
+ info.set_name("test");
+
+ v1::MockResourceProvider resourceProvider(
+ info,
+ v1::createDiskResource(
+ "200", "*", None(), None(), v1::createDiskSourceRaw()));
+
+ // Start and register resource provider.
+ Owned<EndpointDetector> endpointDetector(
+ resource_provider::createEndpointDetector(agent.get()->pid));
+
+ updateAgentMessage = FUTURE_PROTOBUF(UpdateSlaveMessage(), _, _);
+
+ const ContentType contentType = GetParam();
+
+ resourceProvider.start(endpointDetector, contentType, v1::DEFAULT_CREDENTIAL);
+
+ // Wait until the agent's resources have been updated to include the
+ // resource provider resources.
+ AWAIT_READY(updateAgentMessage);
+
+ v1::master::Call v1Call;
+ v1Call.set_type(v1::master::Call::GET_OPERATIONS);
+
+ Future<v1::master::Response> v1Response =
+ post(master.get()->pid, v1Call, contentType);
+
+ AWAIT_READY(v1Response);
+ ASSERT_TRUE(v1Response->IsInitialized());
+ ASSERT_EQ(v1::master::Response::GET_OPERATIONS, v1Response->type());
+ EXPECT_TRUE(v1Response->get_operations().operations().empty());
+
+ // Start a framework to operate on offers.
+ MockScheduler sched;
+ MesosSchedulerDriver driver(
+ &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL);
+
+ EXPECT_CALL(sched, registered(&driver, _, _));
+
+ Future<vector<Offer>> offers;
+ EXPECT_CALL(sched, resourceOffers(&driver, _))
+ .WillOnce(FutureArg<1>(&offers));
+
+ driver.start();
+
+ // We settle here to make sure that the framework has been authenticated
+ // before advancing the clock. Otherwise we would run into a authentication
+ // timeout due to the large allocation interval (1000s) of this fixture.
+ Clock::settle();
+
+ Clock::advance(masterFlags.allocation_interval);
+ Clock::settle();
+
+ AWAIT_READY(offers);
+ ASSERT_FALSE(offers->empty());
+ const Offer& offer = offers->front();
+
+ Option<Resource> rawDisk;
+
+ foreach (const Resource& resource, offer.resources()) {
+ if (resource.has_provider_id() &&
+ resource.has_disk() &&
+ resource.disk().has_source() &&
+ resource.disk().source().type() == Resource::DiskInfo::Source::RAW) {
+ rawDisk = resource;
+ break;
+ }
+ }
+
+ ASSERT_SOME(rawDisk);
+
+ // The operation is still pending when we receive this event.
+ Future<mesos::v1::resource_provider::Event::ApplyOperation> operation;
+ EXPECT_CALL(resourceProvider, applyOperation(_))
+ .WillOnce(FutureArg<0>(&operation));
+
+ // Start an operation.
+ driver.acceptOffers(
+ {offer.id()},
+ {CREATE_VOLUME(rawDisk.get(), Resource::DiskInfo::Source::MOUNT)});
+
+ AWAIT_READY(operation);
+
+ v1Response = post(master.get()->pid, v1Call, contentType);
+
+ AWAIT_READY(v1Response);
+ ASSERT_TRUE(v1Response->IsInitialized());
+ ASSERT_EQ(v1::master::Response::GET_OPERATIONS, v1Response->type());
+ EXPECT_EQ(1, v1Response->get_operations().operations_size());
+ EXPECT_EQ(
+ operation->framework_id(),
+ v1Response->get_operations().operations(0).framework_id());
+ EXPECT_EQ(
+ evolve(updateAgentMessage->slave_id()),
+ v1Response->get_operations().operations(0).agent_id());
+ EXPECT_EQ(
+ operation->info(), v1Response->get_operations().operations(0).info());
+ EXPECT_EQ(
+ operation->operation_uuid(),
+ v1Response->get_operations().operations(0).uuid());
+
+ driver.stop();
+ driver.join();
+}
+
+
TEST_P(MasterAPITest, GetMaster)
{
master::Flags masterFlags = CreateMasterFlags();
@@ -6372,6 +6517,150 @@ TEST_P(AgentAPITest, GetResourceProviders)
}
+TEST_P(AgentAPITest, GetOperations)
+{
+ Clock::pause();
+
+ master::Flags masterFlags = CreateMasterFlags();
+
+ Try<Owned<cluster::Master>> master = this->StartMaster(masterFlags);
+ ASSERT_SOME(master);
+
+ Owned<MasterDetector> detector = master.get()->createDetector();
+
+ Future<UpdateSlaveMessage> updateSlaveMessage =
+ FUTURE_PROTOBUF(UpdateSlaveMessage(), _, _);
+
+ slave::Flags slaveFlags = CreateSlaveFlags();
+
+ // TODO(nfnt): Remove this once 'MockResourceProvider' supports
+ // authentication.
+ slaveFlags.authenticate_http_readwrite = false;
+
+ // Set the resource provider capability.
+ vector<SlaveInfo::Capability> capabilities = slave::AGENT_CAPABILITIES();
+ SlaveInfo::Capability capability;
+ capability.set_type(SlaveInfo::Capability::RESOURCE_PROVIDER);
+ capabilities.push_back(capability);
+
+ slaveFlags.agent_features = SlaveCapabilities();
+ slaveFlags.agent_features->mutable_capabilities()->CopyFrom(
+ {capabilities.begin(), capabilities.end()});
+
+ Try<Owned<cluster::Slave>> agent = StartSlave(detector.get(), slaveFlags);
+ ASSERT_SOME(agent);
+
+ Clock::advance(slaveFlags.registration_backoff_factor);
+ AWAIT_READY(updateSlaveMessage);
+
+ mesos::v1::ResourceProviderInfo info;
+ info.set_type("org.apache.mesos.rp.test");
+ info.set_name("test");
+
+ v1::MockResourceProvider resourceProvider(
+ info,
+ v1::createDiskResource(
+ "200", "*", None(), None(), v1::createDiskSourceRaw()));
+
+ // Start and register a resource provider.
+ Owned<EndpointDetector> endpointDetector(
+ resource_provider::createEndpointDetector(agent.get()->pid));
+
+ updateSlaveMessage = FUTURE_PROTOBUF(UpdateSlaveMessage(), _, _);
+
+ const ContentType contentType = GetParam();
+
+ resourceProvider.start(endpointDetector, contentType, v1::DEFAULT_CREDENTIAL);
+
+ // Wait until the agent's resources have been updated to include the
+ // resource provider resources.
+ AWAIT_READY(updateSlaveMessage);
+
+ v1::agent::Call v1Call;
+ v1Call.set_type(v1::agent::Call::GET_OPERATIONS);
+
+ Future<v1::agent::Response> v1Response =
+ post(agent.get()->pid, v1Call, contentType);
+
+ AWAIT_READY(v1Response);
+ ASSERT_TRUE(v1Response->IsInitialized());
+ ASSERT_EQ(v1::agent::Response::GET_OPERATIONS, v1Response->type());
+ EXPECT_TRUE(v1Response->get_operations().operations().empty());
+
+ // Start a framework to operate on offers.
+ MockScheduler sched;
+ MesosSchedulerDriver driver(
+ &sched, DEFAULT_FRAMEWORK_INFO, master.get()->pid, DEFAULT_CREDENTIAL);
+
+ EXPECT_CALL(sched, registered(&driver, _, _));
+
+ Future<vector<Offer>> offers;
+ EXPECT_CALL(sched, resourceOffers(&driver, _))
+ .WillOnce(FutureArg<1>(&offers));
+
+ driver.start();
+
+ // We settle here to make sure that the framework has been authenticated
+ // before advancing the clock. Otherwise we would run into a authentication
+ // timeout due to the large allocation interval (1000s) of this fixture.
+ Clock::settle();
+
+ Clock::advance(masterFlags.allocation_interval);
+ Clock::settle();
+
+ AWAIT_READY(offers);
+ ASSERT_FALSE(offers->empty());
+ const Offer& offer = offers->front();
+
+ Option<Resource> rawDisk;
+
+ foreach (const Resource& resource, offer.resources()) {
+ if (resource.has_provider_id() &&
+ resource.has_disk() &&
+ resource.disk().has_source() &&
+ resource.disk().source().type() == Resource::DiskInfo::Source::RAW) {
+ rawDisk = resource;
+ break;
+ }
+ }
+
+ ASSERT_SOME(rawDisk);
+
+ // The operation is still pending when we receive this event.
+ Future<mesos::v1::resource_provider::Event::ApplyOperation> operation;
+ EXPECT_CALL(resourceProvider, applyOperation(_))
+ .WillOnce(FutureArg<0>(&operation));
+
+ // Start an operation.
+ driver.acceptOffers(
+ {offer.id()},
+ {CREATE_VOLUME(rawDisk.get(), Resource::DiskInfo::Source::MOUNT)});
+
+ AWAIT_READY(operation);
+
+ v1Response = post(agent.get()->pid, v1Call, contentType);
+
+ AWAIT_READY(v1Response);
+ ASSERT_TRUE(v1Response->IsInitialized());
+ ASSERT_EQ(v1::agent::Response::GET_OPERATIONS, v1Response->type());
+ EXPECT_EQ(1, v1Response->get_operations().operations_size());
+ EXPECT_EQ(
+ operation->framework_id(),
+ v1Response->get_operations().operations(0).framework_id());
+ EXPECT_EQ(
+ evolve(updateSlaveMessage->slave_id()),
+ v1Response->get_operations().operations(0).agent_id());
+ EXPECT_EQ(
+ operation->info(), v1Response->get_operations().operations(0).info());
+ EXPECT_EQ(
+ operation->operation_uuid(),
+ v1Response->get_operations().operations(0).uuid());
+
+ driver.stop();
+ driver.join();
+}
+
+
class AgentAPIStreamingTest
: public MesosTest,
public WithParamInterface<ContentType> {};