You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by bb...@apache.org on 2019/01/18 10:02:48 UTC

[mesos] 02/02: Added an integration test for resource provider removal.

This is an automated email from the ASF dual-hosted git repository.

bbannier pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git

commit 3de5efba936c8b7bd1bf88c2fd05006a93271b73
Author: Benjamin Bannier <be...@mesosphere.io>
AuthorDate: Fri Jan 18 09:39:52 2019 +0100

    Added an integration test for resource provider removal.
    
    Review: https://reviews.apache.org/r/69158/
---
 src/tests/slave_tests.cpp | 247 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 247 insertions(+)

diff --git a/src/tests/slave_tests.cpp b/src/tests/slave_tests.cpp
index dc711de..9168e06 100644
--- a/src/tests/slave_tests.cpp
+++ b/src/tests/slave_tests.cpp
@@ -142,6 +142,7 @@ using testing::_;
 using testing::AtMost;
 using testing::DoAll;
 using testing::Eq;
+using testing::Exactly;
 using testing::StrEq;
 using testing::Invoke;
 using testing::InvokeWithoutArgs;
@@ -10700,6 +10701,252 @@ TEST_F(SlaveTest, ResourceProviderPublishAll)
 }
 
 
+// This test checks that a resource provider gets properly disconnected when
+// being marked gone, and is not able to reconnect. We expect pending operations
+// to be transition to the correct terminal status.
+TEST_F(SlaveTest, RemoveResourceProvider)
+{
+  Clock::pause();
+
+  // Start an agent and a master.
+  Try<Owned<cluster::Master>> master = StartMaster();
+  ASSERT_SOME(master);
+
+  Future<UpdateSlaveMessage> updateSlaveMessage =
+    FUTURE_PROTOBUF(UpdateSlaveMessage(), _, _);
+
+  slave::Flags slaveFlags = CreateSlaveFlags();
+
+  StandaloneMasterDetector detector(master.get()->pid);
+  Try<Owned<cluster::Slave>> slave = StartSlave(&detector, slaveFlags);
+  ASSERT_SOME(slave);
+
+  Clock::advance(slaveFlags.registration_backoff_factor);
+  AWAIT_READY(updateSlaveMessage);
+
+  v1::ResourceProviderInfo resourceProviderInfo;
+  resourceProviderInfo.set_type("org.apache.mesos.rp.test");
+  resourceProviderInfo.set_name("test");
+
+  // Register a local resource provider with the agent.
+  v1::Resource disk = v1::createDiskResource(
+      "200",
+      "storage",
+      None(),
+      None(),
+      v1::createDiskSourceRaw(None(), "profile"));
+
+  v1::MockResourceProvider resourceProvider(resourceProviderInfo, disk);
+
+  Owned<EndpointDetector> endpointDetector(
+      resource_provider::createEndpointDetector(slave.get()->pid));
+
+  updateSlaveMessage = FUTURE_PROTOBUF(UpdateSlaveMessage(), _, _);
+
+  resourceProvider.start(std::move(endpointDetector), ContentType::PROTOBUF);
+
+  AWAIT_READY(updateSlaveMessage);
+
+  // Register a framework and perform a pending operations on the
+  // resource provider resources.
+  v1::FrameworkInfo framework = v1::DEFAULT_FRAMEWORK_INFO;
+  framework.set_roles(0, disk.reservations(0).role());
+
+  auto scheduler = std::make_shared<v1::MockHTTPScheduler>();
+
+  EXPECT_CALL(*scheduler, connected(_))
+    .WillOnce(v1::scheduler::SendSubscribe(framework));
+
+  Future<v1::scheduler::Event::Subscribed> subscribed;
+  EXPECT_CALL(*scheduler, subscribed(_, _))
+    .WillOnce(FutureArg<1>(&subscribed));
+
+  EXPECT_CALL(*scheduler, heartbeat(_))
+    .WillRepeatedly(Return()); // Ignore heartbeats.
+
+  // Since the resources from the resource provider have already reached the
+  // master at this point, the framework will be offered resource provider
+  // resources.
+  Future<v1::scheduler::Event::Offers> offers;
+  EXPECT_CALL(*scheduler, offers(_, _))
+    .WillOnce(FutureArg<1>(&offers));
+
+  v1::scheduler::TestMesos mesos(
+      master.get()->pid,
+      ContentType::PROTOBUF,
+      scheduler);
+
+  AWAIT_READY(subscribed);
+  v1::FrameworkID frameworkId(subscribed->framework_id());
+
+  AWAIT_READY(offers);
+  ASSERT_FALSE(offers->offers().empty());
+  const v1::Offer& offer = offers->offers(0);
+
+  Option<v1::Resource> rawDisk;
+  foreach (const v1::Resource& resource, offer.resources()) {
+    if (resource.has_provider_id() && resource.has_disk() &&
+        resource.disk().has_source() &&
+        resource.disk().source().type() ==
+          v1::Resource::DiskInfo::Source::RAW) {
+      rawDisk = resource;
+      break;
+    }
+  }
+
+  ASSERT_SOME(rawDisk);
+
+  // Create a pending operation.
+  Future<v1::resource_provider::Event::ApplyOperation> applyOperation;
+  EXPECT_CALL(resourceProvider, applyOperation(_))
+    .WillOnce(FutureArg<0>(&applyOperation));
+
+  v1::OperationID operationId;
+  operationId.set_value("operation");
+
+  mesos.send(v1::createCallAccept(
+      frameworkId,
+      offer,
+      {v1::CREATE_DISK(
+          rawDisk.get(),
+          v1::Resource::DiskInfo::Source::MOUNT,
+          None(),
+          operationId)}));
+
+  // Wait for the operation to reach the resource provider.
+  AWAIT_READY(applyOperation);
+
+  // A resource provider cannot be removed while it still has resources.
+  ASSERT_TRUE(resourceProvider.info.has_id());
+
+  const mesos::v1::ResourceProviderID& resourceProviderId =
+    resourceProvider.info.id();
+
+  v1::agent::Call v1Call;
+  v1Call.set_type(v1::agent::Call::MARK_RESOURCE_PROVIDER_GONE);
+  v1Call.mutable_mark_resource_provider_gone()
+    ->mutable_resource_provider_id()
+    ->CopyFrom(resourceProviderId);
+
+  constexpr ContentType contentType = ContentType::PROTOBUF;
+  process::http::Headers headers = createBasicAuthHeaders(DEFAULT_CREDENTIAL);
+  headers["Accept"] = stringify(contentType);
+
+  Future<process::http::Response> response = process::http::post(
+    slave.get()->pid,
+    "api/v1",
+    headers,
+    serialize(contentType, v1Call),
+    stringify(contentType));
+
+  AWAIT_EXPECT_RESPONSE_STATUS_EQ(InternalServerError().status, response);
+
+  // Remove all resources on the resource provider.
+  updateSlaveMessage = FUTURE_PROTOBUF(UpdateSlaveMessage(), _, _);
+
+  {
+    using mesos::v1::resource_provider::Call;
+    using mesos::v1::resource_provider::Event;
+
+    Call call;
+    call.set_type(Call::UPDATE_STATE);
+    call.mutable_resource_provider_id()->CopyFrom(resourceProvider.info.id());
+
+    Call::UpdateState* update = call.mutable_update_state();
+    update->mutable_resources()->Clear();
+    update->mutable_resource_version_uuid()->set_value(
+        id::UUID::random().toBytes());
+
+    // Still report the operation. This allows us to send an operation
+    // status update later on.
+    mesos::v1::Operation* operation = update->add_operations();
+
+    ASSERT_TRUE(applyOperation->has_framework_id());
+    operation->mutable_framework_id()->CopyFrom(applyOperation->framework_id());
+
+    operation->mutable_info()->CopyFrom(applyOperation->info());
+    operation->mutable_uuid()->CopyFrom(applyOperation->operation_uuid());
+    operation->mutable_latest_status()->CopyFrom(
+        evolve(protobuf::createOperationStatus(OPERATION_PENDING)));
+
+    operation->add_statuses()->CopyFrom(operation->latest_status());
+
+    AWAIT_READY(resourceProvider.send(call));
+  }
+
+  // Once the master has seen that there is no resource managed
+  // by the resource provider it can be removed successfully.
+  AWAIT_READY(updateSlaveMessage);
+
+  Future<v1::scheduler::Event::UpdateOperationStatus> updateOperationStatus;
+  EXPECT_CALL(*scheduler, updateOperationStatus(_, _))
+    .WillOnce(FutureArg<1>(&updateOperationStatus));
+
+  // The agent will eventually update the master on its resources
+  // after the resource provider disconnected.
+  updateSlaveMessage = FUTURE_PROTOBUF(UpdateSlaveMessage(), _, _);
+
+  // The resource provider will receive a TEARDOWN event on being marked gone.
+  Future<Nothing> teardown;
+  EXPECT_CALL(resourceProvider, teardown())
+    .WillOnce(FutureSatisfy(&teardown));
+
+  // We expect at least two disconnection events, one initially when the
+  // connected resource provider gets removed, and when the automatic attempt
+  // to resubscribe fails and leads the remote to close the connection.
+  Future<Nothing> disconnected;
+  EXPECT_CALL(resourceProvider, disconnected())
+    .WillOnce(DoDefault())
+    .WillOnce(FutureSatisfy(&disconnected))
+    .WillRepeatedly(Return()); // Ignore additional ddisconnection events.
+
+  // The resource provider will automatically attempt to reconnect.
+  Future<Nothing> connected;
+  EXPECT_CALL(resourceProvider, connected())
+    .WillOnce(DoDefault())
+    .WillRepeatedly(Return());
+
+  // The resource provider should never successfully resubscribe.
+  EXPECT_CALL(resourceProvider, subscribed(_))
+    .Times(Exactly(0));
+
+  response = process::http::post(
+      slave.get()->pid,
+      "api/v1",
+      headers,
+      serialize(contentType, v1Call),
+      stringify(contentType));
+
+  AWAIT_EXPECT_RESPONSE_STATUS_EQ(OK().status, response);
+
+  AWAIT_READY(teardown);
+
+  // On resource provider removal the framework should have received
+  // an operation status update.
+  AWAIT_READY(updateOperationStatus);
+
+  EXPECT_EQ(
+    v1::OperationState::OPERATION_GONE_BY_OPERATOR,
+    updateOperationStatus->status().state());
+
+  // The status update should be generated by the agent and have no
+  // associated resource provider ID.
+  EXPECT_TRUE(updateOperationStatus->status().has_agent_id());
+  EXPECT_FALSE(updateOperationStatus->status().has_resource_provider_id());
+
+  // The agent should also report a change to its resources.
+  AWAIT_READY(updateSlaveMessage);
+
+  // Once we have seen the second disconnection event we know that the
+  // attempt to resubscribe was unsuccessful.
+  AWAIT_READY(disconnected);
+
+  // Settle the clock to ensure no more `subscribed` calls to the
+  // resource provider are enqueued.
+  Clock::settle();
+}
+
+
 // This test checks that the agent correctly updates and sends
 // resource version values when it registers or reregisters.
 TEST_F(SlaveTest, ResourceVersions)