You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by bb...@apache.org on 2019/01/11 08:51:46 UTC
[mesos] branch master updated (50c37a6 -> 7447fbc)
This is an automated email from the ASF dual-hosted git repository.
bbannier pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git.
from 50c37a6 Displayed resource provider information in the Mesos webui.
new 0b6df09 Garbage collected disappeared resource providers from master state.
new 7447fbc Changed agent to not report disconnected resource providers.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
src/master/master.cpp | 355 ++++++++++++++------------
src/messages/messages.proto | 5 +-
src/slave/slave.cpp | 19 +-
src/tests/resource_provider_manager_tests.cpp | 13 +-
4 files changed, 209 insertions(+), 183 deletions(-)
[mesos] 01/02: Garbage collected disappeared resource providers
from master state.
Posted by bb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bbannier pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 0b6df0946eda86be50fb1dd1d6b650e5bfd2eaea
Author: Benjamin Bannier <be...@mesosphere.io>
AuthorDate: Fri Jan 11 09:44:20 2019 +0100
Garbage collected disappeared resource providers from master state.
The master previously kept information on resource providers
indefinitely. This was confusing to API users who saw resource
providers reported which where not present anymore, and also made it
harder to derive actual cluster state.
With this patch we remove resource providers not reported by the agent
from master state. We still need to update the agent to not report
removed resource providers in a follow-up patch.
Review: https://reviews.apache.org/r/69337/
---
src/master/master.cpp | 355 ++++++++++++++++++++++++--------------------
src/messages/messages.proto | 5 +-
2 files changed, 197 insertions(+), 163 deletions(-)
diff --git a/src/master/master.cpp b/src/master/master.cpp
index 454f06a..9624832 100644
--- a/src/master/master.cpp
+++ b/src/master/master.cpp
@@ -8103,187 +8103,220 @@ void Master::updateSlave(UpdateSlaveMessage&& message)
}
}
- foreach (
- const UpdateSlaveMessage::ResourceProvider& resourceProvider,
- message.resource_providers().providers()) {
- CHECK(resourceProvider.has_info());
- CHECK(resourceProvider.info().has_id());
- const ResourceProviderID& providerId = resourceProvider.info().id();
-
- // Below we only add operations to our state from resource
- // providers which are unknown, or possibly remove them for known
- // resource providers. This works since the master should always
- // know more operations of known resource providers than any
- // resource provider itself.
- //
- // NOTE: We do not mutate operation statuses here; that is the
- // responsibility of the `updateOperationStatus` handler.
- //
- // There still exists an edge case where the master might remove a
- // terminal operation from its state when passing an
- // acknowledgement from a framework on to the agent, with the
- // agent failing over before the acknowledgement can be processed.
- // In that case the agent would track an operation unknown to the
- // master.
- //
- // TODO(bbannier): We might want to consider to also learn about
- // new (terminal) operations when observing messages from status
- // update managers to frameworks.
- if (!slave->resourceProviders.contains(providerId)) {
- // If this is a not previously seen resource provider we had a master
- // failover. Add the resources and operations to our state.
- CHECK(
- resourceProvider.total_resources().empty() ||
- !slave->totalResources.contains(resourceProvider.total_resources()));
-
- // We add the resource provider to the master first so
- // that it can be found when e.g., adding operations.
- slave->resourceProviders.put(
- providerId,
- {resourceProvider.info(),
- resourceProvider.total_resources(),
- resourceProvider.resource_version_uuid(),
- {}});
-
- hashmap<FrameworkID, Resources> usedByOperations;
+ if (message.has_resource_providers()) {
+ hashset<ResourceProviderID> receivedResourceProviders;
- foreach (
- const Operation& operation,
- resourceProvider.operations().operations()) {
- // Update to bookkeeping of operations.
- Framework* framework = nullptr;
- if (operation.has_framework_id()) {
- framework = getFramework(operation.framework_id());
- }
+ foreach (
+ const UpdateSlaveMessage::ResourceProvider& resourceProvider,
+ message.resource_providers().providers()) {
+ CHECK(resourceProvider.has_info());
+ CHECK(resourceProvider.info().has_id());
+ const ResourceProviderID& providerId = resourceProvider.info().id();
- addOperation(framework, slave, new Operation(operation));
+ receivedResourceProviders.insert(providerId);
- if (!protobuf::isTerminalState(operation.latest_status().state()) &&
- operation.has_framework_id()) {
- // If we do not yet know the `FrameworkInfo` of the framework the
- // operation originated from, we cannot properly track the operation
- // at this point.
- //
- // TODO(bbannier): Consider introducing ways of making sure an agent
- // always knows the `FrameworkInfo` of operations triggered on its
- // resources, e.g., by adding an explicit `FrameworkInfo` to
- // operations like is already done for `RunTaskMessage`, see
- // MESOS-8582.
- if (framework == nullptr) {
- LOG(WARNING)
- << "Cannot properly account for operation " << operation.uuid()
- << " learnt in reconciliation of agent " << slaveId
- << " since framework " << operation.framework_id()
- << " is unknown; this can lead to assertion failures after the"
- " operation terminates, see MESOS-8536";
- continue;
- }
+ // Below we only add operations to our state from resource
+ // providers which are unknown, or possibly remove them for known
+ // resource providers. This works since the master should always
+ // know more operations of known resource providers than any
+ // resource provider itself.
+ //
+ // NOTE: We do not mutate operation statuses here; that is the
+ // responsibility of the `updateOperationStatus` handler.
+ //
+ // There still exists an edge case where the master might remove a
+ // terminal operation from its state when passing an
+ // acknowledgement from a framework on to the agent, with the
+ // agent failing over before the acknowledgement can be processed.
+ // In that case the agent would track an operation unknown to the
+ // master.
+ //
+ // TODO(bbannier): We might want to consider to also learn about
+ // new (terminal) operations when observing messages from status
+ // update managers to frameworks.
+ if (!slave->resourceProviders.contains(providerId)) {
+ // If this is a not previously seen resource provider we had a master
+ // failover. Add the resources and operations to our state.
+ CHECK(
+ resourceProvider.total_resources().empty() ||
+ !slave->totalResources.contains(
+ resourceProvider.total_resources()));
+
+ // We add the resource provider to the master first so
+ // that it can be found when e.g., adding operations.
+ slave->resourceProviders.put(
+ providerId,
+ {resourceProvider.info(),
+ resourceProvider.total_resources(),
+ resourceProvider.resource_version_uuid(),
+ {}});
+
+ hashmap<FrameworkID, Resources> usedByOperations;
- Try<Resources> consumedResources =
- protobuf::getConsumedResources(operation.info());
+ foreach (
+ const Operation& operation,
+ resourceProvider.operations().operations()) {
+ // Update to bookkeeping of operations.
+ Framework* framework = nullptr;
+ if (operation.has_framework_id()) {
+ framework = getFramework(operation.framework_id());
+ }
- CHECK_SOME(consumedResources)
- << "Could not determine resources consumed by operation "
- << operation.uuid();
+ addOperation(framework, slave, new Operation(operation));
+
+ if (!protobuf::isTerminalState(operation.latest_status().state()) &&
+ operation.has_framework_id()) {
+ // If we do not yet know the `FrameworkInfo` of the framework the
+ // operation originated from, we cannot properly track the operation
+ // at this point.
+ //
+ // TODO(bbannier): Consider introducing ways of making sure an agent
+ // always knows the `FrameworkInfo` of operations triggered on its
+ // resources, e.g., by adding an explicit `FrameworkInfo` to
+ // operations like is already done for `RunTaskMessage`, see
+ // MESOS-8582.
+ if (framework == nullptr) {
+ LOG(WARNING)
+ << "Cannot properly account for operation " << operation.uuid()
+ << " learnt in reconciliation of agent " << slaveId
+ << " since framework " << operation.framework_id()
+ << " is unknown; this can lead to assertion failures after the"
+ " operation terminates, see MESOS-8536";
+ continue;
+ }
- usedByOperations[operation.framework_id()] +=
- consumedResources.get();
- }
- }
+ Try<Resources> consumedResources =
+ protobuf::getConsumedResources(operation.info());
- slave->totalResources += resourceProvider.total_resources();
+ CHECK_SOME(consumedResources)
+ << "Could not determine resources consumed by operation "
+ << operation.uuid();
- allocator->addResourceProvider(
- slaveId, resourceProvider.total_resources(), usedByOperations);
- } else {
- // If this is a known resource provider its total capacity cannot have
- // changed, and it would not know about any non-terminal operations not
- // already known to the master. However, it might not have received an
- // operation for a couple different reasons:
- //
- // - The resource provider or agent could have failed over
- // before the operation's `ApplyOperationMessage` could be
- // received.
- // - The operation's `ApplyOperationMessage` could have raced
- // with this `UpdateSlaveMessage`.
- //
- // In both of these cases, we need to reconcile such operations explicitly
- // with the agent. For operations which the agent or resource provider
- // does not recognize, an OPERATION_DROPPED status update will be
- // generated and the master will remove the operation from its state upon
- // receipt of that update.
- CHECK(slave->resourceProviders.contains(providerId));
+ usedByOperations[operation.framework_id()] +=
+ consumedResources.get();
+ }
+ }
- Slave::ResourceProvider& oldProvider =
- slave->resourceProviders.at(providerId);
+ slave->totalResources += resourceProvider.total_resources();
- hashmap<UUID, const Operation*> newOperations;
- foreach (
- const Operation& operation,
- resourceProvider.operations().operations()) {
- newOperations.put(operation.uuid(), &operation);
- }
+ allocator->addResourceProvider(
+ slaveId, resourceProvider.total_resources(), usedByOperations);
+ } else {
+ // If this is a known resource provider its total capacity cannot have
+ // changed, and it would not know about any non-terminal operations not
+ // already known to the master. However, it might not have received an
+ // operation for a couple different reasons:
+ //
+ // - The resource provider or agent could have failed over
+ // before the operation's `ApplyOperationMessage` could be
+ // received.
+ // - The operation's `ApplyOperationMessage` could have raced
+ // with this `UpdateSlaveMessage`.
+ //
+ // In both of these cases, we need to reconcile such operations
+ // explicitly with the agent. For operations which the agent or resource
+ // provider does not recognize, an OPERATION_DROPPED status update will
+ // be generated and the master will remove the operation from its state
+ // upon receipt of that update.
+ CHECK(slave->resourceProviders.contains(providerId));
- foreachpair (
- const UUID& uuid, Operation* oldOperation, oldProvider.operations) {
- if (!newOperations.contains(uuid)) {
- LOG(WARNING) << "Performing explicit reconciliation with agent for"
- << " known operation " << uuid
- << " since it was not present in original"
- << " reconciliation message from agent";
+ Slave::ResourceProvider& oldProvider =
+ slave->resourceProviders.at(providerId);
- ReconcileOperationsMessage::Operation* reconcileOperation =
- reconcile.add_operations();
+ hashmap<UUID, const Operation*> newOperations;
+ foreach (
+ const Operation& operation,
+ resourceProvider.operations().operations()) {
+ newOperations.put(operation.uuid(), &operation);
+ }
- reconcileOperation->mutable_operation_uuid()->CopyFrom(uuid);
- reconcileOperation->mutable_resource_provider_id()->CopyFrom(
- providerId);
- } else {
- // If a known operation became terminal between any previous offer
- // operation status update and this `UpdateSlaveMessage`, the total
- // resources we were sent already had the operation applied. We need
- // to update the state of the operation to terminal here so that any
- // update sent by the agent later does not cause us to apply the
- // operation again.
-
- const Operation* newOperation = newOperations.at(uuid);
-
- if (!protobuf::isTerminalState(
- oldOperation->latest_status().state()) &&
- protobuf::isTerminalState(
- newOperation->latest_status().state())) {
- Operation* operation = CHECK_NOTNULL(slave->getOperation(uuid));
-
- UpdateOperationStatusMessage update =
- protobuf::createUpdateOperationStatusMessage(
- uuid,
- newOperation->latest_status(),
- newOperation->latest_status(),
- operation->framework_id(),
- operation->slave_id());
-
- updateOperation(
- operation, update, false); // Do not update resources.
+ foreachpair (
+ const UUID& uuid,
+ Operation * oldOperation,
+ oldProvider.operations) {
+ if (!newOperations.contains(uuid)) {
+ LOG(WARNING) << "Performing explicit reconciliation with agent for"
+ << " known operation " << uuid
+ << " since it was not present in original"
+ << " reconciliation message from agent";
+
+ ReconcileOperationsMessage::Operation* reconcileOperation =
+ reconcile.add_operations();
+
+ reconcileOperation->mutable_operation_uuid()->CopyFrom(uuid);
+ reconcileOperation->mutable_resource_provider_id()->CopyFrom(
+ providerId);
+ } else {
+ // If a known operation became terminal between any previous offer
+ // operation status update and this `UpdateSlaveMessage`, the total
+ // resources we were sent already had the operation applied. We need
+ // to update the state of the operation to terminal here so that any
+ // update sent by the agent later does not cause us to apply the
+ // operation again.
+
+ const Operation* newOperation = newOperations.at(uuid);
+
+ if (!protobuf::isTerminalState(
+ oldOperation->latest_status().state()) &&
+ protobuf::isTerminalState(
+ newOperation->latest_status().state())) {
+ Operation* operation = CHECK_NOTNULL(slave->getOperation(uuid));
+
+ UpdateOperationStatusMessage update =
+ protobuf::createUpdateOperationStatusMessage(
+ uuid,
+ newOperation->latest_status(),
+ newOperation->latest_status(),
+ operation->framework_id(),
+ operation->slave_id());
+
+ updateOperation(
+ operation, update, false); // Do not update resources.
+ }
}
}
+
+ // Reconcile the total resources. This includes undoing
+ // speculated operations which are only visible in the total,
+ // but never in the used resources. We explicitly allow for
+ // resource providers to change from or to zero capacity.
+ const Resources oldResources =
+ slave->totalResources.filter([&providerId](const Resource& resource) {
+ return resource.provider_id() == providerId;
+ });
+
+ slave->totalResources -= oldResources;
+ slave->totalResources += resourceProvider.total_resources();
+
+ oldProvider.totalResources = resourceProvider.total_resources();
+
+ // Reconcile resource versions.
+ oldProvider.resourceVersion = resourceProvider.resource_version_uuid();
}
+ }
- // Reconcile the total resources. This includes undoing
- // speculated operations which are only visible in the total,
- // but never in the used resources. We explicitly allow for
- // resource providers to change from or to zero capacity.
- const Resources oldResources =
- slave->totalResources.filter([&providerId](const Resource& resource) {
- return resource.provider_id() == providerId;
- });
+ // Garbage-collect disappeared resource providers disappeared from
+ // the agent. We only perform the cleanup if the agent sent some set
+ // of resource providers as absence of a set resource_providers
+ // field does not communicate absence of resource providers.
+ const hashset<ResourceProviderID> disappearedResourceProviders =
+ slave->resourceProviders.keys() - receivedResourceProviders;
+
+ foreach (
+ const ResourceProviderID& resourceProviderId,
+ disappearedResourceProviders) {
+ LOG(INFO) << "Garbage collecting resource provider " << resourceProviderId
+ << " since it disappeared from the reported agent state";
- slave->totalResources -= oldResources;
- slave->totalResources += resourceProvider.total_resources();
+ // Remove the resource provider resources from the total resources. This
+ // includes undoing speculated operations which are only visible in the
+ // total, but never in the used resources.
+ CHECK(slave->resourceProviders.contains(resourceProviderId));
- oldProvider.totalResources = resourceProvider.total_resources();
+ slave->totalResources -=
+ slave->resourceProviders.at(resourceProviderId).totalResources;
- // Reconcile resource versions.
- oldProvider.resourceVersion = resourceProvider.resource_version_uuid();
+ slave->resourceProviders.erase(resourceProviderId);
}
}
diff --git a/src/messages/messages.proto b/src/messages/messages.proto
index 41e6a8a..633dddb 100644
--- a/src/messages/messages.proto
+++ b/src/messages/messages.proto
@@ -738,7 +738,7 @@ message UpdateSlaveMessage {
// Whether to update oversubscribed resources or not. If we just use
// `oversubscribed_resources`, we don't have a way to tell if the
- // intention is to update the oversubscribed resoruces to empty, or
+ // intention is to update the oversubscribed resources to empty, or
// leave it unchanged. For backwards compatibility, if this field is
// unset (version < 1.5), we treat that as if only
// `oversubscribed_resources` was set.
@@ -777,7 +777,8 @@ message UpdateSlaveMessage {
repeated ResourceProvider providers = 1;
}
- // The list of all resource providers on this agent.
+ // The list of all resource providers on this agent. If this field
+ // is unset the resource providers on the agent remained unchanged.
optional ResourceProviders resource_providers = 8;
}
[mesos] 02/02: Changed agent to not report disconnected resource
providers.
Posted by bb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
bbannier pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 7447fbc45d1375dd64644a3b130007da061fd5dc
Author: Benjamin Bannier <be...@mesosphere.io>
AuthorDate: Fri Jan 11 09:44:24 2019 +0100
Changed agent to not report disconnected resource providers.
This patch changes the agent to not report disconnected resource
providers in the `UpdateSlaveMessage` it emits. We also fix how it
generates resource provider-related updates when no resource providers
are connected to allow the master to react properly when a resource
provider disappears.
Review: https://reviews.apache.org/r/69338/
---
src/slave/slave.cpp | 19 +++++++++----------
src/tests/resource_provider_manager_tests.cpp | 13 +++----------
2 files changed, 12 insertions(+), 20 deletions(-)
diff --git a/src/slave/slave.cpp b/src/slave/slave.cpp
index ad3b693..10cbc19 100644
--- a/src/slave/slave.cpp
+++ b/src/slave/slave.cpp
@@ -7673,6 +7673,11 @@ UpdateSlaveMessage Slave::generateResourceProviderUpdate() const
}
}
+ // Always add a `resource_providers` field so we can distinguish the
+ // empty and unset case.
+ UpdateSlaveMessage::ResourceProviders* providers =
+ message.mutable_resource_providers();
+
foreachvalue (ResourceProvider* resourceProvider, resourceProviders) {
// If the resource provider has not updated its state we do not
// need to and cannot include its information in an
@@ -7681,8 +7686,7 @@ UpdateSlaveMessage Slave::generateResourceProviderUpdate() const
continue;
}
- UpdateSlaveMessage::ResourceProvider* provider =
- message.mutable_resource_providers()->add_providers();
+ UpdateSlaveMessage::ResourceProvider* provider = providers->add_providers();
provider->mutable_info()->CopyFrom(
resourceProvider->info);
@@ -7961,17 +7965,12 @@ void Slave::handleResourceProviderMessage(
break;
}
- // A disconnected resource provider effectively results in its
- // total resources to be set to empty. This will cause offers to
- // be rescinded so that no operation or task can be launched
- // using resources from the disconnected resource provider. If
- // later, the resource provider reconnects, it'll result in
- // an `UpdateSlaveMessage` so that its resources can be offered
- // again by the master.
+ // Remove the resource provider's resources from the agent's
+ // total resources and remove it from our internal tracking.
CHECK(totalResources.contains(resourceProvider->totalResources));
totalResources -= resourceProvider->totalResources;
- resourceProvider->totalResources = Resources();
+ resourceProviders.erase(resourceProviderId);
// Send the updated resources to the master if the agent is running. Note
// that since we have already updated our copy of the latest resource
diff --git a/src/tests/resource_provider_manager_tests.cpp b/src/tests/resource_provider_manager_tests.cpp
index 7049698..20cfb34 100644
--- a/src/tests/resource_provider_manager_tests.cpp
+++ b/src/tests/resource_provider_manager_tests.cpp
@@ -1350,16 +1350,9 @@ TEST_P(ResourceProviderManagerHttpApiTest, ResourceProviderDisconnect)
// Simulate a resource provider disconnection.
resourceProvider.reset();
- {
- AWAIT_READY(updateSlaveMessage);
- ASSERT_TRUE(updateSlaveMessage->has_resource_providers());
- ASSERT_EQ(1, updateSlaveMessage->resource_providers().providers_size());
-
- const Resources& totalResources =
- updateSlaveMessage->resource_providers().providers(0).total_resources();
-
- EXPECT_FALSE(totalResources.contains(devolve(disk)));
- }
+ AWAIT_READY(updateSlaveMessage);
+ ASSERT_TRUE(updateSlaveMessage->has_resource_providers());
+ EXPECT_EQ(0, updateSlaveMessage->resource_providers().providers_size());
}