You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by ch...@apache.org on 2019/03/05 04:50:41 UTC
[mesos] branch master updated (5a2df2c -> 120fab6)
This is an automated email from the ASF dual-hosted git repository.
chhsiao pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git.
from 5a2df2c Removed unnecessary filters in SLRP tests.
new e2b16c1 Changed the `getResourceProviderId` helper to take a `Resources`.
new c2b094f Do not fail a task if it doesn't use resources from a failed provider.
new f19deaf Added MESOS-9607 to the 1.7.3 CHANGELOG.
new 120fab6 Converted `UUID` proto messages to `id::UUID` in the RP manager.
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
CHANGELOG | 1 +
src/common/resources_utils.cpp | 27 +++++-----
src/common/resources_utils.hpp | 11 ++--
src/master/master.cpp | 3 +-
src/resource_provider/manager.cpp | 40 +++++++++------
src/slave/slave.cpp | 104 ++++++++++++++++++++++----------------
src/slave/slave.hpp | 8 +--
7 files changed, 107 insertions(+), 87 deletions(-)
[mesos] 03/04: Added MESOS-9607 to the 1.7.3 CHANGELOG.
Posted by ch...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
chhsiao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit f19deafb774a8a683e746138a84fb32efb9d7012
Author: Chun-Hung Hsiao <ch...@mesosphere.io>
AuthorDate: Mon Mar 4 12:22:57 2019 -0800
Added MESOS-9607 to the 1.7.3 CHANGELOG.
---
CHANGELOG | 1 +
1 file changed, 1 insertion(+)
diff --git a/CHANGELOG b/CHANGELOG
index 06a0bb2..ef88383 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -21,6 +21,7 @@ Release Notes - Mesos - Version 1.7.3 (WIP)
** Bug
* [MESOS-9507] - Agent could not recover due to empty docker volume checkpointed files.
* [MESOS-9549] - nvidia/cuda 10 does not work on GPU isolator.
+ * [MESOS-9607] - Removing a resource provider with consumers breaks resource publishing.
Release Notes - Mesos - Version 1.7.2
[mesos] 04/04: Converted `UUID` proto messages to `id::UUID` in the
RP manager.
Posted by ch...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
chhsiao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 120fab6af265fa65dcfaf0c6ca37a276b9b6714f
Author: Chun-Hung Hsiao <ch...@mesosphere.io>
AuthorDate: Fri Mar 1 12:08:01 2019 -0800
Converted `UUID` proto messages to `id::UUID` in the RP manager.
Since the RP manager might interact with an arbitrary resource provider,
it should convert a `UUID` proto message received in a
`UPDATE_PUBLISH_RESOURCES_STATUS` call into a `id::UUID`, and ignore the
call if its `UUID` is not valid.
Review: https://reviews.apache.org/r/70082
---
src/resource_provider/manager.cpp | 36 +++++++++++++++++++++++-------------
1 file changed, 23 insertions(+), 13 deletions(-)
diff --git a/src/resource_provider/manager.cpp b/src/resource_provider/manager.cpp
index 876a8b2..7d3338e 100644
--- a/src/resource_provider/manager.cpp
+++ b/src/resource_provider/manager.cpp
@@ -173,7 +173,7 @@ struct ResourceProvider
ResourceProviderInfo info;
HttpConnection http;
- hashmap<UUID, Owned<Promise<Nothing>>> publishes;
+ hashmap<id::UUID, Owned<Promise<Nothing>>> publishes;
};
@@ -702,12 +702,13 @@ Future<Nothing> ResourceProviderManagerProcess::publishResources(
foreachpair (const ResourceProviderID& resourceProviderId,
const Resources& resources,
providedResources) {
- UUID uuid = protobuf::createUUID();
+ id::UUID uuid = id::UUID::random();
Event event;
event.set_type(Event::PUBLISH_RESOURCES);
- event.mutable_publish_resources()->mutable_uuid()->CopyFrom(uuid);
- event.mutable_publish_resources()->mutable_resources()->CopyFrom(resources);
+ *event.mutable_publish_resources()->mutable_uuid() =
+ protobuf::createUUID(uuid);
+ *event.mutable_publish_resources()->mutable_resources() = resources;
ResourceProvider* resourceProvider =
resourceProviders.subscribed.at(resourceProviderId).get();
@@ -960,30 +961,39 @@ void ResourceProviderManagerProcess::updatePublishResourcesStatus(
ResourceProvider* resourceProvider,
const Call::UpdatePublishResourcesStatus& update)
{
- const UUID& uuid = update.uuid();
+ Try<id::UUID> uuid = id::UUID::fromBytes(update.uuid().value());
+
+ if (uuid.isError()) {
+ LOG(ERROR)
+ << "Ignoring UpdatePublishResourcesStatus from resource provider "
+ << resourceProvider->info.id() << ": " << uuid.error();
+
+ return;
+ }
+
+ if (!resourceProvider->publishes.contains(uuid.get())) {
+ LOG(ERROR)
+ << "Ignoring UpdatePublishResourcesStatus from resource provider "
+ << resourceProvider->info.id() << ": Unknown UUID " << uuid.get();
- if (!resourceProvider->publishes.contains(uuid)) {
- LOG(ERROR) << "Ignoring UpdatePublishResourcesStatus from resource"
- << " provider " << resourceProvider->info.id()
- << " because UUID " << uuid << " is unknown";
return;
}
LOG(INFO)
<< "Received UPDATE_PUBLISH_RESOURCES_STATUS call for PUBLISH_RESOURCES"
- << " event " << uuid << " with " << update.status()
+ << " event " << uuid.get() << " with " << update.status()
<< " status from resource provider " << resourceProvider->info.id();
if (update.status() == Call::UpdatePublishResourcesStatus::OK) {
- resourceProvider->publishes.at(uuid)->set(Nothing());
+ resourceProvider->publishes.at(uuid.get())->set(Nothing());
} else {
// TODO(jieyu): Consider to include an error message in
// 'UpdatePublishResourcesStatus' and surface that to the caller.
- resourceProvider->publishes.at(uuid)->fail(
+ resourceProvider->publishes.at(uuid.get())->fail(
"Received " + stringify(update.status()) + " status");
}
- resourceProvider->publishes.erase(uuid);
+ resourceProvider->publishes.erase(uuid.get());
}
[mesos] 01/04: Changed the `getResourceProviderId` helper to take a
`Resources`.
Posted by ch...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
chhsiao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit e2b16c1d0066cddaf1f4e5405c8a2e0a431e0197
Author: Chun-Hung Hsiao <ch...@mesosphere.io>
AuthorDate: Thu Feb 28 21:23:13 2019 -0800
Changed the `getResourceProviderId` helper to take a `Resources`.
Review: https://reviews.apache.org/r/70080
---
src/common/resources_utils.cpp | 27 ++++++++++++---------------
src/common/resources_utils.hpp | 11 +++++------
src/master/master.cpp | 3 ++-
3 files changed, 19 insertions(+), 22 deletions(-)
diff --git a/src/common/resources_utils.cpp b/src/common/resources_utils.cpp
index 3b4fcd6..3786bcf 100644
--- a/src/common/resources_utils.cpp
+++ b/src/common/resources_utils.cpp
@@ -334,34 +334,31 @@ Result<ResourceProviderID> getResourceProviderId(
}
-Result<ResourceProviderID> getResourceProviderId(
- const ResourceConversion& conversion)
+Result<ResourceProviderID> getResourceProviderId(const Resources& resources)
{
- if (conversion.consumed.empty()) {
- return Error("Could not determine resource provider");
+ if (resources.empty()) {
+ return Error("Cannot determine resource provider for empty resources");
}
- const Resource& consumed = *conversion.consumed.begin();
-
+ const Resource resource = *resources.begin();
const Option<ResourceProviderID> resourceProviderId =
- consumed.has_provider_id()
- ? consumed.provider_id()
- : Option<ResourceProviderID>::none();
-
+ resource.has_provider_id() ? resource.provider_id()
+ : Option<ResourceProviderID>::none();
- foreach (const Resource& resource, conversion.consumed) {
+ foreach (const Resource& resource_, resources) {
const Option<ResourceProviderID> resourceProviderId_ =
- resource.has_provider_id()
- ? resource.provider_id()
- : Option<ResourceProviderID>::none();
+ resource_.has_provider_id() ? resource_.provider_id()
+ : Option<ResourceProviderID>::none();
+
if (resourceProviderId_ != resourceProviderId) {
- return Error("Conversion works on multiple resource providers");
+ return Error("Resources are from multiple resource providers");
}
}
return resourceProviderId;
}
+
void convertResourceFormat(Resource* resource, ResourceFormat format)
{
switch (format) {
diff --git a/src/common/resources_utils.hpp b/src/common/resources_utils.hpp
index 74cf747..42b6caf 100644
--- a/src/common/resources_utils.hpp
+++ b/src/common/resources_utils.hpp
@@ -50,7 +50,7 @@ Try<Resources> applyCheckpointedResources(
// Returns the resource provider ID associated with the given
-// operation. Returns None() if the operation is for agent default
+// operation. Returns None if the operation is for agent default
// resources. We assume the given operation is validated. Therefore,
// the specified operation should not contain resources from more than
// one resource provider.
@@ -58,12 +58,11 @@ Result<ResourceProviderID> getResourceProviderId(
const Offer::Operation& operation);
-// Returns the ID of the resource provider affected by a resource
-// conversion. Returns None() if the conversion is on agent default
-// resources. We assume a single conversion only being applied on
-// resources from a single resource provider.
+// Returns the resource provider ID associated with the given resources, None if
+// the given resources are agent default resources, or Error if the given
+// resources are from more than one resource providers.
Result<ResourceProviderID> getResourceProviderId(
- const ResourceConversion& conversion);
+ const Resources& resources);
// Returns the resource conversions from the given offer operation.
diff --git a/src/master/master.cpp b/src/master/master.cpp
index 1389e30..3bf84b8 100644
--- a/src/master/master.cpp
+++ b/src/master/master.cpp
@@ -13184,7 +13184,8 @@ void Slave::apply(const vector<ResourceConversion>& conversions)
// Also apply the conversion to the explicitly maintained resource
// provider resources.
foreach (const ResourceConversion& conversion, conversions) {
- Result<ResourceProviderID> providerId = getResourceProviderId(conversion);
+ Result<ResourceProviderID> providerId =
+ getResourceProviderId(conversion.consumed);
if (providerId.isNone()) {
continue;
[mesos] 02/04: Do not fail a task if it doesn't use resources from
a failed provider.
Posted by ch...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
chhsiao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/mesos.git
commit c2b094f1352e8ba9d5fc973269ea83271738d9a9
Author: Chun-Hung Hsiao <ch...@mesosphere.io>
AuthorDate: Fri Mar 1 12:09:06 2019 -0800
Do not fail a task if it doesn't use resources from a failed provider.
`Slave::publishResources` will no longer ask all resource providers to
publish all allocated resources. Instead, it only asks those of the
task's resources to publish resources, so a failed resource provider
would only fail tasks that want to use its resources.
Review: https://reviews.apache.org/r/70081
---
src/resource_provider/manager.cpp | 4 +-
src/slave/slave.cpp | 104 ++++++++++++++++++++++----------------
src/slave/slave.hpp | 8 +--
3 files changed, 64 insertions(+), 52 deletions(-)
diff --git a/src/resource_provider/manager.cpp b/src/resource_provider/manager.cpp
index 2cde62a..876a8b2 100644
--- a/src/resource_provider/manager.cpp
+++ b/src/resource_provider/manager.cpp
@@ -980,9 +980,7 @@ void ResourceProviderManagerProcess::updatePublishResourcesStatus(
// TODO(jieyu): Consider to include an error message in
// 'UpdatePublishResourcesStatus' and surface that to the caller.
resourceProvider->publishes.at(uuid)->fail(
- "Failed to publish resources for resource provider " +
- stringify(resourceProvider->info.id()) + ": Received " +
- stringify(update.status()) + " status");
+ "Received " + stringify(update.status()) + " status");
}
resourceProvider->publishes.erase(uuid);
diff --git a/src/slave/slave.cpp b/src/slave/slave.cpp
index f9b5817..4073d8a 100644
--- a/src/slave/slave.cpp
+++ b/src/slave/slave.cpp
@@ -3069,7 +3069,7 @@ void Slave::__run(
LOG(INFO) << "Queued " << taskOrTaskGroup(task, taskGroup)
<< " for executor " << *executor;
- publishResources()
+ publishResources(executor->containerId, executor->allocatedResources())
.then(defer(self(), [=] {
return containerizer->update(
executor->containerId,
@@ -3531,12 +3531,7 @@ void Slave::launchExecutor(
<< "' of framework " << framework->id();
// Launch the container.
- // NOTE: Since we modify the ExecutorInfo to include the task's
- // resources when launching the executor, these resources need to be
- // published before the containerizer preparing them. This should be
- // revisited after MESOS-600.
- publishResources(
- taskInfo.isSome() ? taskInfo->resources() : Option<Resources>::none())
+ publishResources(executor->containerId, resources)
.then(defer(self(), [=] {
return containerizer->launch(
executor->containerId,
@@ -4963,7 +4958,7 @@ void Slave::subscribe(
}
}
- publishResources()
+ publishResources(executor->containerId, executor->allocatedResources())
.then(defer(self(), [=] {
return containerizer->update(
executor->containerId,
@@ -5120,7 +5115,7 @@ void Slave::registerExecutor(
}
}
- publishResources()
+ publishResources(executor->containerId, executor->allocatedResources())
.then(defer(self(), [=] {
return containerizer->update(
executor->containerId,
@@ -8768,48 +8763,71 @@ void Slave::apply(Operation* operation)
Future<Nothing> Slave::publishResources(
- const Option<Resources>& additionalResources)
+ const ContainerID& containerId, const Resources& resources)
{
- // If the resource provider manager has not been created yet no resource
- // providers have been added and we do not need to publish anything.
- if (resourceProviderManager == nullptr) {
- // We check whether the passed additional resources are compatible
- // with the expectation that no resource provider resources are in
- // use, yet. This is not an exhaustive consistency check.
- if (additionalResources.isSome()) {
- foreach (const Resource& resource, additionalResources.get()) {
- CHECK(!resource.has_provider_id())
- << "Cannot publish resource provider resources "
- << additionalResources.get()
- << " until resource providers have subscribed";
- }
+ hashset<ResourceProviderID> resourceProviderIds;
+ foreach (const Resource& resource, resources) {
+ if (resource.has_provider_id()) {
+ resourceProviderIds.insert(resource.provider_id());
}
-
- return Nothing();
}
- Resources resources;
+ vector<Future<Nothing>> futures;
+ foreach (const ResourceProviderID& resourceProviderId, resourceProviderIds) {
+ auto hasResourceProviderId = [&](const Resource& resource) {
+ return resource.has_provider_id() &&
+ resource.provider_id() == resourceProviderId;
+ };
- // NOTE: For resources providers that serve quantity-based resources
- // without any identifiers (such as memory), it is very hard to keep
- // track of published resources. So instead of implementing diff-based
- // resource publishing, we implement an "ensure-all" semantics, and
- // always calculate the total resources that need to remain published.
- foreachvalue (const Framework* framework, frameworks) {
- // NOTE: We do not call `framework->allocatedResource()` here
- // because we do not want to publsh resources for pending tasks that
- // have not been authorized yet.
- foreachvalue (const Executor* executor, framework->executors) {
- resources += executor->allocatedResources();
+ // NOTE: For resources providers that serve quantity-based resources without
+ // identifier (such as cpus and mem), we cannot achieve idempotency with
+ // diff-based resource publishing, so we have to implement the "ensure-all"
+ // semantics, and always calculate the total resources to publish.
+ Option<Resources> containerResources;
+ Resources complementaryResources;
+ foreachvalue (const Framework* framework, frameworks) {
+ foreachvalue (const Executor* executor, framework->executors) {
+ if (executor->containerId == containerId) {
+ containerResources = resources.filter(hasResourceProviderId);
+ } else {
+ complementaryResources +=
+ executor->allocatedResources().filter(hasResourceProviderId);
+ }
+ }
}
- }
- if (additionalResources.isSome()) {
- resources += additionalResources.get();
- }
+ if (containerResources.isNone()) {
+ // NOTE: This actually should not happen, as the callers have already
+ // ensured the existence of the executor before calling this function
+ // synchronously. However we still treat this as a nonfatal error since
+ // this might change in the future.
+ LOG(WARNING) << "Ignoring publishing resources for container "
+ << containerId << ": Executor cannot be found";
+
+ return Nothing();
+ }
- return CHECK_NOTNULL(resourceProviderManager.get())
- ->publishResources(resources);
+ // Since we already have resources from any resource provider in the
+ // resource pool, the resource provider manager must have been created.
+ futures.push_back(
+ CHECK_NOTNULL(resourceProviderManager.get())
+ ->publishResources(containerResources.get() + complementaryResources)
+ .repair([=](const Future<Nothing>& future) -> Future<Nothing> {
+ // TODO(chhsiao): Consider surfacing the set of published resources
+ // and only fail if `published - complementaryResources` does not
+ // contain `containerResources`.
+ return Failure(
+ "Failed to publish resources '" +
+ stringify(containerResources.get()) + "' for container " +
+ stringify(containerId) + ": " + future.failure());
+ }));
+ }
+
+ // NOTE: Resource cleanups (e.g., unpublishing) are not performed at task
+ // completion, but rather done __lazily__ when necessary. This is not just an
+ // optimization but required because resource allocations are tied to task
+ // lifecycles. As a result, no cleanup is needed here if any future fails.
+ return collect(futures).then([] { return Nothing(); });
}
diff --git a/src/slave/slave.hpp b/src/slave/slave.hpp
index d9dbecd..2715934 100644
--- a/src/slave/slave.hpp
+++ b/src/slave/slave.hpp
@@ -695,13 +695,9 @@ private:
void apply(Operation* operation);
- // Publish all resources that are needed to run the current set of
- // tasks and executors on the agent.
- // NOTE: The `additionalResources` parameter is for publishing
- // additional task resources when launching executors. Consider
- // removing this parameter once we revisited MESOS-600.
+ // Prepare all resources to be consumed by the specified container.
process::Future<Nothing> publishResources(
- const Option<Resources>& additionalResources = None());
+ const ContainerID& containerId, const Resources& resources);
// PullGauge methods.
double _frameworks_active()