You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gobblin.apache.org by zi...@apache.org on 2022/06/17 18:29:04 UTC
[gobblin] branch master updated: make the requestedContainerCountMap correctly update the container count (#3523)
This is an automated email from the ASF dual-hosted git repository.
zihanli58 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/gobblin.git
The following commit(s) were added to refs/heads/master by this push:
new 2414bc6ac make the requestedContainerCountMap correctly update the container count (#3523)
2414bc6ac is described below
commit 2414bc6ac20acdf749ef2de71e2c9ba9fc90b687
Author: Hanghang Nate Liu <na...@gmail.com>
AuthorDate: Fri Jun 17 11:28:59 2022 -0700
make the requestedContainerCountMap correctly update the container count (#3523)
update the place to decrease the requestedContainerCountMap
---
.../src/main/java/org/apache/gobblin/yarn/YarnService.java | 11 +++--------
1 file changed, 3 insertions(+), 8 deletions(-)
diff --git a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnService.java b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnService.java
index d265d0296..a81960fdf 100644
--- a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnService.java
+++ b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnService.java
@@ -448,14 +448,7 @@ public class YarnService extends AbstractIdleService {
for(; requestedContainerCount < desiredContainerCount; requestedContainerCount++) {
requestContainer(Optional.absent(), yarnContainerRequestBundle.getHelixTagResourceMap().get(currentHelixTag));
}
- requestedContainerCountMap.put(currentHelixTag, desiredContainerCount);
- }
-
- // If a requested tag is not presented in the new request, update the requested count to 0 as we should release them
- for(String requestedHelixTag : requestedContainerCountMap.keySet()) {
- if(!yarnContainerRequestBundle.getHelixTagContainerCountMap().containsKey(requestedHelixTag)) {
- requestedContainerCountMap.put(requestedHelixTag, 0);
- }
+ requestedContainerCountMap.put(currentHelixTag, requestedContainerCount);
}
// If the total desired is lower than the currently allocated amount then release free containers.
@@ -473,6 +466,8 @@ public class YarnService extends AbstractIdleService {
ContainerInfo containerInfo = entry.getValue();
if (!inUseInstances.contains(containerInfo.getHelixParticipantId())) {
containersToRelease.add(containerInfo.getContainer());
+ requestedContainerCountMap.put(containerInfo.getHelixTag(),
+ requestedContainerCountMap.get(containerInfo.getHelixTag()) - 1);
}
if (containersToRelease.size() == numToShutdown) {