You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ro...@apache.org on 2018/08/18 05:42:18 UTC
[1/2] hadoop git commit: YARN-7835. Race condition in NM while
publishing events if second attempt is launched on the same node. (Rohith
Sharma K S via Haibo Chen)
Repository: hadoop
Updated Branches:
refs/heads/branch-3.0 a41f18098 -> 675aa2bbc
YARN-7835. Race condition in NM while publishing events if second attempt is launched on the same node. (Rohith Sharma K S via Haibo Chen)
(cherry picked from commit d1274c3b71549cb000868500c293cafd880b3713)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8118b14d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8118b14d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8118b14d
Branch: refs/heads/branch-3.0
Commit: 8118b14db84f8a4b5c955894b65f2f8d22be5254
Parents: a41f180
Author: Haibo Chen <ha...@apache.org>
Authored: Wed Feb 28 21:06:42 2018 -0800
Committer: Rohith Sharma K S <ro...@apache.org>
Committed: Sat Aug 18 11:06:02 2018 +0530
----------------------------------------------------------------------
.../PerNodeTimelineCollectorsAuxService.java | 51 +++++++++--
...TestPerNodeTimelineCollectorsAuxService.java | 93 ++++++++++++++++----
2 files changed, 120 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8118b14d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
index 66f9aab..c15f99d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
@@ -19,7 +19,12 @@
package org.apache.hadoop.yarn.server.timelineservice.collector;
import java.nio.ByteBuffer;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
@@ -31,6 +36,7 @@ import org.apache.hadoop.util.ShutdownHookManager;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.api.ApplicationInitializationContext;
@@ -59,6 +65,8 @@ public class PerNodeTimelineCollectorsAuxService extends AuxiliaryService {
private final NodeTimelineCollectorManager collectorManager;
private long collectorLingerPeriod;
private ScheduledExecutorService scheduler;
+ private Map<ApplicationId, Set<ContainerId>> appIdToContainerId =
+ new ConcurrentHashMap<>();
public PerNodeTimelineCollectorsAuxService() {
this(new NodeTimelineCollectorManager(true));
@@ -148,7 +156,15 @@ public class PerNodeTimelineCollectorsAuxService extends AuxiliaryService {
if (context.getContainerType() == ContainerType.APPLICATION_MASTER) {
ApplicationId appId = context.getContainerId().
getApplicationAttemptId().getApplicationId();
- addApplication(appId, context.getUser());
+ synchronized (appIdToContainerId) {
+ Set<ContainerId> masterContainers = appIdToContainerId.get(appId);
+ if (masterContainers == null) {
+ masterContainers = new HashSet<>();
+ appIdToContainerId.put(appId, masterContainers);
+ }
+ masterContainers.add(context.getContainerId());
+ addApplication(appId, context.getUser());
+ }
}
}
@@ -162,17 +178,36 @@ public class PerNodeTimelineCollectorsAuxService extends AuxiliaryService {
// intercept the event of the AM container being stopped and remove the app
// level collector service
if (context.getContainerType() == ContainerType.APPLICATION_MASTER) {
- final ApplicationId appId =
- context.getContainerId().getApplicationAttemptId().getApplicationId();
- scheduler.schedule(new Runnable() {
- public void run() {
- removeApplication(appId);
- }
- }, collectorLingerPeriod, TimeUnit.MILLISECONDS);
+ final ContainerId containerId = context.getContainerId();
+ removeApplicationCollector(containerId);
}
}
@VisibleForTesting
+ protected Future removeApplicationCollector(final ContainerId containerId) {
+ final ApplicationId appId =
+ containerId.getApplicationAttemptId().getApplicationId();
+ return scheduler.schedule(new Runnable() {
+ public void run() {
+ synchronized (appIdToContainerId) {
+ Set<ContainerId> masterContainers = appIdToContainerId.get(appId);
+ if (masterContainers == null) {
+ LOG.info("Stop container for " + containerId
+ + " is called before initializing container.");
+ return;
+ }
+ masterContainers.remove(containerId);
+ if (masterContainers.size() == 0) {
+ // remove only if it is last master container
+ removeApplication(appId);
+ appIdToContainerId.remove(appId);
+ }
+ }
+ }
+ }, collectorLingerPeriod, TimeUnit.MILLISECONDS);
+ }
+
+ @VisibleForTesting
boolean hasApplication(ApplicationId appId) {
return collectorManager.containsTimelineCollector(appId);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8118b14d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java
index f96466f..04b89d6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java
@@ -29,6 +29,7 @@ import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
import java.io.IOException;
+import java.util.concurrent.Future;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.ExitUtil;
@@ -47,16 +48,17 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.GetTimelineCollectorCon
import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineWriterImpl;
import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineWriter;
import org.junit.After;
+import org.junit.Assert;
import org.junit.Test;
public class TestPerNodeTimelineCollectorsAuxService {
private ApplicationAttemptId appAttemptId;
private PerNodeTimelineCollectorsAuxService auxService;
private Configuration conf;
+ private ApplicationId appId;
public TestPerNodeTimelineCollectorsAuxService() {
- ApplicationId appId =
- ApplicationId.newInstance(System.currentTimeMillis(), 1);
+ appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
conf = new YarnConfiguration();
// enable timeline service v.2
@@ -109,15 +111,6 @@ public class TestPerNodeTimelineCollectorsAuxService {
when(context.getContainerType()).thenReturn(
ContainerType.APPLICATION_MASTER);
auxService.stopContainer(context);
- // auxService should have the app's collector and need to remove only after
- // a configured period
- assertTrue(auxService.hasApplication(appAttemptId.getApplicationId()));
- for (int i = 0; i < 4; i++) {
- Thread.sleep(500L);
- if (!auxService.hasApplication(appAttemptId.getApplicationId())) {
- break;
- }
- }
// auxService should not have that app
assertFalse(auxService.hasApplication(appAttemptId.getApplicationId()));
@@ -157,21 +150,53 @@ public class TestPerNodeTimelineCollectorsAuxService {
private PerNodeTimelineCollectorsAuxService
createCollectorAndAddApplication() {
PerNodeTimelineCollectorsAuxService service = createCollector();
+
+ ContainerInitializationContext context =
+ createContainerInitalizationContext(1);
+ service.initializeContainer(context);
+ return service;
+ }
+
+ ContainerInitializationContext createContainerInitalizationContext(
+ int attempt) {
+ appAttemptId = ApplicationAttemptId.newInstance(appId, attempt);
// create an AM container
ContainerId containerId = getAMContainerId();
ContainerInitializationContext context =
mock(ContainerInitializationContext.class);
when(context.getContainerId()).thenReturn(containerId);
- when(context.getContainerType()).thenReturn(
- ContainerType.APPLICATION_MASTER);
- service.initializeContainer(context);
- return service;
+ when(context.getContainerType())
+ .thenReturn(ContainerType.APPLICATION_MASTER);
+ return context;
+ }
+
+ ContainerTerminationContext createContainerTerminationContext(int attempt) {
+ appAttemptId = ApplicationAttemptId.newInstance(appId, attempt);
+ // create an AM container
+ ContainerId containerId = getAMContainerId();
+ ContainerTerminationContext context =
+ mock(ContainerTerminationContext.class);
+ when(context.getContainerId()).thenReturn(containerId);
+ when(context.getContainerType())
+ .thenReturn(ContainerType.APPLICATION_MASTER);
+ return context;
}
private PerNodeTimelineCollectorsAuxService createCollector() {
NodeTimelineCollectorManager collectorManager = createCollectorManager();
PerNodeTimelineCollectorsAuxService service =
- spy(new PerNodeTimelineCollectorsAuxService(collectorManager));
+ spy(new PerNodeTimelineCollectorsAuxService(collectorManager) {
+ @Override
+ protected Future removeApplicationCollector(ContainerId containerId) {
+ Future future = super.removeApplicationCollector(containerId);
+ try {
+ future.get();
+ } catch (Exception e) {
+ Assert.fail("Expeption thrown while removing collector");
+ }
+ return future;
+ }
+ });
service.init(conf);
service.start();
return service;
@@ -202,4 +227,40 @@ public class TestPerNodeTimelineCollectorsAuxService {
private ContainerId getContainerId(long id) {
return ContainerId.newContainerId(appAttemptId, id);
}
+
+ @Test(timeout = 60000)
+ public void testRemoveAppWhenSecondAttemptAMCotainerIsLaunchedSameNode()
+ throws Exception {
+ // add first attempt collector
+ auxService = createCollectorAndAddApplication();
+ // auxService should have a single app
+ assertTrue(auxService.hasApplication(appAttemptId.getApplicationId()));
+
+ // add second attempt collector before first attempt master container stop
+ ContainerInitializationContext containerInitalizationContext =
+ createContainerInitalizationContext(2);
+ auxService.initializeContainer(containerInitalizationContext);
+
+ assertTrue("Applicatin not found in collectors.",
+ auxService.hasApplication(appAttemptId.getApplicationId()));
+
+ // first attempt stop container
+ ContainerTerminationContext context = createContainerTerminationContext(1);
+ auxService.stopContainer(context);
+
+ // 2nd attempt container removed, still collector should hold application id
+ assertTrue("collector has removed application though 2nd attempt"
+ + " is running this node",
+ auxService.hasApplication(appAttemptId.getApplicationId()));
+
+ // second attempt stop container
+ context = createContainerTerminationContext(2);
+ auxService.stopContainer(context);
+
+ // auxService should not have that app
+ assertFalse("Application is not removed from collector",
+ auxService.hasApplication(appAttemptId.getApplicationId()));
+ auxService.close();
+ }
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[2/2] hadoop git commit: YARN-8679. [ATSv2] If HBase cluster is down
for long time,
high chances that NM ContainerManager dispatcher get blocked. Contributed by
Wangda Tan.
Posted by ro...@apache.org.
YARN-8679. [ATSv2] If HBase cluster is down for long time, high chances that NM ContainerManager dispatcher get blocked. Contributed by Wangda Tan.
(cherry picked from commit 4aacbfff605262aaf3dbd926258afcadc86c72c0)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/675aa2bb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/675aa2bb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/675aa2bb
Branch: refs/heads/branch-3.0
Commit: 675aa2bbc05e2f900403f9b862d38ba0cf77c208
Parents: 8118b14
Author: Rohith Sharma K S <ro...@apache.org>
Authored: Sat Aug 18 10:26:55 2018 +0530
Committer: Rohith Sharma K S <ro...@apache.org>
Committed: Sat Aug 18 11:06:14 2018 +0530
----------------------------------------------------------------------
.../applicationsmanager/TestAMLaunchFailure.java | 2 +-
.../applicationsmanager/TestSchedulerNegotiator.java | 2 +-
.../TestTimelineServiceClientIntegration.java | 3 ++-
.../security/TestTimelineAuthFilterForV2.java | 2 +-
.../collector/PerNodeTimelineCollectorsAuxService.java | 13 +++++++++----
5 files changed, 14 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/675aa2bb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMLaunchFailure.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMLaunchFailure.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMLaunchFailure.java
index c0009dd..ad39099 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMLaunchFailure.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMLaunchFailure.java
@@ -90,7 +90,7 @@ public class TestAMLaunchFailure {
// }
//
// @Override
-// public void addApplication(ApplicationId applicationId,
+// public void addApplicationIfAbsent(ApplicationId applicationId,
// ApplicationMaster master, String user, String queue, Priority priority
// , ApplicationStore appStore)
// throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/675aa2bb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestSchedulerNegotiator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestSchedulerNegotiator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestSchedulerNegotiator.java
index 7d06e55..fedbf2b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestSchedulerNegotiator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestSchedulerNegotiator.java
@@ -67,7 +67,7 @@ public class TestSchedulerNegotiator {
// return null;
// }
// @Override
-// public void addApplication(ApplicationId applicationId,
+// public void addApplicationIfAbsent(ApplicationId applicationId,
// ApplicationMaster master, String user, String queue, Priority priority,
// ApplicationStore store)
// throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/675aa2bb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java
index 6a5ef55..7cf7428 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java
@@ -81,7 +81,8 @@ public class TestTimelineServiceClientIntegration {
auxService =
PerNodeTimelineCollectorsAuxService.launchServer(new String[0],
collectorManager, conf);
- auxService.addApplication(ApplicationId.newInstance(0, 1), "user");
+ auxService
+ .addApplicationIfAbsent(ApplicationId.newInstance(0, 1), "user");
} catch (ExitUtil.ExitException e) {
fail();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/675aa2bb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/security/TestTimelineAuthFilterForV2.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/security/TestTimelineAuthFilterForV2.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/security/TestTimelineAuthFilterForV2.java
index bb511d8..356bfc7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/security/TestTimelineAuthFilterForV2.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/security/TestTimelineAuthFilterForV2.java
@@ -210,7 +210,7 @@ public class TestTimelineAuthFilterForV2 {
YarnConfiguration.TIMELINE_SERVICE_PRINCIPAL, "localhost");
}
ApplicationId appId = ApplicationId.newInstance(0, 1);
- auxService.addApplication(
+ auxService.addApplicationIfAbsent(
appId, UserGroupInformation.getCurrentUser().getUserName());
if (!withKerberosLogin) {
AppLevelTimelineCollector collector =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/675aa2bb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
index c15f99d..82dd793 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
@@ -125,7 +125,7 @@ public class PerNodeTimelineCollectorsAuxService extends AuxiliaryService {
* @param user Application Master container user.
* @return whether it was added successfully
*/
- public boolean addApplication(ApplicationId appId, String user) {
+ public boolean addApplicationIfAbsent(ApplicationId appId, String user) {
AppLevelTimelineCollector collector =
new AppLevelTimelineCollectorWithAgg(appId, user);
return (collectorManager.putIfAbsent(appId, collector)
@@ -156,15 +156,15 @@ public class PerNodeTimelineCollectorsAuxService extends AuxiliaryService {
if (context.getContainerType() == ContainerType.APPLICATION_MASTER) {
ApplicationId appId = context.getContainerId().
getApplicationAttemptId().getApplicationId();
- synchronized (appIdToContainerId) {
+ synchronized (appIdToContainerId){
Set<ContainerId> masterContainers = appIdToContainerId.get(appId);
if (masterContainers == null) {
masterContainers = new HashSet<>();
appIdToContainerId.put(appId, masterContainers);
}
masterContainers.add(context.getContainerId());
- addApplication(appId, context.getUser());
}
+ addApplicationIfAbsent(appId, context.getUser());
}
}
@@ -189,6 +189,7 @@ public class PerNodeTimelineCollectorsAuxService extends AuxiliaryService {
containerId.getApplicationAttemptId().getApplicationId();
return scheduler.schedule(new Runnable() {
public void run() {
+ boolean shouldRemoveApplication = false;
synchronized (appIdToContainerId) {
Set<ContainerId> masterContainers = appIdToContainerId.get(appId);
if (masterContainers == null) {
@@ -199,10 +200,14 @@ public class PerNodeTimelineCollectorsAuxService extends AuxiliaryService {
masterContainers.remove(containerId);
if (masterContainers.size() == 0) {
// remove only if it is last master container
- removeApplication(appId);
+ shouldRemoveApplication = true;
appIdToContainerId.remove(appId);
}
}
+
+ if (shouldRemoveApplication) {
+ removeApplication(appId);
+ }
}
}, collectorLingerPeriod, TimeUnit.MILLISECONDS);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org