You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ju...@apache.org on 2016/01/06 23:29:33 UTC
hadoop git commit: YARN-4380.
TestResourceLocalizationService.testDownloadingResourcesOnContainerKill fails
intermittently. Contributed by Varun Saxena.
Repository: hadoop
Updated Branches:
refs/heads/branch-2.6 cfbb03542 -> 4be18edf2
YARN-4380. TestResourceLocalizationService.testDownloadingResourcesOnContainerKill fails intermittently. Contributed by Varun Saxena.
(cherry picked from commit 0656d2dc83af6a48a8d8d0e37cdf1f813124f366)
(cherry picked from commit d76b523b020fcd76e9abdd661967ca4a931d9863)
Conflicts:
hadoop-yarn-project/CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4be18edf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4be18edf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4be18edf
Branch: refs/heads/branch-2.6
Commit: 4be18edf2f1946e7a6d55799b9db95e01c343d36
Parents: cfbb035
Author: Tsuyoshi Ozawa <oz...@apache.org>
Authored: Thu Nov 26 01:10:02 2015 +0900
Committer: Junping Du <ju...@apache.org>
Committed: Wed Jan 6 14:34:15 2016 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../TestResourceLocalizationService.java | 18 +++++++++++++++++-
2 files changed, 20 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4be18edf/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 684476c..fe921b5 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -37,6 +37,9 @@ Release 2.6.4 - UNRELEASED
YARN-4354. Public resource localization fails with NPE. (Jason Lowe via
junping_du)
+ YARN-4380. TestResourceLocalizationService.testDownloadingResourcesOnContainerKill
+ fails intermittently. (Varun Saxena via ozawa)
+
Release 2.6.3 - 2015-12-17
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4be18edf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
index d19315d..f374f66 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
@@ -61,6 +61,7 @@ import java.util.Set;
import java.util.concurrent.BrokenBarrierException;
import java.util.concurrent.CyclicBarrier;
import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
@@ -1104,16 +1105,23 @@ public class TestResourceLocalizationService {
private static class DummyExecutor extends DefaultContainerExecutor {
private volatile boolean stopLocalization = false;
+ private AtomicInteger numLocalizers = new AtomicInteger(0);
@Override
public void startLocalizer(Path nmPrivateContainerTokensPath,
InetSocketAddress nmAddr, String user, String appId, String locId,
LocalDirsHandlerService dirsHandler)
throws IOException, InterruptedException {
+ numLocalizers.incrementAndGet();
while (!stopLocalization) {
Thread.yield();
}
}
- void setStopLocalization() {
+ private void waitForLocalizers(int num) {
+ while (numLocalizers.intValue() < num) {
+ Thread.yield();
+ }
+ }
+ private void setStopLocalization() {
stopLocalization = true;
}
}
@@ -1256,6 +1264,10 @@ public class TestResourceLocalizationService {
spyService.handle(new ContainerLocalizationRequestEvent(c2, rsrcs1));
dispatcher.await();
+ // Wait for localizers of both container c1 and c2 to begin.
+ exec.waitForLocalizers(2);
+ LocalizerRunner locC1 =
+ spyService.getLocalizerRunner(c1.getContainerId().toString());
final String containerIdStr = c1.getContainerId().toString();
// Heartbeats from container localizer
LocalResourceStatus rsrc1success = mock(LocalResourceStatus.class);
@@ -1323,6 +1335,10 @@ public class TestResourceLocalizationService {
Set<Path> paths =
Sets.newHashSet(new Path(locPath1), new Path(locPath1 + "_tmp"),
new Path(locPath2), new Path(locPath2 + "_tmp"));
+ // Wait for localizer runner thread for container c1 to finish.
+ while (locC1.getState() != Thread.State.TERMINATED) {
+ Thread.sleep(50);
+ }
// Verify if downloading resources were submitted for deletion.
verify(delService).delete(eq(user),
(Path) eq(null), argThat(new DownloadingPathsMatcher(paths)));