You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by mi...@apache.org on 2015/07/10 17:37:50 UTC
[1/2] hadoop git commit: HADOOP-12107. long running apps may have a
huge number of StatisticsData instances under FileSystem (Sangjin Lee via
Ming Ma)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 c1447e654 -> b169889f0
HADOOP-12107. long running apps may have a huge number of StatisticsData instances under FileSystem (Sangjin Lee via Ming Ma)
(cherry picked from commit 8e1bdc17d9134e01115ae7c929503d8ac0325207)
Conflicts:
hadoop-common-project/hadoop-common/CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5e6bbe60
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5e6bbe60
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5e6bbe60
Branch: refs/heads/branch-2
Commit: 5e6bbe603157f3fcc9b5e7baaab5b943ba3df0da
Parents: c1447e6
Author: Ming Ma <mi...@apache.org>
Authored: Mon Jun 29 14:37:38 2015 -0700
Committer: Ming Ma <mi...@apache.org>
Committed: Fri Jul 10 08:32:27 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../java/org/apache/hadoop/fs/FileSystem.java | 140 +++++++++++++------
.../apache/hadoop/fs/FCStatisticsBaseTest.java | 56 +++++++-
3 files changed, 155 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e6bbe60/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 1b8c55d..42dbb55 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -539,6 +539,9 @@ Release 2.7.1 - 2015-07-06
that method doesn't modify the FsPermission (Bibin A Chundatt via Colin P.
McCabe)
+ HADOOP-12107. long running apps may have a huge number of StatisticsData
+ instances under FileSystem (Sangjin Lee via Ming Ma)
+
Release 2.7.0 - 2015-04-20
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e6bbe60/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index 3a589de..b94f65c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -20,7 +20,8 @@ package org.apache.hadoop.fs;
import java.io.Closeable;
import java.io.FileNotFoundException;
import java.io.IOException;
-import java.lang.ref.WeakReference;
+import java.lang.ref.PhantomReference;
+import java.lang.ref.ReferenceQueue;
import java.net.URI;
import java.net.URISyntaxException;
import java.security.PrivilegedExceptionAction;
@@ -32,7 +33,6 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.IdentityHashMap;
import java.util.Iterator;
-import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
@@ -2918,16 +2918,6 @@ public abstract class FileSystem extends Configured implements Closeable {
volatile int readOps;
volatile int largeReadOps;
volatile int writeOps;
- /**
- * Stores a weak reference to the thread owning this StatisticsData.
- * This allows us to remove StatisticsData objects that pertain to
- * threads that no longer exist.
- */
- final WeakReference<Thread> owner;
-
- StatisticsData(WeakReference<Thread> owner) {
- this.owner = owner;
- }
/**
* Add another StatisticsData object to this one.
@@ -2998,17 +2988,37 @@ public abstract class FileSystem extends Configured implements Closeable {
* Thread-local data.
*/
private final ThreadLocal<StatisticsData> threadData;
-
+
/**
- * List of all thread-local data areas. Protected by the Statistics lock.
+ * Set of all thread-local data areas. Protected by the Statistics lock.
+ * The references to the statistics data are kept using phantom references
+ * to the associated threads. Proper clean-up is performed by the cleaner
+ * thread when the threads are garbage collected.
*/
- private LinkedList<StatisticsData> allData;
+ private final Set<StatisticsDataReference> allData;
+
+ /**
+ * Global reference queue and a cleaner thread that manage statistics data
+ * references from all filesystem instances.
+ */
+ private static final ReferenceQueue<Thread> STATS_DATA_REF_QUEUE;
+ private static final Thread STATS_DATA_CLEANER;
+
+ static {
+ STATS_DATA_REF_QUEUE = new ReferenceQueue<Thread>();
+ // start a single daemon cleaner thread
+ STATS_DATA_CLEANER = new Thread(new StatisticsDataReferenceCleaner());
+ STATS_DATA_CLEANER.
+ setName(StatisticsDataReferenceCleaner.class.getName());
+ STATS_DATA_CLEANER.setDaemon(true);
+ STATS_DATA_CLEANER.start();
+ }
public Statistics(String scheme) {
this.scheme = scheme;
- this.rootData = new StatisticsData(null);
+ this.rootData = new StatisticsData();
this.threadData = new ThreadLocal<StatisticsData>();
- this.allData = null;
+ this.allData = new HashSet<StatisticsDataReference>();
}
/**
@@ -3018,7 +3028,7 @@ public abstract class FileSystem extends Configured implements Closeable {
*/
public Statistics(Statistics other) {
this.scheme = other.scheme;
- this.rootData = new StatisticsData(null);
+ this.rootData = new StatisticsData();
other.visitAll(new StatisticsAggregator<Void>() {
@Override
public void accept(StatisticsData data) {
@@ -3030,6 +3040,63 @@ public abstract class FileSystem extends Configured implements Closeable {
}
});
this.threadData = new ThreadLocal<StatisticsData>();
+ this.allData = new HashSet<StatisticsDataReference>();
+ }
+
+ /**
+ * A phantom reference to a thread that also includes the data associated
+ * with that thread. On the thread being garbage collected, it is enqueued
+ * to the reference queue for clean-up.
+ */
+ private class StatisticsDataReference extends PhantomReference<Thread> {
+ private final StatisticsData data;
+
+ public StatisticsDataReference(StatisticsData data, Thread thread) {
+ super(thread, STATS_DATA_REF_QUEUE);
+ this.data = data;
+ }
+
+ public StatisticsData getData() {
+ return data;
+ }
+
+ /**
+ * Performs clean-up action when the associated thread is garbage
+ * collected.
+ */
+ public void cleanUp() {
+ // use the statistics lock for safety
+ synchronized (Statistics.this) {
+ /*
+ * If the thread that created this thread-local data no longer exists,
+ * remove the StatisticsData from our list and fold the values into
+ * rootData.
+ */
+ rootData.add(data);
+ allData.remove(this);
+ }
+ }
+ }
+
+ /**
+ * Background action to act on references being removed.
+ */
+ private static class StatisticsDataReferenceCleaner implements Runnable {
+ @Override
+ public void run() {
+ while (true) {
+ try {
+ StatisticsDataReference ref =
+ (StatisticsDataReference)STATS_DATA_REF_QUEUE.remove();
+ ref.cleanUp();
+ } catch (Throwable th) {
+ // the cleaner thread should continue to run even if there are
+ // exceptions, including InterruptedException
+ LOG.warn("exception in the cleaner thread but it will continue to "
+ + "run", th);
+ }
+ }
+ }
}
/**
@@ -3038,14 +3105,12 @@ public abstract class FileSystem extends Configured implements Closeable {
public StatisticsData getThreadStatistics() {
StatisticsData data = threadData.get();
if (data == null) {
- data = new StatisticsData(
- new WeakReference<Thread>(Thread.currentThread()));
+ data = new StatisticsData();
threadData.set(data);
+ StatisticsDataReference ref =
+ new StatisticsDataReference(data, Thread.currentThread());
synchronized(this) {
- if (allData == null) {
- allData = new LinkedList<StatisticsData>();
- }
- allData.add(data);
+ allData.add(ref);
}
}
return data;
@@ -3103,21 +3168,9 @@ public abstract class FileSystem extends Configured implements Closeable {
*/
private synchronized <T> T visitAll(StatisticsAggregator<T> visitor) {
visitor.accept(rootData);
- if (allData != null) {
- for (Iterator<StatisticsData> iter = allData.iterator();
- iter.hasNext(); ) {
- StatisticsData data = iter.next();
- visitor.accept(data);
- if (data.owner.get() == null) {
- /*
- * If the thread that created this thread-local data no
- * longer exists, remove the StatisticsData from our list
- * and fold the values into rootData.
- */
- rootData.add(data);
- iter.remove();
- }
- }
+ for (StatisticsDataReference ref: allData) {
+ StatisticsData data = ref.getData();
+ visitor.accept(data);
}
return visitor.aggregate();
}
@@ -3224,7 +3277,7 @@ public abstract class FileSystem extends Configured implements Closeable {
@Override
public String toString() {
return visitAll(new StatisticsAggregator<String>() {
- private StatisticsData total = new StatisticsData(null);
+ private StatisticsData total = new StatisticsData();
@Override
public void accept(StatisticsData data) {
@@ -3257,7 +3310,7 @@ public abstract class FileSystem extends Configured implements Closeable {
*/
public void reset() {
visitAll(new StatisticsAggregator<Void>() {
- private StatisticsData total = new StatisticsData(null);
+ private StatisticsData total = new StatisticsData();
@Override
public void accept(StatisticsData data) {
@@ -3279,6 +3332,11 @@ public abstract class FileSystem extends Configured implements Closeable {
public String getScheme() {
return scheme;
}
+
+ @VisibleForTesting
+ synchronized int getAllThreadLocalDataSize() {
+ return allData.size();
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e6bbe60/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java
index 90337a6..3e33362 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java
@@ -18,26 +18,34 @@
package org.apache.hadoop.fs;
+import static org.apache.hadoop.fs.FileContextTestHelper.createFile;
+
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.List;
import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hadoop.fs.FileSystem.Statistics;
+import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Assert;
import org.junit.Test;
+import com.google.common.base.Supplier;
import com.google.common.util.concurrent.Uninterruptibles;
-import static org.apache.hadoop.fs.FileContextTestHelper.*;
-
/**
* <p>
* Base class to test {@link FileContext} Statistics.
* </p>
*/
public abstract class FCStatisticsBaseTest {
-
static protected int blockSize = 512;
static protected int numBlocks = 1;
@@ -102,6 +110,48 @@ public abstract class FCStatisticsBaseTest {
fc.delete(filePath, true);
}
+ @Test(timeout=60000)
+ public void testStatisticsThreadLocalDataCleanUp() throws Exception {
+ final Statistics stats = new Statistics("test");
+ // create a small thread pool to test the statistics
+ final int size = 2;
+ ExecutorService es = Executors.newFixedThreadPool(size);
+ List<Callable<Boolean>> tasks = new ArrayList<Callable<Boolean>>(size);
+ for (int i = 0; i < size; i++) {
+ tasks.add(new Callable<Boolean>() {
+ public Boolean call() {
+ // this populates the data set in statistics
+ stats.incrementReadOps(1);
+ return true;
+ }
+ });
+ }
+ // run the threads
+ es.invokeAll(tasks);
+ // assert that the data size is exactly the number of threads
+ final AtomicInteger allDataSize = new AtomicInteger(0);
+ allDataSize.set(stats.getAllThreadLocalDataSize());
+ Assert.assertEquals(size, allDataSize.get());
+ Assert.assertEquals(size, stats.getReadOps());
+ // force the GC to collect the threads by shutting down the thread pool
+ es.shutdownNow();
+ es.awaitTermination(1, TimeUnit.MINUTES);
+ es = null;
+ System.gc();
+
+ // wait for up to 10 seconds
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ int size = stats.getAllThreadLocalDataSize();
+ allDataSize.set(size);
+ return size == 0;
+ }
+ }, 1000, 10*1000);
+ Assert.assertEquals(0, allDataSize.get());
+ Assert.assertEquals(size, stats.getReadOps());
+ }
+
/**
* Bytes read may be different for different file systems. This method should
* throw assertion error if bytes read are incorrect.
[2/2] hadoop git commit: YARN-3445. Cache runningApps in RMNode for
getting running apps on given NodeId. (Junping Du via mingma)
Posted by mi...@apache.org.
YARN-3445. Cache runningApps in RMNode for getting running apps on given NodeId. (Junping Du via mingma)
(cherry picked from commit 08244264c0583472b9c4e16591cfde72c6db62a2)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b169889f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b169889f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b169889f
Branch: refs/heads/branch-2
Commit: b169889f01309757197a8a27b6244a87c77a3ce3
Parents: 5e6bbe6
Author: Ming Ma <mi...@apache.org>
Authored: Fri Jul 10 08:30:10 2015 -0700
Committer: Ming Ma <mi...@apache.org>
Committed: Fri Jul 10 08:34:01 2015 -0700
----------------------------------------------------------------------
.../hadoop/yarn/sls/nodemanager/NodeInfo.java | 8 +++-
.../yarn/sls/scheduler/RMNodeWrapper.java | 5 +++
hadoop-yarn-project/CHANGES.txt | 3 ++
.../server/resourcemanager/rmnode/RMNode.java | 2 +
.../resourcemanager/rmnode/RMNodeImpl.java | 43 ++++++++++++++++----
.../yarn/server/resourcemanager/MockNodes.java | 5 +++
.../resourcemanager/TestRMNodeTransitions.java | 36 ++++++++++++++--
7 files changed, 91 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b169889f/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
index ee6eb7b..440779c 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
@@ -62,7 +62,8 @@ public class NodeInfo {
private NodeState state;
private List<ContainerId> toCleanUpContainers;
private List<ApplicationId> toCleanUpApplications;
-
+ private List<ApplicationId> runningApplications;
+
public FakeRMNodeImpl(NodeId nodeId, String nodeAddr, String httpAddress,
Resource perNode, String rackName, String healthReport,
int cmdPort, String hostName, NodeState state) {
@@ -77,6 +78,7 @@ public class NodeInfo {
this.state = state;
toCleanUpApplications = new ArrayList<ApplicationId>();
toCleanUpContainers = new ArrayList<ContainerId>();
+ runningApplications = new ArrayList<ApplicationId>();
}
public NodeId getNodeID() {
@@ -135,6 +137,10 @@ public class NodeInfo {
return toCleanUpApplications;
}
+ public List<ApplicationId> getRunningApps() {
+ return runningApplications;
+ }
+
public void updateNodeHeartbeatResponseForCleanup(
NodeHeartbeatResponse response) {
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b169889f/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
index b64be1b..a6633ae 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
@@ -119,6 +119,11 @@ public class RMNodeWrapper implements RMNode {
}
@Override
+ public List<ApplicationId> getRunningApps() {
+ return node.getRunningApps();
+ }
+
+ @Override
public void updateNodeHeartbeatResponseForCleanup(
NodeHeartbeatResponse nodeHeartbeatResponse) {
node.updateNodeHeartbeatResponseForCleanup(nodeHeartbeatResponse);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b169889f/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 2331b67..74705f2 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -1637,6 +1637,9 @@ Release 2.6.0 - 2014-11-18
YARN-2811. In Fair Scheduler, reservation fulfillments shouldn't ignore max
share (Siqi Li via Sandy Ryza)
+ YARN-3445. Cache runningApps in RMNode for getting running apps on given
+ NodeId. (Junping Du via mingma)
+
IMPROVEMENTS
YARN-2242. Improve exception information on AM launch crashes. (Li Lu
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b169889f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
index 95eeaf6..0386be6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
@@ -119,6 +119,8 @@ public interface RMNode {
public List<ApplicationId> getAppsToCleanup();
+ List<ApplicationId> getRunningApps();
+
/**
* Update a {@link NodeHeartbeatResponse} with the list of containers and
* applications to clean up for this node.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b169889f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
index d1e6190..9bc91c7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
@@ -123,11 +123,16 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
new HashSet<ContainerId>();
/* the list of applications that have finished and need to be purged */
- private final List<ApplicationId> finishedApplications = new ArrayList<ApplicationId>();
+ private final List<ApplicationId> finishedApplications =
+ new ArrayList<ApplicationId>();
+
+ /* the list of applications that are running on this node */
+ private final List<ApplicationId> runningApplications =
+ new ArrayList<ApplicationId>();
private NodeHeartbeatResponse latestNodeHeartBeatResponse = recordFactory
.newRecordInstance(NodeHeartbeatResponse.class);
-
+
private static final StateMachineFactory<RMNodeImpl,
NodeState,
RMNodeEventType,
@@ -136,7 +141,7 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
NodeState,
RMNodeEventType,
RMNodeEvent>(NodeState.NEW)
-
+
//Transitions from NEW state
.addTransition(NodeState.NEW, NodeState.RUNNING,
RMNodeEventType.STARTED, new AddNodeTransition())
@@ -383,6 +388,16 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
}
@Override
+ public List<ApplicationId> getRunningApps() {
+ this.readLock.lock();
+ try {
+ return new ArrayList<ApplicationId>(this.runningApplications);
+ } finally {
+ this.readLock.unlock();
+ }
+ }
+
+ @Override
public List<ContainerId> getContainersToCleanUp() {
this.readLock.lock();
@@ -519,9 +534,12 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
LOG.warn("Cannot get RMApp by appId=" + appId
+ ", just added it to finishedApplications list for cleanup");
rmNode.finishedApplications.add(appId);
+ rmNode.runningApplications.remove(appId);
return;
}
+ // Add running applications back due to Node add or Node reconnection.
+ rmNode.runningApplications.add(appId);
context.getDispatcher().getEventHandler()
.handle(new RMAppRunningOnNodeEvent(appId, nodeId));
}
@@ -707,8 +725,9 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
@Override
public void transition(RMNodeImpl rmNode, RMNodeEvent event) {
- rmNode.finishedApplications.add(((
- RMNodeCleanAppEvent) event).getAppId());
+ ApplicationId appId = ((RMNodeCleanAppEvent) event).getAppId();
+ rmNode.finishedApplications.add(appId);
+ rmNode.runningApplications.remove(appId);
}
}
@@ -910,12 +929,22 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
+ "cleanup, no further processing");
continue;
}
- if (finishedApplications.contains(containerId.getApplicationAttemptId()
- .getApplicationId())) {
+
+ ApplicationId containerAppId =
+ containerId.getApplicationAttemptId().getApplicationId();
+
+ if (finishedApplications.contains(containerAppId)) {
LOG.info("Container " + containerId
+ " belongs to an application that is already killed,"
+ " no further processing");
continue;
+ } else if (!runningApplications.contains(containerAppId)) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Container " + containerId
+ + " is the first container get launched for application "
+ + containerAppId);
+ }
+ runningApplications.add(containerAppId);
}
// Process running containers
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b169889f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
index 2d863d1..095fe28 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
@@ -187,6 +187,11 @@ public class MockNodes {
}
@Override
+ public List<ApplicationId> getRunningApps() {
+ return null;
+ }
+
+ @Override
public void updateNodeHeartbeatResponseForCleanup(NodeHeartbeatResponse response) {
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b169889f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
index 01f4357..ece896b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
@@ -33,6 +33,7 @@ import java.util.List;
import org.apache.hadoop.util.HostsFileReader;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
@@ -485,9 +486,9 @@ public class TestRMNodeTransitions {
NodeId nodeId = node.getNodeID();
// Expire a container
- ContainerId completedContainerId = BuilderUtils.newContainerId(
- BuilderUtils.newApplicationAttemptId(
- BuilderUtils.newApplicationId(0, 0), 0), 0);
+ ContainerId completedContainerId = BuilderUtils.newContainerId(
+ BuilderUtils.newApplicationAttemptId(
+ BuilderUtils.newApplicationId(0, 0), 0), 0);
node.handle(new RMNodeCleanContainerEvent(nodeId, completedContainerId));
Assert.assertEquals(1, node.getContainersToCleanUp().size());
@@ -512,6 +513,35 @@ public class TestRMNodeTransitions {
Assert.assertEquals(finishedAppId, hbrsp.getApplicationsToCleanup().get(0));
}
+ @Test(timeout=20000)
+ public void testUpdateHeartbeatResponseForAppLifeCycle() {
+ RMNodeImpl node = getRunningNode();
+ NodeId nodeId = node.getNodeID();
+
+ ApplicationId runningAppId = BuilderUtils.newApplicationId(0, 1);
+ // Create a running container
+ ContainerId runningContainerId = BuilderUtils.newContainerId(
+ BuilderUtils.newApplicationAttemptId(
+ runningAppId, 0), 0);
+
+ ContainerStatus status = ContainerStatus.newInstance(runningContainerId,
+ ContainerState.RUNNING, "", 0);
+ List<ContainerStatus> statusList = new ArrayList<ContainerStatus>();
+ statusList.add(status);
+ NodeHealthStatus nodeHealth = NodeHealthStatus.newInstance(true,
+ "", System.currentTimeMillis());
+ node.handle(new RMNodeStatusEvent(nodeId, nodeHealth,
+ statusList, null, null));
+
+ Assert.assertEquals(1, node.getRunningApps().size());
+
+ // Finish an application
+ ApplicationId finishedAppId = runningAppId;
+ node.handle(new RMNodeCleanAppEvent(nodeId, finishedAppId));
+ Assert.assertEquals(1, node.getAppsToCleanup().size());
+ Assert.assertEquals(0, node.getRunningApps().size());
+ }
+
private RMNodeImpl getRunningNode() {
return getRunningNode(null, 0);
}