You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by cm...@apache.org on 2013/08/13 23:20:01 UTC
svn commit: r1513658 - in
/hadoop/common/branches/HDFS-4949/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/
hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/ap...
Author: cmccabe
Date: Tue Aug 13 21:19:53 2013
New Revision: 1513658
URL: http://svn.apache.org/r1513658
Log:
merge trunk into HDFS-4949 branch
Added:
hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesFairScheduler.java
- copied unchanged from r1512447, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesFairScheduler.java
Modified:
hadoop/common/branches/HDFS-4949/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java
hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/async/impl/TestAMRMClientAsync.java
hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java
hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdater.java
hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java
hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java
hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueMetrics.java
hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerInfo.java
hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerLeafQueueInfo.java
hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java
hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java
hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java
hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/CHANGES.txt?rev=1513658&r1=1513657&r2=1513658&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/CHANGES.txt Tue Aug 13 21:19:53 2013
@@ -38,6 +38,8 @@ Release 2.1.1-beta - UNRELEASED
IMPROVEMENTS
+ YARN-589. Expose a REST API for monitoring the fair scheduler (Sandy Ryza).
+
OPTIMIZATIONS
BUG FIXES
@@ -51,6 +53,18 @@ Release 2.1.1-beta - UNRELEASED
YARN-502. Fixed a state machine issue with RMNode inside ResourceManager
which was crashing scheduler. (Mayank Bansal via vinodkv)
+ YARN-573. Shared data structures in Public Localizer and Private Localizer
+ are not Thread safe. (Omkar Vinit Joshi via jlowe)
+
+ YARN-903. Changed ContainerManager to suppress unnecessary warnings when
+ stopping already stopped containers. (Omkar Vinit Joshi via vinodkv)
+
+ YARN-906. Fixed a bug in NodeManager where cancelling ContainerLaunch at
+ KILLING state causes that the container to hang. (Zhijie Shen via vinodkv)
+
+ YARN-994. HeartBeat thread in AMRMClientAsync does not handle runtime
+ exception correctly (Xuan Gong via bikas)
+
Release 2.1.0-beta - 2013-08-06
INCOMPATIBLE CHANGES
@@ -783,9 +797,6 @@ Release 2.1.0-beta - 2013-08-06
YARN-945. Removed setting of AMRMToken's service from ResourceManager
and changed client libraries do it all the time and correctly. (vinodkv)
- YARN-573. Shared data structures in Public Localizer and Private Localizer
- are not Thread safe. (Omkar Vinit Joshi via jlowe)
-
BREAKDOWN OF HADOOP-8562/YARN-191 SUBTASKS AND RELATED JIRAS
YARN-158. Yarn creating package-info.java must not depend on sh.
@@ -851,6 +862,8 @@ Release 2.1.0-beta - 2013-08-06
YARN-909. Disable TestLinuxContainerExecutorWithMocks on Windows. (Chuan Liu
via cnauroth)
+ YARN-1043. Push all metrics consistently. (Jian He via acmurthy)
+
Release 2.0.5-alpha - 06/06/2013
INCOMPATIBLE CHANGES
@@ -1149,6 +1162,9 @@ Release 0.23.10 - UNRELEASED
IMPROVEMENTS
+ YARN-985. Nodemanager should log where a resource was localized (Ravi
+ Prakash via jeagles)
+
OPTIMIZATIONS
BUG FIXES
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java?rev=1513658&r1=1513657&r2=1513658&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java Tue Aug 13 21:19:53 2013
@@ -65,7 +65,7 @@ extends AMRMClientAsync<T> {
private volatile boolean keepRunning;
private volatile float progress;
- private volatile Exception savedException;
+ private volatile Throwable savedException;
public AMRMClientAsyncImpl(int intervalMs, CallbackHandler callbackHandler) {
this(new AMRMClientImpl<T>(), intervalMs, callbackHandler);
@@ -222,18 +222,12 @@ extends AMRMClientAsync<T> {
try {
response = client.allocate(progress);
- } catch (YarnException ex) {
- LOG.error("Yarn exception on heartbeat", ex);
+ } catch (Throwable ex) {
+ LOG.error("Exception on heartbeat", ex);
savedException = ex;
// interrupt handler thread in case it waiting on the queue
handlerThread.interrupt();
return;
- } catch (IOException e) {
- LOG.error("IO exception on heartbeat", e);
- savedException = e;
- // interrupt handler thread in case it waiting on the queue
- handlerThread.interrupt();
- return;
}
}
if (response != null) {
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java?rev=1513658&r1=1513657&r2=1513658&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java Tue Aug 13 21:19:53 2013
@@ -277,6 +277,8 @@ public class AMRMClientImpl<T extends Co
public void unregisterApplicationMaster(FinalApplicationStatus appStatus,
String appMessage, String appTrackingUrl) throws YarnException,
IOException {
+ Preconditions.checkArgument(appStatus != null,
+ "AppStatus should not be null.");
FinishApplicationMasterRequest request =
FinishApplicationMasterRequest.newInstance(appStatus, appMessage,
appTrackingUrl);
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/async/impl/TestAMRMClientAsync.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/async/impl/TestAMRMClientAsync.java?rev=1513658&r1=1513657&r2=1513658&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/async/impl/TestAMRMClientAsync.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/async/impl/TestAMRMClientAsync.java Tue Aug 13 21:19:53 2013
@@ -159,14 +159,26 @@ public class TestAMRMClientAsync {
@Test(timeout=10000)
public void testAMRMClientAsyncException() throws Exception {
+ String exStr = "TestException";
+ YarnException mockException = mock(YarnException.class);
+ when(mockException.getMessage()).thenReturn(exStr);
+ runHeartBeatThrowOutException(mockException);
+ }
+
+ @Test(timeout=10000)
+ public void testAMRMClientAsyncRunTimeException() throws Exception {
+ String exStr = "TestRunTimeException";
+ RuntimeException mockRunTimeException = mock(RuntimeException.class);
+ when(mockRunTimeException.getMessage()).thenReturn(exStr);
+ runHeartBeatThrowOutException(mockRunTimeException);
+ }
+
+ private void runHeartBeatThrowOutException(Exception ex) throws Exception{
Configuration conf = new Configuration();
TestCallbackHandler callbackHandler = new TestCallbackHandler();
@SuppressWarnings("unchecked")
AMRMClient<ContainerRequest> client = mock(AMRMClientImpl.class);
- String exStr = "TestException";
- YarnException mockException = mock(YarnException.class);
- when(mockException.getMessage()).thenReturn(exStr);
- when(client.allocate(anyFloat())).thenThrow(mockException);
+ when(client.allocate(anyFloat())).thenThrow(ex);
AMRMClientAsync<ContainerRequest> asyncClient =
AMRMClientAsync.createAMRMClientAsync(client, 20, callbackHandler);
@@ -183,14 +195,14 @@ public class TestAMRMClientAsync {
}
}
}
-
- Assert.assertTrue(callbackHandler.savedException.getMessage().contains(exStr));
+ Assert.assertTrue(callbackHandler.savedException.getMessage().contains(
+ ex.getMessage()));
asyncClient.stop();
// stopping should have joined all threads and completed all callbacks
Assert.assertTrue(callbackHandler.callbackCount == 0);
}
-
+
@Test//(timeout=10000)
public void testAMRMClientAsyncReboot() throws Exception {
Configuration conf = new Configuration();
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java?rev=1513658&r1=1513657&r2=1513658&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java Tue Aug 13 21:19:53 2013
@@ -26,6 +26,7 @@ import static org.junit.Assert.fail;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.util.Arrays;
import java.util.List;
import java.util.Set;
import java.util.TreeSet;
@@ -319,7 +320,7 @@ public class TestNMClient {
if (++i < size) {
// NodeManager may still need some time to make the container started
testGetContainerStatus(container, i, ContainerState.RUNNING, "",
- -1000);
+ Arrays.asList(new Integer[] {-1000}));
try {
nmClient.stopContainer(container.getId(), container.getNodeId());
@@ -330,8 +331,21 @@ public class TestNMClient {
}
// getContainerStatus can be called after stopContainer
- testGetContainerStatus(container, i, ContainerState.COMPLETE,
- "Container killed by the ApplicationMaster.", 143);
+ try {
+ // O is possible if CLEANUP_CONTAINER is executed too late
+ testGetContainerStatus(container, i, ContainerState.COMPLETE,
+ "Container killed by the ApplicationMaster.", Arrays.asList(
+ new Integer[] {143, 0}));
+ } catch (YarnException e) {
+ // The exception is possible because, after the container is stopped,
+ // it may be removed from NM's context.
+ if (!e.getMessage()
+ .contains("was recently stopped on node manager")) {
+ throw (AssertionError)
+ (new AssertionError("Exception is not expected: " + e).initCause(
+ e));
+ }
+ }
}
}
}
@@ -345,7 +359,7 @@ public class TestNMClient {
}
private void testGetContainerStatus(Container container, int index,
- ContainerState state, String diagnostics, int exitStatus)
+ ContainerState state, String diagnostics, List<Integer> exitStatuses)
throws YarnException, IOException {
while (true) {
try {
@@ -357,7 +371,7 @@ public class TestNMClient {
assertEquals(container.getId(), status.getContainerId());
assertTrue("" + index + ": " + status.getDiagnostics(),
status.getDiagnostics().contains(diagnostics));
- assertEquals(exitStatus, status.getExitStatus());
+ assertTrue(exitStatuses.contains(status.getExitStatus()));
break;
}
Thread.sleep(100);
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java?rev=1513658&r1=1513657&r2=1513658&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java Tue Aug 13 21:19:53 2013
@@ -29,6 +29,7 @@ import java.util.concurrent.atomic.Atomi
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.security.SecurityUtil;
@@ -456,4 +457,10 @@ public class NodeManager extends Composi
Configuration conf = new YarnConfiguration();
nodeManager.initAndStartNodeManager(conf, false);
}
+
+ @VisibleForTesting
+ @Private
+ public NodeStatusUpdater getNodeStatusUpdater() {
+ return nodeStatusUpdater;
+ }
}
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdater.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdater.java?rev=1513658&r1=1513657&r2=1513658&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdater.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdater.java Tue Aug 13 21:19:53 2013
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.nodemanager;
import org.apache.hadoop.service.Service;
+import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.server.api.records.NodeStatus;
public interface NodeStatusUpdater extends Service {
@@ -28,4 +29,8 @@ public interface NodeStatusUpdater exten
NodeStatus getNodeStatusAndUpdateContainersInContext();
long getRMIdentifier();
+
+ public boolean isContainerRecentlyStopped(ContainerId containerId);
+
+ public void clearFinishedContainersFromCache();
}
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java?rev=1513658&r1=1513657&r2=1513658&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java Tue Aug 13 21:19:53 2013
@@ -24,6 +24,7 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -68,6 +69,9 @@ import com.google.common.annotations.Vis
public class NodeStatusUpdaterImpl extends AbstractService implements
NodeStatusUpdater {
+ public static final String YARN_NODEMANAGER_DURATION_TO_TRACK_STOPPED_CONTAINERS =
+ YarnConfiguration.NM_PREFIX + "duration-to-track-stopped-containers";
+
private static final Log LOG = LogFactory.getLog(NodeStatusUpdaterImpl.class);
private final Object heartbeatMonitor = new Object();
@@ -88,6 +92,10 @@ public class NodeStatusUpdaterImpl exten
private Map<ApplicationId, Long> appTokenKeepAliveMap =
new HashMap<ApplicationId, Long>();
private Random keepAliveDelayRandom = new Random();
+ // It will be used to track recently stopped containers on node manager.
+ private final Map<ContainerId, Long> recentlyStoppedContainers;
+ // Duration for which to track recently stopped container.
+ private long durationToTrackStoppedContainers;
private final NodeHealthCheckerService healthChecker;
private final NodeManagerMetrics metrics;
@@ -103,6 +111,8 @@ public class NodeStatusUpdaterImpl exten
this.context = context;
this.dispatcher = dispatcher;
this.metrics = metrics;
+ this.recentlyStoppedContainers =
+ new LinkedHashMap<ContainerId, Long>();
}
@Override
@@ -129,11 +139,27 @@ public class NodeStatusUpdaterImpl exten
conf.getInt(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS,
YarnConfiguration.DEFAULT_RM_NM_EXPIRY_INTERVAL_MS);
+ // Default duration to track stopped containers on nodemanager is 10Min.
+ // This should not be assigned very large value as it will remember all the
+ // containers stopped during that time.
+ durationToTrackStoppedContainers =
+ conf.getLong(YARN_NODEMANAGER_DURATION_TO_TRACK_STOPPED_CONTAINERS,
+ 600000);
+ if (durationToTrackStoppedContainers < 0) {
+ String message = "Invalid configuration for "
+ + YARN_NODEMANAGER_DURATION_TO_TRACK_STOPPED_CONTAINERS + " default "
+ + "value is 10Min(600000).";
+ LOG.error(message);
+ throw new YarnException(message);
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(YARN_NODEMANAGER_DURATION_TO_TRACK_STOPPED_CONTAINERS + " :"
+ + durationToTrackStoppedContainers);
+ }
+ super.serviceInit(conf);
LOG.info("Initialized nodemanager for " + nodeId + ":" +
" physical-memory=" + memoryMb + " virtual-memory=" + virtualMemoryMb +
" virtual-cores=" + virtualCores);
-
- super.serviceInit(conf);
}
@Override
@@ -290,7 +316,11 @@ public class NodeStatusUpdaterImpl exten
if (containerStatus.getState() == ContainerState.COMPLETE) {
// Remove
i.remove();
-
+ // Adding to finished containers cache. Cache will keep it around at
+ // least for #durationToTrackStoppedContainers duration. In the
+ // subsequent call to stop container it will get removed from cache.
+ addStoppedContainersToCache(containerId);
+
LOG.info("Removed completed container " + containerId);
}
}
@@ -340,6 +370,46 @@ public class NodeStatusUpdaterImpl exten
}
}
+ public boolean isContainerRecentlyStopped(ContainerId containerId) {
+ synchronized (recentlyStoppedContainers) {
+ return recentlyStoppedContainers.containsKey(containerId);
+ }
+ }
+
+ @Private
+ @VisibleForTesting
+ public void addStoppedContainersToCache(ContainerId containerId) {
+ synchronized (recentlyStoppedContainers) {
+ removeVeryOldStoppedContainersFromCache();
+ recentlyStoppedContainers.put(containerId,
+ System.currentTimeMillis() + durationToTrackStoppedContainers);
+ }
+ }
+
+ @Override
+ public void clearFinishedContainersFromCache() {
+ synchronized (recentlyStoppedContainers) {
+ recentlyStoppedContainers.clear();
+ }
+ }
+
+ @Private
+ @VisibleForTesting
+ public void removeVeryOldStoppedContainersFromCache() {
+ synchronized (recentlyStoppedContainers) {
+ long currentTime = System.currentTimeMillis();
+ Iterator<ContainerId> i =
+ recentlyStoppedContainers.keySet().iterator();
+ while (i.hasNext()) {
+ if (recentlyStoppedContainers.get(i.next()) < currentTime) {
+ i.remove();
+ } else {
+ break;
+ }
+ }
+ }
+ }
+
@Override
public long getRMIdentifier() {
return this.rmIdentifier;
@@ -455,4 +525,6 @@ public class NodeStatusUpdaterImpl exten
new Thread(statusUpdaterRunnable, "Node Status Updater");
statusUpdater.start();
}
+
+
}
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java?rev=1513658&r1=1513657&r2=1513658&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java Tue Aug 13 21:19:53 2013
@@ -85,6 +85,7 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger.AuditConstants;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdater;
+import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationContainerInitEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent;
@@ -581,17 +582,24 @@ public class ContainerManagerImpl extend
authorizeGetAndStopContainerRequest(containerID, container, true,
nmTokenIdentifier);
- dispatcher.getEventHandler().handle(
- new ContainerKillEvent(containerID,
- "Container killed by the ApplicationMaster."));
+ if (container == null) {
+ if (!nodeStatusUpdater.isContainerRecentlyStopped(containerID)) {
+ throw RPCUtil.getRemoteException("Container " + containerIDStr
+ + " is not handled by this NodeManager");
+ }
+ } else {
+ dispatcher.getEventHandler().handle(
+ new ContainerKillEvent(containerID,
+ "Container killed by the ApplicationMaster."));
- NMAuditLogger.logSuccess(container.getUser(),
- AuditConstants.STOP_CONTAINER, "ContainerManageImpl", containerID
- .getApplicationAttemptId().getApplicationId(), containerID);
-
- // TODO: Move this code to appropriate place once kill_container is
- // implemented.
- nodeStatusUpdater.sendOutofBandHeartBeat();
+ NMAuditLogger.logSuccess(container.getUser(),
+ AuditConstants.STOP_CONTAINER, "ContainerManageImpl", containerID
+ .getApplicationAttemptId().getApplicationId(), containerID);
+
+ // TODO: Move this code to appropriate place once kill_container is
+ // implemented.
+ nodeStatusUpdater.sendOutofBandHeartBeat();
+ }
}
/**
@@ -627,6 +635,15 @@ public class ContainerManagerImpl extend
authorizeGetAndStopContainerRequest(containerID, container, false,
nmTokenIdentifier);
+ if (container == null) {
+ if (nodeStatusUpdater.isContainerRecentlyStopped(containerID)) {
+ throw RPCUtil.getRemoteException("Container " + containerIDStr
+ + " was recently stopped on node manager.");
+ } else {
+ throw RPCUtil.getRemoteException("Container " + containerIDStr
+ + " is not handled by this NodeManager");
+ }
+ }
ContainerStatus containerStatus = container.cloneAndGetContainerStatus();
LOG.info("Returning " + containerStatus);
return containerStatus;
@@ -658,17 +675,11 @@ public class ContainerManagerImpl extend
container.getContainerId());
} else {
LOG.warn(identifier.getApplicationAttemptId()
- + " attempted to get get status for non-application container : "
+ + " attempted to get status for non-application container : "
+ container.getContainerId().toString());
}
- throw RPCUtil.getRemoteException("Container " + containerId.toString()
- + " is not started by this application attempt.");
}
- if (container == null) {
- throw RPCUtil.getRemoteException("Container " + containerId.toString()
- + " is not handled by this NodeManager");
- }
}
class ContainerEventDispatcher implements EventHandler<ContainerEvent> {
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java?rev=1513658&r1=1513657&r2=1513658&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java Tue Aug 13 21:19:53 2013
@@ -37,12 +37,17 @@ import org.apache.hadoop.yarn.event.Disp
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEventType;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerExitEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
/**
@@ -61,7 +66,8 @@ public class ContainersLauncher extends
private final Dispatcher dispatcher;
private LocalDirsHandlerService dirsHandler;
- private final ExecutorService containerLauncher =
+ @VisibleForTesting
+ public ExecutorService containerLauncher =
Executors.newCachedThreadPool(
new ThreadFactoryBuilder()
.setNameFormat("ContainersLauncher #%d")
@@ -107,6 +113,7 @@ public class ContainersLauncher extends
super.serviceStop();
}
+ @SuppressWarnings("unchecked")
@Override
public void handle(ContainersLauncherEvent event) {
// TODO: ContainersLauncher launches containers one by one!!
@@ -134,9 +141,18 @@ public class ContainersLauncher extends
Future<Integer> rContainer = rContainerDatum.runningcontainer;
if (rContainer != null
&& !rContainer.isDone()) {
- // Cancel the future so that it won't be launched
- // if it isn't already.
- rContainer.cancel(false);
+ // Cancel the future so that it won't be launched if it isn't already.
+ // If it is going to be canceled, make sure CONTAINER_KILLED_ON_REQUEST
+ // will not be missed if the container is already at KILLING
+ if (rContainer.cancel(false)) {
+ if (container.getContainerState() == ContainerState.KILLING) {
+ dispatcher.getEventHandler().handle(
+ new ContainerExitEvent(containerId,
+ ContainerEventType.CONTAINER_KILLED_ON_REQUEST,
+ ExitCode.TERMINATED.getExitCode(),
+ "Container terminated before launch."));
+ }
+ }
}
// Cleanup a container whether it is running/killed/completed, so that
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java?rev=1513658&r1=1513657&r2=1513658&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java Tue Aug 13 21:19:53 2013
@@ -242,6 +242,7 @@ class LocalResourcesTrackerImpl implemen
delService.delete(getUser(), getPathToDelete(rsrc.getLocalPath()));
}
decrementFileCountForLocalCacheDirectory(rem.getRequest(), rsrc);
+ LOG.info("Removed " + rsrc.getLocalPath() + " from localized cache");
return true;
}
}
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java?rev=1513658&r1=1513657&r2=1513658&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java Tue Aug 13 21:19:53 2013
@@ -193,8 +193,8 @@ public class LocalizedResource implement
LOG.warn("Can't handle this event at current state", e);
}
if (oldState != newState) {
- LOG.info("Resource " + resourcePath + " transitioned from "
- + oldState
+ LOG.info("Resource " + resourcePath + (localPath != null ?
+ "(->" + localPath + ")": "") + " transitioned from " + oldState
+ " to " + newState);
}
} finally {
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java?rev=1513658&r1=1513657&r2=1513658&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java Tue Aug 13 21:19:53 2013
@@ -426,7 +426,7 @@ public class TestNodeStatusUpdater {
return this.nodeStatusUpdater;
}
- protected MyNodeStatusUpdater3 getNodeStatusUpdater() {
+ public MyNodeStatusUpdater3 getNodeStatusUpdater() {
return this.nodeStatusUpdater;
}
}
@@ -745,6 +745,40 @@ public class TestNodeStatusUpdater {
lfs.delete(new Path(basedir.getPath()), true);
}
+ @Test(timeout = 90000)
+ public void testRecentlyFinishedContainers() throws Exception {
+ NodeManager nm = new NodeManager();
+ YarnConfiguration conf = new YarnConfiguration();
+ conf.set(
+ NodeStatusUpdaterImpl.YARN_NODEMANAGER_DURATION_TO_TRACK_STOPPED_CONTAINERS,
+ "10000");
+ nm.init(conf);
+ NodeStatusUpdaterImpl nodeStatusUpdater =
+ (NodeStatusUpdaterImpl) nm.getNodeStatusUpdater();
+ ApplicationId appId = ApplicationId.newInstance(0, 0);
+ ApplicationAttemptId appAttemptId =
+ ApplicationAttemptId.newInstance(appId, 0);
+ ContainerId cId = ContainerId.newInstance(appAttemptId, 0);
+
+
+ nodeStatusUpdater.addStoppedContainersToCache(cId);
+ Assert.assertTrue(nodeStatusUpdater.isContainerRecentlyStopped(cId));
+
+ long time1 = System.currentTimeMillis();
+ int waitInterval = 15;
+ while (waitInterval-- > 0
+ && nodeStatusUpdater.isContainerRecentlyStopped(cId)) {
+ nodeStatusUpdater.removeVeryOldStoppedContainersFromCache();
+ Thread.sleep(1000);
+ }
+ long time2 = System.currentTimeMillis();
+ // By this time the container will be removed from cache. need to verify.
+ Assert.assertFalse(nodeStatusUpdater.isContainerRecentlyStopped(cId));
+ Assert.assertTrue((time2 - time1) >= 10000 && (time2 -time1) <= 250000);
+ }
+
+
+
@Test
public void testNMRegistration() throws InterruptedException {
nm = new NodeManager() {
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java?rev=1513658&r1=1513657&r2=1513658&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java Tue Aug 13 21:19:53 2013
@@ -18,9 +18,10 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.container;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.any;
import static org.mockito.Matchers.argThat;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.reset;
@@ -42,11 +43,16 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Random;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
import junit.framework.Assert;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.LocalResource;
@@ -60,10 +66,13 @@ import org.apache.hadoop.yarn.event.Drai
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.AuxServicesEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.AuxServicesEventType;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEventType;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncher;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncherEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncherEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.LocalResourceRequest;
@@ -287,7 +296,8 @@ public class TestContainer {
wc.launchContainer();
reset(wc.localizerBus);
wc.killContainer();
- assertEquals(ContainerState.KILLING, wc.c.getContainerState());
+ assertEquals(ContainerState.CONTAINER_CLEANEDUP_AFTER_KILL,
+ wc.c.getContainerState());
assertNull(wc.c.getLocalizedResources());
wc.containerKilledOnRequest();
@@ -318,6 +328,26 @@ public class TestContainer {
}
}
}
+
+ @Test
+ public void testKillOnLocalized() throws Exception {
+ WrappedContainer wc = null;
+ try {
+ wc = new WrappedContainer(17, 314159265358979L, 4344, "yak");
+ wc.initContainer();
+ wc.localizeResources();
+ assertEquals(ContainerState.LOCALIZED, wc.c.getContainerState());
+ wc.killContainer();
+ assertEquals(ContainerState.CONTAINER_CLEANEDUP_AFTER_KILL,
+ wc.c.getContainerState());
+ assertNull(wc.c.getLocalizedResources());
+ verifyCleanupCall(wc);
+ } finally {
+ if (wc != null) {
+ wc.finished();
+ }
+ }
+ }
@Test
public void testResourceLocalizedOnLocalizationFailed() throws Exception {
@@ -442,10 +472,12 @@ public class TestContainer {
wc.initContainer();
wc.localizeResources();
wc.killContainer();
- assertEquals(ContainerState.KILLING, wc.c.getContainerState());
+ assertEquals(ContainerState.CONTAINER_CLEANEDUP_AFTER_KILL,
+ wc.c.getContainerState());
assertNull(wc.c.getLocalizedResources());
wc.launchContainer();
- assertEquals(ContainerState.KILLING, wc.c.getContainerState());
+ assertEquals(ContainerState.CONTAINER_CLEANEDUP_AFTER_KILL,
+ wc.c.getContainerState());
assertNull(wc.c.getLocalizedResources());
wc.containerKilledOnRequest();
verifyCleanupCall(wc);
@@ -583,6 +615,7 @@ public class TestContainer {
final EventHandler<AuxServicesEvent> auxBus;
final EventHandler<ApplicationEvent> appBus;
final EventHandler<LogHandlerEvent> LogBus;
+ final ContainersLauncher launcher;
final ContainerLaunchContext ctxt;
final ContainerId cId;
@@ -595,6 +628,7 @@ public class TestContainer {
this(appId, timestamp, id, user, true, false);
}
+ @SuppressWarnings("rawtypes")
WrappedContainer(int appId, long timestamp, int id, String user,
boolean withLocalRes, boolean withServiceData) throws IOException {
dispatcher = new DrainDispatcher();
@@ -613,6 +647,22 @@ public class TestContainer {
dispatcher.register(ApplicationEventType.class, appBus);
dispatcher.register(LogHandlerEventType.class, LogBus);
+ Context context = mock(Context.class);
+ when(context.getApplications()).thenReturn(
+ new ConcurrentHashMap<ApplicationId, Application>());
+ launcher = new ContainersLauncher(context, dispatcher, null, null);
+ // create a mock ExecutorService, which will not really launch
+ // ContainerLaunch at all.
+ launcher.containerLauncher = mock(ExecutorService.class);
+ Future future = mock(Future.class);
+ when(launcher.containerLauncher.submit
+ (any(Callable.class))).thenReturn(future);
+ when(future.isDone()).thenReturn(false);
+ when(future.cancel(false)).thenReturn(true);
+ launcher.init(new Configuration());
+ launcher.start();
+ dispatcher.register(ContainersLauncherEventType.class, launcher);
+
ctxt = mock(ContainerLaunchContext.class);
org.apache.hadoop.yarn.api.records.Container mockContainer =
mock(org.apache.hadoop.yarn.api.records.Container.class);
@@ -654,6 +704,13 @@ public class TestContainer {
when(ctxt.getServiceData()).thenReturn(serviceData);
c = new ContainerImpl(conf, dispatcher, ctxt, null, metrics, identifier);
+ dispatcher.register(ContainerEventType.class,
+ new EventHandler<ContainerEvent>() {
+ @Override
+ public void handle(ContainerEvent event) {
+ c.handle(event);
+ }
+ });
dispatcher.start();
}
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java?rev=1513658&r1=1513657&r2=1513658&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java Tue Aug 13 21:19:53 2013
@@ -596,8 +596,9 @@ public class TestContainerLaunch extends
ContainerStatus containerStatus =
containerManager.getContainerStatuses(gcsRequest)
.getContainerStatuses().get(0);
- Assert.assertEquals(ExitCode.FORCE_KILLED.getExitCode(),
- containerStatus.getExitStatus());
+ int expectedExitCode = Shell.WINDOWS ? ExitCode.FORCE_KILLED.getExitCode() :
+ ExitCode.TERMINATED.getExitCode();
+ Assert.assertEquals(expectedExitCode, containerStatus.getExitStatus());
// Now verify the contents of the file. Script generates a message when it
// receives a sigterm so we look for that. We cannot perform this check on
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java?rev=1513658&r1=1513657&r2=1513658&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java Tue Aug 13 21:19:53 2013
@@ -126,18 +126,6 @@ public class QueueMetrics implements Met
enableUserMetrics, conf);
}
- // this method is here because we want to make sure these metrics show up on
- // queue registration.
- public void initMetrics() {
- appsSubmitted.incr(0);
- appsRunning.incr(0);
- appsPending.incr(0);
- appsCompleted.incr(0);
- appsKilled.incr(0);
- appsFailed.incr(0);
- reservedContainers.incr(0);
- }
-
/**
* Helper method to clear cache - used only for unit tests.
*/
@@ -168,7 +156,6 @@ public class QueueMetrics implements Met
ms.register(
sourceName(queueName).toString(),
"Metrics for queue: " + queueName, metrics);
- metrics.initMetrics();
}
queueMetrics.put(queueName, metrics);
}
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueMetrics.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueMetrics.java?rev=1513658&r1=1513657&r2=1513658&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueMetrics.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueMetrics.java Tue Aug 13 21:19:53 2013
@@ -94,7 +94,6 @@ public class FSQueueMetrics extends Queu
metrics = ms.register(
sourceName(queueName).toString(),
"Metrics for queue: " + queueName, metrics);
- metrics.initMetrics();
}
queueMetrics.put(queueName, metrics);
}
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java?rev=1513658&r1=1513657&r2=1513658&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java Tue Aug 13 21:19:53 2013
@@ -52,6 +52,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptsInfo;
@@ -60,6 +61,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FairSchedulerInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FifoSchedulerInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo;
@@ -145,6 +147,9 @@ public class RMWebServices {
CapacityScheduler cs = (CapacityScheduler) rs;
CSQueue root = cs.getRootQueue();
sinfo = new CapacitySchedulerInfo(root);
+ } else if (rs instanceof FairScheduler) {
+ FairScheduler fs = (FairScheduler) rs;
+ sinfo = new FairSchedulerInfo(fs);
} else if (rs instanceof FifoScheduler) {
sinfo = new FifoSchedulerInfo(this.rm);
} else {
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerInfo.java?rev=1513658&r1=1513657&r2=1513658&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerInfo.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerInfo.java Tue Aug 13 21:19:53 2013
@@ -18,14 +18,31 @@
package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlRootElement;
+import javax.xml.bind.annotation.XmlTransient;
+import javax.xml.bind.annotation.XmlType;
+
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
-public class FairSchedulerInfo {
+@XmlRootElement(name = "fairScheduler")
+@XmlType(name = "fairScheduler")
+@XmlAccessorType(XmlAccessType.FIELD)
+public class FairSchedulerInfo extends SchedulerInfo {
+ private FairSchedulerQueueInfo rootQueue;
+
+ @XmlTransient
private FairScheduler scheduler;
+ public FairSchedulerInfo() {
+ } // JAXB needs this
+
public FairSchedulerInfo(FairScheduler fs) {
scheduler = fs;
+ rootQueue = new FairSchedulerQueueInfo(scheduler.getQueueManager().
+ getRootQueue(), scheduler);
}
public int getAppFairShare(ApplicationAttemptId appAttemptId) {
@@ -34,7 +51,6 @@ public class FairSchedulerInfo {
}
public FairSchedulerQueueInfo getRootQueueInfo() {
- return new FairSchedulerQueueInfo(scheduler.getQueueManager().
- getRootQueue(), scheduler);
+ return rootQueue;
}
}
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerLeafQueueInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerLeafQueueInfo.java?rev=1513658&r1=1513657&r2=1513658&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerLeafQueueInfo.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerLeafQueueInfo.java Tue Aug 13 21:19:53 2013
@@ -20,14 +20,23 @@ package org.apache.hadoop.yarn.server.re
import java.util.Collection;
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlRootElement;
+
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AppSchedulable;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue;
+@XmlRootElement
+@XmlAccessorType(XmlAccessType.FIELD)
public class FairSchedulerLeafQueueInfo extends FairSchedulerQueueInfo {
private int numPendingApps;
private int numActiveApps;
+ public FairSchedulerLeafQueueInfo() {
+ }
+
public FairSchedulerLeafQueueInfo(FSLeafQueue queue, FairScheduler scheduler) {
super(queue, scheduler);
Collection<AppSchedulable> apps = queue.getAppSchedulables();
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java?rev=1513658&r1=1513657&r2=1513658&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java Tue Aug 13 21:19:53 2013
@@ -22,6 +22,12 @@ package org.apache.hadoop.yarn.server.re
import java.util.ArrayList;
import java.util.Collection;
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlRootElement;
+import javax.xml.bind.annotation.XmlSeeAlso;
+import javax.xml.bind.annotation.XmlTransient;
+
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue;
@@ -29,39 +35,53 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueueManager;
import org.apache.hadoop.yarn.util.resource.Resources;
+@XmlRootElement
+@XmlAccessorType(XmlAccessType.FIELD)
+@XmlSeeAlso({FairSchedulerLeafQueueInfo.class})
public class FairSchedulerQueueInfo {
private int maxApps;
+ @XmlTransient
private float fractionMemUsed;
+ @XmlTransient
private float fractionMemFairShare;
+ @XmlTransient
private float fractionMemMinShare;
+ @XmlTransient
private float fractionMemMaxShare;
- private Resource minResources;
- private Resource maxResources;
- private Resource usedResources;
- private Resource fairResources;
- private Resource clusterResources;
+ private ResourceInfo minResources;
+ private ResourceInfo maxResources;
+ private ResourceInfo usedResources;
+ private ResourceInfo fairResources;
+ private ResourceInfo clusterResources;
private String queueName;
+ private String schedulingPolicy;
+
+ private Collection<FairSchedulerQueueInfo> childQueues;
- private Collection<FairSchedulerQueueInfo> childInfos;
+ public FairSchedulerQueueInfo() {
+ }
public FairSchedulerQueueInfo(FSQueue queue, FairScheduler scheduler) {
QueueManager manager = scheduler.getQueueManager();
queueName = queue.getName();
-
- clusterResources = scheduler.getClusterCapacity();
+ schedulingPolicy = queue.getPolicy().getName();
+
+ clusterResources = new ResourceInfo(scheduler.getClusterCapacity());
- usedResources = queue.getResourceUsage();
+ usedResources = new ResourceInfo(queue.getResourceUsage());
fractionMemUsed = (float)usedResources.getMemory() /
clusterResources.getMemory();
- fairResources = queue.getFairShare();
- minResources = queue.getMinShare();
- maxResources = queue.getMaxShare();
- maxResources = Resources.componentwiseMin(maxResources, clusterResources);
+ fairResources = new ResourceInfo(queue.getFairShare());
+ minResources = new ResourceInfo(queue.getMinShare());
+ maxResources = new ResourceInfo(queue.getMaxShare());
+ maxResources = new ResourceInfo(
+ Resources.componentwiseMin(queue.getMaxShare(),
+ scheduler.getClusterCapacity()));
fractionMemFairShare = (float)fairResources.getMemory() / clusterResources.getMemory();
fractionMemMinShare = (float)minResources.getMemory() / clusterResources.getMemory();
@@ -69,13 +89,13 @@ public class FairSchedulerQueueInfo {
maxApps = manager.getQueueMaxApps(queueName);
- Collection<FSQueue> childQueues = queue.getChildQueues();
- childInfos = new ArrayList<FairSchedulerQueueInfo>();
- for (FSQueue child : childQueues) {
+ Collection<FSQueue> children = queue.getChildQueues();
+ childQueues = new ArrayList<FairSchedulerQueueInfo>();
+ for (FSQueue child : children) {
if (child instanceof FSLeafQueue) {
- childInfos.add(new FairSchedulerLeafQueueInfo((FSLeafQueue)child, scheduler));
+ childQueues.add(new FairSchedulerLeafQueueInfo((FSLeafQueue)child, scheduler));
} else {
- childInfos.add(new FairSchedulerQueueInfo(child, scheduler));
+ childQueues.add(new FairSchedulerQueueInfo(child, scheduler));
}
}
}
@@ -90,15 +110,15 @@ public class FairSchedulerQueueInfo {
/**
* Returns the fair share of this queue in megabytes.
*/
- public Resource getFairShare() {
+ public ResourceInfo getFairShare() {
return fairResources;
}
- public Resource getMinResources() {
+ public ResourceInfo getMinResources() {
return minResources;
}
- public Resource getMaxResources() {
+ public ResourceInfo getMaxResources() {
return maxResources;
}
@@ -110,7 +130,7 @@ public class FairSchedulerQueueInfo {
return queueName;
}
- public Resource getUsedResources() {
+ public ResourceInfo getUsedResources() {
return usedResources;
}
@@ -138,7 +158,14 @@ public class FairSchedulerQueueInfo {
return fractionMemMaxShare;
}
+ /**
+ * Returns the name of the scheduling policy used by this queue.
+ */
+ public String getSchedulingPolicy() {
+ return schedulingPolicy;
+ }
+
public Collection<FairSchedulerQueueInfo> getChildQueues() {
- return childInfos;
+ return childQueues;
}
}
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java?rev=1513658&r1=1513657&r2=1513658&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java Tue Aug 13 21:19:53 2013
@@ -45,4 +45,9 @@ public class ResourceInfo {
public int getvCores() {
return vCores;
}
+
+ @Override
+ public String toString() {
+ return "<memory:" + memory + ", vCores:" + vCores + ">";
+ }
}
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java?rev=1513658&r1=1513657&r2=1513658&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java Tue Aug 13 21:19:53 2013
@@ -22,7 +22,8 @@ import javax.xml.bind.annotation.XmlRoot
import javax.xml.bind.annotation.XmlSeeAlso;
@XmlRootElement
-@XmlSeeAlso({ CapacitySchedulerInfo.class, FifoSchedulerInfo.class })
+@XmlSeeAlso({ CapacitySchedulerInfo.class, FairSchedulerInfo.class,
+ FifoSchedulerInfo.class })
public class SchedulerInfo {
public SchedulerInfo() {
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java?rev=1513658&r1=1513657&r2=1513658&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java Tue Aug 13 21:19:53 2013
@@ -68,7 +68,7 @@ public class TestQueueMetrics {
metrics.submitApp(user, 1);
MetricsSource userSource = userSource(ms, queueName, user);
- checkApps(queueSource, 1, 1, 0, 0, 0, 0);
+ checkApps(queueSource, 1, 1, 0, 0, 0, 0, true);
metrics.setAvailableResourcesToQueue(Resources.createResource(100*GB, 100));
metrics.incrPendingResources(user, 5, Resources.createResource(15*GB, 15));
@@ -77,7 +77,7 @@ public class TestQueueMetrics {
checkResources(queueSource, 0, 0, 0, 0, 0, 100*GB, 100, 15*GB, 15, 5, 0, 0, 0);
metrics.incrAppsRunning(app, user);
- checkApps(queueSource, 1, 0, 1, 0, 0, 0);
+ checkApps(queueSource, 1, 0, 1, 0, 0, 0, true);
metrics.allocateResources(user, 3, Resources.createResource(2*GB, 2));
checkResources(queueSource, 6*GB, 6, 3, 3, 0, 100*GB, 100, 9*GB, 9, 2, 0, 0, 0);
@@ -86,7 +86,7 @@ public class TestQueueMetrics {
checkResources(queueSource, 4*GB, 4, 2, 3, 1, 100*GB, 100, 9*GB, 9, 2, 0, 0, 0);
metrics.finishApp(app, RMAppAttemptState.FINISHED);
- checkApps(queueSource, 1, 0, 0, 1, 0, 0);
+ checkApps(queueSource, 1, 0, 0, 1, 0, 0, true);
assertNull(userSource);
}
@@ -102,37 +102,37 @@ public class TestQueueMetrics {
metrics.submitApp(user, 1);
MetricsSource userSource = userSource(ms, queueName, user);
- checkApps(queueSource, 1, 1, 0, 0, 0, 0);
+ checkApps(queueSource, 1, 1, 0, 0, 0, 0, true);
metrics.incrAppsRunning(app, user);
- checkApps(queueSource, 1, 0, 1, 0, 0, 0);
+ checkApps(queueSource, 1, 0, 1, 0, 0, 0, true);
metrics.finishApp(app, RMAppAttemptState.FAILED);
- checkApps(queueSource, 1, 0, 0, 0, 1, 0);
+ checkApps(queueSource, 1, 0, 0, 0, 1, 0, true);
// As the application has failed, framework retries the same application
// based on configuration
metrics.submitApp(user, 2);
- checkApps(queueSource, 1, 1, 0, 0, 0, 0);
+ checkApps(queueSource, 1, 1, 0, 0, 0, 0, true);
metrics.incrAppsRunning(app, user);
- checkApps(queueSource, 1, 0, 1, 0, 0, 0);
+ checkApps(queueSource, 1, 0, 1, 0, 0, 0, true);
// Suppose say application has failed this time as well.
metrics.finishApp(app, RMAppAttemptState.FAILED);
- checkApps(queueSource, 1, 0, 0, 0, 1, 0);
+ checkApps(queueSource, 1, 0, 0, 0, 1, 0, true);
// As the application has failed, framework retries the same application
// based on configuration
metrics.submitApp(user, 3);
- checkApps(queueSource, 1, 1, 0, 0, 0, 0);
+ checkApps(queueSource, 1, 1, 0, 0, 0, 0, true);
metrics.incrAppsRunning(app, user);
- checkApps(queueSource, 1, 0, 1, 0, 0, 0);
+ checkApps(queueSource, 1, 0, 1, 0, 0, 0, true);
// Suppose say application has finished.
metrics.finishApp(app, RMAppAttemptState.FINISHED);
- checkApps(queueSource, 1, 0, 0, 1, 0, 0);
+ checkApps(queueSource, 1, 0, 0, 1, 0, 0, true);
assertNull(userSource);
}
@@ -149,8 +149,8 @@ public class TestQueueMetrics {
metrics.submitApp(user, 1);
MetricsSource userSource = userSource(ms, queueName, user);
- checkApps(queueSource, 1, 1, 0, 0, 0, 0);
- checkApps(userSource, 1, 1, 0, 0, 0, 0);
+ checkApps(queueSource, 1, 1, 0, 0, 0, 0, true);
+ checkApps(userSource, 1, 1, 0, 0, 0, 0, true);
metrics.setAvailableResourcesToQueue(Resources.createResource(100*GB, 100));
metrics.setAvailableResourcesToUser(user, Resources.createResource(10*GB, 10));
@@ -161,8 +161,8 @@ public class TestQueueMetrics {
checkResources(userSource, 0, 0, 0, 0, 0, 10*GB, 10, 15*GB, 15, 5, 0, 0, 0);
metrics.incrAppsRunning(app, user);
- checkApps(queueSource, 1, 0, 1, 0, 0, 0);
- checkApps(userSource, 1, 0, 1, 0, 0, 0);
+ checkApps(queueSource, 1, 0, 1, 0, 0, 0, true);
+ checkApps(userSource, 1, 0, 1, 0, 0, 0, true);
metrics.allocateResources(user, 3, Resources.createResource(2*GB, 2));
checkResources(queueSource, 6*GB, 6, 3, 3, 0, 100*GB, 100, 9*GB, 9, 2, 0, 0, 0);
@@ -173,8 +173,8 @@ public class TestQueueMetrics {
checkResources(userSource, 4*GB, 4, 2, 3, 1, 10*GB, 10, 9*GB, 9, 2, 0, 0, 0);
metrics.finishApp(app, RMAppAttemptState.FINISHED);
- checkApps(queueSource, 1, 0, 0, 1, 0, 0);
- checkApps(userSource, 1, 0, 0, 1, 0, 0);
+ checkApps(queueSource, 1, 0, 0, 1, 0, 0, true);
+ checkApps(userSource, 1, 0, 0, 1, 0, 0, true);
}
@Test public void testTwoLevelWithUserMetrics() {
@@ -196,10 +196,10 @@ public class TestQueueMetrics {
MetricsSource userSource = userSource(ms, leafQueueName, user);
MetricsSource parentUserSource = userSource(ms, parentQueueName, user);
- checkApps(queueSource, 1, 1, 0, 0, 0, 0);
- checkApps(parentQueueSource, 1, 1, 0, 0, 0, 0);
- checkApps(userSource, 1, 1, 0, 0, 0, 0);
- checkApps(parentUserSource, 1, 1, 0, 0, 0, 0);
+ checkApps(queueSource, 1, 1, 0, 0, 0, 0, true);
+ checkApps(parentQueueSource, 1, 1, 0, 0, 0, 0, true);
+ checkApps(userSource, 1, 1, 0, 0, 0, 0, true);
+ checkApps(parentUserSource, 1, 1, 0, 0, 0, 0, true);
parentMetrics.setAvailableResourcesToQueue(Resources.createResource(100*GB, 100));
metrics.setAvailableResourcesToQueue(Resources.createResource(100*GB, 100));
@@ -212,8 +212,8 @@ public class TestQueueMetrics {
checkResources(parentUserSource, 0, 0, 0, 0, 0, 10*GB, 10, 15*GB, 15, 5, 0, 0, 0);
metrics.incrAppsRunning(app, user);
- checkApps(queueSource, 1, 0, 1, 0, 0, 0);
- checkApps(userSource, 1, 0, 1, 0, 0, 0);
+ checkApps(queueSource, 1, 0, 1, 0, 0, 0, true);
+ checkApps(userSource, 1, 0, 1, 0, 0, 0, true);
metrics.allocateResources(user, 3, Resources.createResource(2*GB, 2));
metrics.reserveResource(user, Resources.createResource(3*GB, 3));
@@ -232,10 +232,10 @@ public class TestQueueMetrics {
checkResources(parentUserSource, 4*GB, 4, 2, 3, 1, 10*GB, 10, 9*GB, 9, 2, 0, 0, 0);
metrics.finishApp(app, RMAppAttemptState.FINISHED);
- checkApps(queueSource, 1, 0, 0, 1, 0, 0);
- checkApps(parentQueueSource, 1, 0, 0, 1, 0, 0);
- checkApps(userSource, 1, 0, 0, 1, 0, 0);
- checkApps(parentUserSource, 1, 0, 0, 1, 0, 0);
+ checkApps(queueSource, 1, 0, 0, 1, 0, 0, true);
+ checkApps(parentQueueSource, 1, 0, 0, 1, 0, 0, true);
+ checkApps(userSource, 1, 0, 0, 1, 0, 0, true);
+ checkApps(parentUserSource, 1, 0, 0, 1, 0, 0, true);
}
@Test
@@ -275,13 +275,35 @@ public class TestQueueMetrics {
FifoScheduler.class, ResourceScheduler.class);
MockRM rm = new MockRM(conf);
QueueMetrics metrics = rm.getResourceScheduler().getRootQueueMetrics();
- checkApps(metrics, 0, 0, 0, 0, 0, 0);
+ checkApps(metrics, 0, 0, 0, 0, 0, 0, true);
MetricsAsserts.assertGauge("ReservedContainers", 0, metrics);
}
+ // This is to test all metrics can consistently show up if specified true to
+ // collect all metrics, even though they are not modified from last time they
+ // are collected. If not collecting all metrics, only modified metrics will show up.
+ @Test
+ public void testCollectAllMetrics() {
+ String queueName = "single";
+ QueueMetrics.forQueue(ms, queueName, null, false, conf);
+ MetricsSource queueSource = queueSource(ms, queueName);
+
+ checkApps(queueSource, 0, 0, 0, 0, 0, 0, true);
+ try {
+ // do not collect all metrics
+ checkApps(queueSource, 0, 0, 0, 0, 0, 0, false);
+ Assert.fail();
+ } catch (AssertionError e) {
+ Assert.assertTrue(e.getMessage().contains(
+ "Expected exactly one metric for name "));
+ }
+ // collect all metrics
+ checkApps(queueSource, 0, 0, 0, 0, 0, 0, true);
+ }
+
public static void checkApps(MetricsSource source, int submitted, int pending,
- int running, int completed, int failed, int killed) {
- MetricsRecordBuilder rb = getMetrics(source);
+ int running, int completed, int failed, int killed, boolean all) {
+ MetricsRecordBuilder rb = getMetrics(source, all);
assertCounter("AppsSubmitted", submitted, rb);
assertGauge("AppsPending", pending, rb);
assertGauge("AppsRunning", running, rb);
Modified: hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java?rev=1513658&r1=1513657&r2=1513658&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java (original)
+++ hadoop/common/branches/HDFS-4949/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java Tue Aug 13 21:19:53 2013
@@ -43,6 +43,8 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -260,6 +262,11 @@ public class TestContainerManagerSecurit
Assert.assertTrue(testStartContainer(rpc, validAppAttemptId, validNode,
validContainerToken, validNMToken, true).contains(sb.toString()));
+ // Container is removed from node manager's memory by this time.
+ // trying to stop the container. It should not throw any exception.
+ testStopContainer(rpc, validAppAttemptId, validNode, validContainerId,
+ validNMToken, false);
+
// Rolling over master key twice so that we can check whether older keys
// are used for authentication.
rollNMTokenMasterKey(nmTokenSecretManagerRM, nmTokenSecretManagerNM);
@@ -267,13 +274,25 @@ public class TestContainerManagerSecurit
rollNMTokenMasterKey(nmTokenSecretManagerRM, nmTokenSecretManagerNM);
// trying get container status. Now saved nmToken should be used for
- // authentication.
+ // authentication... It should complain saying container was recently
+ // stopped.
+ sb = new StringBuilder("Container ");
+ sb.append(validContainerId);
+ sb.append(" was recently stopped on node manager");
+ Assert.assertTrue(testGetContainer(rpc, validAppAttemptId, validNode,
+ validContainerId, validNMToken, true).contains(sb.toString()));
+
+ // Now lets remove the container from nm-memory
+ nm.getNodeStatusUpdater().clearFinishedContainersFromCache();
+
+ // This should fail as container is removed from recently tracked finished
+ // containers.
sb = new StringBuilder("Container ");
sb.append(validContainerId.toString());
sb.append(" is not handled by this NodeManager");
Assert.assertTrue(testGetContainer(rpc, validAppAttemptId, validNode,
validContainerId, validNMToken, false).contains(sb.toString()));
-
+
}
private void waitForContainerToFinishOnNM(ContainerId containerId) {
@@ -315,6 +334,23 @@ public class TestContainerManagerSecurit
Assert.assertTrue((nmTokenSecretManagerNM.getCurrentKey().getKeyId()
== nmTokenSecretManagerRM.getCurrentKey().getKeyId()));
}
+
+ private String testStopContainer(YarnRPC rpc,
+ ApplicationAttemptId appAttemptId, NodeId nodeId,
+ ContainerId containerId, Token nmToken, boolean isExceptionExpected) {
+ try {
+ stopContainer(rpc, nmToken,
+ Arrays.asList(new ContainerId[] { containerId }), appAttemptId,
+ nodeId);
+ if (isExceptionExpected) {
+ fail("Exception was expected!!");
+ }
+ return "";
+ } catch (Exception e) {
+ e.printStackTrace();
+ return e.getMessage();
+ }
+ }
private String testGetContainer(YarnRPC rpc,
ApplicationAttemptId appAttemptId, NodeId nodeId,
@@ -334,7 +370,7 @@ public class TestContainerManagerSecurit
}
}
- protected String testStartContainer(YarnRPC rpc,
+ private String testStartContainer(YarnRPC rpc,
ApplicationAttemptId appAttemptId, NodeId nodeId,
org.apache.hadoop.yarn.api.records.Token containerToken,
org.apache.hadoop.yarn.api.records.Token nmToken,
@@ -352,6 +388,29 @@ public class TestContainerManagerSecurit
}
}
+ private void stopContainer(YarnRPC rpc, Token nmToken,
+ List<ContainerId> containerId, ApplicationAttemptId appAttemptId,
+ NodeId nodeId) throws Exception {
+ StopContainersRequest request =
+ StopContainersRequest.newInstance(containerId);
+ ContainerManagementProtocol proxy = null;
+ try {
+ proxy =
+ getContainerManagementProtocolProxy(rpc, nmToken, nodeId,
+ appAttemptId.toString());
+ StopContainersResponse response = proxy.stopContainers(request);
+ if (response.getFailedRequests() != null &&
+ response.getFailedRequests().containsKey(containerId)) {
+ parseAndThrowException(response.getFailedRequests().get(containerId)
+ .deSerialize());
+ }
+ } catch (Exception e) {
+ if (proxy != null) {
+ rpc.stopProxy(proxy, conf);
+ }
+ }
+ }
+
private void
getContainerStatus(YarnRPC rpc,
org.apache.hadoop.yarn.api.records.Token nmToken,