You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by sz...@apache.org on 2014/01/31 04:24:31 UTC
svn commit: r1563042 - in
/hadoop/common/branches/HDFS-5535/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/
hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/
hadoop-yarn/...
Author: szetszwo
Date: Fri Jan 31 03:24:24 2014
New Revision: 1563042
URL: http://svn.apache.org/r1563042
Log:
Merge r1555021 through r1563041 from trunk.
Added:
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java
- copied unchanged from r1563041, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java
Modified:
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/RMDelegationTokenIdentifier.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Queue.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/MaxRunningAppsEnforcer.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java
hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/CHANGES.txt?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/CHANGES.txt Fri Jan 31 03:24:24 2014
@@ -9,6 +9,24 @@ Trunk - Unreleased
YARN-1496. Protocol additions to allow moving apps between queues (Sandy
Ryza)
+ YARN-1498. Common scheduler changes for moving apps between queues (Sandy
+ Ryza)
+
+ IMPROVEMENTS
+
+ OPTIMIZATIONS
+
+ BUG FIXES
+
+ YARN-524 TestYarnVersionInfo failing if generated properties doesn't
+ include an SVN URL. (stevel)
+
+Release 2.4.0 - UNRELEASED
+
+ INCOMPATIBLE CHANGES
+
+ NEW FEATURES
+
YARN-930. Bootstrapping ApplicationHistoryService module. (vinodkv)
YARN-947. Implementing the data objects to be used by the History reader
@@ -70,13 +88,16 @@ Trunk - Unreleased
YARN-987. Added ApplicationHistoryManager responsible for exposing reports to
all clients. (Mayank Bansal via vinodkv)
+ YARN-1630. Introduce timeout for async polling operations in YarnClientImpl
+ (Aditya Acharya via Sandy Ryza)
+
+ YARN-1617. Remove ancient comment and surround LOG.debug in
+ AppSchedulingInfo.allocate (Sandy Ryza)
+
OPTIMIZATIONS
BUG FIXES
- YARN-524 TestYarnVersionInfo failing if generated properties doesn't
- include an SVN URL. (stevel)
-
YARN-935. Correcting pom.xml to build applicationhistoryserver module
successfully. (Zhijie Shen via vinodkv)
@@ -112,7 +133,14 @@ Trunk - Unreleased
YARN-1613. Fixed the typo with the configuration name
YARN_HISTORY_SERVICE_ENABLED. (Akira Ajisaka via vinodkv)
-Release 2.4.0 - UNRELEASED
+ YARN-1618. Fix invalid RMApp transition from NEW to FINAL_SAVING (kasha)
+
+ YARN-1600. RM does not startup when security is enabled without spnego
+ configured (Haohui Mai via jlowe)
+
+ YARN-1642. RMDTRenewer#getRMClient should use ClientRMProxy (kasha)
+
+Release 2.3.0 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -169,6 +197,30 @@ Release 2.4.0 - UNRELEASED
IMPROVEMENTS
+ YARN-305. Fair scheduler logs too many "Node offered to app" messages.
+ (Lohit Vijayarenu via Sandy Ryza)
+
+ YARN-1258. Allow configuring the Fair Scheduler root queue (Sandy Ryza)
+
+ YARN-1288. Make Fair Scheduler ACLs more user friendly (Sandy Ryza)
+
+ YARN-1315. TestQueueACLs should also test FairScheduler (Sandy Ryza)
+
+ YARN-1335. Move duplicate code from FSSchedulerApp and FiCaSchedulerApp
+ into SchedulerApplication (Sandy Ryza)
+
+ YARN-1333. Support blacklisting in the Fair Scheduler (Tsuyoshi Ozawa via
+ Sandy Ryza)
+
+ YARN-1109. Demote NodeManager "Sending out status for container" logs to
+ debug (haosdent via Sandy Ryza)
+
+ YARN-1321. Changed NMTokenCache to support both singleton and an instance
+ usage. (Alejandro Abdelnur via vinodkv)
+
+ YARN-1388. Fair Scheduler page always displays blank fair share (Liyin Liang
+ via Sandy Ryza)
+
YARN-7. Support CPU resource for DistributedShell. (Junping Du via llu)
YARN-905. Add state filters to nodes CLI (Wei Yan via Sandy Ryza)
@@ -338,6 +390,62 @@ Release 2.4.0 - UNRELEASED
BUG FIXES
+ YARN-1284. LCE: Race condition leaves dangling cgroups entries for killed
+ containers. (Alejandro Abdelnur via Sandy Ryza)
+
+ YARN-1283. Fixed RM to give a fully-qualified proxy URL for an application
+ so that clients don't need to do scheme-mangling. (Omkar Vinit Joshi via
+ vinodkv)
+
+ YARN-879. Fixed tests w.r.t o.a.h.y.server.resourcemanager.Application.
+ (Junping Du via devaraj)
+
+ YARN-1265. Fair Scheduler chokes on unhealthy node reconnect (Sandy Ryza)
+
+ YARN-1044. used/min/max resources do not display info in the scheduler page
+ (Sangjin Lee via Sandy Ryza)
+
+ YARN-1259. In Fair Scheduler web UI, queue num pending and num active apps
+ switched. (Robert Kanter via Sandy Ryza)
+
+ YARN-1295. In UnixLocalWrapperScriptBuilder, using bash -c can cause Text
+ file busy errors (Sandy Ryza)
+
+ YARN-1185. Fixed FileSystemRMStateStore to not leave partial files that
+ prevent subsequent ResourceManager recovery. (Omkar Vinit Joshi via vinodkv)
+
+ YARN-1331. yarn.cmd exits with NoClassDefFoundError trying to run rmadmin or
+ logs. (cnauroth)
+
+ YARN-1330. Fair Scheduler: defaultQueueSchedulingPolicy does not take effect
+ (Sandy Ryza)
+
+ YARN-1022. Unnecessary INFO logs in AMRMClientAsync (haosdent via bikas)
+
+ YARN-1349. yarn.cmd does not support passthrough to any arbitrary class.
+ (cnauroth)
+
+ YARN-1357. TestContainerLaunch.testContainerEnvVariables fails on Windows.
+ (Chuan Liu via cnauroth)
+
+ YARN-1358. TestYarnCLI fails on Windows due to line endings. (Chuan Liu via
+ cnauroth)
+
+ YARN-1343. NodeManagers additions/restarts are not reported as node updates
+ in AllocateResponse responses to AMs. (tucu)
+
+ YARN-1381. Same relaxLocality appears twice in exception message of
+ AMRMClientImpl#checkLocalityRelaxationConflict() (Ted Yu via Sandy Ryza)
+
+ YARN-1407. RM Web UI and REST APIs should uniformly use
+ YarnApplicationState (Sandy Ryza)
+
+ YARN-1438. Ensure container diagnostics includes exception from container
+ launch. (stevel via acmurthy)
+
+ YARN-1138. yarn.application.classpath is set to point to $HADOOP_CONF_DIR
+ etc., which does not work on Windows. (Chuan Liu via cnauroth)
+
YARN-461. Fair scheduler should not accept apps with empty string queue name.
(ywskycn via tucu)
@@ -463,97 +571,7 @@ Release 2.4.0 - UNRELEASED
YARN-1575. Public localizer crashes with "Localized unkown resource"
(jlowe)
-Release 2.3.0 - UNRELEASED
-
- INCOMPATIBLE CHANGES
-
- NEW FEATURES
-
- IMPROVEMENTS
-
- YARN-305. Fair scheduler logs too many "Node offered to app" messages.
- (Lohit Vijayarenu via Sandy Ryza)
-
- YARN-1258. Allow configuring the Fair Scheduler root queue (Sandy Ryza)
-
- YARN-1288. Make Fair Scheduler ACLs more user friendly (Sandy Ryza)
-
- YARN-1315. TestQueueACLs should also test FairScheduler (Sandy Ryza)
-
- YARN-1335. Move duplicate code from FSSchedulerApp and FiCaSchedulerApp
- into SchedulerApplication (Sandy Ryza)
-
- YARN-1333. Support blacklisting in the Fair Scheduler (Tsuyoshi Ozawa via
- Sandy Ryza)
-
- YARN-1109. Demote NodeManager "Sending out status for container" logs to
- debug (haosdent via Sandy Ryza)
-
- YARN-1321. Changed NMTokenCache to support both singleton and an instance
- usage. (Alejandro Abdelnur via vinodkv)
-
- YARN-1388. Fair Scheduler page always displays blank fair share (Liyin Liang
- via Sandy Ryza)
-
- OPTIMIZATIONS
-
- BUG FIXES
-
- YARN-1284. LCE: Race condition leaves dangling cgroups entries for killed
- containers. (Alejandro Abdelnur via Sandy Ryza)
-
- YARN-1283. Fixed RM to give a fully-qualified proxy URL for an application
- so that clients don't need to do scheme-mangling. (Omkar Vinit Joshi via
- vinodkv)
-
- YARN-879. Fixed tests w.r.t o.a.h.y.server.resourcemanager.Application.
- (Junping Du via devaraj)
-
- YARN-1265. Fair Scheduler chokes on unhealthy node reconnect (Sandy Ryza)
-
- YARN-1044. used/min/max resources do not display info in the scheduler page
- (Sangjin Lee via Sandy Ryza)
-
- YARN-1259. In Fair Scheduler web UI, queue num pending and num active apps
- switched. (Robert Kanter via Sandy Ryza)
-
- YARN-1295. In UnixLocalWrapperScriptBuilder, using bash -c can cause Text
- file busy errors (Sandy Ryza)
-
- YARN-1185. Fixed FileSystemRMStateStore to not leave partial files that
- prevent subsequent ResourceManager recovery. (Omkar Vinit Joshi via vinodkv)
-
- YARN-1331. yarn.cmd exits with NoClassDefFoundError trying to run rmadmin or
- logs. (cnauroth)
-
- YARN-1330. Fair Scheduler: defaultQueueSchedulingPolicy does not take effect
- (Sandy Ryza)
-
- YARN-1022. Unnecessary INFO logs in AMRMClientAsync (haosdent via bikas)
-
- YARN-1349. yarn.cmd does not support passthrough to any arbitrary class.
- (cnauroth)
-
- YARN-1357. TestContainerLaunch.testContainerEnvVariables fails on Windows.
- (Chuan Liu via cnauroth)
-
- YARN-1358. TestYarnCLI fails on Windows due to line endings. (Chuan Liu via
- cnauroth)
-
- YARN-1343. NodeManagers additions/restarts are not reported as node updates
- in AllocateResponse responses to AMs. (tucu)
-
- YARN-1381. Same relaxLocality appears twice in exception message of
- AMRMClientImpl#checkLocalityRelaxationConflict() (Ted Yu via Sandy Ryza)
-
- YARN-1407. RM Web UI and REST APIs should uniformly use
- YarnApplicationState (Sandy Ryza)
-
- YARN-1438. Ensure container diagnostics includes exception from container
- launch. (stevel via acmurthy)
-
- YARN-1138. yarn.application.classpath is set to point to $HADOOP_CONF_DIR
- etc., which does not work on Windows. (Chuan Liu via cnauroth)
+ YARN-1629. IndexOutOfBoundsException in MaxRunningAppsEnforcer (Sandy Ryza)
Release 2.2.0 - 2013-10-13
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java Fri Jan 31 03:24:24 2014
@@ -1020,6 +1020,17 @@ public class YarnConfiguration extends C
YARN_PREFIX + "client.application-client-protocol.poll-interval-ms";
public static final long DEFAULT_YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_INTERVAL_MS =
200;
+
+ /**
+ * The duration that the yarn client library waits, cumulatively across polls,
+ * for an expected state change to occur. Defaults to -1, which indicates no
+ * limit.
+ */
+ public static final String YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_TIMEOUT_MS =
+ YARN_PREFIX + "client.application-client-protocol.poll-timeout-ms";
+ public static final long DEFAULT_YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_TIMEOUT_MS =
+ -1;
+
/**
* Max number of threads in NMClientAsync to process container management
* events
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java Fri Jan 31 03:24:24 2014
@@ -86,6 +86,7 @@ public class YarnClientImpl extends Yarn
protected ApplicationClientProtocol rmClient;
protected long submitPollIntervalMillis;
private long asyncApiPollIntervalMillis;
+ private long asyncApiPollTimeoutMillis;
protected AHSClient historyClient;
private boolean historyServiceEnabled;
@@ -101,6 +102,9 @@ public class YarnClientImpl extends Yarn
asyncApiPollIntervalMillis =
conf.getLong(YarnConfiguration.YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_INTERVAL_MS,
YarnConfiguration.DEFAULT_YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_INTERVAL_MS);
+ asyncApiPollTimeoutMillis =
+ conf.getLong(YarnConfiguration.YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_TIMEOUT_MS,
+ YarnConfiguration.DEFAULT_YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_TIMEOUT_MS);
submitPollIntervalMillis = asyncApiPollIntervalMillis;
if (conf.get(YarnConfiguration.YARN_CLIENT_APP_SUBMISSION_POLL_INTERVAL_MS)
!= null) {
@@ -174,13 +178,24 @@ public class YarnClientImpl extends Yarn
rmClient.submitApplication(request);
int pollCount = 0;
+ long startTime = System.currentTimeMillis();
+
while (true) {
YarnApplicationState state =
getApplicationReport(applicationId).getYarnApplicationState();
if (!state.equals(YarnApplicationState.NEW) &&
!state.equals(YarnApplicationState.NEW_SAVING)) {
+ LOG.info("Submitted application " + applicationId);
break;
}
+
+ long elapsedMillis = System.currentTimeMillis() - startTime;
+ if (enforceAsyncAPITimeout() &&
+ elapsedMillis >= asyncApiPollTimeoutMillis) {
+ throw new YarnException("Timed out while waiting for application " +
+ applicationId + " to be submitted successfully");
+ }
+
// Notify the client through the log every 10 poll, in case the client
// is blocked here too long.
if (++pollCount % 10 == 0) {
@@ -191,10 +206,11 @@ public class YarnClientImpl extends Yarn
try {
Thread.sleep(submitPollIntervalMillis);
} catch (InterruptedException ie) {
+ LOG.error("Interrupted while waiting for application " + applicationId
+ + " to be successfully submitted.");
}
}
- LOG.info("Submitted application " + applicationId);
return applicationId;
}
@@ -207,15 +223,25 @@ public class YarnClientImpl extends Yarn
try {
int pollCount = 0;
+ long startTime = System.currentTimeMillis();
+
while (true) {
KillApplicationResponse response =
rmClient.forceKillApplication(request);
if (response.getIsKillCompleted()) {
+ LOG.info("Killed application " + applicationId);
break;
}
+
+ long elapsedMillis = System.currentTimeMillis() - startTime;
+ if (enforceAsyncAPITimeout() &&
+ elapsedMillis >= this.asyncApiPollTimeoutMillis) {
+ throw new YarnException("Timed out while waiting for application " +
+ applicationId + " to be killed.");
+ }
+
if (++pollCount % 10 == 0) {
- LOG.info("Watiting for application " + applicationId
- + " to be killed.");
+ LOG.info("Waiting for application " + applicationId + " to be killed.");
}
Thread.sleep(asyncApiPollIntervalMillis);
}
@@ -223,7 +249,11 @@ public class YarnClientImpl extends Yarn
LOG.error("Interrupted while waiting for application " + applicationId
+ " to be killed.");
}
- LOG.info("Killed application " + applicationId);
+ }
+
+ @VisibleForTesting
+ boolean enforceAsyncAPITimeout() {
+ return asyncApiPollTimeoutMillis >= 0;
}
@Override
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java Fri Jan 31 03:24:24 2014
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.client.api.impl;
+import static org.junit.Assert.assertFalse;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
@@ -35,6 +36,7 @@ import java.util.Set;
import junit.framework.Assert;
+import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
@@ -474,4 +476,30 @@ public class TestYarnClient {
}
}
+ @Test
+ public void testAsyncAPIPollTimeout() {
+ testAsyncAPIPollTimeoutHelper(null, false);
+ testAsyncAPIPollTimeoutHelper(0L, true);
+ testAsyncAPIPollTimeoutHelper(1L, true);
+ }
+
+ private void testAsyncAPIPollTimeoutHelper(Long valueForTimeout,
+ boolean expectedTimeoutEnforcement) {
+ YarnClientImpl client = new YarnClientImpl();
+ try {
+ Configuration conf = new Configuration();
+ if (valueForTimeout != null) {
+ conf.setLong(
+ YarnConfiguration.YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_TIMEOUT_MS,
+ valueForTimeout);
+ }
+
+ client.init(conf);
+
+ Assert.assertEquals(
+ expectedTimeoutEnforcement, client.enforceAsyncAPITimeout());
+ } finally {
+ IOUtils.closeQuietly(client);
+ }
+ }
}
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/RMDelegationTokenIdentifier.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/RMDelegationTokenIdentifier.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/RMDelegationTokenIdentifier.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/RMDelegationTokenIdentifier.java Fri Jan 31 03:24:24 2014
@@ -37,8 +37,8 @@ import org.apache.hadoop.security.token.
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest;
import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenRequest;
+import org.apache.hadoop.yarn.client.ClientRMProxy;
import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.util.Records;
/**
@@ -138,7 +138,7 @@ public class RMDelegationTokenIdentifier
}
private static ApplicationClientProtocol getRmClient(Token<?> token,
- Configuration conf) {
+ Configuration conf) throws IOException {
InetSocketAddress addr = SecurityUtil.getTokenServiceAddr(token);
if (localSecretManager != null) {
// return null if it's our token
@@ -151,8 +151,7 @@ public class RMDelegationTokenIdentifier
return null;
}
}
- final YarnRPC rpc = YarnRPC.create(conf);
- return (ApplicationClientProtocol)rpc.getProxy(ApplicationClientProtocol.class, addr, conf);
+ return ClientRMProxy.createRMProxy(conf, ApplicationClientProtocol.class);
}
// get renewer so we can always renew our own tokens
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java Fri Jan 31 03:24:24 2014
@@ -28,7 +28,7 @@ import java.util.Map;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.http.HttpServer;
+import org.apache.hadoop.http.HttpServer2;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -59,7 +59,7 @@ public abstract class WebApp extends Ser
private volatile String redirectPath;
private volatile String wsName;
private volatile Configuration conf;
- private volatile HttpServer httpServer;
+ private volatile HttpServer2 httpServer;
private volatile GuiceFilter guiceFilter;
private final Router router = new Router();
@@ -72,11 +72,11 @@ public abstract class WebApp extends Ser
static final Splitter pathSplitter =
Splitter.on('/').trimResults().omitEmptyStrings();
- void setHttpServer(HttpServer server) {
+ void setHttpServer(HttpServer2 server) {
httpServer = checkNotNull(server, "http server");
}
- @Provides public HttpServer httpServer() { return httpServer; }
+ @Provides public HttpServer2 httpServer() { return httpServer; }
/**
* Get the address the http server is bound to
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java Fri Jan 31 03:24:24 2014
@@ -35,7 +35,7 @@ import javax.servlet.http.HttpServlet;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.http.HttpServer;
+import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.security.AdminACLsManager;
import org.slf4j.Logger;
@@ -216,20 +216,22 @@ public class WebApps {
System.exit(1);
}
}
- HttpServer.Builder builder = new HttpServer.Builder().setName(name)
+ HttpServer2.Builder builder = new HttpServer2.Builder().setName(name)
.addEndpoint(URI.create("http://" + bindAddress + ":" + port))
.setConf(conf).setFindPort(findPort)
.setACL(new AdminACLsManager(conf).getAdminAcl())
.setPathSpec(pathList.toArray(new String[0]));
boolean hasSpnegoConf = spnegoPrincipalKey != null
- && spnegoKeytabKey != null;
+ && conf.get(spnegoPrincipalKey) != null && spnegoKeytabKey != null
+ && conf.get(spnegoKeytabKey) != null;
+
if (hasSpnegoConf) {
builder.setUsernameConfKey(spnegoPrincipalKey)
.setKeytabConfKey(spnegoKeytabKey)
.setSecurityEnabled(UserGroupInformation.isSecurityEnabled());
}
- HttpServer server = builder.build();
+ HttpServer2 server = builder.build();
for(ServletStruct struct: servlets) {
server.addServlet(struct.name, struct.spec, struct.clazz);
@@ -237,7 +239,7 @@ public class WebApps {
for(Map.Entry<String, Object> entry : attributes.entrySet()) {
server.setAttribute(entry.getKey(), entry.getValue());
}
- HttpServer.defineFilter(server.getWebAppContext(), "guice",
+ HttpServer2.defineFilter(server.getWebAppContext(), "guice",
GuiceFilter.class.getName(), null, new String[] { "/*" });
webapp.setConf(conf);
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java Fri Jan 31 03:24:24 2014
@@ -24,8 +24,10 @@ public enum RMAppEventType {
RECOVER,
KILL,
- // Source: Scheduler
+ // Source: Scheduler and RMAppManager
APP_REJECTED,
+
+ // Source: Scheduler
APP_ACCEPTED,
// Source: RMAppAttempt
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java Fri Jan 31 03:24:24 2014
@@ -144,13 +144,10 @@ public class RMAppImpl implements RMApp,
RMAppState.ACCEPTED, RMAppState.FINISHED, RMAppState.FAILED,
RMAppState.KILLED, RMAppState.FINAL_SAVING),
RMAppEventType.RECOVER, new RMAppRecoveredTransition())
- .addTransition(RMAppState.NEW, RMAppState.FINAL_SAVING, RMAppEventType.KILL,
- new FinalSavingTransition(
- new AppKilledTransition(), RMAppState.KILLED))
- .addTransition(RMAppState.NEW, RMAppState.FINAL_SAVING,
- RMAppEventType.APP_REJECTED,
- new FinalSavingTransition(
- new AppRejectedTransition(), RMAppState.FAILED))
+ .addTransition(RMAppState.NEW, RMAppState.KILLED, RMAppEventType.KILL,
+ new AppKilledTransition())
+ .addTransition(RMAppState.NEW, RMAppState.FAILED,
+ RMAppEventType.APP_REJECTED, new AppRejectedTransition())
// Transitions from NEW_SAVING state
.addTransition(RMAppState.NEW_SAVING, RMAppState.NEW_SAVING,
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java Fri Jan 31 03:24:24 2014
@@ -64,7 +64,7 @@ public class AppSchedulingInfo {
private Set<String> blacklist = new HashSet<String>();
//private final ApplicationStore store;
- private final ActiveUsersManager activeUsersManager;
+ private ActiveUsersManager activeUsersManager;
/* Allocated by scheduler */
boolean pending = true; // for app metrics
@@ -171,11 +171,10 @@ public class AppSchedulingInfo {
.getNumContainers() : 0;
Resource lastRequestCapability = lastRequest != null ? lastRequest
.getCapability() : Resources.none();
- metrics.incrPendingResources(user, request.getNumContainers()
- - lastRequestContainers, Resources.subtractFrom( // save a clone
- Resources.multiply(request.getCapability(), request
- .getNumContainers()), Resources.multiply(lastRequestCapability,
- lastRequestContainers)));
+ metrics.incrPendingResources(user, request.getNumContainers(),
+ request.getCapability());
+ metrics.decrPendingResources(user, lastRequestContainers,
+ lastRequestCapability);
}
}
}
@@ -262,9 +261,15 @@ public class AppSchedulingInfo {
pending = false;
metrics.runAppAttempt(applicationId, user);
}
- LOG.debug("allocate: user: " + user + ", memory: "
- + request.getCapability());
- metrics.allocateResources(user, 1, request.getCapability());
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("allocate: applicationId=" + applicationId
+ + " container=" + container.getId()
+ + " host=" + container.getNodeId().toString()
+ + " user=" + user
+ + " resource=" + request.getCapability());
+ }
+ metrics.allocateResources(user, 1, request.getCapability(), true);
}
/**
@@ -277,9 +282,6 @@ public class AppSchedulingInfo {
synchronized private void allocateNodeLocal(
SchedulerNode node, Priority priority,
ResourceRequest nodeLocalRequest, Container container) {
- // Update consumption and track allocations
- allocate(container);
-
// Update future requirements
nodeLocalRequest.setNumContainers(nodeLocalRequest.getNumContainers() - 1);
if (nodeLocalRequest.getNumContainers() == 0) {
@@ -306,10 +308,6 @@ public class AppSchedulingInfo {
synchronized private void allocateRackLocal(
SchedulerNode node, Priority priority,
ResourceRequest rackLocalRequest, Container container) {
-
- // Update consumption and track allocations
- allocate(container);
-
// Update future requirements
rackLocalRequest.setNumContainers(rackLocalRequest.getNumContainers() - 1);
if (rackLocalRequest.getNumContainers() == 0) {
@@ -329,10 +327,6 @@ public class AppSchedulingInfo {
synchronized private void allocateOffSwitch(
SchedulerNode node, Priority priority,
ResourceRequest offSwitchRequest, Container container) {
-
- // Update consumption and track allocations
- allocate(container);
-
// Update future requirements
decrementOutstanding(offSwitchRequest);
}
@@ -365,18 +359,24 @@ public class AppSchedulingInfo {
}
}
- synchronized private void allocate(Container container) {
- // Update consumption and track allocations
- //TODO: fixme sharad
- /* try {
- store.storeContainer(container);
- } catch (IOException ie) {
- // TODO fix this. we shouldnt ignore
- }*/
-
- LOG.debug("allocate: applicationId=" + applicationId + " container="
- + container.getId() + " host="
- + container.getNodeId().toString());
+ synchronized public void move(Queue newQueue) {
+ QueueMetrics oldMetrics = queue.getMetrics();
+ QueueMetrics newMetrics = newQueue.getMetrics();
+ for (Map<String, ResourceRequest> asks : requests.values()) {
+ ResourceRequest request = asks.get(ResourceRequest.ANY);
+ if (request != null) {
+ oldMetrics.decrPendingResources(user, request.getNumContainers(),
+ request.getCapability());
+ newMetrics.incrPendingResources(user, request.getNumContainers(),
+ request.getCapability());
+ }
+ }
+ oldMetrics.moveAppFrom(this);
+ newMetrics.moveAppTo(this);
+ activeUsersManager.deactivateApplication(user, applicationId);
+ activeUsersManager = newQueue.getActiveUsersManager();
+ activeUsersManager.activateApplication(user, applicationId);
+ this.queue = newQueue;
}
synchronized public void stop(RMAppAttemptState rmAppAttemptFinalState) {
@@ -386,8 +386,7 @@ public class AppSchedulingInfo {
ResourceRequest request = asks.get(ResourceRequest.ANY);
if (request != null) {
metrics.decrPendingResources(user, request.getNumContainers(),
- Resources.multiply(request.getCapability(), request
- .getNumContainers()));
+ request.getCapability());
}
}
metrics.finishAppAttempt(applicationId, pending, user);
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Queue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Queue.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Queue.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Queue.java Fri Jan 31 03:24:24 2014
@@ -58,4 +58,6 @@ public interface Queue {
List<QueueUserACLInfo> getQueueUserAclInfo(UserGroupInformation user);
boolean hasAccess(QueueACL acl, UserGroupInformation user);
+
+ public ActiveUsersManager getActiveUsersManager();
}
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java Fri Jan 31 03:24:24 2014
@@ -280,6 +280,36 @@ public class QueueMetrics implements Met
parent.finishApp(user, rmAppFinalState);
}
}
+
+ public void moveAppFrom(AppSchedulingInfo app) {
+ if (app.isPending()) {
+ appsPending.decr();
+ } else {
+ appsRunning.decr();
+ }
+ QueueMetrics userMetrics = getUserMetrics(app.getUser());
+ if (userMetrics != null) {
+ userMetrics.moveAppFrom(app);
+ }
+ if (parent != null) {
+ parent.moveAppFrom(app);
+ }
+ }
+
+ public void moveAppTo(AppSchedulingInfo app) {
+ if (app.isPending()) {
+ appsPending.incr();
+ } else {
+ appsRunning.incr();
+ }
+ QueueMetrics userMetrics = getUserMetrics(app.getUser());
+ if (userMetrics != null) {
+ userMetrics.moveAppTo(app);
+ }
+ if (parent != null) {
+ parent.moveAppTo(app);
+ }
+ }
/**
* Set available resources. To be called by scheduler periodically as
@@ -324,8 +354,8 @@ public class QueueMetrics implements Met
private void _incrPendingResources(int containers, Resource res) {
pendingContainers.incr(containers);
- pendingMB.incr(res.getMemory());
- pendingVCores.incr(res.getVirtualCores());
+ pendingMB.incr(res.getMemory() * containers);
+ pendingVCores.incr(res.getVirtualCores() * containers);
}
public void decrPendingResources(String user, int containers, Resource res) {
@@ -341,22 +371,25 @@ public class QueueMetrics implements Met
private void _decrPendingResources(int containers, Resource res) {
pendingContainers.decr(containers);
- pendingMB.decr(res.getMemory());
- pendingVCores.decr(res.getVirtualCores());
+ pendingMB.decr(res.getMemory() * containers);
+ pendingVCores.decr(res.getVirtualCores() * containers);
}
- public void allocateResources(String user, int containers, Resource res) {
+ public void allocateResources(String user, int containers, Resource res,
+ boolean decrPending) {
allocatedContainers.incr(containers);
aggregateContainersAllocated.incr(containers);
allocatedMB.incr(res.getMemory() * containers);
allocatedVCores.incr(res.getVirtualCores() * containers);
- _decrPendingResources(containers, Resources.multiply(res, containers));
+ if (decrPending) {
+ _decrPendingResources(containers, res);
+ }
QueueMetrics userMetrics = getUserMetrics(user);
if (userMetrics != null) {
- userMetrics.allocateResources(user, containers, res);
+ userMetrics.allocateResources(user, containers, res, decrPending);
}
if (parent != null) {
- parent.allocateResources(user, containers, res);
+ parent.allocateResources(user, containers, res, decrPending);
}
}
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java Fri Jan 31 03:24:24 2014
@@ -57,7 +57,7 @@ import com.google.common.collect.Multise
*/
@Private
@Unstable
-public abstract class SchedulerApplicationAttempt {
+public class SchedulerApplicationAttempt {
private static final Log LOG = LogFactory
.getLog(SchedulerApplicationAttempt.class);
@@ -91,7 +91,7 @@ public abstract class SchedulerApplicati
protected Map<Priority, Long> lastScheduledContainer =
new HashMap<Priority, Long>();
- protected final Queue queue;
+ protected Queue queue;
protected boolean isStopped = false;
protected final RMContext rmContext;
@@ -431,4 +431,25 @@ public abstract class SchedulerApplicati
this.appSchedulingInfo
.transferStateFromPreviousAppSchedulingInfo(appAttempt.appSchedulingInfo);
}
+
+ public void move(Queue newQueue) {
+ QueueMetrics oldMetrics = queue.getMetrics();
+ QueueMetrics newMetrics = newQueue.getMetrics();
+ String user = getUser();
+ for (RMContainer liveContainer : liveContainers.values()) {
+ Resource resource = liveContainer.getContainer().getResource();
+ oldMetrics.releaseResources(user, 1, resource);
+ newMetrics.allocateResources(user, 1, resource, false);
+ }
+ for (Map<NodeId, RMContainer> map : reservedContainers.values()) {
+ for (RMContainer reservedContainer : map.values()) {
+ Resource resource = reservedContainer.getReservedResource();
+ oldMetrics.unreserveResource(user, resource);
+ newMetrics.reserveResource(user, resource);
+ }
+ }
+
+ appSchedulingInfo.move(newQueue);
+ this.queue = newQueue;
+ }
}
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java Fri Jan 31 03:24:24 2014
@@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppUtils;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication;
@@ -54,11 +55,14 @@ public class FSLeafQueue extends FSQueue
private long lastTimeAtMinShare;
private long lastTimeAtHalfFairShare;
+ private final ActiveUsersManager activeUsersManager;
+
public FSLeafQueue(String name, FairScheduler scheduler,
FSParentQueue parent) {
super(name, scheduler, parent);
this.lastTimeAtMinShare = scheduler.getClock().getTime();
this.lastTimeAtHalfFairShare = scheduler.getClock().getTime();
+ activeUsersManager = new ActiveUsersManager(getMetrics());
}
public void addApp(FSSchedulerApp app, boolean runnable) {
@@ -91,15 +95,6 @@ public class FSLeafQueue extends FSQueue
}
}
- public void makeAppRunnable(AppSchedulable appSched) {
- if (!nonRunnableAppScheds.remove(appSched)) {
- throw new IllegalStateException("Can't make app runnable that does not " +
- "already exist in queue as non-runnable" + appSched);
- }
-
- runnableAppScheds.add(appSched);
- }
-
public Collection<AppSchedulable> getRunnableAppSchedulables() {
return runnableAppScheds;
}
@@ -254,4 +249,9 @@ public class FSLeafQueue extends FSQueue
public int getNumRunnableApps() {
return runnableAppScheds.size();
}
+
+ @Override
+ public ActiveUsersManager getActiveUsersManager() {
+ return activeUsersManager;
+ }
}
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java Fri Jan 31 03:24:24 2014
@@ -33,7 +33,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.util.resource.Resources;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager;
@Private
@Unstable
@@ -194,4 +194,10 @@ public class FSParentQueue extends FSQue
childQueue.collectSchedulerApplications(apps);
}
}
+
+ @Override
+ public ActiveUsersManager getActiveUsersManager() {
+ // Should never be called since all applications are submitted to LeafQueues
+ return null;
+ }
}
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/MaxRunningAppsEnforcer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/MaxRunningAppsEnforcer.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/MaxRunningAppsEnforcer.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/MaxRunningAppsEnforcer.java Fri Jan 31 03:24:24 2014
@@ -24,6 +24,9 @@ import java.util.List;
import java.util.Map;
import java.util.PriorityQueue;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.ListMultimap;
@@ -33,6 +36,8 @@ import com.google.common.collect.ListMul
* constraints
*/
public class MaxRunningAppsEnforcer {
+ private static final Log LOG = LogFactory.getLog(FairScheduler.class);
+
private final FairScheduler scheduler;
// Tracks the number of running applications by user.
@@ -163,7 +168,7 @@ public class MaxRunningAppsEnforcer {
Iterator<FSSchedulerApp> iter = new MultiListStartTimeIterator(
appsNowMaybeRunnable);
FSSchedulerApp prev = null;
- int numNowRunnable = 0;
+ List<AppSchedulable> noLongerPendingApps = new ArrayList<AppSchedulable>();
while (iter.hasNext()) {
FSSchedulerApp next = iter.next();
if (next == prev) {
@@ -173,21 +178,34 @@ public class MaxRunningAppsEnforcer {
if (canAppBeRunnable(next.getQueue(), next.getUser())) {
trackRunnableApp(next);
AppSchedulable appSched = next.getAppSchedulable();
- next.getQueue().makeAppRunnable(appSched);
- if (!usersNonRunnableApps.remove(next.getUser(), appSched)) {
- throw new IllegalStateException("Waiting app " + next
- + " expected to be in usersNonRunnableApps");
- }
+ next.getQueue().getRunnableAppSchedulables().add(appSched);
+ noLongerPendingApps.add(appSched);
// No more than one app per list will be able to be made runnable, so
// we can stop looking after we've found that many
- if (numNowRunnable >= appsNowMaybeRunnable.size()) {
+ if (noLongerPendingApps.size() >= appsNowMaybeRunnable.size()) {
break;
}
}
prev = next;
}
+
+ // We remove the apps from their pending lists afterwards so that we don't
+ // pull them out from under the iterator. If they are not in these lists
+ // in the first place, there is a bug.
+ for (AppSchedulable appSched : noLongerPendingApps) {
+ if (!appSched.getApp().getQueue().getNonRunnableAppSchedulables()
+ .remove(appSched)) {
+ LOG.error("Can't make app runnable that does not already exist in queue"
+ + " as non-runnable: " + appSched + ". This should never happen.");
+ }
+
+ if (!usersNonRunnableApps.remove(appSched.getApp().getUser(), appSched)) {
+ LOG.error("Waiting app " + appSched + " expected to be in "
+ + "usersNonRunnableApps, but was not. This should never happen.");
+ }
+ }
}
/**
@@ -225,7 +243,7 @@ public class MaxRunningAppsEnforcer {
* This allows us to pick which list to advance in O(log(num lists)) instead
* of O(num lists) time.
*/
- private static class MultiListStartTimeIterator implements
+ static class MultiListStartTimeIterator implements
Iterator<FSSchedulerApp> {
private List<AppSchedulable>[] appLists;
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java Fri Jan 31 03:24:24 2014
@@ -184,6 +184,11 @@ public class FifoScheduler extends Abstr
public boolean hasAccess(QueueACL acl, UserGroupInformation user) {
return getQueueAcls().get(acl).isUserAllowed(user);
}
+
+ @Override
+ public ActiveUsersManager getActiveUsersManager() {
+ return activeUsersManager;
+ }
};
@Override
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java Fri Jan 31 03:24:24 2014
@@ -286,7 +286,7 @@ public class TestRMAppTransitions {
private void assertTimesAtFinish(RMApp application) {
assertStartTimeSet(application);
Assert.assertTrue("application finish time is not greater then 0",
- (application.getFinishTime() > 0));
+ (application.getFinishTime() > 0));
Assert.assertTrue("application finish time is not >= then start time",
(application.getFinishTime() >= application.getStartTime()));
}
@@ -295,6 +295,10 @@ public class TestRMAppTransitions {
verify(store, times(1)).updateApplicationState(any(ApplicationState.class));
}
+ private void assertAppFinalStateNotSaved(RMApp application){
+ verify(store, times(0)).updateApplicationState(any(ApplicationState.class));
+ }
+
private void assertKilled(RMApp application) {
assertTimesAtFinish(application);
assertAppState(RMAppState.KILLED, application);
@@ -321,7 +325,6 @@ public class TestRMAppTransitions {
StringBuilder diag = application.getDiagnostics();
Assert.assertTrue("application diagnostics is not correct",
diag.toString().matches(regex));
- assertAppFinalStateSaved(application);
}
private void sendAppUpdateSavedEvent(RMApp application) {
@@ -477,6 +480,7 @@ public class TestRMAppTransitions {
sendAppUpdateSavedEvent(application);
assertFailed(application,
".*Unmanaged application.*Failing the application.*");
+ assertAppFinalStateSaved(application);
}
@Test
@@ -506,6 +510,7 @@ public class TestRMAppTransitions {
rmDispatcher.await();
sendAppUpdateSavedEvent(application);
assertKilled(application);
+ assertAppFinalStateNotSaved(application);
verify(writer).applicationFinished(any(RMApp.class));
}
@@ -522,6 +527,7 @@ public class TestRMAppTransitions {
rmDispatcher.await();
sendAppUpdateSavedEvent(application);
assertFailed(application, rejectedText);
+ assertAppFinalStateNotSaved(application);
verify(writer).applicationFinished(any(RMApp.class));
}
@@ -553,6 +559,7 @@ public class TestRMAppTransitions {
rmDispatcher.await();
sendAppUpdateSavedEvent(application);
assertFailed(application, rejectedText);
+ assertAppFinalStateSaved(application);
verify(writer).applicationFinished(any(RMApp.class));
}
@@ -569,6 +576,7 @@ public class TestRMAppTransitions {
rmDispatcher.await();
sendAppUpdateSavedEvent(application);
assertFailed(application, rejectedText);
+ assertAppFinalStateSaved(application);
verify(writer).applicationFinished(any(RMApp.class));
}
@@ -618,6 +626,7 @@ public class TestRMAppTransitions {
rmDispatcher.await();
sendAppUpdateSavedEvent(application);
assertFailed(application, ".*" + message + ".*Failing the application.*");
+ assertAppFinalStateSaved(application);
verify(writer).applicationFinished(any(RMApp.class));
}
@@ -703,12 +712,14 @@ public class TestRMAppTransitions {
rmDispatcher.await();
sendAppUpdateSavedEvent(application);
assertFailed(application, ".*Failing the application.*");
+ assertAppFinalStateSaved(application);
// FAILED => FAILED event RMAppEventType.KILL
event = new RMAppEvent(application.getApplicationId(), RMAppEventType.KILL);
application.handle(event);
rmDispatcher.await();
assertFailed(application, ".*Failing the application.*");
+ assertAppFinalStateSaved(application);
verify(writer).applicationFinished(any(RMApp.class));
}
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java Fri Jan 31 03:24:24 2014
@@ -73,7 +73,7 @@ public class TestQueueMetrics {
checkApps(queueSource, 1, 1, 0, 0, 0, 0, true);
metrics.setAvailableResourcesToQueue(Resources.createResource(100*GB, 100));
- metrics.incrPendingResources(user, 5, Resources.createResource(15*GB, 15));
+ metrics.incrPendingResources(user, 5, Resources.createResource(3*GB, 3));
// Available resources is set externally, as it depends on dynamic
// configurable cluster/queue resources
checkResources(queueSource, 0, 0, 0, 0, 0, 100*GB, 100, 15*GB, 15, 5, 0, 0, 0);
@@ -81,7 +81,7 @@ public class TestQueueMetrics {
metrics.runAppAttempt(app.getApplicationId(), user);
checkApps(queueSource, 1, 0, 1, 0, 0, 0, true);
- metrics.allocateResources(user, 3, Resources.createResource(2*GB, 2));
+ metrics.allocateResources(user, 3, Resources.createResource(2*GB, 2), true);
checkResources(queueSource, 6*GB, 6, 3, 3, 0, 100*GB, 100, 9*GB, 9, 2, 0, 0, 0);
metrics.releaseResources(user, 1, Resources.createResource(2*GB, 2));
@@ -171,7 +171,7 @@ public class TestQueueMetrics {
metrics.setAvailableResourcesToQueue(Resources.createResource(100*GB, 100));
metrics.setAvailableResourcesToUser(user, Resources.createResource(10*GB, 10));
- metrics.incrPendingResources(user, 5, Resources.createResource(15*GB, 15));
+ metrics.incrPendingResources(user, 5, Resources.createResource(3*GB, 3));
// Available resources is set externally, as it depends on dynamic
// configurable cluster/queue resources
checkResources(queueSource, 0, 0, 0, 0, 0, 100*GB, 100, 15*GB, 15, 5, 0, 0, 0);
@@ -181,7 +181,7 @@ public class TestQueueMetrics {
checkApps(queueSource, 1, 0, 1, 0, 0, 0, true);
checkApps(userSource, 1, 0, 1, 0, 0, 0, true);
- metrics.allocateResources(user, 3, Resources.createResource(2*GB, 2));
+ metrics.allocateResources(user, 3, Resources.createResource(2*GB, 2), true);
checkResources(queueSource, 6*GB, 6, 3, 3, 0, 100*GB, 100, 9*GB, 9, 2, 0, 0, 0);
checkResources(userSource, 6*GB, 6, 3, 3, 0, 10*GB, 10, 9*GB, 9, 2, 0, 0, 0);
@@ -232,7 +232,7 @@ public class TestQueueMetrics {
metrics.setAvailableResourcesToQueue(Resources.createResource(100*GB, 100));
parentMetrics.setAvailableResourcesToUser(user, Resources.createResource(10*GB, 10));
metrics.setAvailableResourcesToUser(user, Resources.createResource(10*GB, 10));
- metrics.incrPendingResources(user, 5, Resources.createResource(15*GB, 15));
+ metrics.incrPendingResources(user, 5, Resources.createResource(3*GB, 3));
checkResources(queueSource, 0, 0, 0, 0, 0, 100*GB, 100, 15*GB, 15, 5, 0, 0, 0);
checkResources(parentQueueSource, 0, 0, 0, 0, 0, 100*GB, 100, 15*GB, 15, 5, 0, 0, 0);
checkResources(userSource, 0, 0, 0, 0, 0, 10*GB, 10, 15*GB, 15, 5, 0, 0, 0);
@@ -242,7 +242,7 @@ public class TestQueueMetrics {
checkApps(queueSource, 1, 0, 1, 0, 0, 0, true);
checkApps(userSource, 1, 0, 1, 0, 0, 0, true);
- metrics.allocateResources(user, 3, Resources.createResource(2*GB, 2));
+ metrics.allocateResources(user, 3, Resources.createResource(2*GB, 2), true);
metrics.reserveResource(user, Resources.createResource(3*GB, 3));
// Available resources is set externally, as it depends on dynamic
// configurable cluster/queue resources
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java Fri Jan 31 03:24:24 2014
@@ -21,6 +21,10 @@ import static org.junit.Assert.assertEqu
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
@@ -152,4 +156,41 @@ public class TestMaxRunningAppsEnforcer
assertEquals(0, leaf2.getNonRunnableAppSchedulables().size());
}
+ @Test
+ public void testMultipleAppsWaitingOnCousinQueue() {
+ FSLeafQueue leaf1 = queueManager.getLeafQueue("root.queue1.subqueue1.leaf1", true);
+ FSLeafQueue leaf2 = queueManager.getLeafQueue("root.queue1.subqueue2.leaf2", true);
+ queueMaxApps.put("root.queue1", 2);
+ FSSchedulerApp app1 = addApp(leaf1, "user");
+ addApp(leaf2, "user");
+ addApp(leaf2, "user");
+ addApp(leaf2, "user");
+ assertEquals(1, leaf1.getRunnableAppSchedulables().size());
+ assertEquals(1, leaf2.getRunnableAppSchedulables().size());
+ assertEquals(2, leaf2.getNonRunnableAppSchedulables().size());
+ removeApp(app1);
+ assertEquals(0, leaf1.getRunnableAppSchedulables().size());
+ assertEquals(2, leaf2.getRunnableAppSchedulables().size());
+ assertEquals(1, leaf2.getNonRunnableAppSchedulables().size());
+ }
+
+ @Test
+ public void testMultiListStartTimeIteratorEmptyAppLists() {
+ List<List<AppSchedulable>> lists = new ArrayList<List<AppSchedulable>>();
+ lists.add(Arrays.asList(mockAppSched(1)));
+ lists.add(Arrays.asList(mockAppSched(2)));
+ Iterator<FSSchedulerApp> iter =
+ new MaxRunningAppsEnforcer.MultiListStartTimeIterator(lists);
+ assertEquals(1, iter.next().getAppSchedulable().getStartTime());
+ assertEquals(2, iter.next().getAppSchedulable().getStartTime());
+ }
+
+ private AppSchedulable mockAppSched(long startTime) {
+ AppSchedulable appSched = mock(AppSchedulable.class);
+ when(appSched.getStartTime()).thenReturn(startTime);
+ FSSchedulerApp schedApp = mock(FSSchedulerApp.class);
+ when(schedApp.getAppSchedulable()).thenReturn(appSched);
+ when(appSched.getApp()).thenReturn(schedApp);
+ return appSched;
+ }
}
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java Fri Jan 31 03:24:24 2014
@@ -24,7 +24,7 @@ import java.net.URI;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.http.HttpServer;
+import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.util.StringUtils;
@@ -41,7 +41,7 @@ public class WebAppProxy extends Abstrac
public static final String PROXY_HOST_ATTRIBUTE = "proxyHost";
private static final Log LOG = LogFactory.getLog(WebAppProxy.class);
- private HttpServer proxyServer = null;
+ private HttpServer2 proxyServer = null;
private String bindAddress = null;
private int port = 0;
private AccessControlList acl = null;
@@ -90,7 +90,7 @@ public class WebAppProxy extends Abstrac
@Override
protected void serviceStart() throws Exception {
try {
- proxyServer = new HttpServer.Builder().setName("proxy")
+ proxyServer = new HttpServer2.Builder().setName("proxy")
.addEndpoint(URI.create("http://" + bindAddress + ":" + port))
.setFindPort(port == 0)
.setConf(getConfig()).setACL(acl).build();
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java?rev=1563042&r1=1563041&r2=1563042&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java Fri Jan 31 03:24:24 2014
@@ -43,7 +43,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
-import org.apache.hadoop.http.HttpServer;
+import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.service.CompositeService;
import org.apache.hadoop.util.StringUtils;
@@ -274,7 +274,7 @@ public class TestWebAppProxyServlet {
private class WebAppProxyForTest extends WebAppProxy {
- HttpServer proxyServer;
+ HttpServer2 proxyServer;
AppReportFetcherForTest appReportFetcher;
@Override
@@ -286,7 +286,7 @@ public class TestWebAppProxyServlet {
AccessControlList acl = new AccessControlList(
conf.get(YarnConfiguration.YARN_ADMIN_ACL,
YarnConfiguration.DEFAULT_YARN_ADMIN_ACL));
- proxyServer = new HttpServer.Builder()
+ proxyServer = new HttpServer2.Builder()
.setName("proxy")
.addEndpoint(URI.create("http://" + bindAddress + ":0"))
.setFindPort(true)