You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by cn...@apache.org on 2014/01/30 02:55:22 UTC
svn commit: r1562670 - in
/hadoop/common/branches/HDFS-4685/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/
hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/
hadoop-yarn/...
Author: cnauroth
Date: Thu Jan 30 01:55:14 2014
New Revision: 1562670
URL: http://svn.apache.org/r1562670
Log:
Merge trunk to HDFS-4685.
Modified:
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/RMDelegationTokenIdentifier.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/MaxRunningAppsEnforcer.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/CHANGES.txt?rev=1562670&r1=1562669&r2=1562670&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/CHANGES.txt Thu Jan 30 01:55:14 2014
@@ -334,6 +334,9 @@ Release 2.4.0 - UNRELEASED
YARN-1573. ZK store should use a private password for root-node-acls.
(kasha).
+ YARN-1630. Introduce timeout for async polling operations in YarnClientImpl
+ (Aditya Acharya via Sandy Ryza)
+
OPTIMIZATIONS
BUG FIXES
@@ -463,6 +466,12 @@ Release 2.4.0 - UNRELEASED
YARN-1575. Public localizer crashes with "Localized unkown resource"
(jlowe)
+ YARN-1642. RMDTRenewer#getRMClient should use ClientRMProxy (kasha)
+
+ YARN-1629. IndexOutOfBoundsException in MaxRunningAppsEnforcer (Sandy Ryza)
+
+ YARN-1618. Fix invalid RMApp transition from NEW to FINAL_SAVING (kasha)
+
Release 2.3.0 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -555,6 +564,9 @@ Release 2.3.0 - UNRELEASED
YARN-1138. yarn.application.classpath is set to point to $HADOOP_CONF_DIR
etc., which does not work on Windows. (Chuan Liu via cnauroth)
+ YARN-1600. RM does not startup when security is enabled without spnego
+ configured (Haohui Mai via jlowe)
+
Release 2.2.0 - 2013-10-13
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1562670&r1=1562669&r2=1562670&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java Thu Jan 30 01:55:14 2014
@@ -1020,6 +1020,17 @@ public class YarnConfiguration extends C
YARN_PREFIX + "client.application-client-protocol.poll-interval-ms";
public static final long DEFAULT_YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_INTERVAL_MS =
200;
+
+ /**
+ * The duration that the yarn client library waits, cumulatively across polls,
+ * for an expected state change to occur. Defaults to -1, which indicates no
+ * limit.
+ */
+ public static final String YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_TIMEOUT_MS =
+ YARN_PREFIX + "client.application-client-protocol.poll-timeout-ms";
+ public static final long DEFAULT_YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_TIMEOUT_MS =
+ -1;
+
/**
* Max number of threads in NMClientAsync to process container management
* events
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java?rev=1562670&r1=1562669&r2=1562670&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java Thu Jan 30 01:55:14 2014
@@ -86,6 +86,7 @@ public class YarnClientImpl extends Yarn
protected ApplicationClientProtocol rmClient;
protected long submitPollIntervalMillis;
private long asyncApiPollIntervalMillis;
+ private long asyncApiPollTimeoutMillis;
protected AHSClient historyClient;
private boolean historyServiceEnabled;
@@ -101,6 +102,9 @@ public class YarnClientImpl extends Yarn
asyncApiPollIntervalMillis =
conf.getLong(YarnConfiguration.YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_INTERVAL_MS,
YarnConfiguration.DEFAULT_YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_INTERVAL_MS);
+ asyncApiPollTimeoutMillis =
+ conf.getLong(YarnConfiguration.YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_TIMEOUT_MS,
+ YarnConfiguration.DEFAULT_YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_TIMEOUT_MS);
submitPollIntervalMillis = asyncApiPollIntervalMillis;
if (conf.get(YarnConfiguration.YARN_CLIENT_APP_SUBMISSION_POLL_INTERVAL_MS)
!= null) {
@@ -174,13 +178,24 @@ public class YarnClientImpl extends Yarn
rmClient.submitApplication(request);
int pollCount = 0;
+ long startTime = System.currentTimeMillis();
+
while (true) {
YarnApplicationState state =
getApplicationReport(applicationId).getYarnApplicationState();
if (!state.equals(YarnApplicationState.NEW) &&
!state.equals(YarnApplicationState.NEW_SAVING)) {
+ LOG.info("Submitted application " + applicationId);
break;
}
+
+ long elapsedMillis = System.currentTimeMillis() - startTime;
+ if (enforceAsyncAPITimeout() &&
+ elapsedMillis >= asyncApiPollTimeoutMillis) {
+ throw new YarnException("Timed out while waiting for application " +
+ applicationId + " to be submitted successfully");
+ }
+
// Notify the client through the log every 10 poll, in case the client
// is blocked here too long.
if (++pollCount % 10 == 0) {
@@ -191,10 +206,11 @@ public class YarnClientImpl extends Yarn
try {
Thread.sleep(submitPollIntervalMillis);
} catch (InterruptedException ie) {
+ LOG.error("Interrupted while waiting for application " + applicationId
+ + " to be successfully submitted.");
}
}
- LOG.info("Submitted application " + applicationId);
return applicationId;
}
@@ -207,15 +223,25 @@ public class YarnClientImpl extends Yarn
try {
int pollCount = 0;
+ long startTime = System.currentTimeMillis();
+
while (true) {
KillApplicationResponse response =
rmClient.forceKillApplication(request);
if (response.getIsKillCompleted()) {
+ LOG.info("Killed application " + applicationId);
break;
}
+
+ long elapsedMillis = System.currentTimeMillis() - startTime;
+ if (enforceAsyncAPITimeout() &&
+ elapsedMillis >= this.asyncApiPollTimeoutMillis) {
+ throw new YarnException("Timed out while waiting for application " +
+ applicationId + " to be killed.");
+ }
+
if (++pollCount % 10 == 0) {
- LOG.info("Watiting for application " + applicationId
- + " to be killed.");
+ LOG.info("Waiting for application " + applicationId + " to be killed.");
}
Thread.sleep(asyncApiPollIntervalMillis);
}
@@ -223,7 +249,11 @@ public class YarnClientImpl extends Yarn
LOG.error("Interrupted while waiting for application " + applicationId
+ " to be killed.");
}
- LOG.info("Killed application " + applicationId);
+ }
+
+ @VisibleForTesting
+ boolean enforceAsyncAPITimeout() {
+ return asyncApiPollTimeoutMillis >= 0;
}
@Override
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java?rev=1562670&r1=1562669&r2=1562670&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java Thu Jan 30 01:55:14 2014
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.client.api.impl;
+import static org.junit.Assert.assertFalse;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
@@ -35,6 +36,7 @@ import java.util.Set;
import junit.framework.Assert;
+import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
@@ -474,4 +476,30 @@ public class TestYarnClient {
}
}
+ @Test
+ public void testAsyncAPIPollTimeout() {
+ testAsyncAPIPollTimeoutHelper(null, false);
+ testAsyncAPIPollTimeoutHelper(0L, true);
+ testAsyncAPIPollTimeoutHelper(1L, true);
+ }
+
+ private void testAsyncAPIPollTimeoutHelper(Long valueForTimeout,
+ boolean expectedTimeoutEnforcement) {
+ YarnClientImpl client = new YarnClientImpl();
+ try {
+ Configuration conf = new Configuration();
+ if (valueForTimeout != null) {
+ conf.setLong(
+ YarnConfiguration.YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_TIMEOUT_MS,
+ valueForTimeout);
+ }
+
+ client.init(conf);
+
+ Assert.assertEquals(
+ expectedTimeoutEnforcement, client.enforceAsyncAPITimeout());
+ } finally {
+ IOUtils.closeQuietly(client);
+ }
+ }
}
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/RMDelegationTokenIdentifier.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/RMDelegationTokenIdentifier.java?rev=1562670&r1=1562669&r2=1562670&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/RMDelegationTokenIdentifier.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/RMDelegationTokenIdentifier.java Thu Jan 30 01:55:14 2014
@@ -37,8 +37,8 @@ import org.apache.hadoop.security.token.
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest;
import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenRequest;
+import org.apache.hadoop.yarn.client.ClientRMProxy;
import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.util.Records;
/**
@@ -138,7 +138,7 @@ public class RMDelegationTokenIdentifier
}
private static ApplicationClientProtocol getRmClient(Token<?> token,
- Configuration conf) {
+ Configuration conf) throws IOException {
InetSocketAddress addr = SecurityUtil.getTokenServiceAddr(token);
if (localSecretManager != null) {
// return null if it's our token
@@ -151,8 +151,7 @@ public class RMDelegationTokenIdentifier
return null;
}
}
- final YarnRPC rpc = YarnRPC.create(conf);
- return (ApplicationClientProtocol)rpc.getProxy(ApplicationClientProtocol.class, addr, conf);
+ return ClientRMProxy.createRMProxy(conf, ApplicationClientProtocol.class);
}
// get renewer so we can always renew our own tokens
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java?rev=1562670&r1=1562669&r2=1562670&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java Thu Jan 30 01:55:14 2014
@@ -28,7 +28,7 @@ import java.util.Map;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.http.HttpServer;
+import org.apache.hadoop.http.HttpServer2;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -59,7 +59,7 @@ public abstract class WebApp extends Ser
private volatile String redirectPath;
private volatile String wsName;
private volatile Configuration conf;
- private volatile HttpServer httpServer;
+ private volatile HttpServer2 httpServer;
private volatile GuiceFilter guiceFilter;
private final Router router = new Router();
@@ -72,11 +72,11 @@ public abstract class WebApp extends Ser
static final Splitter pathSplitter =
Splitter.on('/').trimResults().omitEmptyStrings();
- void setHttpServer(HttpServer server) {
+ void setHttpServer(HttpServer2 server) {
httpServer = checkNotNull(server, "http server");
}
- @Provides public HttpServer httpServer() { return httpServer; }
+ @Provides public HttpServer2 httpServer() { return httpServer; }
/**
* Get the address the http server is bound to
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java?rev=1562670&r1=1562669&r2=1562670&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java Thu Jan 30 01:55:14 2014
@@ -35,7 +35,7 @@ import javax.servlet.http.HttpServlet;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.http.HttpServer;
+import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.security.AdminACLsManager;
import org.slf4j.Logger;
@@ -216,20 +216,22 @@ public class WebApps {
System.exit(1);
}
}
- HttpServer.Builder builder = new HttpServer.Builder().setName(name)
+ HttpServer2.Builder builder = new HttpServer2.Builder().setName(name)
.addEndpoint(URI.create("http://" + bindAddress + ":" + port))
.setConf(conf).setFindPort(findPort)
.setACL(new AdminACLsManager(conf).getAdminAcl())
.setPathSpec(pathList.toArray(new String[0]));
boolean hasSpnegoConf = spnegoPrincipalKey != null
- && spnegoKeytabKey != null;
+ && conf.get(spnegoPrincipalKey) != null && spnegoKeytabKey != null
+ && conf.get(spnegoKeytabKey) != null;
+
if (hasSpnegoConf) {
builder.setUsernameConfKey(spnegoPrincipalKey)
.setKeytabConfKey(spnegoKeytabKey)
.setSecurityEnabled(UserGroupInformation.isSecurityEnabled());
}
- HttpServer server = builder.build();
+ HttpServer2 server = builder.build();
for(ServletStruct struct: servlets) {
server.addServlet(struct.name, struct.spec, struct.clazz);
@@ -237,7 +239,7 @@ public class WebApps {
for(Map.Entry<String, Object> entry : attributes.entrySet()) {
server.setAttribute(entry.getKey(), entry.getValue());
}
- HttpServer.defineFilter(server.getWebAppContext(), "guice",
+ HttpServer2.defineFilter(server.getWebAppContext(), "guice",
GuiceFilter.class.getName(), null, new String[] { "/*" });
webapp.setConf(conf);
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java?rev=1562670&r1=1562669&r2=1562670&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java Thu Jan 30 01:55:14 2014
@@ -24,8 +24,10 @@ public enum RMAppEventType {
RECOVER,
KILL,
- // Source: Scheduler
+ // Source: Scheduler and RMAppManager
APP_REJECTED,
+
+ // Source: Scheduler
APP_ACCEPTED,
// Source: RMAppAttempt
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java?rev=1562670&r1=1562669&r2=1562670&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java Thu Jan 30 01:55:14 2014
@@ -144,13 +144,10 @@ public class RMAppImpl implements RMApp,
RMAppState.ACCEPTED, RMAppState.FINISHED, RMAppState.FAILED,
RMAppState.KILLED, RMAppState.FINAL_SAVING),
RMAppEventType.RECOVER, new RMAppRecoveredTransition())
- .addTransition(RMAppState.NEW, RMAppState.FINAL_SAVING, RMAppEventType.KILL,
- new FinalSavingTransition(
- new AppKilledTransition(), RMAppState.KILLED))
- .addTransition(RMAppState.NEW, RMAppState.FINAL_SAVING,
- RMAppEventType.APP_REJECTED,
- new FinalSavingTransition(
- new AppRejectedTransition(), RMAppState.FAILED))
+ .addTransition(RMAppState.NEW, RMAppState.KILLED, RMAppEventType.KILL,
+ new AppKilledTransition())
+ .addTransition(RMAppState.NEW, RMAppState.FAILED,
+ RMAppEventType.APP_REJECTED, new AppRejectedTransition())
// Transitions from NEW_SAVING state
.addTransition(RMAppState.NEW_SAVING, RMAppState.NEW_SAVING,
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java?rev=1562670&r1=1562669&r2=1562670&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java Thu Jan 30 01:55:14 2014
@@ -91,15 +91,6 @@ public class FSLeafQueue extends FSQueue
}
}
- public void makeAppRunnable(AppSchedulable appSched) {
- if (!nonRunnableAppScheds.remove(appSched)) {
- throw new IllegalStateException("Can't make app runnable that does not " +
- "already exist in queue as non-runnable" + appSched);
- }
-
- runnableAppScheds.add(appSched);
- }
-
public Collection<AppSchedulable> getRunnableAppSchedulables() {
return runnableAppScheds;
}
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/MaxRunningAppsEnforcer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/MaxRunningAppsEnforcer.java?rev=1562670&r1=1562669&r2=1562670&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/MaxRunningAppsEnforcer.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/MaxRunningAppsEnforcer.java Thu Jan 30 01:55:14 2014
@@ -24,6 +24,9 @@ import java.util.List;
import java.util.Map;
import java.util.PriorityQueue;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.ListMultimap;
@@ -33,6 +36,8 @@ import com.google.common.collect.ListMul
* constraints
*/
public class MaxRunningAppsEnforcer {
+ private static final Log LOG = LogFactory.getLog(FairScheduler.class);
+
private final FairScheduler scheduler;
// Tracks the number of running applications by user.
@@ -163,7 +168,7 @@ public class MaxRunningAppsEnforcer {
Iterator<FSSchedulerApp> iter = new MultiListStartTimeIterator(
appsNowMaybeRunnable);
FSSchedulerApp prev = null;
- int numNowRunnable = 0;
+ List<AppSchedulable> noLongerPendingApps = new ArrayList<AppSchedulable>();
while (iter.hasNext()) {
FSSchedulerApp next = iter.next();
if (next == prev) {
@@ -173,21 +178,34 @@ public class MaxRunningAppsEnforcer {
if (canAppBeRunnable(next.getQueue(), next.getUser())) {
trackRunnableApp(next);
AppSchedulable appSched = next.getAppSchedulable();
- next.getQueue().makeAppRunnable(appSched);
- if (!usersNonRunnableApps.remove(next.getUser(), appSched)) {
- throw new IllegalStateException("Waiting app " + next
- + " expected to be in usersNonRunnableApps");
- }
+ next.getQueue().getRunnableAppSchedulables().add(appSched);
+ noLongerPendingApps.add(appSched);
// No more than one app per list will be able to be made runnable, so
// we can stop looking after we've found that many
- if (numNowRunnable >= appsNowMaybeRunnable.size()) {
+ if (noLongerPendingApps.size() >= appsNowMaybeRunnable.size()) {
break;
}
}
prev = next;
}
+
+ // We remove the apps from their pending lists afterwards so that we don't
+ // pull them out from under the iterator. If they are not in these lists
+ // in the first place, there is a bug.
+ for (AppSchedulable appSched : noLongerPendingApps) {
+ if (!appSched.getApp().getQueue().getNonRunnableAppSchedulables()
+ .remove(appSched)) {
+ LOG.error("Can't make app runnable that does not already exist in queue"
+ + " as non-runnable: " + appSched + ". This should never happen.");
+ }
+
+ if (!usersNonRunnableApps.remove(appSched.getApp().getUser(), appSched)) {
+ LOG.error("Waiting app " + appSched + " expected to be in "
+ + "usersNonRunnableApps, but was not. This should never happen.");
+ }
+ }
}
/**
@@ -225,7 +243,7 @@ public class MaxRunningAppsEnforcer {
* This allows us to pick which list to advance in O(log(num lists)) instead
* of O(num lists) time.
*/
- private static class MultiListStartTimeIterator implements
+ static class MultiListStartTimeIterator implements
Iterator<FSSchedulerApp> {
private List<AppSchedulable>[] appLists;
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java?rev=1562670&r1=1562669&r2=1562670&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java Thu Jan 30 01:55:14 2014
@@ -286,7 +286,7 @@ public class TestRMAppTransitions {
private void assertTimesAtFinish(RMApp application) {
assertStartTimeSet(application);
Assert.assertTrue("application finish time is not greater then 0",
- (application.getFinishTime() > 0));
+ (application.getFinishTime() > 0));
Assert.assertTrue("application finish time is not >= then start time",
(application.getFinishTime() >= application.getStartTime()));
}
@@ -295,6 +295,10 @@ public class TestRMAppTransitions {
verify(store, times(1)).updateApplicationState(any(ApplicationState.class));
}
+ private void assertAppFinalStateNotSaved(RMApp application){
+ verify(store, times(0)).updateApplicationState(any(ApplicationState.class));
+ }
+
private void assertKilled(RMApp application) {
assertTimesAtFinish(application);
assertAppState(RMAppState.KILLED, application);
@@ -321,7 +325,6 @@ public class TestRMAppTransitions {
StringBuilder diag = application.getDiagnostics();
Assert.assertTrue("application diagnostics is not correct",
diag.toString().matches(regex));
- assertAppFinalStateSaved(application);
}
private void sendAppUpdateSavedEvent(RMApp application) {
@@ -477,6 +480,7 @@ public class TestRMAppTransitions {
sendAppUpdateSavedEvent(application);
assertFailed(application,
".*Unmanaged application.*Failing the application.*");
+ assertAppFinalStateSaved(application);
}
@Test
@@ -506,6 +510,7 @@ public class TestRMAppTransitions {
rmDispatcher.await();
sendAppUpdateSavedEvent(application);
assertKilled(application);
+ assertAppFinalStateNotSaved(application);
verify(writer).applicationFinished(any(RMApp.class));
}
@@ -522,6 +527,7 @@ public class TestRMAppTransitions {
rmDispatcher.await();
sendAppUpdateSavedEvent(application);
assertFailed(application, rejectedText);
+ assertAppFinalStateNotSaved(application);
verify(writer).applicationFinished(any(RMApp.class));
}
@@ -553,6 +559,7 @@ public class TestRMAppTransitions {
rmDispatcher.await();
sendAppUpdateSavedEvent(application);
assertFailed(application, rejectedText);
+ assertAppFinalStateSaved(application);
verify(writer).applicationFinished(any(RMApp.class));
}
@@ -569,6 +576,7 @@ public class TestRMAppTransitions {
rmDispatcher.await();
sendAppUpdateSavedEvent(application);
assertFailed(application, rejectedText);
+ assertAppFinalStateSaved(application);
verify(writer).applicationFinished(any(RMApp.class));
}
@@ -618,6 +626,7 @@ public class TestRMAppTransitions {
rmDispatcher.await();
sendAppUpdateSavedEvent(application);
assertFailed(application, ".*" + message + ".*Failing the application.*");
+ assertAppFinalStateSaved(application);
verify(writer).applicationFinished(any(RMApp.class));
}
@@ -703,12 +712,14 @@ public class TestRMAppTransitions {
rmDispatcher.await();
sendAppUpdateSavedEvent(application);
assertFailed(application, ".*Failing the application.*");
+ assertAppFinalStateSaved(application);
// FAILED => FAILED event RMAppEventType.KILL
event = new RMAppEvent(application.getApplicationId(), RMAppEventType.KILL);
application.handle(event);
rmDispatcher.await();
assertFailed(application, ".*Failing the application.*");
+ assertAppFinalStateSaved(application);
verify(writer).applicationFinished(any(RMApp.class));
}
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java?rev=1562670&r1=1562669&r2=1562670&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java Thu Jan 30 01:55:14 2014
@@ -21,6 +21,10 @@ import static org.junit.Assert.assertEqu
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
@@ -152,4 +156,41 @@ public class TestMaxRunningAppsEnforcer
assertEquals(0, leaf2.getNonRunnableAppSchedulables().size());
}
+ @Test
+ public void testMultipleAppsWaitingOnCousinQueue() {
+ FSLeafQueue leaf1 = queueManager.getLeafQueue("root.queue1.subqueue1.leaf1", true);
+ FSLeafQueue leaf2 = queueManager.getLeafQueue("root.queue1.subqueue2.leaf2", true);
+ queueMaxApps.put("root.queue1", 2);
+ FSSchedulerApp app1 = addApp(leaf1, "user");
+ addApp(leaf2, "user");
+ addApp(leaf2, "user");
+ addApp(leaf2, "user");
+ assertEquals(1, leaf1.getRunnableAppSchedulables().size());
+ assertEquals(1, leaf2.getRunnableAppSchedulables().size());
+ assertEquals(2, leaf2.getNonRunnableAppSchedulables().size());
+ removeApp(app1);
+ assertEquals(0, leaf1.getRunnableAppSchedulables().size());
+ assertEquals(2, leaf2.getRunnableAppSchedulables().size());
+ assertEquals(1, leaf2.getNonRunnableAppSchedulables().size());
+ }
+
+ @Test
+ public void testMultiListStartTimeIteratorEmptyAppLists() {
+ List<List<AppSchedulable>> lists = new ArrayList<List<AppSchedulable>>();
+ lists.add(Arrays.asList(mockAppSched(1)));
+ lists.add(Arrays.asList(mockAppSched(2)));
+ Iterator<FSSchedulerApp> iter =
+ new MaxRunningAppsEnforcer.MultiListStartTimeIterator(lists);
+ assertEquals(1, iter.next().getAppSchedulable().getStartTime());
+ assertEquals(2, iter.next().getAppSchedulable().getStartTime());
+ }
+
+ private AppSchedulable mockAppSched(long startTime) {
+ AppSchedulable appSched = mock(AppSchedulable.class);
+ when(appSched.getStartTime()).thenReturn(startTime);
+ FSSchedulerApp schedApp = mock(FSSchedulerApp.class);
+ when(schedApp.getAppSchedulable()).thenReturn(appSched);
+ when(appSched.getApp()).thenReturn(schedApp);
+ return appSched;
+ }
}
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java?rev=1562670&r1=1562669&r2=1562670&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxy.java Thu Jan 30 01:55:14 2014
@@ -24,7 +24,7 @@ import java.net.URI;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.http.HttpServer;
+import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.util.StringUtils;
@@ -41,7 +41,7 @@ public class WebAppProxy extends Abstrac
public static final String PROXY_HOST_ATTRIBUTE = "proxyHost";
private static final Log LOG = LogFactory.getLog(WebAppProxy.class);
- private HttpServer proxyServer = null;
+ private HttpServer2 proxyServer = null;
private String bindAddress = null;
private int port = 0;
private AccessControlList acl = null;
@@ -90,7 +90,7 @@ public class WebAppProxy extends Abstrac
@Override
protected void serviceStart() throws Exception {
try {
- proxyServer = new HttpServer.Builder().setName("proxy")
+ proxyServer = new HttpServer2.Builder().setName("proxy")
.addEndpoint(URI.create("http://" + bindAddress + ":" + port))
.setFindPort(port == 0)
.setConf(getConfig()).setACL(acl).build();
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java?rev=1562670&r1=1562669&r2=1562670&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java Thu Jan 30 01:55:14 2014
@@ -43,7 +43,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
-import org.apache.hadoop.http.HttpServer;
+import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.service.CompositeService;
import org.apache.hadoop.util.StringUtils;
@@ -274,7 +274,7 @@ public class TestWebAppProxyServlet {
private class WebAppProxyForTest extends WebAppProxy {
- HttpServer proxyServer;
+ HttpServer2 proxyServer;
AppReportFetcherForTest appReportFetcher;
@Override
@@ -286,7 +286,7 @@ public class TestWebAppProxyServlet {
AccessControlList acl = new AccessControlList(
conf.get(YarnConfiguration.YARN_ADMIN_ACL,
YarnConfiguration.DEFAULT_YARN_ADMIN_ACL));
- proxyServer = new HttpServer.Builder()
+ proxyServer = new HttpServer2.Builder()
.setName("proxy")
.addEndpoint(URI.create("http://" + bindAddress + ":0"))
.setFindPort(true)