You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by sz...@apache.org on 2014/08/16 23:02:28 UTC
svn commit: r1618417 [1/3] - in
/hadoop/common/branches/HDFS-6584/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/
hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/jav...
Author: szetszwo
Date: Sat Aug 16 21:02:21 2014
New Revision: 1618417
URL: http://svn.apache.org/r1618417
Log:
Merge r1609845 through r1618416 from trunk.
Added:
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
- copied unchanged from r1618416, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilterInitializer.java
- copied unchanged from r1618416, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilterInitializer.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilter.java
- copied unchanged from r1618416, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilter.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilterInitializer.java
- copied unchanged from r1618416, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilterInitializer.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
- copied unchanged from r1618416, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppAttempt.java
- copied unchanged from r1618416, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppAttempt.java
Removed:
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerApp.java
Modified:
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/http/RMAuthenticationFilterInitializer.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FifoAppComparator.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/MaxRunningAppsEnforcer.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/NewAppWeightBooster.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/Schedulable.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/WeightAdjuster.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerInfo.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerLeafQueueInfo.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesDelegationTokenAuthentication.java
hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WritingYarnApplications.apt.vm
Modified: hadoop/common/branches/HDFS-6584/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/CHANGES.txt?rev=1618417&r1=1618416&r2=1618417&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-yarn-project/CHANGES.txt Sat Aug 16 21:02:21 2014
@@ -41,8 +41,20 @@ Release 2.6.0 - UNRELEASED
YARN-1337. Recover containers upon nodemanager restart. (Jason Lowe via
junping_du)
+ YARN-2277. Added cross-origin support for the timeline server web services.
+ (Jonathan Eagles via zjshen)
+
+ YARN-2378. Added support for moving applications across queues in
+ CapacityScheduler. (Subramaniam Venkatraman Krishnan via jianhe)
+
IMPROVEMENTS
+ YARN-2197. Add a link to YARN CHANGES.txt in the left side of doc
+ (Akira AJISAKA via aw)
+
+ YARN-1918. Typo in description and error message for
+ 'yarn.resourcemanager.cluster-id' (Anandha L Ranganathan via aw)
+
YARN-2242. Improve exception information on AM launch crashes. (Li Lu
via junping_du)
@@ -119,6 +131,18 @@ Release 2.6.0 - UNRELEASED
YARN-2373. Changed WebAppUtils to use Configuration#getPassword for
accessing SSL passwords. (Larry McCay via jianhe)
+ YARN-2317. Updated the document about how to write YARN applications. (Li Lu via
+ zjshen)
+
+ YARN-2399. FairScheduler: Merge AppSchedulable and FSSchedulerApp into
+ FSAppAttempt. (kasha)
+
+ YARN-1370. Fair scheduler to re-populate container allocation state.
+ (Anubhav Dhoot via kasha)
+
+ YARN-2389. Added functionality for schedulers to kill all applications in a
+ queue. (Subramaniam Venkatraman Krishnan via jianhe)
+
OPTIMIZATIONS
BUG FIXES
@@ -178,6 +202,12 @@ Release 2.6.0 - UNRELEASED
YARN-2361. RMAppAttempt state machine entries for KILLED state has duplicate
event entries. (Zhihai Xu via kasha)
+ YARN-2070. Made DistributedShell publish the short user name to the timeline
+ server. (Robert Kanter via zjshen)
+
+ YARN-2397. Avoided loading two authentication filters for RM and TS web
+ interfaces. (Varun Vasudev via zjshen)
+
Release 2.5.0 - UNRELEASED
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1618417&r1=1618416&r2=1618417&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java Sat Aug 16 21:02:21 2014
@@ -1370,7 +1370,7 @@ public class YarnConfiguration extends C
public static String getClusterId(Configuration conf) {
String clusterId = conf.get(YarnConfiguration.RM_CLUSTER_ID);
if (clusterId == null) {
- throw new HadoopIllegalArgumentException("Configuration doesn't specify" +
+ throw new HadoopIllegalArgumentException("Configuration doesn't specify " +
YarnConfiguration.RM_CLUSTER_ID);
}
return clusterId;
Modified: hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java?rev=1618417&r1=1618416&r2=1618417&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java Sat Aug 16 21:02:21 2014
@@ -1054,8 +1054,8 @@ public class ApplicationMaster {
TimelineEntity entity = new TimelineEntity();
entity.setEntityId(container.getId().toString());
entity.setEntityType(DSEntity.DS_CONTAINER.toString());
- entity.addPrimaryFilter("user", UserGroupInformation.getCurrentUser()
- .toString());
+ entity.addPrimaryFilter("user",
+ UserGroupInformation.getCurrentUser().getShortUserName());
TimelineEvent event = new TimelineEvent();
event.setTimestamp(System.currentTimeMillis());
event.setEventType(DSEvent.DS_CONTAINER_START.toString());
@@ -1071,8 +1071,8 @@ public class ApplicationMaster {
TimelineEntity entity = new TimelineEntity();
entity.setEntityId(container.getContainerId().toString());
entity.setEntityType(DSEntity.DS_CONTAINER.toString());
- entity.addPrimaryFilter("user", UserGroupInformation.getCurrentUser()
- .toString());
+ entity.addPrimaryFilter("user",
+ UserGroupInformation.getCurrentUser().getShortUserName());
TimelineEvent event = new TimelineEvent();
event.setTimestamp(System.currentTimeMillis());
event.setEventType(DSEvent.DS_CONTAINER_END.toString());
@@ -1089,8 +1089,8 @@ public class ApplicationMaster {
TimelineEntity entity = new TimelineEntity();
entity.setEntityId(appAttemptId);
entity.setEntityType(DSEntity.DS_APP_ATTEMPT.toString());
- entity.addPrimaryFilter("user", UserGroupInformation.getCurrentUser()
- .toString());
+ entity.addPrimaryFilter("user",
+ UserGroupInformation.getCurrentUser().getShortUserName());
TimelineEvent event = new TimelineEvent();
event.setEventType(appEvent.toString());
event.setTimestamp(System.currentTimeMillis());
Modified: hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml?rev=1618417&r1=1618416&r2=1618417&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml Sat Aug 16 21:02:21 2014
@@ -474,7 +474,7 @@
<property>
<description>Name of the cluster. In a HA setting,
this is used to ensure the RM participates in leader
- election fo this cluster and ensures it does not affect
+ election for this cluster and ensures it does not affect
other clusters</description>
<name>yarn.resourcemanager.cluster-id</name>
<!--value>yarn-cluster</value-->
Modified: hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java?rev=1618417&r1=1618416&r2=1618417&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java Sat Aug 16 21:02:21 2014
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.ap
import java.io.IOException;
import java.net.InetSocketAddress;
+import java.util.ArrayList;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -27,6 +28,7 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.metrics2.source.JvmMetrics;
+import org.apache.hadoop.security.AuthenticationFilterInitializer;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.service.CompositeService;
import org.apache.hadoop.service.Service;
@@ -195,14 +197,31 @@ public class ApplicationHistoryServer ex
// the customized filter will be loaded by the timeline server to do Kerberos
// + DT authentication.
String initializers = conf.get("hadoop.http.filter.initializers");
+
initializers =
- initializers == null || initializers.length() == 0 ? "" : ","
- + initializers;
- if (!initializers.contains(
- TimelineAuthenticationFilterInitializer.class.getName())) {
- conf.set("hadoop.http.filter.initializers",
- TimelineAuthenticationFilterInitializer.class.getName()
- + initializers);
+ initializers == null || initializers.length() == 0 ? "" : initializers;
+
+ if (!initializers.contains(TimelineAuthenticationFilterInitializer.class
+ .getName())) {
+ initializers =
+ TimelineAuthenticationFilterInitializer.class.getName() + ","
+ + initializers;
+ }
+
+ String[] parts = initializers.split(",");
+ ArrayList<String> target = new ArrayList<String>();
+ for (String filterInitializer : parts) {
+ filterInitializer = filterInitializer.trim();
+ if (filterInitializer.equals(AuthenticationFilterInitializer.class
+ .getName())) {
+ continue;
+ }
+ target.add(filterInitializer);
+ }
+ String actualInitializers =
+ org.apache.commons.lang.StringUtils.join(target, ",");
+ if (!actualInitializers.equals(initializers)) {
+ conf.set("hadoop.http.filter.initializers", actualInitializers);
}
String bindAddress = WebAppUtils.getWebAppBindURL(conf,
YarnConfiguration.TIMELINE_SERVICE_BIND_HOST,
Modified: hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java?rev=1618417&r1=1618416&r2=1618417&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java Sat Aug 16 21:02:21 2014
@@ -23,11 +23,14 @@ import static org.junit.Assert.assertNot
import static org.junit.Assert.fail;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.AuthenticationFilterInitializer;
import org.apache.hadoop.service.Service.STATE;
import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp;
+import org.apache.hadoop.yarn.server.timeline.security.TimelineAuthenticationFilterInitializer;
import org.junit.After;
+import org.junit.Assert;
import org.junit.Test;
public class TestApplicationHistoryServer {
@@ -69,6 +72,31 @@ public class TestApplicationHistoryServe
}
}
+ @Test(timeout = 50000)
+ public void testFilteOverrides() throws Exception {
+
+ String[] filterInitializers =
+ {
+ AuthenticationFilterInitializer.class.getName(),
+ TimelineAuthenticationFilterInitializer.class.getName(),
+ AuthenticationFilterInitializer.class.getName() + ","
+ + TimelineAuthenticationFilterInitializer.class.getName(),
+ AuthenticationFilterInitializer.class.getName() + ", "
+ + TimelineAuthenticationFilterInitializer.class.getName() };
+ for (String filterInitializer : filterInitializers) {
+ historyServer = new ApplicationHistoryServer();
+ Configuration config = new YarnConfiguration();
+ config.set("hadoop.http.filter.initializers", filterInitializer);
+ historyServer.init(config);
+ historyServer.start();
+ Configuration tmp = historyServer.getConfig();
+ assertEquals(TimelineAuthenticationFilterInitializer.class.getName(),
+ tmp.get("hadoop.http.filter.initializers"));
+ historyServer.stop();
+ AHSWebApp.resetInstance();
+ }
+ }
+
@After
public void stop() {
if (historyServer != null) {
Modified: hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/http/RMAuthenticationFilterInitializer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/http/RMAuthenticationFilterInitializer.java?rev=1618417&r1=1618416&r2=1618417&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/http/RMAuthenticationFilterInitializer.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/http/RMAuthenticationFilterInitializer.java Sat Aug 16 21:02:21 2014
@@ -114,7 +114,7 @@ public class RMAuthenticationFilterIniti
public void initFilter(FilterContainer container, Configuration conf) {
Map<String, String> filterConfig = createFilterConfig(conf);
- container.addFilter("YARNAuthenticationFilter",
+ container.addFilter("RMAuthenticationFilter",
RMAuthenticationFilter.class.getName(), filterConfig);
}
Modified: hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java?rev=1618417&r1=1618416&r2=1618417&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java Sat Aug 16 21:02:21 2014
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.net.InetSocketAddress;
import java.security.PrivilegedExceptionAction;
+import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
@@ -35,6 +36,7 @@ import org.apache.hadoop.ha.HAServicePro
import org.apache.hadoop.http.lib.StaticUserWebFilter;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.metrics2.source.JvmMetrics;
+import org.apache.hadoop.security.AuthenticationFilterInitializer;
import org.apache.hadoop.security.Groups;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
@@ -799,10 +801,11 @@ public class ResourceManager extends Com
// Use the customized yarn filter instead of the standard kerberos filter to
// allow users to authenticate using delegation tokens
- // 3 conditions need to be satisfied -
+ // 4 conditions need to be satisfied -
// 1. security is enabled
// 2. http auth type is set to kerberos
// 3. "yarn.resourcemanager.webapp.use-yarn-filter" override is set to true
+ // 4. hadoop.http.filter.initializers container AuthenticationFilterInitializer
Configuration conf = getConfig();
boolean useYarnAuthenticationFilter =
@@ -811,41 +814,66 @@ public class ResourceManager extends Com
YarnConfiguration.DEFAULT_RM_WEBAPP_DELEGATION_TOKEN_AUTH_FILTER);
String authPrefix = "hadoop.http.authentication.";
String authTypeKey = authPrefix + "type";
- String initializers = conf.get("hadoop.http.filter.initializers");
- if (UserGroupInformation.isSecurityEnabled()
- && useYarnAuthenticationFilter
- && conf.get(authTypeKey, "").equalsIgnoreCase(
- KerberosAuthenticationHandler.TYPE)) {
- LOG.info("Using RM authentication filter(kerberos/delegation-token)"
- + " for RM webapp authentication");
- RMAuthenticationHandler
- .setSecretManager(getClientRMService().rmDTSecretManager);
- String yarnAuthKey =
- authPrefix + RMAuthenticationFilter.AUTH_HANDLER_PROPERTY;
- conf.setStrings(yarnAuthKey, RMAuthenticationHandler.class.getName());
-
- initializers =
- initializers == null || initializers.isEmpty() ? "" : ","
- + initializers;
- if (!initializers.contains(RMAuthenticationFilterInitializer.class
- .getName())) {
- conf.set("hadoop.http.filter.initializers",
- RMAuthenticationFilterInitializer.class.getName() + initializers);
+ String filterInitializerConfKey = "hadoop.http.filter.initializers";
+ String actualInitializers = "";
+ Class<?>[] initializersClasses =
+ conf.getClasses(filterInitializerConfKey);
+
+ boolean hasHadoopAuthFilterInitializer = false;
+ boolean hasRMAuthFilterInitializer = false;
+ if (initializersClasses != null) {
+ for (Class<?> initializer : initializersClasses) {
+ if (initializer.getName().equals(
+ AuthenticationFilterInitializer.class.getName())) {
+ hasHadoopAuthFilterInitializer = true;
+ }
+ if (initializer.getName().equals(
+ RMAuthenticationFilterInitializer.class.getName())) {
+ hasRMAuthFilterInitializer = true;
+ }
+ }
+ if (UserGroupInformation.isSecurityEnabled()
+ && useYarnAuthenticationFilter
+ && hasHadoopAuthFilterInitializer
+ && conf.get(authTypeKey, "").equals(
+ KerberosAuthenticationHandler.TYPE)) {
+ ArrayList<String> target = new ArrayList<String>();
+ for (Class<?> filterInitializer : initializersClasses) {
+ if (filterInitializer.getName().equals(
+ AuthenticationFilterInitializer.class.getName())) {
+ if (hasRMAuthFilterInitializer == false) {
+ target.add(RMAuthenticationFilterInitializer.class.getName());
+ }
+ continue;
+ }
+ target.add(filterInitializer.getName());
+ }
+ actualInitializers = StringUtils.join(",", target);
+
+ LOG.info("Using RM authentication filter(kerberos/delegation-token)"
+ + " for RM webapp authentication");
+ RMAuthenticationHandler
+ .setSecretManager(getClientRMService().rmDTSecretManager);
+ String yarnAuthKey =
+ authPrefix + RMAuthenticationFilter.AUTH_HANDLER_PROPERTY;
+ conf.setStrings(yarnAuthKey, RMAuthenticationHandler.class.getName());
+ conf.set(filterInitializerConfKey, actualInitializers);
}
}
- // if security is not enabled and the default filter initializer has been
- // set, set the initializer to include the
+ // if security is not enabled and the default filter initializer has not
+ // been set, set the initializer to include the
// RMAuthenticationFilterInitializer which in turn will set up the simple
// auth filter.
+ String initializers = conf.get(filterInitializerConfKey);
if (!UserGroupInformation.isSecurityEnabled()) {
- if (initializers == null || initializers.isEmpty()) {
- conf.set("hadoop.http.filter.initializers",
+ if (initializersClasses == null || initializersClasses.length == 0) {
+ conf.set(filterInitializerConfKey,
RMAuthenticationFilterInitializer.class.getName());
conf.set(authTypeKey, "simple");
} else if (initializers.equals(StaticUserWebFilter.class.getName())) {
- conf.set("hadoop.http.filter.initializers",
+ conf.set(filterInitializerConfKey,
RMAuthenticationFilterInitializer.class.getName() + ","
+ initializers);
conf.set(authTypeKey, "simple");
Modified: hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java?rev=1618417&r1=1618416&r2=1618417&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java Sat Aug 16 21:02:21 2014
@@ -166,6 +166,8 @@ public class RMAppImpl implements RMApp,
RMAppEventType.APP_REJECTED,
new FinalSavingTransition(new AppRejectedTransition(),
RMAppState.FAILED))
+ .addTransition(RMAppState.NEW_SAVING, RMAppState.NEW_SAVING,
+ RMAppEventType.MOVE, new RMAppMoveTransition())
// Transitions from SUBMITTED state
.addTransition(RMAppState.SUBMITTED, RMAppState.SUBMITTED,
@@ -243,7 +245,7 @@ public class RMAppImpl implements RMApp,
// ignorable transitions
.addTransition(RMAppState.FINAL_SAVING, RMAppState.FINAL_SAVING,
EnumSet.of(RMAppEventType.NODE_UPDATE, RMAppEventType.KILL,
- RMAppEventType.APP_NEW_SAVED))
+ RMAppEventType.APP_NEW_SAVED, RMAppEventType.MOVE))
// Transitions from FINISHING state
.addTransition(RMAppState.FINISHING, RMAppState.FINISHED,
@@ -254,9 +256,9 @@ public class RMAppImpl implements RMApp,
// ignorable transitions
.addTransition(RMAppState.FINISHING, RMAppState.FINISHING,
EnumSet.of(RMAppEventType.NODE_UPDATE,
- // ignore Kill as we have already saved the final Finished state in
- // state store.
- RMAppEventType.KILL))
+ // ignore Kill/Move as we have already saved the final Finished state
+ // in state store.
+ RMAppEventType.KILL, RMAppEventType.MOVE))
// Transitions from KILLING state
.addTransition(RMAppState.KILLING, RMAppState.KILLING,
@@ -274,7 +276,7 @@ public class RMAppImpl implements RMApp,
RMAppEventType.ATTEMPT_FINISHED,
RMAppEventType.ATTEMPT_FAILED,
RMAppEventType.APP_UPDATE_SAVED,
- RMAppEventType.KILL))
+ RMAppEventType.KILL, RMAppEventType.MOVE))
// Transitions from FINISHED state
// ignorable transitions
@@ -286,7 +288,7 @@ public class RMAppImpl implements RMApp,
RMAppEventType.NODE_UPDATE,
RMAppEventType.ATTEMPT_UNREGISTERED,
RMAppEventType.ATTEMPT_FINISHED,
- RMAppEventType.KILL))
+ RMAppEventType.KILL, RMAppEventType.MOVE))
// Transitions from FAILED state
// ignorable transitions
@@ -294,7 +296,8 @@ public class RMAppImpl implements RMApp,
RMAppEventType.APP_RUNNING_ON_NODE,
new AppRunningOnNodeTransition())
.addTransition(RMAppState.FAILED, RMAppState.FAILED,
- EnumSet.of(RMAppEventType.KILL, RMAppEventType.NODE_UPDATE))
+ EnumSet.of(RMAppEventType.KILL, RMAppEventType.NODE_UPDATE,
+ RMAppEventType.MOVE))
// Transitions from KILLED state
// ignorable transitions
@@ -307,7 +310,7 @@ public class RMAppImpl implements RMApp,
EnumSet.of(RMAppEventType.APP_ACCEPTED,
RMAppEventType.APP_REJECTED, RMAppEventType.KILL,
RMAppEventType.ATTEMPT_FINISHED, RMAppEventType.ATTEMPT_FAILED,
- RMAppEventType.NODE_UPDATE))
+ RMAppEventType.NODE_UPDATE, RMAppEventType.MOVE))
.installTopology();
Modified: hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java?rev=1618417&r1=1618416&r2=1618417&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java Sat Aug 16 21:02:21 2014
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
@@ -40,6 +41,9 @@ import org.apache.hadoop.yarn.exceptions
import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMoveEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl;
@@ -48,6 +52,8 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeCleanContainerEvent;
import org.apache.hadoop.yarn.util.resource.Resources;
+import com.google.common.util.concurrent.SettableFuture;
+
@SuppressWarnings("unchecked")
public abstract class AbstractYarnScheduler
<T extends SchedulerApplicationAttempt, N extends SchedulerNode>
@@ -317,4 +323,50 @@ public abstract class AbstractYarnSchedu
public SchedulerNode getSchedulerNode(NodeId nodeId) {
return nodes.get(nodeId);
}
+
+ @Override
+ public synchronized void moveAllApps(String sourceQueue, String destQueue)
+ throws YarnException {
+ // check if destination queue is a valid leaf queue
+ try {
+ getQueueInfo(destQueue, false, false);
+ } catch (IOException e) {
+ LOG.warn(e);
+ throw new YarnException(e);
+ }
+ // check if source queue is a valid
+ List<ApplicationAttemptId> apps = getAppsInQueue(sourceQueue);
+ if (apps == null) {
+ String errMsg = "The specified Queue: " + sourceQueue + " doesn't exist";
+ LOG.warn(errMsg);
+ throw new YarnException(errMsg);
+ }
+ // generate move events for each pending/running app
+ for (ApplicationAttemptId app : apps) {
+ SettableFuture<Object> future = SettableFuture.create();
+ this.rmContext
+ .getDispatcher()
+ .getEventHandler()
+ .handle(new RMAppMoveEvent(app.getApplicationId(), destQueue, future));
+ }
+ }
+
+ @Override
+ public synchronized void killAllAppsInQueue(String queueName)
+ throws YarnException {
+ // check if queue is a valid
+ List<ApplicationAttemptId> apps = getAppsInQueue(queueName);
+ if (apps == null) {
+ String errMsg = "The specified Queue: " + queueName + " doesn't exist";
+ LOG.warn(errMsg);
+ throw new YarnException(errMsg);
+ }
+ // generate kill events for each pending/running app
+ for (ApplicationAttemptId app : apps) {
+ this.rmContext
+ .getDispatcher()
+ .getEventHandler()
+ .handle(new RMAppEvent(app.getApplicationId(), RMAppEventType.KILL));
+ }
+ }
}
Modified: hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java?rev=1618417&r1=1618416&r2=1618417&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java Sat Aug 16 21:02:21 2014
@@ -54,7 +54,7 @@ public class AppSchedulingInfo {
private static final Log LOG = LogFactory.getLog(AppSchedulingInfo.class);
private final ApplicationAttemptId applicationAttemptId;
final ApplicationId applicationId;
- private final String queueName;
+ private String queueName;
Queue queue;
final String user;
// TODO making containerIdCounter long
@@ -410,6 +410,7 @@ public class AppSchedulingInfo {
activeUsersManager = newQueue.getActiveUsersManager();
activeUsersManager.activateApplication(user, applicationId);
this.queue = newQueue;
+ this.queueName = newQueue.getQueueName();
}
synchronized public void stop(RMAppAttemptState rmAppAttemptFinalState) {
Modified: hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java?rev=1618417&r1=1618416&r2=1618417&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java Sat Aug 16 21:02:21 2014
@@ -202,4 +202,22 @@ public interface YarnScheduler extends E
@Evolving
public String moveApplication(ApplicationId appId, String newQueue)
throws YarnException;
+
+ /**
+ * Completely drain sourceQueue of applications, by moving all of them to
+ * destQueue.
+ *
+ * @param sourceQueue
+ * @param destQueue
+ * @throws YarnException
+ */
+ void moveAllApps(String sourceQueue, String destQueue) throws YarnException;
+
+ /**
+ * Terminate all applications in the specified queue.
+ *
+ * @param queueName the name of queue to be drained
+ * @throws YarnException
+ */
+ void killAllAppsInQueue(String queueName) throws YarnException;
}
Modified: hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java?rev=1618417&r1=1618416&r2=1618417&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java Sat Aug 16 21:02:21 2014
@@ -238,4 +238,22 @@ extends org.apache.hadoop.yarn.server.re
* @param apps the collection to add the applications to
*/
public void collectSchedulerApplications(Collection<ApplicationAttemptId> apps);
+
+ /**
+ * Detach a container from this queue
+ * @param clusterResource the current cluster resource
+ * @param application application to which the container was assigned
+ * @param container the container to detach
+ */
+ public void detachContainer(Resource clusterResource,
+ FiCaSchedulerApp application, RMContainer container);
+
+ /**
+ * Attach a container to this queue
+ * @param clusterResource the current cluster resource
+ * @param application application to which the container was assigned
+ * @param container the container to attach
+ */
+ public void attachContainer(Resource clusterResource,
+ FiCaSchedulerApp application, RMContainer container);
}
Modified: hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java?rev=1618417&r1=1618416&r2=1618417&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java Sat Aug 16 21:02:21 2014
@@ -53,6 +53,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger;
import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants;
@@ -547,6 +548,8 @@ public class CapacityScheduler extends
.handle(new RMAppRejectedEvent(applicationId, ace.toString()));
return;
}
+ // update the metrics
+ queue.getMetrics().submitApp(user);
SchedulerApplication<FiCaSchedulerApp> application =
new SchedulerApplication<FiCaSchedulerApp>(queue, user);
applications.put(applicationId, application);
@@ -1131,4 +1134,59 @@ public class CapacityScheduler extends
throw new IOException(e);
}
}
+
+ @Override
+ public synchronized String moveApplication(ApplicationId appId,
+ String targetQueueName) throws YarnException {
+ FiCaSchedulerApp app =
+ getApplicationAttempt(ApplicationAttemptId.newInstance(appId, 0));
+ String sourceQueueName = app.getQueue().getQueueName();
+ LeafQueue source = getAndCheckLeafQueue(sourceQueueName);
+ LeafQueue dest = getAndCheckLeafQueue(targetQueueName);
+ // Validation check - ACLs, submission limits for user & queue
+ String user = app.getUser();
+ try {
+ dest.submitApplication(appId, user, targetQueueName);
+ } catch (AccessControlException e) {
+ throw new YarnException(e);
+ }
+ // Move all live containers
+ for (RMContainer rmContainer : app.getLiveContainers()) {
+ source.detachContainer(clusterResource, app, rmContainer);
+ // attach the Container to another queue
+ dest.attachContainer(clusterResource, app, rmContainer);
+ }
+ // Detach the application..
+ source.finishApplicationAttempt(app, sourceQueueName);
+ source.getParent().finishApplication(appId, app.getUser());
+ // Finish app & update metrics
+ app.move(dest);
+ // Submit to a new queue
+ dest.submitApplicationAttempt(app, user);
+ applications.get(appId).setQueue(dest);
+ LOG.info("App: " + app.getApplicationId() + " successfully moved from "
+ + sourceQueueName + " to: " + targetQueueName);
+ return targetQueueName;
+ }
+
+ /**
+ * Check that the String provided in input is the name of an existing,
+ * LeafQueue, if successful returns the queue.
+ *
+ * @param queue
+ * @return the LeafQueue
+ * @throws YarnException
+ */
+ private LeafQueue getAndCheckLeafQueue(String queue) throws YarnException {
+ CSQueue ret = this.getQueue(queue);
+ if (ret == null) {
+ throw new YarnException("The specified Queue: " + queue
+ + " doesn't exist");
+ }
+ if (!(ret instanceof LeafQueue)) {
+ throw new YarnException("The specified Queue: " + queue
+ + " is not a Leaf Queue. Move is supported only for Leaf Queues.");
+ }
+ return (LeafQueue) ret;
+ }
}
Modified: hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java?rev=1618417&r1=1618416&r2=1618417&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java Sat Aug 16 21:02:21 2014
@@ -643,7 +643,10 @@ public class LeafQueue implements CSQueu
addApplicationAttempt(application, user);
}
- metrics.submitAppAttempt(userName);
+ // We don't want to update metrics for move app
+ if (application.isPending()) {
+ metrics.submitAppAttempt(userName);
+ }
getParent().submitApplicationAttempt(application, userName);
}
@@ -701,7 +704,6 @@ public class LeafQueue implements CSQueu
throw ace;
}
- metrics.submitApp(userName);
}
private synchronized void activateApplications() {
@@ -1620,8 +1622,43 @@ public class LeafQueue implements CSQueu
@Override
public void collectSchedulerApplications(
Collection<ApplicationAttemptId> apps) {
+ for (FiCaSchedulerApp pendingApp : pendingApplications) {
+ apps.add(pendingApp.getApplicationAttemptId());
+ }
for (FiCaSchedulerApp app : activeApplications) {
apps.add(app.getApplicationAttemptId());
}
}
+
+ @Override
+ public void attachContainer(Resource clusterResource,
+ FiCaSchedulerApp application, RMContainer rmContainer) {
+ if (application != null) {
+ allocateResource(clusterResource, application, rmContainer.getContainer()
+ .getResource());
+ LOG.info("movedContainer" + " container=" + rmContainer.getContainer()
+ + " resource=" + rmContainer.getContainer().getResource()
+ + " queueMoveIn=" + this + " usedCapacity=" + getUsedCapacity()
+ + " absoluteUsedCapacity=" + getAbsoluteUsedCapacity() + " used="
+ + usedResources + " cluster=" + clusterResource);
+ // Inform the parent queue
+ getParent().attachContainer(clusterResource, application, rmContainer);
+ }
+ }
+
+ @Override
+ public void detachContainer(Resource clusterResource,
+ FiCaSchedulerApp application, RMContainer rmContainer) {
+ if (application != null) {
+ releaseResource(clusterResource, application, rmContainer.getContainer()
+ .getResource());
+ LOG.info("movedContainer" + " container=" + rmContainer.getContainer()
+ + " resource=" + rmContainer.getContainer().getResource()
+ + " queueMoveOut=" + this + " usedCapacity=" + getUsedCapacity()
+ + " absoluteUsedCapacity=" + getAbsoluteUsedCapacity() + " used="
+ + usedResources + " cluster=" + clusterResource);
+ // Inform the parent queue
+ getParent().detachContainer(clusterResource, application, rmContainer);
+ }
+ }
}
Modified: hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java?rev=1618417&r1=1618416&r2=1618417&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java Sat Aug 16 21:02:21 2014
@@ -791,4 +791,37 @@ public class ParentQueue implements CSQu
queue.collectSchedulerApplications(apps);
}
}
+
+ @Override
+ public void attachContainer(Resource clusterResource,
+ FiCaSchedulerApp application, RMContainer rmContainer) {
+ if (application != null) {
+ allocateResource(clusterResource, rmContainer.getContainer()
+ .getResource());
+ LOG.info("movedContainer" + " queueMoveIn=" + getQueueName()
+ + " usedCapacity=" + getUsedCapacity() + " absoluteUsedCapacity="
+ + getAbsoluteUsedCapacity() + " used=" + usedResources + " cluster="
+ + clusterResource);
+ // Inform the parent
+ if (parent != null) {
+ parent.attachContainer(clusterResource, application, rmContainer);
+ }
+ }
+ }
+
+ @Override
+ public void detachContainer(Resource clusterResource,
+ FiCaSchedulerApp application, RMContainer rmContainer) {
+ if (application != null) {
+ releaseResource(clusterResource, rmContainer.getContainer().getResource());
+ LOG.info("movedContainer" + " queueMoveOut=" + getQueueName()
+ + " usedCapacity=" + getUsedCapacity() + " absoluteUsedCapacity="
+ + getAbsoluteUsedCapacity() + " used=" + usedResources + " cluster="
+ + clusterResource);
+ // Inform the parent
+ if (parent != null) {
+ parent.detachContainer(clusterResource, application, rmContainer);
+ }
+ }
+ }
}
Modified: hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java?rev=1618417&r1=1618416&r2=1618417&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java Sat Aug 16 21:02:21 2014
@@ -44,11 +44,11 @@ import org.apache.hadoop.yarn.util.resou
public class FSLeafQueue extends FSQueue {
private static final Log LOG = LogFactory.getLog(
FSLeafQueue.class.getName());
-
- private final List<AppSchedulable> runnableAppScheds = // apps that are runnable
- new ArrayList<AppSchedulable>();
- private final List<AppSchedulable> nonRunnableAppScheds =
- new ArrayList<AppSchedulable>();
+
+ private final List<FSAppAttempt> runnableApps = // apps that are runnable
+ new ArrayList<FSAppAttempt>();
+ private final List<FSAppAttempt> nonRunnableApps =
+ new ArrayList<FSAppAttempt>();
private Resource demand = Resources.createResource(0);
@@ -70,33 +70,31 @@ public class FSLeafQueue extends FSQueue
amResourceUsage = Resource.newInstance(0, 0);
}
- public void addApp(FSSchedulerApp app, boolean runnable) {
- AppSchedulable appSchedulable = new AppSchedulable(scheduler, app, this);
- app.setAppSchedulable(appSchedulable);
+ public void addApp(FSAppAttempt app, boolean runnable) {
if (runnable) {
- runnableAppScheds.add(appSchedulable);
+ runnableApps.add(app);
} else {
- nonRunnableAppScheds.add(appSchedulable);
+ nonRunnableApps.add(app);
}
}
// for testing
- void addAppSchedulable(AppSchedulable appSched) {
- runnableAppScheds.add(appSched);
+ void addAppSchedulable(FSAppAttempt appSched) {
+ runnableApps.add(appSched);
}
/**
* Removes the given app from this queue.
* @return whether or not the app was runnable
*/
- public boolean removeApp(FSSchedulerApp app) {
- if (runnableAppScheds.remove(app.getAppSchedulable())) {
+ public boolean removeApp(FSAppAttempt app) {
+ if (runnableApps.remove(app)) {
// Update AM resource usage
if (app.isAmRunning() && app.getAMResource() != null) {
Resources.subtractFrom(amResourceUsage, app.getAMResource());
}
return true;
- } else if (nonRunnableAppScheds.remove(app.getAppSchedulable())) {
+ } else if (nonRunnableApps.remove(app)) {
return false;
} else {
throw new IllegalStateException("Given app to remove " + app +
@@ -104,22 +102,22 @@ public class FSLeafQueue extends FSQueue
}
}
- public Collection<AppSchedulable> getRunnableAppSchedulables() {
- return runnableAppScheds;
+ public Collection<FSAppAttempt> getRunnableAppSchedulables() {
+ return runnableApps;
}
- public List<AppSchedulable> getNonRunnableAppSchedulables() {
- return nonRunnableAppScheds;
+ public List<FSAppAttempt> getNonRunnableAppSchedulables() {
+ return nonRunnableApps;
}
@Override
public void collectSchedulerApplications(
Collection<ApplicationAttemptId> apps) {
- for (AppSchedulable appSched : runnableAppScheds) {
- apps.add(appSched.getApp().getApplicationAttemptId());
+ for (FSAppAttempt appSched : runnableApps) {
+ apps.add(appSched.getApplicationAttemptId());
}
- for (AppSchedulable appSched : nonRunnableAppScheds) {
- apps.add(appSched.getApp().getApplicationAttemptId());
+ for (FSAppAttempt appSched : nonRunnableApps) {
+ apps.add(appSched.getApplicationAttemptId());
}
}
@@ -145,10 +143,10 @@ public class FSLeafQueue extends FSQueue
@Override
public Resource getResourceUsage() {
Resource usage = Resources.createResource(0);
- for (AppSchedulable app : runnableAppScheds) {
+ for (FSAppAttempt app : runnableApps) {
Resources.addTo(usage, app.getResourceUsage());
}
- for (AppSchedulable app : nonRunnableAppScheds) {
+ for (FSAppAttempt app : nonRunnableApps) {
Resources.addTo(usage, app.getResourceUsage());
}
return usage;
@@ -165,13 +163,13 @@ public class FSLeafQueue extends FSQueue
Resource maxRes = scheduler.getAllocationConfiguration()
.getMaxResources(getName());
demand = Resources.createResource(0);
- for (AppSchedulable sched : runnableAppScheds) {
+ for (FSAppAttempt sched : runnableApps) {
if (Resources.equals(demand, maxRes)) {
break;
}
updateDemandForApp(sched, maxRes);
}
- for (AppSchedulable sched : nonRunnableAppScheds) {
+ for (FSAppAttempt sched : nonRunnableApps) {
if (Resources.equals(demand, maxRes)) {
break;
}
@@ -183,7 +181,7 @@ public class FSLeafQueue extends FSQueue
}
}
- private void updateDemandForApp(AppSchedulable sched, Resource maxRes) {
+ private void updateDemandForApp(FSAppAttempt sched, Resource maxRes) {
sched.updateDemand();
Resource toAdd = sched.getDemand();
if (LOG.isDebugEnabled()) {
@@ -207,9 +205,9 @@ public class FSLeafQueue extends FSQueue
}
Comparator<Schedulable> comparator = policy.getComparator();
- Collections.sort(runnableAppScheds, comparator);
- for (AppSchedulable sched : runnableAppScheds) {
- if (SchedulerAppUtils.isBlacklisted(sched.getApp(), node, LOG)) {
+ Collections.sort(runnableApps, comparator);
+ for (FSAppAttempt sched : runnableApps) {
+ if (SchedulerAppUtils.isBlacklisted(sched, node, LOG)) {
continue;
}
@@ -237,8 +235,8 @@ public class FSLeafQueue extends FSQueue
// Choose the app that is most over fair share
Comparator<Schedulable> comparator = policy.getComparator();
- AppSchedulable candidateSched = null;
- for (AppSchedulable sched : runnableAppScheds) {
+ FSAppAttempt candidateSched = null;
+ for (FSAppAttempt sched : runnableApps) {
if (candidateSched == null ||
comparator.compare(sched, candidateSched) > 0) {
candidateSched = sched;
@@ -291,7 +289,7 @@ public class FSLeafQueue extends FSQueue
@Override
public int getNumRunnableApps() {
- return runnableAppScheds.size();
+ return runnableApps.size();
}
@Override
Modified: hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java?rev=1618417&r1=1618416&r2=1618417&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java Sat Aug 16 21:02:21 2014
@@ -39,7 +39,8 @@ import org.apache.hadoop.yarn.util.resou
@Private
@Unstable
-public abstract class FSQueue extends Schedulable implements Queue {
+public abstract class FSQueue implements Queue, Schedulable {
+ private Resource fairShare = Resources.createResource(0, 0);
private final String name;
protected final FairScheduler scheduler;
private final FSQueueMetrics metrics;
@@ -139,10 +140,15 @@ public abstract class FSQueue extends Sc
public FSQueueMetrics getMetrics() {
return metrics;
}
-
+
+ /** Get the fair share assigned to this Schedulable. */
+ public Resource getFairShare() {
+ return fairShare;
+ }
+
@Override
public void setFairShare(Resource fairShare) {
- super.setFairShare(fairShare);
+ this.fairShare = fairShare;
metrics.setFairShare(fairShare);
}
@@ -187,4 +193,16 @@ public abstract class FSQueue extends Sc
}
return true;
}
+
+ @Override
+ public boolean isActive() {
+ return getNumRunnableApps() > 0;
+ }
+
+ /** Convenient toString implementation for debugging. */
+ @Override
+ public String toString() {
+ return String.format("[%s, demand=%s, running=%s, share=%s, w=%s]",
+ getName(), getDemand(), getResourceUsage(), fairShare, getWeights());
+ }
}
Modified: hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java?rev=1618417&r1=1618416&r2=1618417&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java Sat Aug 16 21:02:21 2014
@@ -35,7 +35,7 @@ public class FSSchedulerNode extends Sch
private static final Log LOG = LogFactory.getLog(FSSchedulerNode.class);
- private AppSchedulable reservedAppSchedulable;
+ private FSAppAttempt reservedAppSchedulable;
public FSSchedulerNode(RMNode node, boolean usePortForNodeName) {
super(node, usePortForNodeName);
@@ -76,7 +76,7 @@ public class FSSchedulerNode extends Sch
" on node " + this + " for application " + application);
}
setReservedContainer(container);
- this.reservedAppSchedulable = ((FSSchedulerApp) application).getAppSchedulable();
+ this.reservedAppSchedulable = (FSAppAttempt) application;
}
@Override
@@ -98,7 +98,7 @@ public class FSSchedulerNode extends Sch
this.reservedAppSchedulable = null;
}
- public synchronized AppSchedulable getReservedAppSchedulable() {
+ public synchronized FSAppAttempt getReservedAppSchedulable() {
return reservedAppSchedulable;
}
}
Modified: hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java?rev=1618417&r1=1618416&r2=1618417&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java Sat Aug 16 21:02:21 2014
@@ -117,7 +117,7 @@ import com.google.common.base.Preconditi
@Unstable
@SuppressWarnings("unchecked")
public class FairScheduler extends
- AbstractYarnScheduler<FSSchedulerApp, FSSchedulerNode> {
+ AbstractYarnScheduler<FSAppAttempt, FSSchedulerNode> {
private FairSchedulerConfiguration conf;
private Resource incrAllocation;
@@ -432,8 +432,8 @@ public class FairScheduler extends
try {
// Reset preemptedResource for each app
for (FSLeafQueue queue : getQueueManager().getLeafQueues()) {
- for (AppSchedulable app : queue.getRunnableAppSchedulables()) {
- app.getApp().resetPreemptedResources();
+ for (FSAppAttempt app : queue.getRunnableAppSchedulables()) {
+ app.resetPreemptedResources();
}
}
@@ -453,8 +453,8 @@ public class FairScheduler extends
} finally {
// Clear preemptedResources for each app
for (FSLeafQueue queue : getQueueManager().getLeafQueues()) {
- for (AppSchedulable app : queue.getRunnableAppSchedulables()) {
- app.getApp().clearPreemptedResources();
+ for (FSAppAttempt app : queue.getRunnableAppSchedulables()) {
+ app.clearPreemptedResources();
}
}
}
@@ -465,7 +465,7 @@ public class FairScheduler extends
protected void warnOrKillContainer(RMContainer container) {
ApplicationAttemptId appAttemptId = container.getApplicationAttemptId();
- FSSchedulerApp app = getSchedulerApp(appAttemptId);
+ FSAppAttempt app = getSchedulerApp(appAttemptId);
FSLeafQueue queue = app.getQueue();
LOG.info("Preempting container (prio=" + container.getContainer().getPriority() +
"res=" + container.getContainer().getResource() +
@@ -490,7 +490,7 @@ public class FairScheduler extends
(getClock().getTime() - time) + "ms)");
}
} else {
- // track the request in the FSSchedulerApp itself
+ // track the request in the FSAppAttempt itself
app.addPreemption(container, getClock().getTime());
}
}
@@ -541,7 +541,7 @@ public class FairScheduler extends
}
// synchronized for sizeBasedWeight
- public synchronized ResourceWeights getAppWeight(AppSchedulable app) {
+ public synchronized ResourceWeights getAppWeight(FSAppAttempt app) {
double weight = 1.0;
if (sizeBasedWeight) {
// Set weight based on current memory demand
@@ -636,8 +636,8 @@ public class FairScheduler extends
return;
}
- SchedulerApplication<FSSchedulerApp> application =
- new SchedulerApplication<FSSchedulerApp>(queue, user);
+ SchedulerApplication<FSAppAttempt> application =
+ new SchedulerApplication<FSAppAttempt>(queue, user);
applications.put(applicationId, application);
queue.getMetrics().submitApp(user);
@@ -661,13 +661,13 @@ public class FairScheduler extends
ApplicationAttemptId applicationAttemptId,
boolean transferStateFromPreviousAttempt,
boolean isAttemptRecovering) {
- SchedulerApplication<FSSchedulerApp> application =
+ SchedulerApplication<FSAppAttempt> application =
applications.get(applicationAttemptId.getApplicationId());
String user = application.getUser();
FSLeafQueue queue = (FSLeafQueue) application.getQueue();
- FSSchedulerApp attempt =
- new FSSchedulerApp(applicationAttemptId, user,
+ FSAppAttempt attempt =
+ new FSAppAttempt(this, applicationAttemptId, user,
queue, new ActiveUsersManager(getRootQueueMetrics()),
rmContext);
if (transferStateFromPreviousAttempt) {
@@ -742,7 +742,7 @@ public class FairScheduler extends
private synchronized void removeApplication(ApplicationId applicationId,
RMAppState finalState) {
- SchedulerApplication<FSSchedulerApp> application =
+ SchedulerApplication<FSAppAttempt> application =
applications.get(applicationId);
if (application == null){
LOG.warn("Couldn't find application " + applicationId);
@@ -757,9 +757,9 @@ public class FairScheduler extends
RMAppAttemptState rmAppAttemptFinalState, boolean keepContainers) {
LOG.info("Application " + applicationAttemptId + " is done." +
" finalState=" + rmAppAttemptFinalState);
- SchedulerApplication<FSSchedulerApp> application =
+ SchedulerApplication<FSAppAttempt> application =
applications.get(applicationAttemptId.getApplicationId());
- FSSchedulerApp attempt = getSchedulerApp(applicationAttemptId);
+ FSAppAttempt attempt = getSchedulerApp(applicationAttemptId);
if (attempt == null || application == null) {
LOG.info("Unknown application " + applicationAttemptId + " has completed!");
@@ -820,7 +820,7 @@ public class FairScheduler extends
Container container = rmContainer.getContainer();
// Get the application for the finished container
- FSSchedulerApp application =
+ FSAppAttempt application =
getCurrentAttemptForContainer(container.getId());
ApplicationId appId =
container.getId().getApplicationAttemptId().getApplicationId();
@@ -835,8 +835,7 @@ public class FairScheduler extends
FSSchedulerNode node = getFSSchedulerNode(container.getNodeId());
if (rmContainer.getState() == RMContainerState.RESERVED) {
- application.unreserve(node, rmContainer.getReservedPriority());
- node.unreserveResource(application);
+ application.unreserve(rmContainer.getReservedPriority(), node);
} else {
application.containerCompleted(rmContainer, containerStatus, event);
node.releaseContainer(container);
@@ -896,7 +895,7 @@ public class FairScheduler extends
List<ResourceRequest> ask, List<ContainerId> release, List<String> blacklistAdditions, List<String> blacklistRemovals) {
// Make sure this application exists
- FSSchedulerApp application = getSchedulerApp(appAttemptId);
+ FSAppAttempt application = getSchedulerApp(appAttemptId);
if (application == null) {
LOG.info("Calling allocate on removed " +
"or non existant application " + appAttemptId);
@@ -1066,13 +1065,13 @@ public class FairScheduler extends
// 1. Check for reserved applications
// 2. Schedule if there are no reservations
- AppSchedulable reservedAppSchedulable = node.getReservedAppSchedulable();
+ FSAppAttempt reservedAppSchedulable = node.getReservedAppSchedulable();
if (reservedAppSchedulable != null) {
Priority reservedPriority = node.getReservedContainer().getReservedPriority();
if (!reservedAppSchedulable.hasContainerForNode(reservedPriority, node)) {
// Don't hold the reservation if app can no longer use it
LOG.info("Releasing reservation that cannot be satisfied for application "
- + reservedAppSchedulable.getApp().getApplicationAttemptId()
+ + reservedAppSchedulable.getApplicationAttemptId()
+ " on node " + node);
reservedAppSchedulable.unreserve(reservedPriority, node);
reservedAppSchedulable = null;
@@ -1080,7 +1079,7 @@ public class FairScheduler extends
// Reservation exists; try to fulfill the reservation
if (LOG.isDebugEnabled()) {
LOG.debug("Trying to fulfill reservation for application "
- + reservedAppSchedulable.getApp().getApplicationAttemptId()
+ + reservedAppSchedulable.getApplicationAttemptId()
+ " on node: " + node);
}
@@ -1105,8 +1104,8 @@ public class FairScheduler extends
updateRootQueueMetrics();
}
- public FSSchedulerApp getSchedulerApp(ApplicationAttemptId appAttemptId) {
- return (FSSchedulerApp) super.getApplicationAttempt(appAttemptId);
+ public FSAppAttempt getSchedulerApp(ApplicationAttemptId appAttemptId) {
+ return super.getApplicationAttempt(appAttemptId);
}
/**
@@ -1151,6 +1150,8 @@ public class FairScheduler extends
}
NodeAddedSchedulerEvent nodeAddedEvent = (NodeAddedSchedulerEvent)event;
addNode(nodeAddedEvent.getAddedRMNode());
+ recoverContainersOnNode(nodeAddedEvent.getContainerReports(),
+ nodeAddedEvent.getAddedRMNode());
break;
case NODE_REMOVED:
if (!(event instanceof NodeRemovedSchedulerEvent)) {
@@ -1268,8 +1269,8 @@ public class FairScheduler extends
fsOpDurations = FSOpDurations.getInstance(true);
// This stores per-application scheduling information
- this.applications =
- new ConcurrentHashMap<ApplicationId,SchedulerApplication<FSSchedulerApp>>();
+ this.applications = new ConcurrentHashMap<
+ ApplicationId, SchedulerApplication<FSAppAttempt>>();
this.eventLog = new FairSchedulerEventLog();
eventLog.init(this.conf);
@@ -1369,7 +1370,7 @@ public class FairScheduler extends
@Override
public List<QueueUserACLInfo> getQueueUserAclInfo() {
- UserGroupInformation user = null;
+ UserGroupInformation user;
try {
user = UserGroupInformation.getCurrentUser();
} catch (IOException ioe) {
@@ -1431,11 +1432,11 @@ public class FairScheduler extends
@Override
public synchronized String moveApplication(ApplicationId appId,
String queueName) throws YarnException {
- SchedulerApplication<FSSchedulerApp> app = applications.get(appId);
+ SchedulerApplication<FSAppAttempt> app = applications.get(appId);
if (app == null) {
throw new YarnException("App to be moved " + appId + " not found.");
}
- FSSchedulerApp attempt = (FSSchedulerApp) app.getCurrentAppAttempt();
+ FSAppAttempt attempt = (FSAppAttempt) app.getCurrentAppAttempt();
// To serialize with FairScheduler#allocate, synchronize on app attempt
synchronized (attempt) {
FSLeafQueue oldQueue = (FSLeafQueue) app.getQueue();
@@ -1448,8 +1449,7 @@ public class FairScheduler extends
return oldQueue.getQueueName();
}
- if (oldQueue.getRunnableAppSchedulables().contains(
- attempt.getAppSchedulable())) {
+ if (oldQueue.getRunnableAppSchedulables().contains(attempt)) {
verifyMoveDoesNotViolateConstraints(attempt, oldQueue, targetQueue);
}
@@ -1458,7 +1458,7 @@ public class FairScheduler extends
}
}
- private void verifyMoveDoesNotViolateConstraints(FSSchedulerApp app,
+ private void verifyMoveDoesNotViolateConstraints(FSAppAttempt app,
FSLeafQueue oldQueue, FSLeafQueue targetQueue) throws YarnException {
String queueName = targetQueue.getQueueName();
ApplicationAttemptId appAttId = app.getApplicationAttemptId();
@@ -1495,8 +1495,8 @@ public class FairScheduler extends
* Helper for moveApplication, which has appropriate synchronization, so all
* operations will be atomic.
*/
- private void executeMove(SchedulerApplication<FSSchedulerApp> app,
- FSSchedulerApp attempt, FSLeafQueue oldQueue, FSLeafQueue newQueue) {
+ private void executeMove(SchedulerApplication<FSAppAttempt> app,
+ FSAppAttempt attempt, FSLeafQueue oldQueue, FSLeafQueue newQueue) {
boolean wasRunnable = oldQueue.removeApp(attempt);
// if app was not runnable before, it may be runnable now
boolean nowRunnable = maxRunningEnforcer.canAppBeRunnable(newQueue,
Modified: hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FifoAppComparator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FifoAppComparator.java?rev=1618417&r1=1618416&r2=1618417&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FifoAppComparator.java (original)
+++ hadoop/common/branches/HDFS-6584/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FifoAppComparator.java Sat Aug 16 21:02:21 2014
@@ -25,15 +25,15 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.classification.InterfaceStability.Unstable;
/**
- * Order {@link AppSchedulable} objects by priority and then by submit time, as
+ * Order {@link FSAppAttempt} objects by priority and then by submit time, as
* in the default scheduler in Hadoop.
*/
@Private
@Unstable
-public class FifoAppComparator implements Comparator<AppSchedulable>, Serializable {
+public class FifoAppComparator implements Comparator<FSAppAttempt>, Serializable {
private static final long serialVersionUID = 3428835083489547918L;
- public int compare(AppSchedulable a1, AppSchedulable a2) {
+ public int compare(FSAppAttempt a1, FSAppAttempt a2) {
int res = a1.getPriority().compareTo(a2.getPriority());
if (res == 0) {
if (a1.getStartTime() < a2.getStartTime()) {
@@ -44,7 +44,7 @@ public class FifoAppComparator implement
}
if (res == 0) {
// If there is a tie, break it by app ID to get a deterministic order
- res = a1.getApp().getApplicationId().compareTo(a2.getApp().getApplicationId());
+ res = a1.getApplicationId().compareTo(a2.getApplicationId());
}
return res;
}