You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by cn...@apache.org on 2014/01/07 02:07:41 UTC
svn commit: r1556097 [1/4] - in
/hadoop/common/branches/HDFS-4685/hadoop-yarn-project: ./
hadoop-yarn/dev-support/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/...
Author: cnauroth
Date: Tue Jan 7 01:07:33 2014
New Revision: 1556097
URL: http://svn.apache.org/r1556097
Log:
Merge trunk to HDFS-4685.
Added:
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMSecretManagerService.java
- copied unchanged from r1556096, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMSecretManagerService.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
- copied unchanged from r1556096, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAddedSchedulerEvent.java
- copied unchanged from r1556096, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAddedSchedulerEvent.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppRemovedSchedulerEvent.java
- copied unchanged from r1556096, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppRemovedSchedulerEvent.java
Removed:
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/RMNotYetActiveException.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/event/RMAppAttemptRejectedEvent.java
Modified:
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/ClientRMProxy.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMAdminCLI.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ServerRMProxy.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptEventType.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ActiveUsersManager.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerAppReport.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerAppUtils.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplication.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAttemptAddedSchedulerEvent.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/SchedulerEventType.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/QueueACLsTestBase.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationCleanup.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestRMDelegationTokens.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestRMNMSecretKeys.java
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/CHANGES.txt?rev=1556097&r1=1556096&r2=1556097&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/CHANGES.txt Tue Jan 7 01:07:33 2014
@@ -187,6 +187,16 @@ Release 2.4.0 - UNRELEASED
YARN-1307. Redesign znode structure for Zookeeper based RM state-store for
better organization and scalability. (Tsuyoshi OZAWA via vinodkv)
+ YARN-1172. Convert SecretManagers in RM to services (Tsuyoshi OZAWA via kasha)
+
+ YARN-1523. Use StandbyException instead of RMNotYetReadyException (kasha)
+
+ YARN-1541. Changed ResourceManager to invalidate ApplicationMaster host/port
+ information once an AM crashes. (Jian He via vinodkv)
+
+ YARN-1493. Changed ResourceManager and Scheduler interfacing to recognize
+ app-attempts separately from apps. (Jian He via vinodkv)
+
OPTIMIZATIONS
BUG FIXES
@@ -267,6 +277,21 @@ Release 2.4.0 - UNRELEASED
YARN-1451. TestResourceManager relies on the scheduler assigning multiple
containers in a single node update. (Sandy Ryza via kasha)
+ YARN-1527. Fix yarn rmadmin command to print the correct usage info.
+ (Akira AJISAKA via jianhe)
+
+ YARN-1522. Fixed a race condition in the test TestApplicationCleanup that was
+ causing it to randomly fail. (Liyin Liang via vinodkv)
+
+ YARN-1549. Fixed a bug in ResourceManager's ApplicationMasterService that
+ was causing unamanged AMs to not finish correctly. (haosdent via vinodkv)
+
+ YARN-1559. Race between ServerRMProxy and ClientRMProxy setting
+ RMProxy#INSTANCE. (kasha and vinodkv via kasha)
+
+ YARN-1560. Fixed TestYarnClient#testAMMRTokens failure with null AMRM token.
+ (Ted Yu via jianhe)
+
Release 2.3.0 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -473,6 +498,9 @@ Release 2.2.0 - 2013-10-13
YARN-1278. Fixed NodeManager to not delete local resources for apps on resync
command from RM - a bug caused by YARN-1149. (Hitesh Shah via vinodkv)
+ YARN-1463. Tests should avoid starting http-server where possible or creates
+ spnego keytab/principals (vinodkv via kasha)
+
Release 2.1.1-beta - 2013-09-23
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml?rev=1556097&r1=1556096&r2=1556097&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml Tue Jan 7 01:07:33 2014
@@ -309,13 +309,4 @@
<Class name="org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore" />
<Bug pattern="IS2_INCONSISTENT_SYNC" />
</Match>
-
- <!-- Ignore INSTANCE not being final as it is created in sub-classes -->
- <Match>
- <Class name="org.apache.hadoop.yarn.client.RMProxy" />
- <Field name="INSTANCE" />
- <Bug pattern="MS_SHOULD_BE_FINAL"/>
- </Match>
-
-
</FindBugsFilter>
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java?rev=1556097&r1=1556096&r2=1556097&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java Tue Jan 7 01:07:33 2014
@@ -24,10 +24,10 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.classification.InterfaceStability.Stable;
+import org.apache.hadoop.ipc.StandbyException;
import org.apache.hadoop.tools.GetUserMappingsProtocol;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.ResourceOption;
-import org.apache.hadoop.yarn.exceptions.RMNotYetActiveException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsResponse;
@@ -51,25 +51,25 @@ public interface ResourceManagerAdminist
@Public
@Stable
public RefreshQueuesResponse refreshQueues(RefreshQueuesRequest request)
- throws RMNotYetActiveException, YarnException, IOException;
+ throws StandbyException, YarnException, IOException;
@Public
@Stable
public RefreshNodesResponse refreshNodes(RefreshNodesRequest request)
- throws RMNotYetActiveException, YarnException, IOException;
+ throws StandbyException, YarnException, IOException;
@Public
@Stable
public RefreshSuperUserGroupsConfigurationResponse
refreshSuperUserGroupsConfiguration(
RefreshSuperUserGroupsConfigurationRequest request)
- throws RMNotYetActiveException, YarnException, IOException;
+ throws StandbyException, YarnException, IOException;
@Public
@Stable
public RefreshUserToGroupsMappingsResponse refreshUserToGroupsMappings(
RefreshUserToGroupsMappingsRequest request)
- throws RMNotYetActiveException, YarnException, IOException;
+ throws StandbyException, YarnException, IOException;
@Public
@Stable
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java?rev=1556097&r1=1556096&r2=1556097&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/test/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/TestUnmanagedAMLauncher.java Tue Jan 7 01:07:33 2014
@@ -29,12 +29,14 @@ import java.io.OutputStream;
import java.net.URL;
import junit.framework.Assert;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterResponse;
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.client.ClientRMProxy;
@@ -193,8 +195,10 @@ public class TestUnmanagedAMLauncher {
client.registerApplicationMaster(RegisterApplicationMasterRequest
.newInstance(NetUtils.getHostname(), -1, ""));
Thread.sleep(1000);
- client.finishApplicationMaster(FinishApplicationMasterRequest
- .newInstance(FinalApplicationStatus.SUCCEEDED, "success", null));
+ FinishApplicationMasterResponse resp =
+ client.finishApplicationMaster(FinishApplicationMasterRequest
+ .newInstance(FinalApplicationStatus.SUCCEEDED, "success", null));
+ assertTrue(resp.getIsUnregistered());
System.exit(0);
} else {
System.exit(1);
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/ClientRMProxy.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/ClientRMProxy.java?rev=1556097&r1=1556096&r2=1556097&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/ClientRMProxy.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/ClientRMProxy.java Tue Jan 7 01:07:33 2014
@@ -39,16 +39,13 @@ import com.google.common.base.Preconditi
public class ClientRMProxy<T> extends RMProxy<T> {
private static final Log LOG = LogFactory.getLog(ClientRMProxy.class);
+ private static final ClientRMProxy INSTANCE = new ClientRMProxy();
private interface ClientRMProtocols extends ApplicationClientProtocol,
ApplicationMasterProtocol, ResourceManagerAdministrationProtocol {
// Add nothing
}
- static {
- INSTANCE = new ClientRMProxy();
- }
-
private ClientRMProxy(){
super();
}
@@ -63,9 +60,7 @@ public class ClientRMProxy<T> extends RM
*/
public static <T> T createRMProxy(final Configuration configuration,
final Class<T> protocol) throws IOException {
- // This method exists only to initiate this class' static INSTANCE. TODO:
- // FIX if possible
- return RMProxy.createRMProxy(configuration, protocol);
+ return createRMProxy(configuration, protocol, INSTANCE);
}
private static void setupTokens(InetSocketAddress resourceManagerAddress)
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java?rev=1556097&r1=1556096&r2=1556097&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java Tue Jan 7 01:07:33 2014
@@ -122,13 +122,13 @@ public class RMAdminCLI extends HAAdmin
}
}
String space = (usageInfo.args == "") ? "" : " ";
- builder.append("Usage: java RMAdmin ["
+ builder.append("Usage: yarn rmadmin ["
+ cmd + space + usageInfo.args
+ "]\n");
}
private static void buildUsageMsg(StringBuilder builder) {
- builder.append("Usage: java RMAdmin");
+ builder.append("Usage: yarn rmadmin\n");
for (String cmdKey : ADMIN_USAGE.keySet()) {
UsageInfo usageInfo = ADMIN_USAGE.get(cmdKey);
builder.append(" " + cmdKey + " " + usageInfo.args + "\n");
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMAdminCLI.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMAdminCLI.java?rev=1556097&r1=1556096&r2=1556097&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMAdminCLI.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMAdminCLI.java Tue Jan 7 01:07:33 2014
@@ -241,37 +241,37 @@ public class TestRMAdminCLI {
"commands if none"));
testError(new String[] { "-help", "-refreshQueues" },
- "Usage: java RMAdmin [-refreshQueues]", dataErr, 0);
+ "Usage: yarn rmadmin [-refreshQueues]", dataErr, 0);
testError(new String[] { "-help", "-refreshNodes" },
- "Usage: java RMAdmin [-refreshNodes]", dataErr, 0);
+ "Usage: yarn rmadmin [-refreshNodes]", dataErr, 0);
testError(new String[] { "-help", "-refreshUserToGroupsMappings" },
- "Usage: java RMAdmin [-refreshUserToGroupsMappings]", dataErr, 0);
+ "Usage: yarn rmadmin [-refreshUserToGroupsMappings]", dataErr, 0);
testError(
new String[] { "-help", "-refreshSuperUserGroupsConfiguration" },
- "Usage: java RMAdmin [-refreshSuperUserGroupsConfiguration]",
+ "Usage: yarn rmadmin [-refreshSuperUserGroupsConfiguration]",
dataErr, 0);
testError(new String[] { "-help", "-refreshAdminAcls" },
- "Usage: java RMAdmin [-refreshAdminAcls]", dataErr, 0);
+ "Usage: yarn rmadmin [-refreshAdminAcls]", dataErr, 0);
testError(new String[] { "-help", "-refreshServiceAcl" },
- "Usage: java RMAdmin [-refreshServiceAcl]", dataErr, 0);
+ "Usage: yarn rmadmin [-refreshServiceAcl]", dataErr, 0);
testError(new String[] { "-help", "-getGroups" },
- "Usage: java RMAdmin [-getGroups [username]]", dataErr, 0);
+ "Usage: yarn rmadmin [-getGroups [username]]", dataErr, 0);
testError(new String[] { "-help", "-transitionToActive" },
- "Usage: java RMAdmin [-transitionToActive <serviceId>]", dataErr, 0);
+ "Usage: yarn rmadmin [-transitionToActive <serviceId>]", dataErr, 0);
testError(new String[] { "-help", "-transitionToStandby" },
- "Usage: java RMAdmin [-transitionToStandby <serviceId>]", dataErr, 0);
+ "Usage: yarn rmadmin [-transitionToStandby <serviceId>]", dataErr, 0);
testError(new String[] { "-help", "-getServiceState" },
- "Usage: java RMAdmin [-getServiceState <serviceId>]", dataErr, 0);
+ "Usage: yarn rmadmin [-getServiceState <serviceId>]", dataErr, 0);
testError(new String[] { "-help", "-checkHealth" },
- "Usage: java RMAdmin [-checkHealth <serviceId>]", dataErr, 0);
+ "Usage: yarn rmadmin [-checkHealth <serviceId>]", dataErr, 0);
testError(new String[] { "-help", "-failover" },
- "Usage: java RMAdmin " +
+ "Usage: yarn rmadmin " +
"[-failover [--forcefence] [--forceactive] " +
"<serviceId> <serviceId>]",
dataErr, 0);
testError(new String[] { "-help", "-badParameter" },
- "Usage: java RMAdmin", dataErr, 0);
+ "Usage: yarn rmadmin", dataErr, 0);
testError(new String[] { "-badParameter" },
"badParameter: Unknown command", dataErr, -1);
} finally {
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java?rev=1556097&r1=1556096&r2=1556097&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java Tue Jan 7 01:07:33 2014
@@ -378,6 +378,13 @@ public class TestYarnClient {
appId = createApp(rmClient, true);
waitTillAccepted(rmClient, appId);
+ long start = System.currentTimeMillis();
+ while (rmClient.getAMRMToken(appId) == null) {
+ if (System.currentTimeMillis() - start > 20 * 1000) {
+ Assert.fail("AMRM token is null");
+ }
+ Thread.sleep(100);
+ }
//unmanaged AMs do return AMRM token
Assert.assertNotNull(rmClient.getAMRMToken(appId));
@@ -392,6 +399,13 @@ public class TestYarnClient {
rmClient.start();
ApplicationId appId = createApp(rmClient, true);
waitTillAccepted(rmClient, appId);
+ long start = System.currentTimeMillis();
+ while (rmClient.getAMRMToken(appId) == null) {
+ if (System.currentTimeMillis() - start > 20 * 1000) {
+ Assert.fail("AMRM token is null");
+ }
+ Thread.sleep(100);
+ }
//unmanaged AMs do return AMRM token
Assert.assertNotNull(rmClient.getAMRMToken(appId));
return appId;
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java?rev=1556097&r1=1556096&r2=1556097&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java Tue Jan 7 01:07:33 2014
@@ -50,7 +50,6 @@ import com.google.common.annotations.Vis
public class RMProxy<T> {
private static final Log LOG = LogFactory.getLog(RMProxy.class);
- protected static RMProxy INSTANCE;
protected RMProxy() {}
@@ -79,17 +78,17 @@ public class RMProxy<T> {
*/
@Private
protected static <T> T createRMProxy(final Configuration configuration,
- final Class<T> protocol) throws IOException {
+ final Class<T> protocol, RMProxy instance) throws IOException {
YarnConfiguration conf = (configuration instanceof YarnConfiguration)
? (YarnConfiguration) configuration
: new YarnConfiguration(configuration);
RetryPolicy retryPolicy = createRetryPolicy(conf);
if (HAUtil.isHAEnabled(conf)) {
RMFailoverProxyProvider<T> provider =
- INSTANCE.createRMFailoverProxyProvider(conf, protocol);
+ instance.createRMFailoverProxyProvider(conf, protocol);
return (T) RetryProxy.create(protocol, provider, retryPolicy);
} else {
- InetSocketAddress rmAddress = INSTANCE.getRMAddress(conf, protocol);
+ InetSocketAddress rmAddress = instance.getRMAddress(conf, protocol);
LOG.info("Connecting to ResourceManager at " + rmAddress);
T proxy = RMProxy.<T>getProxy(conf, protocol, rmAddress);
return (T) RetryProxy.create(protocol, proxy, retryPolicy);
@@ -160,25 +159,6 @@ public class RMProxy<T> {
}
/**
- * A RetryPolicy to allow failing over upto the specified maximum time.
- */
- private static class FailoverUptoMaximumTimePolicy implements RetryPolicy {
- private long maxTime;
-
- FailoverUptoMaximumTimePolicy(long maxTime) {
- this.maxTime = maxTime;
- }
-
- @Override
- public RetryAction shouldRetry(Exception e, int retries, int failovers,
- boolean isIdempotentOrAtMostOnce) throws Exception {
- return System.currentTimeMillis() < maxTime
- ? RetryAction.FAILOVER_AND_RETRY
- : RetryAction.FAIL;
- }
- }
-
- /**
* Fetch retry policy from Configuration
*/
@Private
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java?rev=1556097&r1=1556096&r2=1556097&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java Tue Jan 7 01:07:33 2014
@@ -56,6 +56,7 @@ public class AsyncDispatcher extends Abs
// Indicates all the remaining dispatcher's events on stop have been drained
// and processed.
private volatile boolean drained = true;
+ private Object waitForDrained = new Object();
// For drainEventsOnStop enabled only, block newly coming events into the
// queue while stopping.
@@ -82,6 +83,16 @@ public class AsyncDispatcher extends Abs
public void run() {
while (!stopped && !Thread.currentThread().isInterrupted()) {
drained = eventQueue.isEmpty();
+ // blockNewEvents is only set when dispatcher is draining to stop,
+ // adding this check is to avoid the overhead of acquiring the lock
+ // and calling notify every time in the normal run of the loop.
+ if (blockNewEvents) {
+ synchronized (waitForDrained) {
+ if (drained) {
+ waitForDrained.notify();
+ }
+ }
+ }
Event event;
try {
event = eventQueue.take();
@@ -125,8 +136,11 @@ public class AsyncDispatcher extends Abs
if (drainEventsOnStop) {
blockNewEvents = true;
LOG.info("AsyncDispatcher is draining to stop, igonring any new events.");
- while(!drained) {
- Thread.yield();
+ synchronized (waitForDrained) {
+ while (!drained && eventHandlingThread.isAlive()) {
+ waitForDrained.wait(1000);
+ LOG.info("Waiting for AsyncDispatcher to drain.");
+ }
}
}
stopped = true;
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java?rev=1556097&r1=1556096&r2=1556097&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java Tue Jan 7 01:07:33 2014
@@ -225,8 +225,8 @@ public class WebApps {
boolean hasSpnegoConf = spnegoPrincipalKey != null
&& spnegoKeytabKey != null;
if (hasSpnegoConf) {
- builder.setUsernameConfKey(conf.get(spnegoPrincipalKey))
- .setKeytabConfKey(conf.get(spnegoKeytabKey))
+ builder.setUsernameConfKey(spnegoPrincipalKey)
+ .setKeytabConfKey(spnegoKeytabKey)
.setSecurityEnabled(UserGroupInformation.isSecurityEnabled());
}
HttpServer server = builder.build();
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ServerRMProxy.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ServerRMProxy.java?rev=1556097&r1=1556096&r2=1556097&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ServerRMProxy.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ServerRMProxy.java Tue Jan 7 01:07:33 2014
@@ -32,10 +32,7 @@ import com.google.common.base.Preconditi
public class ServerRMProxy<T> extends RMProxy<T> {
private static final Log LOG = LogFactory.getLog(ServerRMProxy.class);
-
- static {
- INSTANCE = new ServerRMProxy();
- }
+ private static final ServerRMProxy INSTANCE = new ServerRMProxy();
private ServerRMProxy() {
super();
@@ -51,10 +48,8 @@ public class ServerRMProxy<T> extends RM
*/
public static <T> T createRMProxy(final Configuration configuration,
final Class<T> protocol) throws IOException {
- // This method exists only to initiate this class' static INSTANCE. TODO:
- // FIX if possible
- return RMProxy.createRMProxy(configuration, protocol);
- }
+ return createRMProxy(configuration, protocol, INSTANCE);
+}
@InterfaceAudience.Private
@Override
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java?rev=1556097&r1=1556096&r2=1556097&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java Tue Jan 7 01:07:33 2014
@@ -40,6 +40,7 @@ import org.apache.hadoop.ha.protocolPB.H
import org.apache.hadoop.ipc.ProtobufRpcEngine;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.ipc.RPC.Server;
+import org.apache.hadoop.ipc.StandbyException;
import org.apache.hadoop.security.Groups;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AccessControlList;
@@ -49,7 +50,6 @@ import org.apache.hadoop.service.Abstrac
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.ResourceOption;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.RMNotYetActiveException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
@@ -82,6 +82,7 @@ public class AdminService extends Abstra
private final RMContext rmContext;
private final ResourceManager rm;
+ private String rmId;
private Server server;
private InetSocketAddress masterServiceAddress;
@@ -105,6 +106,7 @@ public class AdminService extends Abstra
adminAcl = new AccessControlList(conf.get(
YarnConfiguration.YARN_ADMIN_ACL,
YarnConfiguration.DEFAULT_YARN_ADMIN_ACL));
+ rmId = conf.get(YarnConfiguration.RM_HA_ID);
super.serviceInit(conf);
}
@@ -176,6 +178,10 @@ public class AdminService extends Abstra
return HAServiceState.ACTIVE == rmContext.getHAServiceState();
}
+ private void throwStandbyException() throws StandbyException {
+ throw new StandbyException("ResourceManager " + rmId + " is not Active!");
+ }
+
@Override
public synchronized void monitorHealth()
throws IOException {
@@ -239,14 +245,14 @@ public class AdminService extends Abstra
@Override
public RefreshQueuesResponse refreshQueues(RefreshQueuesRequest request)
- throws YarnException {
+ throws YarnException, StandbyException {
UserGroupInformation user = checkAcls("refreshQueues");
if (!isRMActive()) {
RMAuditLogger.logFailure(user.getShortUserName(), "refreshQueues",
adminAcl.toString(), "AdminService",
"ResourceManager is not active. Can not refresh queues.");
- throw new RMNotYetActiveException();
+ throwStandbyException();
}
try {
@@ -265,14 +271,14 @@ public class AdminService extends Abstra
@Override
public RefreshNodesResponse refreshNodes(RefreshNodesRequest request)
- throws YarnException {
+ throws YarnException, StandbyException {
UserGroupInformation user = checkAcls("refreshNodes");
if (!isRMActive()) {
RMAuditLogger.logFailure(user.getShortUserName(), "refreshNodes",
adminAcl.toString(), "AdminService",
"ResourceManager is not active. Can not refresh nodes.");
- throw new RMNotYetActiveException();
+ throwStandbyException();
}
try {
@@ -291,7 +297,7 @@ public class AdminService extends Abstra
@Override
public RefreshSuperUserGroupsConfigurationResponse refreshSuperUserGroupsConfiguration(
RefreshSuperUserGroupsConfigurationRequest request)
- throws YarnException {
+ throws YarnException, StandbyException {
UserGroupInformation user = checkAcls("refreshSuperUserGroupsConfiguration");
// TODO (YARN-1459): Revisit handling super-user-groups on Standby RM
@@ -300,7 +306,7 @@ public class AdminService extends Abstra
"refreshSuperUserGroupsConfiguration",
adminAcl.toString(), "AdminService",
"ResourceManager is not active. Can not refresh super-user-groups.");
- throw new RMNotYetActiveException();
+ throwStandbyException();
}
ProxyUsers.refreshSuperUserGroupsConfiguration(new Configuration());
@@ -313,7 +319,8 @@ public class AdminService extends Abstra
@Override
public RefreshUserToGroupsMappingsResponse refreshUserToGroupsMappings(
- RefreshUserToGroupsMappingsRequest request) throws YarnException {
+ RefreshUserToGroupsMappingsRequest request)
+ throws YarnException, StandbyException {
UserGroupInformation user = checkAcls("refreshUserToGroupsMappings");
// TODO (YARN-1459): Revisit handling user-groups on Standby RM
@@ -322,7 +329,7 @@ public class AdminService extends Abstra
"refreshUserToGroupsMapping",
adminAcl.toString(), "AdminService",
"ResourceManager is not active. Can not refresh user-groups.");
- throw new RMNotYetActiveException();
+ throwStandbyException();
}
Groups.getUserToGroupsMappingService().refresh();
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java?rev=1556097&r1=1556096&r2=1556097&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java Tue Jan 7 01:07:33 2014
@@ -292,8 +292,21 @@ public class ApplicationMasterService ex
this.amLivelinessMonitor.receivedPing(applicationAttemptId);
- if (rmContext.getRMApps().get(applicationAttemptId.getApplicationId())
- .isAppSafeToTerminate()) {
+ RMApp rmApp =
+ rmContext.getRMApps().get(applicationAttemptId.getApplicationId());
+
+ if (rmApp.getApplicationSubmissionContext().getUnmanagedAM()) {
+ // No recovery supported yet for unmanaged AM. Send the unregister event
+ // and (falsely) acknowledge state-store write immediately.
+ rmContext.getDispatcher().getEventHandler().handle(
+ new RMAppAttemptUnregistrationEvent(applicationAttemptId, request
+ .getTrackingUrl(), request.getFinalApplicationStatus(), request
+ .getDiagnostics()));
+ return FinishApplicationMasterResponse.newInstance(true);
+ }
+
+ // Not an unmanaged-AM.
+ if (rmApp.isAppSafeToTerminate()) {
return FinishApplicationMasterResponse.newInstance(true);
} else {
// keep sending the unregister event as RM may crash in the meanwhile.
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java?rev=1556097&r1=1556096&r2=1556097&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java Tue Jan 7 01:07:33 2014
@@ -81,7 +81,6 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager;
-import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer;
import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager;
@@ -134,13 +133,7 @@ public class ResourceManager extends Com
* in Active state.
*/
protected RMActiveServices activeServices;
- protected ClientToAMTokenSecretManagerInRM clientToAMSecretManager;
-
- protected RMContainerTokenSecretManager containerTokenSecretManager;
- protected NMTokenSecretManagerInRM nmTokenSecretManager;
-
- protected AMRMTokenSecretManager amRmTokenSecretManager;
-
+ protected RMSecretManagerService rmSecretManagerService;
private Dispatcher rmDispatcher;
protected ResourceScheduler scheduler;
@@ -154,7 +147,6 @@ public class ResourceManager extends Com
protected RMAppManager rmAppManager;
protected ApplicationACLsManager applicationACLsManager;
protected QueueACLsManager queueACLsManager;
- protected RMDelegationTokenSecretManager rmDTSecretManager;
private DelegationTokenRenewer delegationTokenRenewer;
private WebApp webApp;
protected ResourceTrackerService resourceTracker;
@@ -211,16 +203,6 @@ public class ResourceManager extends Com
rmContext.setStateStore(rmStore);
}
- protected RMContainerTokenSecretManager createContainerTokenSecretManager(
- Configuration conf) {
- return new RMContainerTokenSecretManager(conf);
- }
-
- protected NMTokenSecretManagerInRM createNMTokenSecretManager(
- Configuration conf) {
- return new NMTokenSecretManagerInRM(conf);
- }
-
protected EventHandler<SchedulerEvent> createSchedulerEventDispatcher() {
return new SchedulerEventDispatcher(this.scheduler);
}
@@ -234,11 +216,6 @@ public class ResourceManager extends Com
return new AsyncDispatcher();
}
- protected AMRMTokenSecretManager createAMRMTokenSecretManager(
- Configuration conf) {
- return new AMRMTokenSecretManager(conf);
- }
-
protected ResourceScheduler createScheduler() {
String schedulerClassName = conf.get(YarnConfiguration.RM_SCHEDULER,
YarnConfiguration.DEFAULT_RM_SCHEDULER);
@@ -324,11 +301,8 @@ public class ResourceManager extends Com
addIfService(rmDispatcher);
rmContext.setDispatcher(rmDispatcher);
- clientToAMSecretManager = new ClientToAMTokenSecretManagerInRM();
- rmContext.setClientToAMTokenSecretManager(clientToAMSecretManager);
-
- amRmTokenSecretManager = createAMRMTokenSecretManager(conf);
- rmContext.setAMRMTokenSecretManager(amRmTokenSecretManager);
+ rmSecretManagerService = createRMSecretManagerService();
+ addService(rmSecretManagerService);
containerAllocationExpirer = new ContainerAllocationExpirer(rmDispatcher);
addService(containerAllocationExpirer);
@@ -342,12 +316,6 @@ public class ResourceManager extends Com
addService(amFinishingMonitor);
rmContext.setAMFinishingMonitor(amFinishingMonitor);
- containerTokenSecretManager = createContainerTokenSecretManager(conf);
- rmContext.setContainerTokenSecretManager(containerTokenSecretManager);
-
- nmTokenSecretManager = createNMTokenSecretManager(conf);
- rmContext.setNMTokenSecretManager(nmTokenSecretManager);
-
boolean isRecoveryEnabled = conf.getBoolean(
YarnConfiguration.RECOVERY_ENABLED,
YarnConfiguration.DEFAULT_RM_RECOVERY_ENABLED);
@@ -435,8 +403,6 @@ public class ResourceManager extends Com
rmAppManager = createRMAppManager();
// Register event handler for RMAppManagerEvents
rmDispatcher.register(RMAppManagerEventType.class, rmAppManager);
- rmDTSecretManager = createRMDelegationTokenSecretManager(rmContext);
- rmContext.setRMDelegationTokenSecretManager(rmDTSecretManager);
clientRM = createClientRMService();
rmContext.setClientRMService(clientRM);
@@ -460,10 +426,6 @@ public class ResourceManager extends Com
@Override
protected void serviceStart() throws Exception {
- amRmTokenSecretManager.start();
- containerTokenSecretManager.start();
- nmTokenSecretManager.start();
-
RMStateStore rmStore = rmContext.getStateStore();
// The state store needs to start irrespective of recoveryEnabled as apps
// need events to move to further states.
@@ -481,13 +443,7 @@ public class ResourceManager extends Com
throw e;
}
}
-
startWepApp();
- try {
- rmDTSecretManager.startThreads();
- } catch(IOException ie) {
- throw new YarnRuntimeException("Failed to start secret manager threads", ie);
- }
if (getConfig().getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) {
int port = webApp.port();
@@ -502,19 +458,7 @@ public class ResourceManager extends Com
if (webApp != null) {
webApp.stop();
}
- if (rmDTSecretManager != null) {
- rmDTSecretManager.stopThreads();
- }
- if (amRmTokenSecretManager != null) {
- amRmTokenSecretManager.stop();
- }
- if (containerTokenSecretManager != null) {
- containerTokenSecretManager.stop();
- }
- if(nmTokenSecretManager != null) {
- nmTokenSecretManager.stop();
- }
DefaultMetricsSystem.shutdown();
@@ -939,30 +883,15 @@ public class ResourceManager extends Com
protected ResourceTrackerService createResourceTrackerService() {
return new ResourceTrackerService(this.rmContext, this.nodesListManager,
- this.nmLivelinessMonitor, this.containerTokenSecretManager,
- this.nmTokenSecretManager);
- }
-
- protected RMDelegationTokenSecretManager
- createRMDelegationTokenSecretManager(RMContext rmContext) {
- long secretKeyInterval =
- conf.getLong(YarnConfiguration.DELEGATION_KEY_UPDATE_INTERVAL_KEY,
- YarnConfiguration.DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT);
- long tokenMaxLifetime =
- conf.getLong(YarnConfiguration.DELEGATION_TOKEN_MAX_LIFETIME_KEY,
- YarnConfiguration.DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT);
- long tokenRenewInterval =
- conf.getLong(YarnConfiguration.DELEGATION_TOKEN_RENEW_INTERVAL_KEY,
- YarnConfiguration.DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT);
-
- return new RMDelegationTokenSecretManager(secretKeyInterval,
- tokenMaxLifetime, tokenRenewInterval, 3600000, rmContext);
+ this.nmLivelinessMonitor,
+ this.rmContext.getContainerTokenSecretManager(),
+ this.rmContext.getNMTokenSecretManager());
}
protected ClientRMService createClientRMService() {
return new ClientRMService(this.rmContext, scheduler, this.rmAppManager,
this.applicationACLsManager, this.queueACLsManager,
- this.rmDTSecretManager);
+ getRMDTSecretManager());
}
protected ApplicationMasterService createApplicationMasterService() {
@@ -973,6 +902,10 @@ public class ResourceManager extends Com
return new AdminService(this, rmContext);
}
+ protected RMSecretManagerService createRMSecretManagerService() {
+ return new RMSecretManagerService(conf, rmContext);
+ }
+
@Private
public ClientRMService getClientRMService() {
return this.clientRM;
@@ -1013,23 +946,28 @@ public class ResourceManager extends Com
@Private
public RMContainerTokenSecretManager getRMContainerTokenSecretManager() {
- return this.containerTokenSecretManager;
+ return this.rmContext.getContainerTokenSecretManager();
}
@Private
public NMTokenSecretManagerInRM getRMNMTokenSecretManager() {
- return this.nmTokenSecretManager;
+ return this.rmContext.getNMTokenSecretManager();
}
@Private
public AMRMTokenSecretManager getAMRMTokenSecretManager(){
- return this.amRmTokenSecretManager;
+ return this.rmContext.getAMRMTokenSecretManager();
+ }
+
+ @Private
+ public RMDelegationTokenSecretManager getRMDTSecretManager(){
+ return this.rmContext.getRMDelegationTokenSecretManager();
}
@Override
public void recover(RMState state) throws Exception {
// recover RMdelegationTokenSecretManager
- rmDTSecretManager.recover(state);
+ getRMDTSecretManager().recover(state);
// recover applications
rmAppManager.recover(state);
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java?rev=1556097&r1=1556096&r2=1556097&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java Tue Jan 7 01:07:33 2014
@@ -24,9 +24,11 @@ public enum RMAppEventType {
RECOVER,
KILL,
- // Source: RMAppAttempt
+ // Source: Scheduler
APP_REJECTED,
APP_ACCEPTED,
+
+ // Source: RMAppAttempt
ATTEMPT_REGISTERED,
ATTEMPT_UNREGISTERED,
ATTEMPT_FINISHED, // Will send the final state
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java?rev=1556097&r1=1556096&r2=1556097&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java Tue Jan 7 01:07:33 2014
@@ -66,6 +66,8 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeCleanAppEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.state.InvalidStateTransitonException;
import org.apache.hadoop.yarn.state.MultipleArcTransition;
@@ -136,7 +138,7 @@ public class RMAppImpl implements RMApp,
.addTransition(RMAppState.NEW, RMAppState.NEW_SAVING,
RMAppEventType.START, new RMAppNewlySavingTransition())
.addTransition(RMAppState.NEW, EnumSet.of(RMAppState.SUBMITTED,
- RMAppState.RUNNING, RMAppState.FINISHED, RMAppState.FAILED,
+ RMAppState.ACCEPTED, RMAppState.FINISHED, RMAppState.FAILED,
RMAppState.KILLED, RMAppState.FINAL_SAVING),
RMAppEventType.RECOVER, new RMAppRecoveredTransition())
.addTransition(RMAppState.NEW, RMAppState.FINAL_SAVING, RMAppEventType.KILL,
@@ -151,7 +153,7 @@ public class RMAppImpl implements RMApp,
.addTransition(RMAppState.NEW_SAVING, RMAppState.NEW_SAVING,
RMAppEventType.NODE_UPDATE, new RMAppNodeUpdateTransition())
.addTransition(RMAppState.NEW_SAVING, RMAppState.SUBMITTED,
- RMAppEventType.APP_NEW_SAVED, new StartAppAttemptTransition())
+ RMAppEventType.APP_NEW_SAVED, new AddApplicationToSchedulerTransition())
.addTransition(RMAppState.NEW_SAVING, RMAppState.FINAL_SAVING,
RMAppEventType.KILL,
new FinalSavingTransition(
@@ -169,9 +171,12 @@ public class RMAppImpl implements RMApp,
new FinalSavingTransition(
new AppRejectedTransition(), RMAppState.FAILED))
.addTransition(RMAppState.SUBMITTED, RMAppState.ACCEPTED,
- RMAppEventType.APP_ACCEPTED)
- .addTransition(RMAppState.SUBMITTED, RMAppState.KILLING,
- RMAppEventType.KILL,new KillAttemptTransition())
+ RMAppEventType.APP_ACCEPTED, new StartAppAttemptTransition())
+ .addTransition(RMAppState.SUBMITTED, RMAppState.FINAL_SAVING,
+ RMAppEventType.KILL,
+ new FinalSavingTransition(
+ new AppKilledTransition(), RMAppState.KILLED))
+
// Transitions from ACCEPTED state
.addTransition(RMAppState.ACCEPTED, RMAppState.ACCEPTED,
@@ -179,11 +184,22 @@ public class RMAppImpl implements RMApp,
.addTransition(RMAppState.ACCEPTED, RMAppState.RUNNING,
RMAppEventType.ATTEMPT_REGISTERED)
.addTransition(RMAppState.ACCEPTED,
- EnumSet.of(RMAppState.SUBMITTED, RMAppState.FINAL_SAVING),
+ EnumSet.of(RMAppState.ACCEPTED, RMAppState.FINAL_SAVING),
+ // ACCEPTED state is possible to receive ATTEMPT_FAILED event because
+ // RMAppRecoveredTransition is returning ACCEPTED state directly and
+ // waiting for the previous AM to exit.
RMAppEventType.ATTEMPT_FAILED,
- new AttemptFailedTransition(RMAppState.SUBMITTED))
- .addTransition(RMAppState.ACCEPTED, RMAppState.KILLING,
- RMAppEventType.KILL,new KillAttemptTransition())
+ new AttemptFailedTransition(RMAppState.ACCEPTED))
+ .addTransition(RMAppState.ACCEPTED, RMAppState.FINAL_SAVING,
+ RMAppEventType.KILL,
+ new FinalSavingTransition(
+ new AppKilledTransition(), RMAppState.KILLED))
+ // ACCECPTED state can once again receive APP_ACCEPTED event, because on
+ // recovery the app returns ACCEPTED state and the app once again go
+ // through the scheduler and triggers one more APP_ACCEPTED event at
+ // ACCEPTED state.
+ .addTransition(RMAppState.ACCEPTED, RMAppState.ACCEPTED,
+ RMAppEventType.APP_ACCEPTED)
// Transitions from RUNNING state
.addTransition(RMAppState.RUNNING, RMAppState.RUNNING,
@@ -197,9 +213,9 @@ public class RMAppImpl implements RMApp,
// UnManagedAM directly jumps to finished
RMAppEventType.ATTEMPT_FINISHED, FINISHED_TRANSITION)
.addTransition(RMAppState.RUNNING,
- EnumSet.of(RMAppState.SUBMITTED, RMAppState.FINAL_SAVING),
+ EnumSet.of(RMAppState.ACCEPTED, RMAppState.FINAL_SAVING),
RMAppEventType.ATTEMPT_FAILED,
- new AttemptFailedTransition(RMAppState.SUBMITTED))
+ new AttemptFailedTransition(RMAppState.ACCEPTED))
.addTransition(RMAppState.RUNNING, RMAppState.KILLING,
RMAppEventType.KILL, new KillAttemptTransition())
@@ -641,7 +657,7 @@ public class RMAppImpl implements RMApp,
ApplicationAttemptId.newInstance(applicationId, attempts.size() + 1);
RMAppAttempt attempt =
new RMAppAttemptImpl(appAttemptId, rmContext, scheduler, masterService,
- submissionContext, conf, user);
+ submissionContext, conf);
attempts.put(appAttemptId, attempt);
currentAttempt = attempt;
if(startAttempt) {
@@ -695,29 +711,46 @@ public class RMAppImpl implements RMApp,
return app.recoveredFinalState;
}
+ // Notify scheduler about the app on recovery
+ new AddApplicationToSchedulerTransition().transition(app, event);
+
// No existent attempts means the attempt associated with this app was not
// started or started but not yet saved.
if (app.attempts.isEmpty()) {
- app.createNewAttempt(true);
return RMAppState.SUBMITTED;
}
- return RMAppState.RUNNING;
+ // YARN-1507 is saving the application state after the application is
+ // accepted. So after YARN-1507, an app is saved meaning it is accepted.
+ // Thus we return ACCECPTED state on recovery.
+ return RMAppState.ACCEPTED;
}
}
- private static final class StartAppAttemptTransition extends RMAppTransition {
+ private static final class AddApplicationToSchedulerTransition extends
+ RMAppTransition {
+ @SuppressWarnings("unchecked")
@Override
public void transition(RMAppImpl app, RMAppEvent event) {
- RMAppNewSavedEvent storeEvent = (RMAppNewSavedEvent) event;
- if (storeEvent.getStoredException() != null) {
+ if (event instanceof RMAppNewSavedEvent) {
+ RMAppNewSavedEvent storeEvent = (RMAppNewSavedEvent) event;
// For HA this exception needs to be handled by giving up
// master status if we got fenced
- LOG.error(
- "Failed to store application: " + storeEvent.getApplicationId(),
- storeEvent.getStoredException());
- ExitUtil.terminate(1, storeEvent.getStoredException());
+ if (((RMAppNewSavedEvent) event).getStoredException() != null) {
+ LOG.error(
+ "Failed to store application: " + storeEvent.getApplicationId(),
+ storeEvent.getStoredException());
+ ExitUtil.terminate(1, storeEvent.getStoredException());
+ }
}
+ app.handler.handle(new AppAddedSchedulerEvent(app.applicationId,
+ app.submissionContext.getQueue(), app.user));
+ }
+ }
+
+ private static final class StartAppAttemptTransition extends RMAppTransition {
+ @Override
+ public void transition(RMAppImpl app, RMAppEvent event) {
app.createNewAttempt(true);
};
}
@@ -965,6 +998,8 @@ public class RMAppImpl implements RMApp,
if (app.finishTime == 0 ) {
app.finishTime = System.currentTimeMillis();
}
+ app.handler.handle(new AppRemovedSchedulerEvent(app.applicationId, app
+ .getState()));
app.handler.handle(
new RMAppManagerEvent(app.applicationId,
RMAppManagerEventType.APP_COMPLETED));
@@ -993,7 +1028,6 @@ public class RMAppImpl implements RMApp,
return RMAppState.FINAL_SAVING;
}
}
-
}
@Override
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptEventType.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptEventType.java?rev=1556097&r1=1556096&r2=1556097&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptEventType.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptEventType.java Tue Jan 7 01:07:33 2014
@@ -45,8 +45,7 @@ public enum RMAppAttemptEventType {
ATTEMPT_UPDATE_SAVED,
// Source: Scheduler
- APP_REJECTED,
- APP_ACCEPTED,
+ ATTEMPT_ADDED,
// Source: RMAttemptImpl.recover
RECOVER
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java?rev=1556097&r1=1556096&r2=1556097&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java Tue Jan 7 01:07:33 2014
@@ -75,13 +75,11 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppFailedAttemptEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppFinishedAttemptEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl;
-import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppRejectedEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptContainerAcquiredEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptContainerFinishedEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptLaunchFailedEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptNewSavedEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptRegistrationEvent;
-import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptRejectedEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptStatusupdateEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptUnregistrationEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptUpdateSavedEvent;
@@ -139,7 +137,7 @@ public class RMAppAttemptImpl implements
private float progress = 0;
private String host = "N/A";
- private int rpcPort;
+ private int rpcPort = -1;
private String originalTrackingUrl = "N/A";
private String proxiedTrackingUrl = "N/A";
private long startTime = 0;
@@ -150,7 +148,6 @@ public class RMAppAttemptImpl implements
private final StringBuilder diagnostics = new StringBuilder();
private Configuration conf;
- private String user;
private static final ExpiredTransition EXPIRED_TRANSITION =
new ExpiredTransition();
@@ -186,14 +183,10 @@ public class RMAppAttemptImpl implements
RMAppAttemptEventType.RECOVER, new AttemptRecoveredTransition())
// Transitions from SUBMITTED state
- .addTransition(RMAppAttemptState.SUBMITTED, RMAppAttemptState.FINAL_SAVING,
- RMAppAttemptEventType.APP_REJECTED,
- new FinalSavingTransition(new AppRejectedTransition(),
- RMAppAttemptState.FAILED))
.addTransition(RMAppAttemptState.SUBMITTED,
EnumSet.of(RMAppAttemptState.LAUNCHED_UNMANAGED_SAVING,
RMAppAttemptState.SCHEDULED),
- RMAppAttemptEventType.APP_ACCEPTED,
+ RMAppAttemptEventType.ATTEMPT_ADDED,
new ScheduleTransition())
.addTransition(RMAppAttemptState.SUBMITTED, RMAppAttemptState.FINAL_SAVING,
RMAppAttemptEventType.KILL,
@@ -380,8 +373,7 @@ public class RMAppAttemptImpl implements
.addTransition(
RMAppAttemptState.KILLED,
RMAppAttemptState.KILLED,
- EnumSet.of(RMAppAttemptEventType.APP_ACCEPTED,
- RMAppAttemptEventType.APP_REJECTED,
+ EnumSet.of(RMAppAttemptEventType.ATTEMPT_ADDED,
RMAppAttemptEventType.EXPIRE,
RMAppAttemptEventType.LAUNCHED,
RMAppAttemptEventType.LAUNCH_FAILED,
@@ -398,7 +390,7 @@ public class RMAppAttemptImpl implements
RMContext rmContext, YarnScheduler scheduler,
ApplicationMasterService masterService,
ApplicationSubmissionContext submissionContext,
- Configuration conf, String user) {
+ Configuration conf) {
this.conf = conf;
this.applicationAttemptId = appAttemptId;
this.rmContext = rmContext;
@@ -414,7 +406,6 @@ public class RMAppAttemptImpl implements
this.proxiedTrackingUrl = generateProxyUriWithScheme(null);
this.stateMachine = stateMachineFactory.make(this);
- this.user = user;
}
@Override
@@ -526,6 +517,11 @@ public class RMAppAttemptImpl implements
proxiedTrackingUrl = originalTrackingUrl;
}
+ private void invalidateAMHostAndPort() {
+ this.host = "N/A";
+ this.rpcPort = -1;
+ }
+
// This is only used for RMStateStore. Normal operation must invoke the secret
// manager to get the key and not use the local key directly.
@Override
@@ -745,35 +741,8 @@ public class RMAppAttemptImpl implements
appAttempt.rmContext.getAMRMTokenSecretManager());
// Add the applicationAttempt to the scheduler
- appAttempt.eventHandler.handle(
- new AppAttemptAddedSchedulerEvent(appAttempt.applicationAttemptId,
- appAttempt.submissionContext.getQueue(), appAttempt.user));
- }
- }
-
- private static final class AppRejectedTransition extends BaseTransition {
- @Override
- public void transition(RMAppAttemptImpl appAttempt,
- RMAppAttemptEvent event) {
-
- RMAppAttemptRejectedEvent rejectedEvent = (RMAppAttemptRejectedEvent) event;
-
- // Tell the AMS. Unregister from the ApplicationMasterService
- appAttempt.masterService
- .unregisterAttempt(appAttempt.applicationAttemptId);
-
- // Save the diagnostic message
- String message = rejectedEvent.getMessage();
- appAttempt.diagnostics.append(message);
-
- // Send the rejection event to app
- appAttempt.eventHandler.handle(
- new RMAppRejectedEvent(
- rejectedEvent.getApplicationAttemptId().getApplicationId(),
- message)
- );
-
- appAttempt.removeCredentials(appAttempt);
+ appAttempt.eventHandler.handle(new AppAttemptAddedSchedulerEvent(
+ appAttempt.applicationAttemptId));
}
}
@@ -789,11 +758,6 @@ public class RMAppAttemptImpl implements
public RMAppAttemptState transition(RMAppAttemptImpl appAttempt,
RMAppAttemptEvent event) {
if (!appAttempt.submissionContext.getUnmanagedAM()) {
- // Send the acceptance to the app
- appAttempt.eventHandler.handle(new RMAppEvent(event
- .getApplicationAttemptId().getApplicationId(),
- RMAppEventType.APP_ACCEPTED));
-
// Request a container for the AM.
ResourceRequest request =
BuilderUtils.newResourceRequest(
@@ -913,11 +877,6 @@ public class RMAppAttemptImpl implements
FinalApplicationStatus finalStatus = null;
switch (event.getType()) {
- case APP_REJECTED:
- RMAppAttemptRejectedEvent rejectedEvent =
- (RMAppAttemptRejectedEvent) event;
- diags = rejectedEvent.getMessage();
- break;
case LAUNCH_FAILED:
RMAppAttemptLaunchFailedEvent launchFaileEvent =
(RMAppAttemptLaunchFailedEvent) event;
@@ -1033,6 +992,7 @@ public class RMAppAttemptImpl implements
{
// don't leave the tracking URL pointing to a non-existent AM
appAttempt.setTrackingUrlToRMAppPage();
+ appAttempt.invalidateAMHostAndPort();
appEvent =
new RMAppFailedAttemptEvent(applicationId,
RMAppEventType.ATTEMPT_KILLED,
@@ -1043,6 +1003,7 @@ public class RMAppAttemptImpl implements
{
// don't leave the tracking URL pointing to a non-existent AM
appAttempt.setTrackingUrlToRMAppPage();
+ appAttempt.invalidateAMHostAndPort();
appEvent =
new RMAppFailedAttemptEvent(applicationId,
RMAppEventType.ATTEMPT_FAILED,
@@ -1059,7 +1020,6 @@ public class RMAppAttemptImpl implements
appAttempt.eventHandler.handle(appEvent);
appAttempt.eventHandler.handle(new AppAttemptRemovedSchedulerEvent(
appAttemptId, finalAttemptState));
-
appAttempt.removeCredentials(appAttempt);
}
}
@@ -1085,16 +1045,6 @@ public class RMAppAttemptImpl implements
public void transition(RMAppAttemptImpl appAttempt,
RMAppAttemptEvent event) {
appAttempt.checkAttemptStoreError(event);
- // Send the acceptance to the app
- // Ideally this should have been done when the scheduler accepted the app.
- // But its here because until the attempt is saved the client should not
- // launch the unmanaged AM. Client waits for the app status to be accepted
- // before doing so. So we have to delay the accepted state until we have
- // completed storing the attempt
- appAttempt.eventHandler.handle(new RMAppEvent(event
- .getApplicationAttemptId().getApplicationId(),
- RMAppEventType.APP_ACCEPTED));
-
super.transition(appAttempt, event);
}
}
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ActiveUsersManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ActiveUsersManager.java?rev=1556097&r1=1556096&r2=1556097&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ActiveUsersManager.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ActiveUsersManager.java Tue Jan 7 01:07:33 2014
@@ -56,7 +56,7 @@ public class ActiveUsersManager {
* @param user application user
* @param applicationId activated application
*/
- @Lock({Queue.class, SchedulerApplication.class})
+ @Lock({Queue.class, SchedulerApplicationAttempt.class})
synchronized public void activateApplication(
String user, ApplicationId applicationId) {
Set<ApplicationId> userApps = usersApplications.get(user);
@@ -79,7 +79,7 @@ public class ActiveUsersManager {
* @param user application user
* @param applicationId deactivated application
*/
- @Lock({Queue.class, SchedulerApplication.class})
+ @Lock({Queue.class, SchedulerApplicationAttempt.class})
synchronized public void deactivateApplication(
String user, ApplicationId applicationId) {
Set<ApplicationId> userApps = usersApplications.get(user);
@@ -102,7 +102,7 @@ public class ActiveUsersManager {
* resource requests.
* @return number of active users
*/
- @Lock({Queue.class, SchedulerApplication.class})
+ @Lock({Queue.class, SchedulerApplicationAttempt.class})
synchronized public int getNumActiveUsers() {
return activeUsers;
}
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerAppReport.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerAppReport.java?rev=1556097&r1=1556096&r2=1556097&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerAppReport.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerAppReport.java Tue Jan 7 01:07:33 2014
@@ -36,7 +36,7 @@ public class SchedulerAppReport {
private final Collection<RMContainer> reserved;
private final boolean pending;
- public SchedulerAppReport(SchedulerApplication app) {
+ public SchedulerAppReport(SchedulerApplicationAttempt app) {
this.live = app.getLiveContainers();
this.reserved = app.getReservedContainers();
this.pending = app.isPending();
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerAppUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerAppUtils.java?rev=1556097&r1=1556096&r2=1556097&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerAppUtils.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerAppUtils.java Tue Jan 7 01:07:33 2014
@@ -22,7 +22,7 @@ import org.apache.commons.logging.Log;
public class SchedulerAppUtils {
- public static boolean isBlacklisted(SchedulerApplication application,
+ public static boolean isBlacklisted(SchedulerApplicationAttempt application,
SchedulerNode node, Log LOG) {
if (application.isBlacklisted(node.getNodeName())) {
if (LOG.isDebugEnabled()) {