You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by ar...@apache.org on 2013/10/31 19:49:56 UTC
svn commit: r1537584 [1/4] - in
/hadoop/common/branches/HDFS-2832/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-api/src/main/proto/server/
hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/
hadoop-yarn/hadoop-yarn-client/...
Author: arp
Date: Thu Oct 31 18:49:54 2013
New Revision: 1537584
URL: http://svn.apache.org/r1537584
Log:
Merging r1537252 through r1537583 from trunk to branch HDFS-2832
Added:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateUpdateAppAttemptEvent.java
- copied unchanged from r1537583, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateUpdateAppAttemptEvent.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateUpdateAppEvent.java
- copied unchanged from r1537583, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateUpdateAppEvent.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppNewSavedEvent.java
- copied unchanged from r1537583, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppNewSavedEvent.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppUpdateSavedEvent.java
- copied unchanged from r1537583, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppUpdateSavedEvent.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/event/RMAppAttemptNewSavedEvent.java
- copied unchanged from r1537583, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/event/RMAppAttemptNewSavedEvent.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/event/RMAppAttemptUpdateSavedEvent.java
- copied unchanged from r1537583, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/event/RMAppAttemptUpdateSavedEvent.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMReconnect.java
- copied unchanged from r1537583, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMReconnect.java
Removed:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppStoredEvent.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/event/RMAppAttemptStoredEvent.java
Modified:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/NMClient.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/NMTokenCache.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/ContainerManagementProtocolProxy.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/NMClientImpl.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreEventType.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationStateData.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationStateDataPBImpl.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppState.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptEventType.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptState.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/event/RMAppAttemptUnregistrationEvent.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/CHANGES.txt?rev=1537584&r1=1537583&r2=1537584&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/CHANGES.txt Thu Oct 31 18:49:54 2013
@@ -73,6 +73,10 @@ Release 2.3.0 - UNRELEASED
YARN-1306. Clean up hadoop-sls sample-conf according to YARN-1228 (Wei Yan
via Sandy Ryza)
+ YARN-891. Modified ResourceManager state-store to remember completed
+ applications so that clients can get information about them post RM-restart.
+ (Jian He via vinodkv)
+
OPTIMIZATIONS
BUG FIXES
@@ -123,6 +127,9 @@ Release 2.2.1 - UNRELEASED
YARN-1109. Demote NodeManager "Sending out status for container" logs to
debug (haosdent via Sandy Ryza)
+ YARN-1321. Changed NMTokenCache to support both singleton and an instance
+ usage. (Alejandro Abdelnur via vinodkv)
+
OPTIMIZATIONS
BUG FIXES
@@ -161,6 +168,15 @@ Release 2.2.1 - UNRELEASED
YARN-1349. yarn.cmd does not support passthrough to any arbitrary class.
(cnauroth)
+
+ YARN-1357. TestContainerLaunch.testContainerEnvVariables fails on Windows.
+ (Chuan Liu via cnauroth)
+
+ YARN-1358. TestYarnCLI fails on Windows due to line endings. (Chuan Liu via
+ cnauroth)
+
+ YARN-1343. NodeManagers additions/restarts are not reported as node updates
+ in AllocateResponse responses to AMs. (tucu)
Release 2.2.0 - 2013-10-13
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto?rev=1537584&r1=1537583&r2=1537584&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto Thu Oct 31 18:49:54 2013
@@ -71,14 +71,53 @@ message GetGroupsForUserResponseProto {
////////////////////////////////////////////////////////////////////////
////// RM recovery related records /////////////////////////////////////
////////////////////////////////////////////////////////////////////////
+enum RMAppAttemptStateProto {
+ RMATTEMPT_NEW = 1;
+ RMATTEMPT_SUBMITTED = 2;
+ RMATTEMPT_SCHEDULED = 3;
+ RMATTEMPT_ALLOCATED = 4;
+ RMATTEMPT_LAUNCHED = 5;
+ RMATTEMPT_FAILED = 6;
+ RMATTEMPT_RUNNING = 7;
+ RMATTEMPT_FINISHING = 8;
+ RMATTEMPT_FINISHED = 9;
+ RMATTEMPT_KILLED = 10;
+ RMATTEMPT_ALLOCATED_SAVING = 11;
+ RMATTEMPT_LAUNCHED_UNMANAGED_SAVING = 12;
+ RMATTEMPT_RECOVERED = 13;
+ RMATTEMPT_FINAL_SAVING = 14;
+}
+
+enum RMAppStateProto {
+ RMAPP_NEW = 1;
+ RMAPP_NEW_SAVING = 2;
+ RMAPP_SUBMITTED = 3;
+ RMAPP_ACCEPTED = 4;
+ RMAPP_RUNNING = 5;
+ RMAPP_FINAL_SAVING = 6;
+ RMAPP_FINISHING = 7;
+ RMAPP_FINISHED = 8;
+ RMAPP_FAILED = 9;
+ RMAPP_KILLED = 10;
+}
+
message ApplicationStateDataProto {
optional int64 submit_time = 1;
- optional ApplicationSubmissionContextProto application_submission_context = 2;
- optional string user = 3;
+ optional int64 start_time = 2;
+ optional ApplicationSubmissionContextProto application_submission_context = 3;
+ optional string user = 4;
+ optional RMAppStateProto application_state = 5;
+ optional string diagnostics = 6 [default = "N/A"];
+ optional int64 finish_time = 7;
}
message ApplicationAttemptStateDataProto {
optional ApplicationAttemptIdProto attemptId = 1;
optional ContainerProto master_container = 2;
optional bytes app_attempt_tokens = 3;
+ optional RMAppAttemptStateProto app_attempt_state = 4;
+ optional string final_tracking_url = 5;
+ optional string diagnostics = 6 [default = "N/A"];
+ optional int64 start_time = 7;
+ optional FinalApplicationStatusProto final_application_status = 8;
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java?rev=1537584&r1=1537583&r2=1537584&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java Thu Oct 31 18:49:54 2013
@@ -59,9 +59,12 @@ public abstract class AMRMClient<T exten
return client;
}
+ private NMTokenCache nmTokenCache;
+
@Private
protected AMRMClient(String name) {
super(name);
+ nmTokenCache = NMTokenCache.getSingleton();
}
/**
@@ -297,4 +300,33 @@ public abstract class AMRMClient<T exten
*/
public abstract void updateBlacklist(List<String> blacklistAdditions,
List<String> blacklistRemovals);
+
+ /**
+ * Set the NM token cache for the <code>AMRMClient</code>. This cache must
+ * be shared with the {@link NMClient} used to manage containers for the
+ * <code>AMRMClient</code>
+ * <p/>
+ * If a NM token cache is not set, the {@link NMTokenCache#getSingleton()}
+ * singleton instance will be used.
+ *
+ * @param nmTokenCache the NM token cache to use.
+ */
+ public void setNMTokenCache(NMTokenCache nmTokenCache) {
+ this.nmTokenCache = nmTokenCache;
+ }
+
+ /**
+ * Get the NM token cache of the <code>AMRMClient</code>. This cache must be
+ * shared with the {@link NMClient} used to manage containers for the
+ * <code>AMRMClient</code>.
+ * <p/>
+ * If a NM token cache is not set, the {@link NMTokenCache#getSingleton()}
+ * singleton instance will be used.
+ *
+ * @return the NM token cache.
+ */
+ public NMTokenCache getNMTokenCache() {
+ return nmTokenCache;
+ }
+
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/NMClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/NMClient.java?rev=1537584&r1=1537583&r2=1537584&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/NMClient.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/NMClient.java Thu Oct 31 18:49:54 2013
@@ -58,6 +58,8 @@ public abstract class NMClient extends A
return client;
}
+ private NMTokenCache nmTokenCache = NMTokenCache.getSingleton();
+
@Private
protected NMClient(String name) {
super(name);
@@ -118,4 +120,33 @@ public abstract class NMClient extends A
* @param enabled whether the feature is enabled or not
*/
public abstract void cleanupRunningContainersOnStop(boolean enabled);
+
+ /**
+ * Set the NM Token cache of the <code>NMClient</code>. This cache must be
+ * shared with the {@link AMRMClient} that requested the containers managed
+ * by this <code>NMClient</code>
+ * <p/>
+ * If a NM token cache is not set, the {@link NMTokenCache#getSingleton()}
+ * singleton instance will be used.
+ *
+ * @param nmTokenCache the NM token cache to use.
+ */
+ public void setNMTokenCache(NMTokenCache nmTokenCache) {
+ this.nmTokenCache = nmTokenCache;
+ }
+
+ /**
+ * Get the NM token cache of the <code>NMClient</code>. This cache must be
+ * shared with the {@link AMRMClient} that requested the containers managed
+ * by this <code>NMClient</code>
+ * <p/>
+ * If a NM token cache is not set, the {@link NMTokenCache#getSingleton()}
+ * singleton instance will be used.
+ *
+ * @return the NM token cache
+ */
+ public NMTokenCache getNMTokenCache() {
+ return nmTokenCache;
+ }
+
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/NMTokenCache.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/NMTokenCache.java?rev=1537584&r1=1537583&r2=1537584&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/NMTokenCache.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/NMTokenCache.java Thu Oct 31 18:49:54 2013
@@ -23,21 +23,139 @@ import java.util.concurrent.ConcurrentHa
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
+import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
+import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
import org.apache.hadoop.yarn.api.records.Token;
+import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
+import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
import com.google.common.annotations.VisibleForTesting;
/**
- * It manages NMTokens required for communicating with Node manager. Its a
- * static token cache.
+ * NMTokenCache manages NMTokens required for an Application Master
+ * communicating with individual NodeManagers.
+ * <p/>
+ * By default Yarn client libraries {@link AMRMClient} and {@link NMClient} use
+ * {@link #getSingleton()} instance of the cache.
+ * <ul>
+ * <li>Using the singleton instance of the cache is appropriate when running a
+ * single ApplicationMaster in the same JVM.</li>
+ * <li>When using the singleton, users don't need to do anything special,
+ * {@link AMRMClient} and {@link NMClient} are already set up to use the default
+ * singleton {@link NMTokenCache}</li>
+ * </ul>
+ * <p/>
+ * If running multiple Application Masters in the same JVM, a different cache
+ * instance should be used for each Application Master.
+ * <p/>
+ * <ul>
+ * <li>
+ * If using the {@link AMRMClient} and the {@link NMClient}, setting up and using
+ * an instance cache is as follows:
+ * <p/>
+ *
+ * <pre>
+ * NMTokenCache nmTokenCache = new NMTokenCache();
+ * AMRMClient rmClient = AMRMClient.createAMRMClient();
+ * NMClient nmClient = NMClient.createNMClient();
+ * nmClient.setNMTokenCache(nmTokenCache);
+ * ...
+ * </pre>
+ * </li>
+ * <li>
+ * If using the {@link AMRMClientAsync} and the {@link NMClientAsync}, setting up
+ * and using an instance cache is as follows:
+ * <p/>
+ *
+ * <pre>
+ * NMTokenCache nmTokenCache = new NMTokenCache();
+ * AMRMClient rmClient = AMRMClient.createAMRMClient();
+ * NMClient nmClient = NMClient.createNMClient();
+ * nmClient.setNMTokenCache(nmTokenCache);
+ * AMRMClientAsync rmClientAsync = new AMRMClientAsync(rmClient, 1000, [AMRM_CALLBACK]);
+ * NMClientAsync nmClientAsync = new NMClientAsync("nmClient", nmClient, [NM_CALLBACK]);
+ * ...
+ * </pre>
+ * </li>
+ * <li>
+ * If using {@link ApplicationMasterProtocol} and
+ * {@link ContainerManagementProtocol} directly, setting up and using an
+ * instance cache is as follows:
+ * <p/>
+ *
+ * <pre>
+ * NMTokenCache nmTokenCache = new NMTokenCache();
+ * ...
+ * ApplicationMasterProtocol amPro = ClientRMProxy.createRMProxy(conf, ApplicationMasterProtocol.class);
+ * ...
+ * AllocateRequest allocateRequest = ...
+ * ...
+ * AllocateResponse allocateResponse = rmClient.allocate(allocateRequest);
+ * for (NMToken token : allocateResponse.getNMTokens()) {
+ * nmTokenCache.setToken(token.getNodeId().toString(), token.getToken());
+ * }
+ * ...
+ * ContainerManagementProtocolProxy nmPro = ContainerManagementProtocolProxy(conf, nmTokenCache);
+ * ...
+ * nmPro.startContainer(container, containerContext);
+ * ...
+ * </pre>
+ * </li>
+ * </ul>
+ * It is also possible to mix the usage of a client (<code>AMRMClient</code> or
+ * <code>NMClient</code>, or the async versions of them) with a protocol proxy (
+ * <code>ContainerManagementProtocolProxy</code> or
+ * <code>ApplicationMasterProtocol</code>).
*/
@Public
@Evolving
public class NMTokenCache {
- private static ConcurrentHashMap<String, Token> nmTokens;
+ private static final NMTokenCache NM_TOKEN_CACHE = new NMTokenCache();
+ /**
+ * Returns the singleton NM token cache.
+ *
+ * @return the singleton NM token cache.
+ */
+ public static NMTokenCache getSingleton() {
+ return NM_TOKEN_CACHE;
+ }
- static {
+ /**
+ * Returns NMToken, null if absent. Only the singleton obtained from
+ * {@link #getSingleton()} is looked at for the tokens. If you are using your
+ * own NMTokenCache that is different from the singleton, use
+ * {@link #getToken(String) }
+ *
+ * @param nodeAddr
+ * @return {@link Token} NMToken required for communicating with node manager
+ */
+ @Public
+ public static Token getNMToken(String nodeAddr) {
+ return NM_TOKEN_CACHE.getToken(nodeAddr);
+ }
+
+ /**
+ * Sets the NMToken for node address only in the singleton obtained from
+ * {@link #getSingleton()}. If you are using your own NMTokenCache that is
+ * different from the singleton, use {@link #setToken(String, Token) }
+ *
+ * @param nodeAddr
+ * node address (host:port)
+ * @param token
+ * NMToken
+ */
+ @Public
+ public static void setNMToken(String nodeAddr, Token token) {
+ NM_TOKEN_CACHE.setToken(nodeAddr, token);
+ }
+
+ private ConcurrentHashMap<String, Token> nmTokens;
+
+ /**
+ * Creates a NM token cache instance.
+ */
+ public NMTokenCache() {
nmTokens = new ConcurrentHashMap<String, Token>();
}
@@ -45,11 +163,11 @@ public class NMTokenCache {
* Returns NMToken, null if absent
* @param nodeAddr
* @return {@link Token} NMToken required for communicating with node
- * manager
+ * manager
*/
@Public
@Evolving
- public static Token getNMToken(String nodeAddr) {
+ public Token getToken(String nodeAddr) {
return nmTokens.get(nodeAddr);
}
@@ -60,7 +178,7 @@ public class NMTokenCache {
*/
@Public
@Evolving
- public static void setNMToken(String nodeAddr, Token token) {
+ public void setToken(String nodeAddr, Token token) {
nmTokens.put(nodeAddr, token);
}
@@ -69,7 +187,7 @@ public class NMTokenCache {
*/
@Private
@VisibleForTesting
- public static boolean containsNMToken(String nodeAddr) {
+ public boolean containsToken(String nodeAddr) {
return nmTokens.containsKey(nodeAddr);
}
@@ -78,7 +196,7 @@ public class NMTokenCache {
*/
@Private
@VisibleForTesting
- public static int numberOfNMTokensInCache() {
+ public int numberOfTokensInCache() {
return nmTokens.size();
}
@@ -88,7 +206,7 @@ public class NMTokenCache {
*/
@Private
@VisibleForTesting
- public static void removeNMToken(String nodeAddr) {
+ public void removeToken(String nodeAddr) {
nmTokens.remove(nodeAddr);
}
@@ -97,7 +215,7 @@ public class NMTokenCache {
*/
@Private
@VisibleForTesting
- public static void clearCache() {
+ public void clearCache() {
nmTokens.clear();
}
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java?rev=1537584&r1=1537583&r2=1537584&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java Thu Oct 31 18:49:54 2013
@@ -58,7 +58,6 @@ import org.apache.hadoop.yarn.client.Cli
import org.apache.hadoop.yarn.client.api.AMRMClient;
import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
import org.apache.hadoop.yarn.client.api.InvalidContainerRequestException;
-import org.apache.hadoop.yarn.client.api.NMTokenCache;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
@@ -288,12 +287,12 @@ public class AMRMClientImpl<T extends Co
protected void populateNMTokens(AllocateResponse allocateResponse) {
for (NMToken token : allocateResponse.getNMTokens()) {
String nodeId = token.getNodeId().toString();
- if (NMTokenCache.containsNMToken(nodeId)) {
+ if (getNMTokenCache().containsToken(nodeId)) {
LOG.debug("Replacing token for : " + nodeId);
} else {
LOG.debug("Received new token for : " + nodeId);
}
- NMTokenCache.setNMToken(nodeId, token.getToken());
+ getNMTokenCache().setToken(nodeId, token.getToken());
}
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/ContainerManagementProtocolProxy.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/ContainerManagementProtocolProxy.java?rev=1537584&r1=1537583&r2=1537584&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/ContainerManagementProtocolProxy.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/ContainerManagementProtocolProxy.java Thu Oct 31 18:49:54 2013
@@ -56,9 +56,16 @@ public class ContainerManagementProtocol
private final LinkedHashMap<String, ContainerManagementProtocolProxyData> cmProxy;
private final Configuration conf;
private final YarnRPC rpc;
+ private NMTokenCache nmTokenCache;
public ContainerManagementProtocolProxy(Configuration conf) {
+ this(conf, NMTokenCache.getSingleton());
+ }
+
+ public ContainerManagementProtocolProxy(Configuration conf,
+ NMTokenCache nmTokenCache) {
this.conf = conf;
+ this.nmTokenCache = nmTokenCache;
maxConnectedNMs =
conf.getInt(YarnConfiguration.NM_CLIENT_MAX_NM_PROXIES,
@@ -86,7 +93,7 @@ public class ContainerManagementProtocol
while (proxy != null
&& !proxy.token.getIdentifier().equals(
- NMTokenCache.getNMToken(containerManagerBindAddr).getIdentifier())) {
+ nmTokenCache.getToken(containerManagerBindAddr).getIdentifier())) {
LOG.info("Refreshing proxy as NMToken got updated for node : "
+ containerManagerBindAddr);
// Token is updated. check if anyone has already tried closing it.
@@ -109,7 +116,7 @@ public class ContainerManagementProtocol
if (proxy == null) {
proxy =
new ContainerManagementProtocolProxyData(rpc, containerManagerBindAddr,
- containerId, NMTokenCache.getNMToken(containerManagerBindAddr));
+ containerId, nmTokenCache.getToken(containerManagerBindAddr));
if (cmProxy.size() > maxConnectedNMs) {
// Number of existing proxy exceed the limit.
String cmAddr = cmProxy.keySet().iterator().next();
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/NMClientImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/NMClientImpl.java?rev=1537584&r1=1537583&r2=1537584&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/NMClientImpl.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/NMClientImpl.java Thu Oct 31 18:49:54 2013
@@ -130,7 +130,10 @@ public class NMClientImpl extends NMClie
@Override
protected void serviceInit(Configuration conf) throws Exception {
super.serviceInit(conf);
- cmProxy = new ContainerManagementProtocolProxy(conf);
+ if (getNMTokenCache() == null) {
+ throw new IllegalStateException("NMTokenCache has not been set");
+ }
+ cmProxy = new ContainerManagementProtocolProxy(conf, getNMTokenCache());
}
@Override
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java?rev=1537584&r1=1537583&r2=1537584&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java Thu Oct 31 18:49:54 2013
@@ -626,6 +626,13 @@ public class TestAMRMClient {
try {
// start am rm client
amClient = AMRMClient.<ContainerRequest>createAMRMClient();
+
+ //setting an instance NMTokenCache
+ amClient.setNMTokenCache(new NMTokenCache());
+ //asserting we are not using the singleton instance cache
+ Assert.assertNotSame(NMTokenCache.getSingleton(),
+ amClient.getNMTokenCache());
+
amClient.init(conf);
amClient.start();
@@ -681,8 +688,8 @@ public class TestAMRMClient {
int iterationsLeft = 3;
Set<ContainerId> releases = new TreeSet<ContainerId>();
- NMTokenCache.clearCache();
- Assert.assertEquals(0, NMTokenCache.numberOfNMTokensInCache());
+ amClient.getNMTokenCache().clearCache();
+ Assert.assertEquals(0, amClient.getNMTokenCache().numberOfTokensInCache());
HashMap<String, Token> receivedNMTokens = new HashMap<String, Token>();
while (allocatedContainerCount < containersRequestedAny
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java?rev=1537584&r1=1537583&r2=1537584&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java Thu Oct 31 18:49:54 2013
@@ -78,6 +78,7 @@ public class TestNMClient {
List<NodeReport> nodeReports = null;
ApplicationAttemptId attemptId = null;
int nodeCount = 3;
+ NMTokenCache nmTokenCache = null;
@Before
public void setup() throws YarnException, IOException {
@@ -155,10 +156,16 @@ public class TestNMClient {
.createRemoteUser(UserGroupInformation.getCurrentUser().getUserName()));
UserGroupInformation.getCurrentUser().addToken(appAttempt.getAMRMToken());
+ //creating an instance NMTokenCase
+ nmTokenCache = new NMTokenCache();
+
// start am rm client
rmClient =
(AMRMClientImpl<ContainerRequest>) AMRMClient
.<ContainerRequest> createAMRMClient();
+
+ //setting an instance NMTokenCase
+ rmClient.setNMTokenCache(nmTokenCache);
rmClient.init(conf);
rmClient.start();
assertNotNull(rmClient);
@@ -166,6 +173,9 @@ public class TestNMClient {
// start am nm client
nmClient = (NMClientImpl) NMClient.createNMClient();
+
+ //propagating the AMRMClient NMTokenCache instance
+ nmClient.setNMTokenCache(rmClient.getNMTokenCache());
nmClient.init(conf);
nmClient.start();
assertNotNull(nmClient);
@@ -258,7 +268,7 @@ public class TestNMClient {
}
if (!allocResponse.getNMTokens().isEmpty()) {
for (NMToken token : allocResponse.getNMTokens()) {
- NMTokenCache.setNMToken(token.getNodeId().toString(),
+ rmClient.getNMTokenCache().setToken(token.getNodeId().toString(),
token.getToken());
}
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java?rev=1537584&r1=1537583&r2=1537584&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java Thu Oct 31 18:49:54 2013
@@ -875,8 +875,8 @@ public class TestYarnCLI {
ApplicationCLI cli = createAndGetAppCLI();
int result = cli.run(new String[] { "-status" });
Assert.assertEquals(result, -1);
- Assert.assertEquals("Missing argument for options\n"
- + createApplicationCLIHelpMessage(), sysOutStream.toString());
+ Assert.assertEquals(String.format("Missing argument for options%n%1s",
+ createApplicationCLIHelpMessage()), sysOutStream.toString());
sysOutStream.reset();
NodeCLI nodeCLI = new NodeCLI();
@@ -885,8 +885,8 @@ public class TestYarnCLI {
nodeCLI.setSysErrPrintStream(sysErr);
result = nodeCLI.run(new String[] { "-status" });
Assert.assertEquals(result, -1);
- Assert.assertEquals("Missing argument for options\n"
- + createNodeCLIHelpMessage(), sysOutStream.toString());
+ Assert.assertEquals(String.format("Missing argument for options%n%1s",
+ createNodeCLIHelpMessage()), sysOutStream.toString());
}
private void verifyUsageInfo(YarnCLI cli) throws Exception {
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java?rev=1537584&r1=1537583&r2=1537584&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java Thu Oct 31 18:49:54 2013
@@ -388,8 +388,8 @@ public class TestContainerLaunch extends
+ processStartFile);
for (String serviceName : containerManager.getAuxServiceMetaData()
.keySet()) {
- fileWriter.println("@echo" + AuxiliaryServiceHelper.NM_AUX_SERVICE
- + serviceName + " >> "
+ fileWriter.println("@echo %" + AuxiliaryServiceHelper.NM_AUX_SERVICE
+ + serviceName + "%>> "
+ processStartFile);
}
fileWriter.println("@echo " + cId + ">> " + processStartFile);
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java?rev=1537584&r1=1537583&r2=1537584&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java Thu Oct 31 18:49:54 2013
@@ -160,17 +160,14 @@ public class NodesListManager extends Ab
if (unusableRMNodesConcurrentSet.contains(eventNode)) {
LOG.debug(eventNode + " reported usable");
unusableRMNodesConcurrentSet.remove(eventNode);
- for (RMApp app : rmContext.getRMApps().values()) {
- this.rmContext
- .getDispatcher()
- .getEventHandler()
- .handle(
- new RMAppNodeUpdateEvent(app.getApplicationId(), eventNode,
- RMAppNodeUpdateType.NODE_USABLE));
- }
- } else {
- LOG.warn(eventNode
- + " reported usable without first reporting unusable");
+ }
+ for (RMApp app : rmContext.getRMApps().values()) {
+ this.rmContext
+ .getDispatcher()
+ .getEventHandler()
+ .handle(
+ new RMAppNodeUpdateEvent(app.getApplicationId(), eventNode,
+ RMAppNodeUpdateType.NODE_USABLE));
}
break;
default:
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java?rev=1537584&r1=1537583&r2=1537584&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java Thu Oct 31 18:49:54 2013
@@ -63,7 +63,6 @@ public class RMAppManager implements Eve
private static final Log LOG = LogFactory.getLog(RMAppManager.class);
private int completedAppsMax = YarnConfiguration.DEFAULT_RM_MAX_COMPLETED_APPLICATIONS;
- private int globalMaxAppAttempts;
private LinkedList<ApplicationId> completedApps = new LinkedList<ApplicationId>();
private final RMContext rmContext;
@@ -83,8 +82,6 @@ public class RMAppManager implements Eve
setCompletedAppsMax(conf.getInt(
YarnConfiguration.RM_MAX_COMPLETED_APPLICATIONS,
YarnConfiguration.DEFAULT_RM_MAX_COMPLETED_APPLICATIONS));
- globalMaxAppAttempts = conf.getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
- YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
}
/**
@@ -302,10 +299,11 @@ public class RMAppManager implements Eve
throw RPCUtil.getRemoteException(ie);
}
- // All done, start the RMApp
- this.rmContext.getDispatcher().getEventHandler().handle(
- new RMAppEvent(applicationId, isRecovered ? RMAppEventType.RECOVER:
- RMAppEventType.START));
+ if (!isRecovered) {
+ // All done, start the RMApp
+ this.rmContext.getDispatcher().getEventHandler()
+ .handle(new RMAppEvent(applicationId, RMAppEventType.START));
+ }
}
private Credentials parseCredentials(ApplicationSubmissionContext application)
@@ -328,53 +326,19 @@ public class RMAppManager implements Eve
// recover applications
Map<ApplicationId, ApplicationState> appStates = state.getApplicationState();
LOG.info("Recovering " + appStates.size() + " applications");
- for(ApplicationState appState : appStates.values()) {
- boolean shouldRecover = true;
- if(appState.getApplicationSubmissionContext().getUnmanagedAM()) {
- // do not recover unmanaged applications since current recovery
- // mechanism of restarting attempts does not work for them.
- // This will need to be changed in work preserving recovery in which
- // RM will re-connect with the running AM's instead of restarting them
- LOG.info("Not recovering unmanaged application " + appState.getAppId());
- shouldRecover = false;
- }
- int individualMaxAppAttempts = appState.getApplicationSubmissionContext()
- .getMaxAppAttempts();
- int maxAppAttempts;
- if (individualMaxAppAttempts <= 0 ||
- individualMaxAppAttempts > globalMaxAppAttempts) {
- maxAppAttempts = globalMaxAppAttempts;
- LOG.warn("The specific max attempts: " + individualMaxAppAttempts
- + " for application: " + appState.getAppId()
- + " is invalid, because it is out of the range [1, "
- + globalMaxAppAttempts + "]. Use the global max attempts instead.");
- } else {
- maxAppAttempts = individualMaxAppAttempts;
- }
- // In work-preserve restart, if attemptCount == maxAttempts, the job still
- // needs to be recovered because the last attempt may still be running.
- if(appState.getAttemptCount() >= maxAppAttempts) {
- LOG.info("Not recovering application " + appState.getAppId() +
- " due to recovering attempt is beyond maxAppAttempt limit");
- shouldRecover = false;
- }
-
- // re-submit the application
- // this is going to send an app start event but since the async dispatcher
- // has not started that event will be queued until we have completed re
- // populating the state
- if(shouldRecover) {
- LOG.info("Recovering application " + appState.getAppId());
- submitApplication(appState.getApplicationSubmissionContext(),
- appState.getSubmitTime(), true, appState.getUser());
- // re-populate attempt information in application
- RMAppImpl appImpl = (RMAppImpl) rmContext.getRMApps().get(
- appState.getAppId());
- appImpl.recover(state);
- }
- else {
- store.removeApplication(appState);
- }
+ for (ApplicationState appState : appStates.values()) {
+ LOG.info("Recovering application " + appState.getAppId());
+ submitApplication(appState.getApplicationSubmissionContext(),
+ appState.getSubmitTime(), true, appState.getUser());
+ // re-populate attempt information in application
+ RMAppImpl appImpl =
+ (RMAppImpl) rmContext.getRMApps().get(appState.getAppId());
+ appImpl.recover(state);
+ // Recover the app synchronously, as otherwise client is possible to see
+ // the application not recovered before it is actually recovered because
+ // ClientRMService is already started at this point of time.
+ appImpl.handle(new RMAppEvent(appImpl.getApplicationId(),
+ RMAppEventType.RECOVER));
}
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java?rev=1537584&r1=1537583&r2=1537584&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java Thu Oct 31 18:49:54 2013
@@ -133,8 +133,11 @@ public class FileSystemRMStateStore exte
ApplicationStateDataProto.parseFrom(childData));
ApplicationState appState =
new ApplicationState(appStateData.getSubmitTime(),
+ appStateData.getStartTime(),
appStateData.getApplicationSubmissionContext(),
- appStateData.getUser());
+ appStateData.getUser(),
+ appStateData.getState(),
+ appStateData.getDiagnostics(), appStateData.getFinishTime());
// assert child node name is same as actual applicationId
assert appId.equals(appState.context.getApplicationId());
rmState.appState.put(appId, appState);
@@ -156,7 +159,12 @@ public class FileSystemRMStateStore exte
}
ApplicationAttemptState attemptState =
new ApplicationAttemptState(attemptId,
- attemptStateData.getMasterContainer(), credentials);
+ attemptStateData.getMasterContainer(), credentials,
+ attemptStateData.getStartTime(),
+ attemptStateData.getState(),
+ attemptStateData.getFinalTrackingUrl(),
+ attemptStateData.getDiagnostics(),
+ attemptStateData.getFinalApplicationStatus());
// assert child node name is same as application attempt id
assert attemptId.equals(attemptState.getAttemptId());
@@ -232,7 +240,7 @@ public class FileSystemRMStateStore exte
}
@Override
- public synchronized void storeApplicationState(String appId,
+ public synchronized void storeApplicationStateInternal(String appId,
ApplicationStateDataPBImpl appStateDataPB) throws Exception {
Path appDirPath = getAppDir(rmAppRoot, appId);
fs.mkdirs(appDirPath);
@@ -251,15 +259,34 @@ public class FileSystemRMStateStore exte
}
@Override
- public synchronized void storeApplicationAttemptState(String attemptId,
- ApplicationAttemptStateDataPBImpl attemptStateDataPB) throws Exception {
+ public synchronized void updateApplicationStateInternal(String appId,
+ ApplicationStateDataPBImpl appStateDataPB) throws Exception {
+ Path appDirPath = getAppDir(rmAppRoot, appId);
+ Path nodeCreatePath = getNodePath(appDirPath, appId);
+
+ LOG.info("Updating info for app: " + appId + " at: " + nodeCreatePath);
+ byte[] appStateData = appStateDataPB.getProto().toByteArray();
+ try {
+ // currently throw all exceptions. May need to respond differently for HA
+ // based on whether we have lost the right to write to FS
+ updateFile(nodeCreatePath, appStateData);
+ } catch (Exception e) {
+ LOG.info("Error updating info for app: " + appId, e);
+ throw e;
+ }
+ }
+
+ @Override
+ public synchronized void storeApplicationAttemptStateInternal(
+ String attemptId, ApplicationAttemptStateDataPBImpl attemptStateDataPB)
+ throws Exception {
ApplicationAttemptId appAttemptId =
ConverterUtils.toApplicationAttemptId(attemptId);
Path appDirPath =
getAppDir(rmAppRoot, appAttemptId.getApplicationId().toString());
Path nodeCreatePath = getNodePath(appDirPath, attemptId);
- LOG.info("Storing info for attempt: " + attemptId
- + " at: " + nodeCreatePath);
+ LOG.info("Storing info for attempt: " + attemptId + " at: "
+ + nodeCreatePath);
byte[] attemptStateData = attemptStateDataPB.getProto().toByteArray();
try {
// currently throw all exceptions. May need to respond differently for HA
@@ -272,6 +299,28 @@ public class FileSystemRMStateStore exte
}
@Override
+ public synchronized void updateApplicationAttemptStateInternal(
+ String attemptId, ApplicationAttemptStateDataPBImpl attemptStateDataPB)
+ throws Exception {
+ ApplicationAttemptId appAttemptId =
+ ConverterUtils.toApplicationAttemptId(attemptId);
+ Path appDirPath =
+ getAppDir(rmAppRoot, appAttemptId.getApplicationId().toString());
+ Path nodeCreatePath = getNodePath(appDirPath, attemptId);
+ LOG.info("Updating info for attempt: " + attemptId + " at: "
+ + nodeCreatePath);
+ byte[] attemptStateData = attemptStateDataPB.getProto().toByteArray();
+ try {
+ // currently throw all exceptions. May need to respond differently for HA
+ // based on whether we have lost the right to write to FS
+ updateFile(nodeCreatePath, attemptStateData);
+ } catch (Exception e) {
+ LOG.info("Error updating info for attempt: " + attemptId, e);
+ throw e;
+ }
+ }
+
+ @Override
public synchronized void removeApplicationState(ApplicationState appState)
throws Exception {
String appId = appState.getAppId().toString();
@@ -373,12 +422,21 @@ public class FileSystemRMStateStore exte
Path tempPath =
new Path(outputPath.getParent(), outputPath.getName() + ".tmp");
FSDataOutputStream fsOut = null;
- fsOut = fs.create(tempPath, false);
+ // This file will be overwritten when app/attempt finishes for saving the
+ // final status.
+ fsOut = fs.create(tempPath, true);
fsOut.write(data);
fsOut.close();
fs.rename(tempPath, outputPath);
}
+ private void updateFile(Path outputPath, byte[] data) throws Exception {
+ if (fs.exists(outputPath)) {
+ deleteFile(outputPath);
+ }
+ writeFile(outputPath, data);
+ }
+
private boolean renameFile(Path src, Path dst) throws Exception {
return fs.rename(src, dst);
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java?rev=1537584&r1=1537583&r2=1537584&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java Thu Oct 31 18:49:54 2013
@@ -76,22 +76,39 @@ public class MemoryRMStateStore extends
}
@Override
- public void storeApplicationState(String appId,
+ public void storeApplicationStateInternal(String appId,
ApplicationStateDataPBImpl appStateData)
throws Exception {
- ApplicationState appState = new ApplicationState(
- appStateData.getSubmitTime(),
- appStateData.getApplicationSubmissionContext(), appStateData.getUser());
- if (state.appState.containsKey(appState.getAppId())) {
- Exception e = new IOException("App: " + appId + " is already stored.");
- LOG.info("Error storing info for app: " + appId, e);
- throw e;
- }
+ ApplicationState appState =
+ new ApplicationState(appStateData.getSubmitTime(),
+ appStateData.getStartTime(),
+ appStateData.getApplicationSubmissionContext(),
+ appStateData.getUser());
state.appState.put(appState.getAppId(), appState);
}
@Override
- public synchronized void storeApplicationAttemptState(String attemptIdStr,
+ public void updateApplicationStateInternal(String appId,
+ ApplicationStateDataPBImpl appStateData) throws Exception {
+ ApplicationState updatedAppState =
+ new ApplicationState(appStateData.getSubmitTime(),
+ appStateData.getStartTime(),
+ appStateData.getApplicationSubmissionContext(),
+ appStateData.getUser(), appStateData.getState(),
+ appStateData.getDiagnostics(), appStateData.getFinishTime());
+ LOG.info("Updating final state " + appStateData.getState() + " for app: "
+ + appId);
+ ApplicationId applicationId = updatedAppState.getAppId();
+ if (state.appState.get(applicationId) != null) {
+ // add the earlier attempts back
+ updatedAppState.attempts
+ .putAll(state.appState.get(applicationId).attempts);
+ }
+ state.appState.put(applicationId, updatedAppState);
+ }
+
+ @Override
+ public synchronized void storeApplicationAttemptStateInternal(String attemptIdStr,
ApplicationAttemptStateDataPBImpl attemptStateData)
throws Exception {
ApplicationAttemptId attemptId = ConverterUtils
@@ -105,22 +122,48 @@ public class MemoryRMStateStore extends
}
ApplicationAttemptState attemptState =
new ApplicationAttemptState(attemptId,
- attemptStateData.getMasterContainer(), credentials);
+ attemptStateData.getMasterContainer(), credentials,
+ attemptStateData.getStartTime());
ApplicationState appState = state.getApplicationState().get(
attemptState.getAttemptId().getApplicationId());
if (appState == null) {
throw new YarnRuntimeException("Application doesn't exist");
}
+ appState.attempts.put(attemptState.getAttemptId(), attemptState);
+ }
- if (appState.attempts.containsKey(attemptState.getAttemptId())) {
- Exception e = new IOException("Attempt: " +
- attemptState.getAttemptId() + " is already stored.");
- LOG.info("Error storing info for attempt: " +
- attemptState.getAttemptId(), e);
- throw e;
+ @Override
+ public synchronized void updateApplicationAttemptStateInternal(
+ String attemptIdStr, ApplicationAttemptStateDataPBImpl attemptStateData)
+ throws Exception {
+ ApplicationAttemptId attemptId =
+ ConverterUtils.toApplicationAttemptId(attemptIdStr);
+ Credentials credentials = null;
+ if (attemptStateData.getAppAttemptTokens() != null) {
+ DataInputByteBuffer dibb = new DataInputByteBuffer();
+ credentials = new Credentials();
+ dibb.reset(attemptStateData.getAppAttemptTokens());
+ credentials.readTokenStorageStream(dibb);
}
- appState.attempts.put(attemptState.getAttemptId(), attemptState);
+ ApplicationAttemptState updatedAttemptState =
+ new ApplicationAttemptState(attemptId,
+ attemptStateData.getMasterContainer(), credentials,
+ attemptStateData.getStartTime(), attemptStateData.getState(),
+ attemptStateData.getFinalTrackingUrl(),
+ attemptStateData.getDiagnostics(),
+ attemptStateData.getFinalApplicationStatus());
+
+ ApplicationState appState =
+ state.getApplicationState().get(
+ updatedAttemptState.getAttemptId().getApplicationId());
+ if (appState == null) {
+ throw new YarnRuntimeException("Application doesn't exist");
+ }
+ LOG.info("Updating final state " + updatedAttemptState.getState()
+ + " for attempt: " + updatedAttemptState.getAttemptId());
+ appState.attempts.put(updatedAttemptState.getAttemptId(),
+ updatedAttemptState);
}
@Override
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java?rev=1537584&r1=1537583&r2=1537584&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java Thu Oct 31 18:49:54 2013
@@ -50,13 +50,13 @@ public class NullRMStateStore extends RM
}
@Override
- protected void storeApplicationState(String appId,
+ protected void storeApplicationStateInternal(String appId,
ApplicationStateDataPBImpl appStateData) throws Exception {
// Do nothing
}
@Override
- protected void storeApplicationAttemptState(String attemptId,
+ protected void storeApplicationAttemptStateInternal(String attemptId,
ApplicationAttemptStateDataPBImpl attemptStateData) throws Exception {
// Do nothing
}
@@ -89,4 +89,16 @@ public class NullRMStateStore extends RM
public void removeRMDTMasterKeyState(DelegationKey delegationKey) throws Exception {
// Do nothing
}
+
+ @Override
+ protected void updateApplicationStateInternal(String appId,
+ ApplicationStateDataPBImpl appStateData) throws Exception {
+ // Do nothing
+ }
+
+ @Override
+ protected void updateApplicationAttemptStateInternal(String attemptId,
+ ApplicationAttemptStateDataPBImpl attemptStateData) throws Exception {
+ // Do nothing
+ }
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java?rev=1537584&r1=1537583&r2=1537584&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java Thu Oct 31 18:49:54 2013
@@ -41,6 +41,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationSubmissionContextPBImpl;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.Dispatcher;
@@ -50,10 +51,14 @@ import org.apache.hadoop.yarn.security.c
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationAttemptStateDataPBImpl;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationStateDataPBImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
-import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppStoredEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppRemovedEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppNewSavedEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppUpdateSavedEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
-import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptStoredEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptNewSavedEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptUpdateSavedEvent;
@Private
@Unstable
@@ -86,13 +91,32 @@ public abstract class RMStateStore exten
final ApplicationAttemptId attemptId;
final Container masterContainer;
final Credentials appAttemptCredentials;
+ long startTime = 0;
+ // fields set when attempt completes
+ RMAppAttemptState state;
+ String finalTrackingUrl = "N/A";
+ String diagnostics;
+ FinalApplicationStatus amUnregisteredFinalStatus;
public ApplicationAttemptState(ApplicationAttemptId attemptId,
- Container masterContainer,
- Credentials appAttemptCredentials) {
+ Container masterContainer, Credentials appAttemptCredentials,
+ long startTime) {
+ this(attemptId, masterContainer, appAttemptCredentials, startTime, null,
+ null, "", null);
+ }
+
+ public ApplicationAttemptState(ApplicationAttemptId attemptId,
+ Container masterContainer, Credentials appAttemptCredentials,
+ long startTime, RMAppAttemptState state, String finalTrackingUrl,
+ String diagnostics, FinalApplicationStatus amUnregisteredFinalStatus) {
this.attemptId = attemptId;
this.masterContainer = masterContainer;
this.appAttemptCredentials = appAttemptCredentials;
+ this.startTime = startTime;
+ this.state = state;
+ this.finalTrackingUrl = finalTrackingUrl;
+ this.diagnostics = diagnostics == null ? "" : diagnostics;
+ this.amUnregisteredFinalStatus = amUnregisteredFinalStatus;
}
public Container getMasterContainer() {
@@ -104,6 +128,21 @@ public abstract class RMStateStore exten
public Credentials getAppAttemptCredentials() {
return appAttemptCredentials;
}
+ public RMAppAttemptState getState(){
+ return state;
+ }
+ public String getFinalTrackingUrl() {
+ return finalTrackingUrl;
+ }
+ public String getDiagnostics() {
+ return diagnostics;
+ }
+ public long getStartTime() {
+ return startTime;
+ }
+ public FinalApplicationStatus getFinalApplicationStatus() {
+ return amUnregisteredFinalStatus;
+ }
}
/**
@@ -112,15 +151,30 @@ public abstract class RMStateStore exten
public static class ApplicationState {
final ApplicationSubmissionContext context;
final long submitTime;
+ final long startTime;
final String user;
Map<ApplicationAttemptId, ApplicationAttemptState> attempts =
new HashMap<ApplicationAttemptId, ApplicationAttemptState>();
-
- ApplicationState(long submitTime, ApplicationSubmissionContext context,
- String user) {
+ // fields set when application completes.
+ RMAppState state;
+ String diagnostics;
+ long finishTime;
+
+ public ApplicationState(long submitTime,
+ long startTime, ApplicationSubmissionContext context, String user) {
+ this(submitTime, startTime, context, user, null, "", 0);
+ }
+
+ public ApplicationState(long submitTime,
+ long startTime,ApplicationSubmissionContext context,
+ String user, RMAppState state, String diagnostics, long finishTime) {
this.submitTime = submitTime;
+ this.startTime = startTime;
this.context = context;
this.user = user;
+ this.state = state;
+ this.diagnostics = diagnostics == null ? "" : diagnostics;
+ this.finishTime = finishTime;
}
public ApplicationId getAppId() {
@@ -129,6 +183,9 @@ public abstract class RMStateStore exten
public long getSubmitTime() {
return submitTime;
}
+ public long getStartTime() {
+ return startTime;
+ }
public int getAttemptCount() {
return attempts.size();
}
@@ -141,6 +198,15 @@ public abstract class RMStateStore exten
public String getUser() {
return user;
}
+ public RMAppState getState() {
+ return state;
+ }
+ public String getDiagnostics() {
+ return diagnostics;
+ }
+ public long getFinishTime() {
+ return finishTime;
+ }
}
public static class RMDTSecretManagerState {
@@ -249,23 +315,31 @@ public abstract class RMStateStore exten
* RMAppStoredEvent will be sent on completion to notify the RMApp
*/
@SuppressWarnings("unchecked")
- public synchronized void storeApplication(RMApp app) {
+ public synchronized void storeNewApplication(RMApp app) {
ApplicationSubmissionContext context = app
.getApplicationSubmissionContext();
assert context instanceof ApplicationSubmissionContextPBImpl;
- ApplicationState appState = new ApplicationState(
- app.getSubmitTime(), context, app.getUser());
+ ApplicationState appState =
+ new ApplicationState(app.getSubmitTime(), app.getStartTime(), context,
+ app.getUser());
dispatcher.getEventHandler().handle(new RMStateStoreAppEvent(appState));
}
-
+
+ @SuppressWarnings("unchecked")
+ public synchronized void updateApplicationState(ApplicationState appState) {
+ dispatcher.getEventHandler().handle(new RMStateUpdateAppEvent(appState));
+ }
+
/**
* Blocking API
* Derived classes must implement this method to store the state of an
* application.
*/
- protected abstract void storeApplicationState(String appId,
- ApplicationStateDataPBImpl appStateData)
- throws Exception;
+ protected abstract void storeApplicationStateInternal(String appId,
+ ApplicationStateDataPBImpl appStateData) throws Exception;
+
+ protected abstract void updateApplicationStateInternal(String appId,
+ ApplicationStateDataPBImpl appStateData) throws Exception;
@SuppressWarnings("unchecked")
/**
@@ -274,26 +348,35 @@ public abstract class RMStateStore exten
* This does not block the dispatcher threads
* RMAppAttemptStoredEvent will be sent on completion to notify the RMAppAttempt
*/
- public synchronized void storeApplicationAttempt(RMAppAttempt appAttempt) {
+ public synchronized void storeNewApplicationAttempt(RMAppAttempt appAttempt) {
Credentials credentials = getCredentialsFromAppAttempt(appAttempt);
ApplicationAttemptState attemptState =
new ApplicationAttemptState(appAttempt.getAppAttemptId(),
- appAttempt.getMasterContainer(), credentials);
+ appAttempt.getMasterContainer(), credentials,
+ appAttempt.getStartTime());
dispatcher.getEventHandler().handle(
new RMStateStoreAppAttemptEvent(attemptState));
}
-
+
+ @SuppressWarnings("unchecked")
+ public synchronized void updateApplicationAttemptState(
+ ApplicationAttemptState attemptState) {
+ dispatcher.getEventHandler().handle(
+ new RMStateUpdateAppAttemptEvent(attemptState));
+ }
+
/**
* Blocking API
* Derived classes must implement this method to store the state of an
* application attempt
*/
- protected abstract void storeApplicationAttemptState(String attemptId,
- ApplicationAttemptStateDataPBImpl attemptStateData)
- throws Exception;
+ protected abstract void storeApplicationAttemptStateInternal(String attemptId,
+ ApplicationAttemptStateDataPBImpl attemptStateData) throws Exception;
+ protected abstract void updateApplicationAttemptStateInternal(String attemptId,
+ ApplicationAttemptStateDataPBImpl attemptStateData) throws Exception;
/**
* RMDTSecretManager call this to store the state of a delegation token
@@ -372,13 +455,14 @@ public abstract class RMStateStore exten
*/
public synchronized void removeApplication(RMApp app) {
ApplicationState appState = new ApplicationState(
- app.getSubmitTime(), app.getApplicationSubmissionContext(),
- app.getUser());
+ app.getSubmitTime(), app.getStartTime(),
+ app.getApplicationSubmissionContext(), app.getUser());
for(RMAppAttempt appAttempt : app.getAppAttempts().values()) {
Credentials credentials = getCredentialsFromAppAttempt(appAttempt);
ApplicationAttemptState attemptState =
new ApplicationAttemptState(appAttempt.getAppAttemptId(),
- appAttempt.getMasterContainer(), credentials);
+ appAttempt.getMasterContainer(), credentials,
+ appAttempt.getStartTime());
appState.attempts.put(attemptState.getAttemptId(), attemptState);
}
@@ -409,7 +493,7 @@ public abstract class RMStateStore exten
public static final Text AM_CLIENT_TOKEN_MASTER_KEY_NAME =
new Text("YARN_CLIENT_TOKEN_MASTER_KEY");
- private Credentials getCredentialsFromAppAttempt(RMAppAttempt appAttempt) {
+ public Credentials getCredentialsFromAppAttempt(RMAppAttempt appAttempt) {
Credentials credentials = new Credentials();
Token<AMRMTokenIdentifier> appToken = appAttempt.getAMRMToken();
if(appToken != null){
@@ -427,92 +511,123 @@ public abstract class RMStateStore exten
// Dispatcher related code
private synchronized void handleStoreEvent(RMStateStoreEvent event) {
- switch(event.getType()) {
- case STORE_APP:
- {
- ApplicationState apptState =
- ((RMStateStoreAppEvent) event).getAppState();
- Exception storedException = null;
- ApplicationStateDataPBImpl appStateData =
- new ApplicationStateDataPBImpl();
- appStateData.setSubmitTime(apptState.getSubmitTime());
- appStateData.setApplicationSubmissionContext(
- apptState.getApplicationSubmissionContext());
- appStateData.setUser(apptState.getUser());
- ApplicationId appId =
- apptState.getApplicationSubmissionContext().getApplicationId();
-
- LOG.info("Storing info for app: " + appId);
- try {
- storeApplicationState(appId.toString(), appStateData);
- } catch (Exception e) {
- LOG.error("Error storing app: " + appId, e);
- storedException = e;
- } finally {
- notifyDoneStoringApplication(appId, storedException);
- }
+ if (event.getType().equals(RMStateStoreEventType.STORE_APP)
+ || event.getType().equals(RMStateStoreEventType.UPDATE_APP)) {
+ ApplicationState appState = null;
+ if (event.getType().equals(RMStateStoreEventType.STORE_APP)) {
+ appState = ((RMStateStoreAppEvent) event).getAppState();
+ } else {
+ assert event.getType().equals(RMStateStoreEventType.UPDATE_APP);
+ appState = ((RMStateUpdateAppEvent) event).getAppState();
+ }
+
+ Exception storedException = null;
+ ApplicationStateDataPBImpl appStateData =
+ (ApplicationStateDataPBImpl) ApplicationStateDataPBImpl
+ .newApplicationStateData(appState.getSubmitTime(),
+ appState.getStartTime(), appState.getUser(),
+ appState.getApplicationSubmissionContext(), appState.getState(),
+ appState.getDiagnostics(), appState.getFinishTime());
+
+ ApplicationId appId =
+ appState.getApplicationSubmissionContext().getApplicationId();
+
+ LOG.info("Storing info for app: " + appId);
+ try {
+ if (event.getType().equals(RMStateStoreEventType.STORE_APP)) {
+ storeApplicationStateInternal(appId.toString(), appStateData);
+ } else {
+ assert event.getType().equals(RMStateStoreEventType.UPDATE_APP);
+ updateApplicationStateInternal(appId.toString(), appStateData);
+ }
+ } catch (Exception e) {
+ LOG.error("Error storing app: " + appId, e);
+ storedException = e;
+ } finally {
+ if (event.getType().equals(RMStateStoreEventType.STORE_APP)) {
+ notifyDoneStoringApplication(appId, storedException);
+ } else {
+ notifyDoneUpdatingApplication(appId, storedException);
+ }
+ }
+ } else if (event.getType().equals(RMStateStoreEventType.STORE_APP_ATTEMPT)
+ || event.getType().equals(RMStateStoreEventType.UPDATE_APP_ATTEMPT)) {
+
+ ApplicationAttemptState attemptState = null;
+ if (event.getType().equals(RMStateStoreEventType.STORE_APP_ATTEMPT)) {
+ attemptState =
+ ((RMStateStoreAppAttemptEvent) event).getAppAttemptState();
+ } else {
+ assert event.getType().equals(RMStateStoreEventType.UPDATE_APP_ATTEMPT);
+ attemptState =
+ ((RMStateUpdateAppAttemptEvent) event).getAppAttemptState();
+ }
+
+ Exception storedException = null;
+ Credentials credentials = attemptState.getAppAttemptCredentials();
+ ByteBuffer appAttemptTokens = null;
+ try {
+ if (credentials != null) {
+ DataOutputBuffer dob = new DataOutputBuffer();
+ credentials.writeTokenStorageToStream(dob);
+ appAttemptTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
+ }
+ ApplicationAttemptStateDataPBImpl attemptStateData =
+ (ApplicationAttemptStateDataPBImpl) ApplicationAttemptStateDataPBImpl
+ .newApplicationAttemptStateData(attemptState.getAttemptId(),
+ attemptState.getMasterContainer(), appAttemptTokens,
+ attemptState.getStartTime(), attemptState.getState(),
+ attemptState.getFinalTrackingUrl(),
+ attemptState.getDiagnostics(),
+ attemptState.getFinalApplicationStatus());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Storing info for attempt: " + attemptState.getAttemptId());
}
- break;
- case STORE_APP_ATTEMPT:
- {
- ApplicationAttemptState attemptState =
- ((RMStateStoreAppAttemptEvent) event).getAppAttemptState();
- Exception storedException = null;
-
- Credentials credentials = attemptState.getAppAttemptCredentials();
- ByteBuffer appAttemptTokens = null;
- try {
- if(credentials != null){
- DataOutputBuffer dob = new DataOutputBuffer();
- credentials.writeTokenStorageToStream(dob);
- appAttemptTokens =
- ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
- }
- ApplicationAttemptStateDataPBImpl attemptStateData =
- (ApplicationAttemptStateDataPBImpl) ApplicationAttemptStateDataPBImpl
- .newApplicationAttemptStateData(attemptState.getAttemptId(),
- attemptState.getMasterContainer(), appAttemptTokens);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Storing info for attempt: " + attemptState.getAttemptId());
- }
- storeApplicationAttemptState(attemptState.getAttemptId().toString(),
- attemptStateData);
- } catch (Exception e) {
- LOG.error("Error storing appAttempt: "
- + attemptState.getAttemptId(), e);
- storedException = e;
- } finally {
- notifyDoneStoringApplicationAttempt(attemptState.getAttemptId(),
- storedException);
- }
+ if (event.getType().equals(RMStateStoreEventType.STORE_APP_ATTEMPT)) {
+ storeApplicationAttemptStateInternal(attemptState.getAttemptId()
+ .toString(), attemptStateData);
+ } else {
+ assert event.getType().equals(
+ RMStateStoreEventType.UPDATE_APP_ATTEMPT);
+ updateApplicationAttemptStateInternal(attemptState.getAttemptId()
+ .toString(), attemptStateData);
}
- break;
- case REMOVE_APP:
- {
- ApplicationState appState =
- ((RMStateStoreRemoveAppEvent) event).getAppState();
- ApplicationId appId = appState.getAppId();
- Exception removedException = null;
- LOG.info("Removing info for app: " + appId);
- try {
- removeApplicationState(appState);
- } catch (Exception e) {
- LOG.error("Error removing app: " + appId, e);
- removedException = e;
- } finally {
- notifyDoneRemovingApplcation(appId, removedException);
- }
+ } catch (Exception e) {
+ LOG
+ .error("Error storing appAttempt: " + attemptState.getAttemptId(), e);
+ storedException = e;
+ } finally {
+ if (event.getType().equals(RMStateStoreEventType.STORE_APP_ATTEMPT)) {
+ notifyDoneStoringApplicationAttempt(attemptState.getAttemptId(),
+ storedException);
+ } else {
+ notifyDoneUpdatingApplicationAttempt(attemptState.getAttemptId(),
+ storedException);
}
- break;
- default:
- LOG.error("Unknown RMStateStoreEvent type: " + event.getType());
+ }
+ } else if (event.getType().equals(RMStateStoreEventType.REMOVE_APP)) {
+ ApplicationState appState =
+ ((RMStateStoreRemoveAppEvent) event).getAppState();
+ ApplicationId appId = appState.getAppId();
+ Exception removedException = null;
+ LOG.info("Removing info for app: " + appId);
+ try {
+ removeApplicationState(appState);
+ } catch (Exception e) {
+ LOG.error("Error removing app: " + appId, e);
+ removedException = e;
+ } finally {
+ notifyDoneRemovingApplcation(appId, removedException);
+ }
+ } else {
+ LOG.error("Unknown RMStateStoreEvent type: " + event.getType());
}
}
@SuppressWarnings("unchecked")
/**
* In (@link handleStoreEvent}, this method is called to notify the
- * application about operation completion
+ * application that new application is stored in state store
* @param appId id of the application that has been saved
* @param storedException the exception that is thrown when storing the
* application
@@ -520,19 +635,33 @@ public abstract class RMStateStore exten
private void notifyDoneStoringApplication(ApplicationId appId,
Exception storedException) {
rmDispatcher.getEventHandler().handle(
- new RMAppStoredEvent(appId, storedException));
+ new RMAppNewSavedEvent(appId, storedException));
}
-
+
+ @SuppressWarnings("unchecked")
+ private void notifyDoneUpdatingApplication(ApplicationId appId,
+ Exception storedException) {
+ rmDispatcher.getEventHandler().handle(
+ new RMAppUpdateSavedEvent(appId, storedException));
+ }
+
@SuppressWarnings("unchecked")
/**
* In (@link handleStoreEvent}, this method is called to notify the
- * application attempt about operation completion
+ * application attempt that new attempt is stored in state store
* @param appAttempt attempt that has been saved
*/
private void notifyDoneStoringApplicationAttempt(ApplicationAttemptId attemptId,
Exception storedException) {
rmDispatcher.getEventHandler().handle(
- new RMAppAttemptStoredEvent(attemptId, storedException));
+ new RMAppAttemptNewSavedEvent(attemptId, storedException));
+ }
+
+ @SuppressWarnings("unchecked")
+ private void notifyDoneUpdatingApplicationAttempt(ApplicationAttemptId attemptId,
+ Exception updatedException) {
+ rmDispatcher.getEventHandler().handle(
+ new RMAppAttemptUpdateSavedEvent(attemptId, updatedException));
}
@SuppressWarnings("unchecked")
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreEventType.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreEventType.java?rev=1537584&r1=1537583&r2=1537584&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreEventType.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreEventType.java Thu Oct 31 18:49:54 2013
@@ -21,5 +21,7 @@ package org.apache.hadoop.yarn.server.re
public enum RMStateStoreEventType {
STORE_APP_ATTEMPT,
STORE_APP,
+ UPDATE_APP,
+ UPDATE_APP_ATTEMPT,
REMOVE_APP
}