You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by vi...@apache.org on 2013/11/10 21:09:16 UTC
svn commit: r1540535 [1/5] - in
/hadoop/common/branches/YARN-321/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/
hadoop-yarn/ha...
Author: vinodkv
Date: Sun Nov 10 20:09:09 2013
New Revision: 1540535
URL: http://svn.apache.org/r1540535
Log:
Forwarding YARN-321 branch to latest branch-2.
svn merge ../branch-2
Added:
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceOption.java
- copied unchanged from r1540532, hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceOption.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceOptionPBImpl.java
- copied unchanged from r1540532, hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceOptionPBImpl.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateUpdateAppAttemptEvent.java
- copied unchanged from r1540532, hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateUpdateAppAttemptEvent.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateUpdateAppEvent.java
- copied unchanged from r1540532, hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateUpdateAppEvent.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppNewSavedEvent.java
- copied unchanged from r1540532, hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppNewSavedEvent.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppUpdateSavedEvent.java
- copied unchanged from r1540532, hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppUpdateSavedEvent.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/event/RMAppAttemptNewSavedEvent.java
- copied unchanged from r1540532, hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/event/RMAppAttemptNewSavedEvent.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/event/RMAppAttemptUpdateSavedEvent.java
- copied unchanged from r1540532, hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/event/RMAppAttemptUpdateSavedEvent.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/TestSchedulingMonitor.java
- copied unchanged from r1540532, hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/TestSchedulingMonitor.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMReconnect.java
- copied unchanged from r1540532, hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMReconnect.java
Removed:
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppStoredEvent.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/event/RMAppAttemptStoredEvent.java
Modified:
hadoop/common/branches/YARN-321/hadoop-yarn-project/ (props changed)
hadoop/common/branches/YARN-321/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/NMClient.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/NMTokenCache.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/ContainerManagementProtocolProxy.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/NMClientImpl.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreEventType.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationStateData.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationStateDataPBImpl.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppState.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptEventType.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptState.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/event/RMAppAttemptUnregistrationEvent.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplication.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
Propchange: hadoop/common/branches/YARN-321/hadoop-yarn-project/
------------------------------------------------------------------------------
Merged /hadoop/common/branches/branch-2/hadoop-yarn-project:r1537327-1540532
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/CHANGES.txt?rev=1540535&r1=1540534&r2=1540535&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/CHANGES.txt Sun Nov 10 20:09:09 2013
@@ -19,6 +19,9 @@ Release 2.3.0 - UNRELEASED
YARN-1068. Add admin support for HA operations (Karthik Kambatla via
bikas)
+ YARN-311. RM/scheduler support for dynamic resource configuration.
+ (Junping Du via llu)
+
IMPROVEMENTS
YARN-7. Support CPU resource for DistributedShell. (Junping Du via llu)
@@ -55,6 +58,21 @@ Release 2.3.0 - UNRELEASED
YARN-1306. Clean up hadoop-sls sample-conf according to YARN-1228 (Wei Yan
via Sandy Ryza)
+ YARN-891. Modified ResourceManager state-store to remember completed
+ applications so that clients can get information about them post RM-restart.
+ (Jian He via vinodkv)
+
+ YARN-1290. Let continuous scheduling achieve more balanced task assignment
+ (Wei Yan via Sandy Ryza)
+
+ YARN-786. Expose application resource usage in RM REST API (Sandy Ryza)
+
+ YARN-1323. Set HTTPS webapp address along with other RPC addresses in HAUtil
+ (Karthik Kambatla via Sandy Ryza)
+
+ YARN-1121. Changed ResourceManager's state-store to drain all events on
+ shut-down. (Jian He via vinodkv)
+
OPTIMIZATIONS
BUG FIXES
@@ -79,6 +97,9 @@ Release 2.3.0 - UNRELEASED
YARN-1305. RMHAProtocolService#serviceInit should handle HAUtil's
IllegalArgumentException (Tsuyoshi Ozawa via bikas)
+ YARN-1374. Changed ResourceManager to start the preemption policy monitors
+ as active services. (Karthik Kambatla via vinodkv)
+
Release 2.2.1 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -105,6 +126,12 @@ Release 2.2.1 - UNRELEASED
YARN-1109. Demote NodeManager "Sending out status for container" logs to
debug (haosdent via Sandy Ryza)
+ YARN-1321. Changed NMTokenCache to support both singleton and an instance
+ usage. (Alejandro Abdelnur via vinodkv)
+
+ YARN-1388. Fair Scheduler page always displays blank fair share (Liyin Liang
+ via Sandy Ryza)
+
OPTIMIZATIONS
BUG FIXES
@@ -153,6 +180,12 @@ Release 2.2.1 - UNRELEASED
YARN-1358. TestYarnCLI fails on Windows due to line endings. (Chuan Liu via
cnauroth)
+ YARN-1343. NodeManagers additions/restarts are not reported as node updates
+ in AllocateResponse responses to AMs. (tucu)
+
+ YARN-1381. Same relaxLocality appears twice in exception message of
+ AMRMClientImpl#checkLocalityRelaxationConflict() (Ted Yu via Sandy Ryza)
+
Release 2.2.0 - 2013-10-13
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java?rev=1540535&r1=1540534&r2=1540535&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java Sun Nov 10 20:09:09 2013
@@ -42,6 +42,7 @@ public class HAUtil {
YarnConfiguration.RM_ADMIN_ADDRESS,
YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
YarnConfiguration.RM_WEBAPP_ADDRESS,
+ YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS,
// TODO Remove after YARN-1318
YarnConfiguration.RM_HA_ADMIN_ADDRESS));
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto?rev=1540535&r1=1540534&r2=1540535&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto Sun Nov 10 20:09:09 2013
@@ -71,14 +71,53 @@ message GetGroupsForUserResponseProto {
////////////////////////////////////////////////////////////////////////
////// RM recovery related records /////////////////////////////////////
////////////////////////////////////////////////////////////////////////
+enum RMAppAttemptStateProto {
+ RMATTEMPT_NEW = 1;
+ RMATTEMPT_SUBMITTED = 2;
+ RMATTEMPT_SCHEDULED = 3;
+ RMATTEMPT_ALLOCATED = 4;
+ RMATTEMPT_LAUNCHED = 5;
+ RMATTEMPT_FAILED = 6;
+ RMATTEMPT_RUNNING = 7;
+ RMATTEMPT_FINISHING = 8;
+ RMATTEMPT_FINISHED = 9;
+ RMATTEMPT_KILLED = 10;
+ RMATTEMPT_ALLOCATED_SAVING = 11;
+ RMATTEMPT_LAUNCHED_UNMANAGED_SAVING = 12;
+ RMATTEMPT_RECOVERED = 13;
+ RMATTEMPT_FINAL_SAVING = 14;
+}
+
+enum RMAppStateProto {
+ RMAPP_NEW = 1;
+ RMAPP_NEW_SAVING = 2;
+ RMAPP_SUBMITTED = 3;
+ RMAPP_ACCEPTED = 4;
+ RMAPP_RUNNING = 5;
+ RMAPP_FINAL_SAVING = 6;
+ RMAPP_FINISHING = 7;
+ RMAPP_FINISHED = 8;
+ RMAPP_FAILED = 9;
+ RMAPP_KILLED = 10;
+}
+
message ApplicationStateDataProto {
optional int64 submit_time = 1;
- optional ApplicationSubmissionContextProto application_submission_context = 2;
- optional string user = 3;
+ optional int64 start_time = 2;
+ optional ApplicationSubmissionContextProto application_submission_context = 3;
+ optional string user = 4;
+ optional RMAppStateProto application_state = 5;
+ optional string diagnostics = 6 [default = "N/A"];
+ optional int64 finish_time = 7;
}
message ApplicationAttemptStateDataProto {
optional ApplicationAttemptIdProto attemptId = 1;
optional ContainerProto master_container = 2;
optional bytes app_attempt_tokens = 3;
+ optional RMAppAttemptStateProto app_attempt_state = 4;
+ optional string final_tracking_url = 5;
+ optional string diagnostics = 6 [default = "N/A"];
+ optional int64 start_time = 7;
+ optional FinalApplicationStatusProto final_application_status = 8;
}
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto?rev=1540535&r1=1540534&r2=1540535&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto Sun Nov 10 20:09:09 2013
@@ -58,6 +58,11 @@ message ResourceProto {
optional int32 virtual_cores = 2;
}
+message ResourceOptionProto {
+ optional ResourceProto resource = 1;
+ optional int32 over_commit_timeout = 2;
+}
+
message PriorityProto {
optional int32 priority = 1;
}
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java?rev=1540535&r1=1540534&r2=1540535&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java Sun Nov 10 20:09:09 2013
@@ -59,9 +59,12 @@ public abstract class AMRMClient<T exten
return client;
}
+ private NMTokenCache nmTokenCache;
+
@Private
protected AMRMClient(String name) {
super(name);
+ nmTokenCache = NMTokenCache.getSingleton();
}
/**
@@ -297,4 +300,33 @@ public abstract class AMRMClient<T exten
*/
public abstract void updateBlacklist(List<String> blacklistAdditions,
List<String> blacklistRemovals);
+
+ /**
+ * Set the NM token cache for the <code>AMRMClient</code>. This cache must
+ * be shared with the {@link NMClient} used to manage containers for the
+ * <code>AMRMClient</code>
+ * <p/>
+ * If a NM token cache is not set, the {@link NMTokenCache#getSingleton()}
+ * singleton instance will be used.
+ *
+ * @param nmTokenCache the NM token cache to use.
+ */
+ public void setNMTokenCache(NMTokenCache nmTokenCache) {
+ this.nmTokenCache = nmTokenCache;
+ }
+
+ /**
+ * Get the NM token cache of the <code>AMRMClient</code>. This cache must be
+ * shared with the {@link NMClient} used to manage containers for the
+ * <code>AMRMClient</code>.
+ * <p/>
+ * If a NM token cache is not set, the {@link NMTokenCache#getSingleton()}
+ * singleton instance will be used.
+ *
+ * @return the NM token cache.
+ */
+ public NMTokenCache getNMTokenCache() {
+ return nmTokenCache;
+ }
+
}
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/NMClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/NMClient.java?rev=1540535&r1=1540534&r2=1540535&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/NMClient.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/NMClient.java Sun Nov 10 20:09:09 2013
@@ -58,6 +58,8 @@ public abstract class NMClient extends A
return client;
}
+ private NMTokenCache nmTokenCache = NMTokenCache.getSingleton();
+
@Private
protected NMClient(String name) {
super(name);
@@ -118,4 +120,33 @@ public abstract class NMClient extends A
* @param enabled whether the feature is enabled or not
*/
public abstract void cleanupRunningContainersOnStop(boolean enabled);
+
+ /**
+ * Set the NM Token cache of the <code>NMClient</code>. This cache must be
+ * shared with the {@link AMRMClient} that requested the containers managed
+ * by this <code>NMClient</code>
+ * <p/>
+ * If a NM token cache is not set, the {@link NMTokenCache#getSingleton()}
+ * singleton instance will be used.
+ *
+ * @param nmTokenCache the NM token cache to use.
+ */
+ public void setNMTokenCache(NMTokenCache nmTokenCache) {
+ this.nmTokenCache = nmTokenCache;
+ }
+
+ /**
+ * Get the NM token cache of the <code>NMClient</code>. This cache must be
+ * shared with the {@link AMRMClient} that requested the containers managed
+ * by this <code>NMClient</code>
+ * <p/>
+ * If a NM token cache is not set, the {@link NMTokenCache#getSingleton()}
+ * singleton instance will be used.
+ *
+ * @return the NM token cache
+ */
+ public NMTokenCache getNMTokenCache() {
+ return nmTokenCache;
+ }
+
}
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/NMTokenCache.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/NMTokenCache.java?rev=1540535&r1=1540534&r2=1540535&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/NMTokenCache.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/NMTokenCache.java Sun Nov 10 20:09:09 2013
@@ -23,21 +23,139 @@ import java.util.concurrent.ConcurrentHa
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
+import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
+import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
import org.apache.hadoop.yarn.api.records.Token;
+import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
+import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
import com.google.common.annotations.VisibleForTesting;
/**
- * It manages NMTokens required for communicating with Node manager. Its a
- * static token cache.
+ * NMTokenCache manages NMTokens required for an Application Master
+ * communicating with individual NodeManagers.
+ * <p/>
+ * By default Yarn client libraries {@link AMRMClient} and {@link NMClient} use
+ * {@link #getSingleton()} instance of the cache.
+ * <ul>
+ * <li>Using the singleton instance of the cache is appropriate when running a
+ * single ApplicationMaster in the same JVM.</li>
+ * <li>When using the singleton, users don't need to do anything special,
+ * {@link AMRMClient} and {@link NMClient} are already set up to use the default
+ * singleton {@link NMTokenCache}</li>
+ * </ul>
+ * <p/>
+ * If running multiple Application Masters in the same JVM, a different cache
+ * instance should be used for each Application Master.
+ * <p/>
+ * <ul>
+ * <li>
+ * If using the {@link AMRMClient} and the {@link NMClient}, setting up and using
+ * an instance cache is as follows:
+ * <p/>
+ *
+ * <pre>
+ * NMTokenCache nmTokenCache = new NMTokenCache();
+ * AMRMClient rmClient = AMRMClient.createAMRMClient();
+ * NMClient nmClient = NMClient.createNMClient();
+ * nmClient.setNMTokenCache(nmTokenCache);
+ * ...
+ * </pre>
+ * </li>
+ * <li>
+ * If using the {@link AMRMClientAsync} and the {@link NMClientAsync}, setting up
+ * and using an instance cache is as follows:
+ * <p/>
+ *
+ * <pre>
+ * NMTokenCache nmTokenCache = new NMTokenCache();
+ * AMRMClient rmClient = AMRMClient.createAMRMClient();
+ * NMClient nmClient = NMClient.createNMClient();
+ * nmClient.setNMTokenCache(nmTokenCache);
+ * AMRMClientAsync rmClientAsync = new AMRMClientAsync(rmClient, 1000, [AMRM_CALLBACK]);
+ * NMClientAsync nmClientAsync = new NMClientAsync("nmClient", nmClient, [NM_CALLBACK]);
+ * ...
+ * </pre>
+ * </li>
+ * <li>
+ * If using {@link ApplicationMasterProtocol} and
+ * {@link ContainerManagementProtocol} directly, setting up and using an
+ * instance cache is as follows:
+ * <p/>
+ *
+ * <pre>
+ * NMTokenCache nmTokenCache = new NMTokenCache();
+ * ...
+ * ApplicationMasterProtocol amPro = ClientRMProxy.createRMProxy(conf, ApplicationMasterProtocol.class);
+ * ...
+ * AllocateRequest allocateRequest = ...
+ * ...
+ * AllocateResponse allocateResponse = rmClient.allocate(allocateRequest);
+ * for (NMToken token : allocateResponse.getNMTokens()) {
+ * nmTokenCache.setToken(token.getNodeId().toString(), token.getToken());
+ * }
+ * ...
+ * ContainerManagementProtocolProxy nmPro = ContainerManagementProtocolProxy(conf, nmTokenCache);
+ * ...
+ * nmPro.startContainer(container, containerContext);
+ * ...
+ * </pre>
+ * </li>
+ * </ul>
+ * It is also possible to mix the usage of a client (<code>AMRMClient</code> or
+ * <code>NMClient</code>, or the async versions of them) with a protocol proxy (
+ * <code>ContainerManagementProtocolProxy</code> or
+ * <code>ApplicationMasterProtocol</code>).
*/
@Public
@Evolving
public class NMTokenCache {
- private static ConcurrentHashMap<String, Token> nmTokens;
+ private static final NMTokenCache NM_TOKEN_CACHE = new NMTokenCache();
+ /**
+ * Returns the singleton NM token cache.
+ *
+ * @return the singleton NM token cache.
+ */
+ public static NMTokenCache getSingleton() {
+ return NM_TOKEN_CACHE;
+ }
- static {
+ /**
+ * Returns NMToken, null if absent. Only the singleton obtained from
+ * {@link #getSingleton()} is looked at for the tokens. If you are using your
+ * own NMTokenCache that is different from the singleton, use
+ * {@link #getToken(String) }
+ *
+ * @param nodeAddr
+ * @return {@link Token} NMToken required for communicating with node manager
+ */
+ @Public
+ public static Token getNMToken(String nodeAddr) {
+ return NM_TOKEN_CACHE.getToken(nodeAddr);
+ }
+
+ /**
+ * Sets the NMToken for node address only in the singleton obtained from
+ * {@link #getSingleton()}. If you are using your own NMTokenCache that is
+ * different from the singleton, use {@link #setToken(String, Token) }
+ *
+ * @param nodeAddr
+ * node address (host:port)
+ * @param token
+ * NMToken
+ */
+ @Public
+ public static void setNMToken(String nodeAddr, Token token) {
+ NM_TOKEN_CACHE.setToken(nodeAddr, token);
+ }
+
+ private ConcurrentHashMap<String, Token> nmTokens;
+
+ /**
+ * Creates a NM token cache instance.
+ */
+ public NMTokenCache() {
nmTokens = new ConcurrentHashMap<String, Token>();
}
@@ -45,11 +163,11 @@ public class NMTokenCache {
* Returns NMToken, null if absent
* @param nodeAddr
* @return {@link Token} NMToken required for communicating with node
- * manager
+ * manager
*/
@Public
@Evolving
- public static Token getNMToken(String nodeAddr) {
+ public Token getToken(String nodeAddr) {
return nmTokens.get(nodeAddr);
}
@@ -60,7 +178,7 @@ public class NMTokenCache {
*/
@Public
@Evolving
- public static void setNMToken(String nodeAddr, Token token) {
+ public void setToken(String nodeAddr, Token token) {
nmTokens.put(nodeAddr, token);
}
@@ -69,7 +187,7 @@ public class NMTokenCache {
*/
@Private
@VisibleForTesting
- public static boolean containsNMToken(String nodeAddr) {
+ public boolean containsToken(String nodeAddr) {
return nmTokens.containsKey(nodeAddr);
}
@@ -78,7 +196,7 @@ public class NMTokenCache {
*/
@Private
@VisibleForTesting
- public static int numberOfNMTokensInCache() {
+ public int numberOfTokensInCache() {
return nmTokens.size();
}
@@ -88,7 +206,7 @@ public class NMTokenCache {
*/
@Private
@VisibleForTesting
- public static void removeNMToken(String nodeAddr) {
+ public void removeToken(String nodeAddr) {
nmTokens.remove(nodeAddr);
}
@@ -97,7 +215,7 @@ public class NMTokenCache {
*/
@Private
@VisibleForTesting
- public static void clearCache() {
+ public void clearCache() {
nmTokens.clear();
}
}
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java?rev=1540535&r1=1540534&r2=1540535&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java Sun Nov 10 20:09:09 2013
@@ -58,7 +58,6 @@ import org.apache.hadoop.yarn.client.Cli
import org.apache.hadoop.yarn.client.api.AMRMClient;
import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
import org.apache.hadoop.yarn.client.api.InvalidContainerRequestException;
-import org.apache.hadoop.yarn.client.api.NMTokenCache;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
@@ -288,12 +287,12 @@ public class AMRMClientImpl<T extends Co
protected void populateNMTokens(AllocateResponse allocateResponse) {
for (NMToken token : allocateResponse.getNMTokens()) {
String nodeId = token.getNodeId().toString();
- if (NMTokenCache.containsNMToken(nodeId)) {
+ if (getNMTokenCache().containsToken(nodeId)) {
LOG.debug("Replacing token for : " + nodeId);
} else {
LOG.debug("Received new token for : " + nodeId);
}
- NMTokenCache.setNMToken(nodeId, token.getToken());
+ getNMTokenCache().setToken(nodeId, token.getToken());
}
}
@@ -500,13 +499,15 @@ public class AMRMClientImpl<T extends Co
for (String location : locations) {
TreeMap<Resource, ResourceRequestInfo> reqs =
remoteRequests.get(location);
- if (reqs != null && !reqs.isEmpty()
- && reqs.values().iterator().next().remoteRequest.getRelaxLocality()
- != relaxLocality) {
- throw new InvalidContainerRequestException("Cannot submit a "
- + "ContainerRequest asking for location " + location
- + " with locality relaxation " + relaxLocality + " when it has "
- + "already been requested with locality relaxation " + relaxLocality);
+ if (reqs != null && !reqs.isEmpty()) {
+ boolean existingRelaxLocality =
+ reqs.values().iterator().next().remoteRequest.getRelaxLocality();
+ if (relaxLocality != existingRelaxLocality) {
+ throw new InvalidContainerRequestException("Cannot submit a "
+ + "ContainerRequest asking for location " + location
+ + " with locality relaxation " + relaxLocality + " when it has "
+ + "already been requested with locality relaxation " + existingRelaxLocality);
+ }
}
}
}
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/ContainerManagementProtocolProxy.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/ContainerManagementProtocolProxy.java?rev=1540535&r1=1540534&r2=1540535&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/ContainerManagementProtocolProxy.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/ContainerManagementProtocolProxy.java Sun Nov 10 20:09:09 2013
@@ -56,9 +56,16 @@ public class ContainerManagementProtocol
private final LinkedHashMap<String, ContainerManagementProtocolProxyData> cmProxy;
private final Configuration conf;
private final YarnRPC rpc;
+ private NMTokenCache nmTokenCache;
public ContainerManagementProtocolProxy(Configuration conf) {
+ this(conf, NMTokenCache.getSingleton());
+ }
+
+ public ContainerManagementProtocolProxy(Configuration conf,
+ NMTokenCache nmTokenCache) {
this.conf = conf;
+ this.nmTokenCache = nmTokenCache;
maxConnectedNMs =
conf.getInt(YarnConfiguration.NM_CLIENT_MAX_NM_PROXIES,
@@ -86,7 +93,7 @@ public class ContainerManagementProtocol
while (proxy != null
&& !proxy.token.getIdentifier().equals(
- NMTokenCache.getNMToken(containerManagerBindAddr).getIdentifier())) {
+ nmTokenCache.getToken(containerManagerBindAddr).getIdentifier())) {
LOG.info("Refreshing proxy as NMToken got updated for node : "
+ containerManagerBindAddr);
// Token is updated. check if anyone has already tried closing it.
@@ -109,7 +116,7 @@ public class ContainerManagementProtocol
if (proxy == null) {
proxy =
new ContainerManagementProtocolProxyData(rpc, containerManagerBindAddr,
- containerId, NMTokenCache.getNMToken(containerManagerBindAddr));
+ containerId, nmTokenCache.getToken(containerManagerBindAddr));
if (cmProxy.size() > maxConnectedNMs) {
// Number of existing proxy exceed the limit.
String cmAddr = cmProxy.keySet().iterator().next();
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/NMClientImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/NMClientImpl.java?rev=1540535&r1=1540534&r2=1540535&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/NMClientImpl.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/NMClientImpl.java Sun Nov 10 20:09:09 2013
@@ -130,7 +130,10 @@ public class NMClientImpl extends NMClie
@Override
protected void serviceInit(Configuration conf) throws Exception {
super.serviceInit(conf);
- cmProxy = new ContainerManagementProtocolProxy(conf);
+ if (getNMTokenCache() == null) {
+ throw new IllegalStateException("NMTokenCache has not been set");
+ }
+ cmProxy = new ContainerManagementProtocolProxy(conf, getNMTokenCache());
}
@Override
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java?rev=1540535&r1=1540534&r2=1540535&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java Sun Nov 10 20:09:09 2013
@@ -626,6 +626,13 @@ public class TestAMRMClient {
try {
// start am rm client
amClient = AMRMClient.<ContainerRequest>createAMRMClient();
+
+ //setting an instance NMTokenCache
+ amClient.setNMTokenCache(new NMTokenCache());
+ //asserting we are not using the singleton instance cache
+ Assert.assertNotSame(NMTokenCache.getSingleton(),
+ amClient.getNMTokenCache());
+
amClient.init(conf);
amClient.start();
@@ -681,8 +688,8 @@ public class TestAMRMClient {
int iterationsLeft = 3;
Set<ContainerId> releases = new TreeSet<ContainerId>();
- NMTokenCache.clearCache();
- Assert.assertEquals(0, NMTokenCache.numberOfNMTokensInCache());
+ amClient.getNMTokenCache().clearCache();
+ Assert.assertEquals(0, amClient.getNMTokenCache().numberOfTokensInCache());
HashMap<String, Token> receivedNMTokens = new HashMap<String, Token>();
while (allocatedContainerCount < containersRequestedAny
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java?rev=1540535&r1=1540534&r2=1540535&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java Sun Nov 10 20:09:09 2013
@@ -78,6 +78,7 @@ public class TestNMClient {
List<NodeReport> nodeReports = null;
ApplicationAttemptId attemptId = null;
int nodeCount = 3;
+ NMTokenCache nmTokenCache = null;
@Before
public void setup() throws YarnException, IOException {
@@ -155,10 +156,16 @@ public class TestNMClient {
.createRemoteUser(UserGroupInformation.getCurrentUser().getUserName()));
UserGroupInformation.getCurrentUser().addToken(appAttempt.getAMRMToken());
+ //creating an instance NMTokenCase
+ nmTokenCache = new NMTokenCache();
+
// start am rm client
rmClient =
(AMRMClientImpl<ContainerRequest>) AMRMClient
.<ContainerRequest> createAMRMClient();
+
+ //setting an instance NMTokenCase
+ rmClient.setNMTokenCache(nmTokenCache);
rmClient.init(conf);
rmClient.start();
assertNotNull(rmClient);
@@ -166,6 +173,9 @@ public class TestNMClient {
// start am nm client
nmClient = (NMClientImpl) NMClient.createNMClient();
+
+ //propagating the AMRMClient NMTokenCache instance
+ nmClient.setNMTokenCache(rmClient.getNMTokenCache());
nmClient.init(conf);
nmClient.start();
assertNotNull(nmClient);
@@ -258,7 +268,7 @@ public class TestNMClient {
}
if (!allocResponse.getNMTokens().isEmpty()) {
for (NMToken token : allocResponse.getNMTokens()) {
- NMTokenCache.setNMToken(token.getNodeId().toString(),
+ rmClient.getNMTokenCache().setToken(token.getNodeId().toString(),
token.getToken());
}
}
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java?rev=1540535&r1=1540534&r2=1540535&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java Sun Nov 10 20:09:09 2013
@@ -49,6 +49,19 @@ public class AsyncDispatcher extends Abs
private final BlockingQueue<Event> eventQueue;
private volatile boolean stopped = false;
+ // Configuration flag for enabling/disabling draining dispatcher's events on
+ // stop functionality.
+ private volatile boolean drainEventsOnStop = false;
+
+ // Indicates all the remaining dispatcher's events on stop have been drained
+ // and processed.
+ private volatile boolean drained = true;
+
+ // For drainEventsOnStop enabled only, block newly coming events into the
+ // queue while stopping.
+ private volatile boolean blockNewEvents = false;
+ private EventHandler handlerInstance = null;
+
private Thread eventHandlingThread;
protected final Map<Class<? extends Enum>, EventHandler> eventDispatchers;
private boolean exitOnDispatchException;
@@ -68,6 +81,7 @@ public class AsyncDispatcher extends Abs
@Override
public void run() {
while (!stopped && !Thread.currentThread().isInterrupted()) {
+ drained = eventQueue.isEmpty();
Event event;
try {
event = eventQueue.take();
@@ -102,8 +116,19 @@ public class AsyncDispatcher extends Abs
eventHandlingThread.start();
}
+ public void setDrainEventsOnStop() {
+ drainEventsOnStop = true;
+ }
+
@Override
protected void serviceStop() throws Exception {
+ if (drainEventsOnStop) {
+ blockNewEvents = true;
+ LOG.info("AsyncDispatcher is draining to stop, igonring any new events.");
+ while(!drained) {
+ Thread.yield();
+ }
+ }
stopped = true;
if (eventHandlingThread != null) {
eventHandlingThread.interrupt();
@@ -173,11 +198,19 @@ public class AsyncDispatcher extends Abs
@Override
public EventHandler getEventHandler() {
- return new GenericEventHandler();
+ if (handlerInstance == null) {
+ handlerInstance = new GenericEventHandler();
+ }
+ return handlerInstance;
}
class GenericEventHandler implements EventHandler<Event> {
public void handle(Event event) {
+ if (blockNewEvents) {
+ return;
+ }
+ drained = false;
+
/* all this method does is enqueue all the events onto the queue */
int qSize = eventQueue.size();
if (qSize !=0 && qSize %1000 == 0) {
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java?rev=1540535&r1=1540534&r2=1540535&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java Sun Nov 10 20:09:09 2013
@@ -160,17 +160,14 @@ public class NodesListManager extends Ab
if (unusableRMNodesConcurrentSet.contains(eventNode)) {
LOG.debug(eventNode + " reported usable");
unusableRMNodesConcurrentSet.remove(eventNode);
- for (RMApp app : rmContext.getRMApps().values()) {
- this.rmContext
- .getDispatcher()
- .getEventHandler()
- .handle(
- new RMAppNodeUpdateEvent(app.getApplicationId(), eventNode,
- RMAppNodeUpdateType.NODE_USABLE));
- }
- } else {
- LOG.warn(eventNode
- + " reported usable without first reporting unusable");
+ }
+ for (RMApp app : rmContext.getRMApps().values()) {
+ this.rmContext
+ .getDispatcher()
+ .getEventHandler()
+ .handle(
+ new RMAppNodeUpdateEvent(app.getApplicationId(), eventNode,
+ RMAppNodeUpdateType.NODE_USABLE));
}
break;
default:
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java?rev=1540535&r1=1540534&r2=1540535&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java Sun Nov 10 20:09:09 2013
@@ -63,7 +63,6 @@ public class RMAppManager implements Eve
private static final Log LOG = LogFactory.getLog(RMAppManager.class);
private int completedAppsMax = YarnConfiguration.DEFAULT_RM_MAX_COMPLETED_APPLICATIONS;
- private int globalMaxAppAttempts;
private LinkedList<ApplicationId> completedApps = new LinkedList<ApplicationId>();
private final RMContext rmContext;
@@ -83,8 +82,6 @@ public class RMAppManager implements Eve
setCompletedAppsMax(conf.getInt(
YarnConfiguration.RM_MAX_COMPLETED_APPLICATIONS,
YarnConfiguration.DEFAULT_RM_MAX_COMPLETED_APPLICATIONS));
- globalMaxAppAttempts = conf.getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
- YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
}
/**
@@ -302,10 +299,11 @@ public class RMAppManager implements Eve
throw RPCUtil.getRemoteException(ie);
}
- // All done, start the RMApp
- this.rmContext.getDispatcher().getEventHandler().handle(
- new RMAppEvent(applicationId, isRecovered ? RMAppEventType.RECOVER:
- RMAppEventType.START));
+ if (!isRecovered) {
+ // All done, start the RMApp
+ this.rmContext.getDispatcher().getEventHandler()
+ .handle(new RMAppEvent(applicationId, RMAppEventType.START));
+ }
}
private Credentials parseCredentials(ApplicationSubmissionContext application)
@@ -328,53 +326,19 @@ public class RMAppManager implements Eve
// recover applications
Map<ApplicationId, ApplicationState> appStates = state.getApplicationState();
LOG.info("Recovering " + appStates.size() + " applications");
- for(ApplicationState appState : appStates.values()) {
- boolean shouldRecover = true;
- if(appState.getApplicationSubmissionContext().getUnmanagedAM()) {
- // do not recover unmanaged applications since current recovery
- // mechanism of restarting attempts does not work for them.
- // This will need to be changed in work preserving recovery in which
- // RM will re-connect with the running AM's instead of restarting them
- LOG.info("Not recovering unmanaged application " + appState.getAppId());
- shouldRecover = false;
- }
- int individualMaxAppAttempts = appState.getApplicationSubmissionContext()
- .getMaxAppAttempts();
- int maxAppAttempts;
- if (individualMaxAppAttempts <= 0 ||
- individualMaxAppAttempts > globalMaxAppAttempts) {
- maxAppAttempts = globalMaxAppAttempts;
- LOG.warn("The specific max attempts: " + individualMaxAppAttempts
- + " for application: " + appState.getAppId()
- + " is invalid, because it is out of the range [1, "
- + globalMaxAppAttempts + "]. Use the global max attempts instead.");
- } else {
- maxAppAttempts = individualMaxAppAttempts;
- }
- // In work-preserve restart, if attemptCount == maxAttempts, the job still
- // needs to be recovered because the last attempt may still be running.
- if(appState.getAttemptCount() >= maxAppAttempts) {
- LOG.info("Not recovering application " + appState.getAppId() +
- " due to recovering attempt is beyond maxAppAttempt limit");
- shouldRecover = false;
- }
-
- // re-submit the application
- // this is going to send an app start event but since the async dispatcher
- // has not started that event will be queued until we have completed re
- // populating the state
- if(shouldRecover) {
- LOG.info("Recovering application " + appState.getAppId());
- submitApplication(appState.getApplicationSubmissionContext(),
- appState.getSubmitTime(), true, appState.getUser());
- // re-populate attempt information in application
- RMAppImpl appImpl = (RMAppImpl) rmContext.getRMApps().get(
- appState.getAppId());
- appImpl.recover(state);
- }
- else {
- store.removeApplication(appState);
- }
+ for (ApplicationState appState : appStates.values()) {
+ LOG.info("Recovering application " + appState.getAppId());
+ submitApplication(appState.getApplicationSubmissionContext(),
+ appState.getSubmitTime(), true, appState.getUser());
+ // re-populate attempt information in application
+ RMAppImpl appImpl =
+ (RMAppImpl) rmContext.getRMApps().get(appState.getAppId());
+ appImpl.recover(state);
+ // Recover the app synchronously, as otherwise client is possible to see
+ // the application not recovered before it is actually recovered because
+ // ClientRMService is already started at this point of time.
+ appImpl.handle(new RMAppEvent(appImpl.getApplicationId(),
+ RMAppEventType.RECOVER));
}
}
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java?rev=1540535&r1=1540534&r2=1540535&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java Sun Nov 10 20:09:09 2013
@@ -501,6 +501,36 @@ public class ResourceManager extends Com
super.serviceStop();
}
+
+ protected void createPolicyMonitors() {
+ if (scheduler instanceof PreemptableResourceScheduler
+ && conf.getBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_ENABLE_MONITORS)) {
+ LOG.info("Loading policy monitors");
+ List<SchedulingEditPolicy> policies = conf.getInstances(
+ YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES,
+ SchedulingEditPolicy.class);
+ if (policies.size() > 0) {
+ rmDispatcher.register(ContainerPreemptEventType.class,
+ new RMContainerPreemptEventDispatcher(
+ (PreemptableResourceScheduler) scheduler));
+ for (SchedulingEditPolicy policy : policies) {
+ LOG.info("LOADING SchedulingEditPolicy:" + policy.getPolicyName());
+ policy.init(conf, rmContext.getDispatcher().getEventHandler(),
+ (PreemptableResourceScheduler) scheduler);
+ // periodically check whether we need to take action to guarantee
+ // constraints
+ SchedulingMonitor mon = new SchedulingMonitor(policy);
+ addService(mon);
+ }
+ } else {
+ LOG.warn("Policy monitors configured (" +
+ YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS +
+ ") but none specified (" +
+ YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES + ")");
+ }
+ }
+ }
}
@Private
@@ -829,37 +859,6 @@ public class ResourceManager extends Com
return new ApplicationMasterService(this.rmContext, scheduler);
}
- protected void createPolicyMonitors() {
- if (scheduler instanceof PreemptableResourceScheduler
- && conf.getBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS,
- YarnConfiguration.DEFAULT_RM_SCHEDULER_ENABLE_MONITORS)) {
- LOG.info("Loading policy monitors");
- List<SchedulingEditPolicy> policies = conf.getInstances(
- YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES,
- SchedulingEditPolicy.class);
- if (policies.size() > 0) {
- this.rmDispatcher.register(ContainerPreemptEventType.class,
- new RMContainerPreemptEventDispatcher(
- (PreemptableResourceScheduler) scheduler));
- for (SchedulingEditPolicy policy : policies) {
- LOG.info("LOADING SchedulingEditPolicy:" + policy.getPolicyName());
- policy.init(conf, this.rmContext.getDispatcher().getEventHandler(),
- (PreemptableResourceScheduler) scheduler);
- // periodically check whether we need to take action to guarantee
- // constraints
- SchedulingMonitor mon = new SchedulingMonitor(policy);
- addService(mon);
-
- }
- } else {
- LOG.warn("Policy monitors configured (" +
- YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS +
- ") but none specified (" +
- YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES + ")");
- }
- }
- }
-
protected AdminService createAdminService(
ClientRMService clientRMService,
ApplicationMasterService applicationMasterService,
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java?rev=1540535&r1=1540534&r2=1540535&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java Sun Nov 10 20:09:09 2013
@@ -31,6 +31,7 @@ import org.apache.hadoop.service.Abstrac
import org.apache.hadoop.util.VersionUtil;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceOption;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
@@ -234,7 +235,8 @@ public class ResourceTrackerService exte
.getCurrentKey());
RMNode rmNode = new RMNodeImpl(nodeId, rmContext, host, cmPort, httpPort,
- resolve(host), capability, nodeManagerVersion);
+ resolve(host), ResourceOption.newInstance(capability, RMNode.OVER_COMMIT_TIMEOUT_MILLIS_DEFAULT),
+ nodeManagerVersion);
RMNode oldNode = this.rmContext.getRMNodes().putIfAbsent(nodeId, rmNode);
if (oldNode == null) {
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java?rev=1540535&r1=1540534&r2=1540535&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java Sun Nov 10 20:09:09 2013
@@ -133,8 +133,11 @@ public class FileSystemRMStateStore exte
ApplicationStateDataProto.parseFrom(childData));
ApplicationState appState =
new ApplicationState(appStateData.getSubmitTime(),
+ appStateData.getStartTime(),
appStateData.getApplicationSubmissionContext(),
- appStateData.getUser());
+ appStateData.getUser(),
+ appStateData.getState(),
+ appStateData.getDiagnostics(), appStateData.getFinishTime());
// assert child node name is same as actual applicationId
assert appId.equals(appState.context.getApplicationId());
rmState.appState.put(appId, appState);
@@ -156,7 +159,12 @@ public class FileSystemRMStateStore exte
}
ApplicationAttemptState attemptState =
new ApplicationAttemptState(attemptId,
- attemptStateData.getMasterContainer(), credentials);
+ attemptStateData.getMasterContainer(), credentials,
+ attemptStateData.getStartTime(),
+ attemptStateData.getState(),
+ attemptStateData.getFinalTrackingUrl(),
+ attemptStateData.getDiagnostics(),
+ attemptStateData.getFinalApplicationStatus());
// assert child node name is same as application attempt id
assert attemptId.equals(attemptState.getAttemptId());
@@ -232,7 +240,7 @@ public class FileSystemRMStateStore exte
}
@Override
- public synchronized void storeApplicationState(String appId,
+ public synchronized void storeApplicationStateInternal(String appId,
ApplicationStateDataPBImpl appStateDataPB) throws Exception {
Path appDirPath = getAppDir(rmAppRoot, appId);
fs.mkdirs(appDirPath);
@@ -251,15 +259,34 @@ public class FileSystemRMStateStore exte
}
@Override
- public synchronized void storeApplicationAttemptState(String attemptId,
- ApplicationAttemptStateDataPBImpl attemptStateDataPB) throws Exception {
+ public synchronized void updateApplicationStateInternal(String appId,
+ ApplicationStateDataPBImpl appStateDataPB) throws Exception {
+ Path appDirPath = getAppDir(rmAppRoot, appId);
+ Path nodeCreatePath = getNodePath(appDirPath, appId);
+
+ LOG.info("Updating info for app: " + appId + " at: " + nodeCreatePath);
+ byte[] appStateData = appStateDataPB.getProto().toByteArray();
+ try {
+ // currently throw all exceptions. May need to respond differently for HA
+ // based on whether we have lost the right to write to FS
+ updateFile(nodeCreatePath, appStateData);
+ } catch (Exception e) {
+ LOG.info("Error updating info for app: " + appId, e);
+ throw e;
+ }
+ }
+
+ @Override
+ public synchronized void storeApplicationAttemptStateInternal(
+ String attemptId, ApplicationAttemptStateDataPBImpl attemptStateDataPB)
+ throws Exception {
ApplicationAttemptId appAttemptId =
ConverterUtils.toApplicationAttemptId(attemptId);
Path appDirPath =
getAppDir(rmAppRoot, appAttemptId.getApplicationId().toString());
Path nodeCreatePath = getNodePath(appDirPath, attemptId);
- LOG.info("Storing info for attempt: " + attemptId
- + " at: " + nodeCreatePath);
+ LOG.info("Storing info for attempt: " + attemptId + " at: "
+ + nodeCreatePath);
byte[] attemptStateData = attemptStateDataPB.getProto().toByteArray();
try {
// currently throw all exceptions. May need to respond differently for HA
@@ -272,6 +299,28 @@ public class FileSystemRMStateStore exte
}
@Override
+ public synchronized void updateApplicationAttemptStateInternal(
+ String attemptId, ApplicationAttemptStateDataPBImpl attemptStateDataPB)
+ throws Exception {
+ ApplicationAttemptId appAttemptId =
+ ConverterUtils.toApplicationAttemptId(attemptId);
+ Path appDirPath =
+ getAppDir(rmAppRoot, appAttemptId.getApplicationId().toString());
+ Path nodeCreatePath = getNodePath(appDirPath, attemptId);
+ LOG.info("Updating info for attempt: " + attemptId + " at: "
+ + nodeCreatePath);
+ byte[] attemptStateData = attemptStateDataPB.getProto().toByteArray();
+ try {
+ // currently throw all exceptions. May need to respond differently for HA
+ // based on whether we have lost the right to write to FS
+ updateFile(nodeCreatePath, attemptStateData);
+ } catch (Exception e) {
+ LOG.info("Error updating info for attempt: " + attemptId, e);
+ throw e;
+ }
+ }
+
+ @Override
public synchronized void removeApplicationState(ApplicationState appState)
throws Exception {
String appId = appState.getAppId().toString();
@@ -373,12 +422,21 @@ public class FileSystemRMStateStore exte
Path tempPath =
new Path(outputPath.getParent(), outputPath.getName() + ".tmp");
FSDataOutputStream fsOut = null;
- fsOut = fs.create(tempPath, false);
+ // This file will be overwritten when app/attempt finishes for saving the
+ // final status.
+ fsOut = fs.create(tempPath, true);
fsOut.write(data);
fsOut.close();
fs.rename(tempPath, outputPath);
}
+ private void updateFile(Path outputPath, byte[] data) throws Exception {
+ if (fs.exists(outputPath)) {
+ deleteFile(outputPath);
+ }
+ writeFile(outputPath, data);
+ }
+
private boolean renameFile(Path src, Path dst) throws Exception {
return fs.rename(src, dst);
}
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java?rev=1540535&r1=1540534&r2=1540535&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java Sun Nov 10 20:09:09 2013
@@ -76,22 +76,39 @@ public class MemoryRMStateStore extends
}
@Override
- public void storeApplicationState(String appId,
+ public void storeApplicationStateInternal(String appId,
ApplicationStateDataPBImpl appStateData)
throws Exception {
- ApplicationState appState = new ApplicationState(
- appStateData.getSubmitTime(),
- appStateData.getApplicationSubmissionContext(), appStateData.getUser());
- if (state.appState.containsKey(appState.getAppId())) {
- Exception e = new IOException("App: " + appId + " is already stored.");
- LOG.info("Error storing info for app: " + appId, e);
- throw e;
- }
+ ApplicationState appState =
+ new ApplicationState(appStateData.getSubmitTime(),
+ appStateData.getStartTime(),
+ appStateData.getApplicationSubmissionContext(),
+ appStateData.getUser());
state.appState.put(appState.getAppId(), appState);
}
@Override
- public synchronized void storeApplicationAttemptState(String attemptIdStr,
+ public void updateApplicationStateInternal(String appId,
+ ApplicationStateDataPBImpl appStateData) throws Exception {
+ ApplicationState updatedAppState =
+ new ApplicationState(appStateData.getSubmitTime(),
+ appStateData.getStartTime(),
+ appStateData.getApplicationSubmissionContext(),
+ appStateData.getUser(), appStateData.getState(),
+ appStateData.getDiagnostics(), appStateData.getFinishTime());
+ LOG.info("Updating final state " + appStateData.getState() + " for app: "
+ + appId);
+ ApplicationId applicationId = updatedAppState.getAppId();
+ if (state.appState.get(applicationId) != null) {
+ // add the earlier attempts back
+ updatedAppState.attempts
+ .putAll(state.appState.get(applicationId).attempts);
+ }
+ state.appState.put(applicationId, updatedAppState);
+ }
+
+ @Override
+ public synchronized void storeApplicationAttemptStateInternal(String attemptIdStr,
ApplicationAttemptStateDataPBImpl attemptStateData)
throws Exception {
ApplicationAttemptId attemptId = ConverterUtils
@@ -105,22 +122,48 @@ public class MemoryRMStateStore extends
}
ApplicationAttemptState attemptState =
new ApplicationAttemptState(attemptId,
- attemptStateData.getMasterContainer(), credentials);
+ attemptStateData.getMasterContainer(), credentials,
+ attemptStateData.getStartTime());
ApplicationState appState = state.getApplicationState().get(
attemptState.getAttemptId().getApplicationId());
if (appState == null) {
throw new YarnRuntimeException("Application doesn't exist");
}
+ appState.attempts.put(attemptState.getAttemptId(), attemptState);
+ }
- if (appState.attempts.containsKey(attemptState.getAttemptId())) {
- Exception e = new IOException("Attempt: " +
- attemptState.getAttemptId() + " is already stored.");
- LOG.info("Error storing info for attempt: " +
- attemptState.getAttemptId(), e);
- throw e;
+ @Override
+ public synchronized void updateApplicationAttemptStateInternal(
+ String attemptIdStr, ApplicationAttemptStateDataPBImpl attemptStateData)
+ throws Exception {
+ ApplicationAttemptId attemptId =
+ ConverterUtils.toApplicationAttemptId(attemptIdStr);
+ Credentials credentials = null;
+ if (attemptStateData.getAppAttemptTokens() != null) {
+ DataInputByteBuffer dibb = new DataInputByteBuffer();
+ credentials = new Credentials();
+ dibb.reset(attemptStateData.getAppAttemptTokens());
+ credentials.readTokenStorageStream(dibb);
}
- appState.attempts.put(attemptState.getAttemptId(), attemptState);
+ ApplicationAttemptState updatedAttemptState =
+ new ApplicationAttemptState(attemptId,
+ attemptStateData.getMasterContainer(), credentials,
+ attemptStateData.getStartTime(), attemptStateData.getState(),
+ attemptStateData.getFinalTrackingUrl(),
+ attemptStateData.getDiagnostics(),
+ attemptStateData.getFinalApplicationStatus());
+
+ ApplicationState appState =
+ state.getApplicationState().get(
+ updatedAttemptState.getAttemptId().getApplicationId());
+ if (appState == null) {
+ throw new YarnRuntimeException("Application doesn't exist");
+ }
+ LOG.info("Updating final state " + updatedAttemptState.getState()
+ + " for attempt: " + updatedAttemptState.getAttemptId());
+ appState.attempts.put(updatedAttemptState.getAttemptId(),
+ updatedAttemptState);
}
@Override
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java?rev=1540535&r1=1540534&r2=1540535&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/NullRMStateStore.java Sun Nov 10 20:09:09 2013
@@ -50,13 +50,13 @@ public class NullRMStateStore extends RM
}
@Override
- protected void storeApplicationState(String appId,
+ protected void storeApplicationStateInternal(String appId,
ApplicationStateDataPBImpl appStateData) throws Exception {
// Do nothing
}
@Override
- protected void storeApplicationAttemptState(String attemptId,
+ protected void storeApplicationAttemptStateInternal(String attemptId,
ApplicationAttemptStateDataPBImpl attemptStateData) throws Exception {
// Do nothing
}
@@ -89,4 +89,16 @@ public class NullRMStateStore extends RM
public void removeRMDTMasterKeyState(DelegationKey delegationKey) throws Exception {
// Do nothing
}
+
+ @Override
+ protected void updateApplicationStateInternal(String appId,
+ ApplicationStateDataPBImpl appStateData) throws Exception {
+ // Do nothing
+ }
+
+ @Override
+ protected void updateApplicationAttemptStateInternal(String attemptId,
+ ApplicationAttemptStateDataPBImpl attemptStateData) throws Exception {
+ // Do nothing
+ }
}