You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by sz...@apache.org on 2013/04/19 04:42:18 UTC
svn commit: r1469669 [1/2] - in
/hadoop/common/branches/HDFS-2802/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/
hadoop-yarn/hadoop-yarn-api/src/main/proto/
hadoop-yarn/hadoop-yarn-common/src/main/...
Author: szetszwo
Date: Fri Apr 19 02:42:16 2013
New Revision: 1469669
URL: http://svn.apache.org/r1469669
Log:
Merge r1469042 through r1469643 from trunk.
Added:
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreAppEvent.java
- copied unchanged from r1469643, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreAppEvent.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppStoredEvent.java
- copied unchanged from r1469643, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppStoredEvent.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java
- copied unchanged from r1469643, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/
- copied from r1469643, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java
- copied unchanged from r1469643, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java
- copied unchanged from r1469643, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestSchedulingPolicy.java
- copied unchanged from r1469643, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestSchedulingPolicy.java
Removed:
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingMode.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/modes/FairSchedulingMode.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/modes/FifoSchedulingMode.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestSchedulingMode.java
Modified:
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/YarnApplicationState.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreEventType.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppState.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/Schedulable.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestComputeFairShares.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/index.apt.vm
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt?rev=1469669&r1=1469668&r2=1469669&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt Fri Apr 19 02:42:16 2013
@@ -87,6 +87,9 @@ Release 2.0.5-beta - UNRELEASED
NEW FEATURES
+ YARN-482. FS: Extend SchedulingMode to intermediate queues.
+ (kkambatl via tucu)
+
IMPROVEMENTS
YARN-365. Change NM heartbeat handling to not generate a scheduler event
@@ -145,6 +148,9 @@ Release 2.0.5-beta - UNRELEASED
YARN-495. Changed NM reboot behaviour to be a simple resync - kill all
containers and re-register with RM. (Jian He via vinodkv)
+ YARN-514. Delayed store operations should not result in RM unavailability
+ for app submission (Zhijie Shen via bikas)
+
OPTIMIZATIONS
BUG FIXES
@@ -241,6 +247,15 @@ Release 2.0.5-beta - UNRELEASED
YARN-500. Fixed YARN webapps to not roll-over ports when explicitly asked
to use non-ephemeral ports. (Kenji Kikushima via vinodkv)
+ YARN-518. Fair Scheduler's document link could be added to the hadoop 2.x
+ main doc page. (sandyr via tucu)
+
+ YARN-476. ProcfsBasedProcessTree info message confuses users.
+ (sandyr via tucu)
+
+ YARN-585. Fix failure in TestFairScheduler#testNotAllowSubmitApplication
+ caused by YARN-514. (Zhijie Shen via vinodkv)
+
Release 2.0.4-alpha - UNRELEASED
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/YarnApplicationState.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/YarnApplicationState.java?rev=1469669&r1=1469668&r2=1469669&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/YarnApplicationState.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/YarnApplicationState.java Fri Apr 19 02:42:16 2013
@@ -30,9 +30,15 @@ public enum YarnApplicationState {
/** Application which was just created. */
NEW,
+ /** Application which is being saved. */
+ NEW_SAVING,
+
/** Application which has been submitted. */
SUBMITTED,
-
+
+ /** Application has been accepted by the scheduler */
+ ACCEPTED,
+
/** Application which is currently running. */
RUNNING,
@@ -43,8 +49,5 @@ public enum YarnApplicationState {
FAILED,
/** Application which was terminated by a user or admin. */
- KILLED,
-
- /** Application has been accepted by the scheduler */
- ACCEPTED
+ KILLED
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto?rev=1469669&r1=1469668&r2=1469669&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto Fri Apr 19 02:42:16 2013
@@ -72,12 +72,13 @@ message ContainerProto {
enum YarnApplicationStateProto {
NEW = 1;
- SUBMITTED = 2;
- RUNNING = 3;
- FINISHED = 4;
- FAILED = 5;
- KILLED = 6;
- ACCEPTED = 7;
+ NEW_SAVING = 2;
+ SUBMITTED = 3;
+ ACCEPTED = 4;
+ RUNNING = 5;
+ FINISHED = 6;
+ FAILED = 7;
+ KILLED = 8;
}
enum FinalApplicationStatusProto {
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java?rev=1469669&r1=1469668&r2=1469669&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java Fri Apr 19 02:42:16 2013
@@ -382,8 +382,6 @@ public class ProcfsBasedProcessTree exte
in = new BufferedReader(fReader);
} catch (FileNotFoundException f) {
// The process vanished in the interim!
- LOG.info("The process " + pinfo.getPid()
- + " may have finished in the interim.");
return ret;
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java?rev=1469669&r1=1469668&r2=1469669&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java Fri Apr 19 02:42:16 2013
@@ -297,20 +297,6 @@ public class ClientRMService extends Abs
// So call handle directly and do not send an event.
rmAppManager.handle(new RMAppManagerSubmitEvent(submissionContext, System
.currentTimeMillis()));
-
- // If recovery is enabled then store the application information in a
- // blocking call so make sure that RM has stored the information needed
- // to restart the AM after RM restart without further client communication
- RMStateStore stateStore = rmContext.getStateStore();
- LOG.info("Storing Application with id " + applicationId);
- try {
- stateStore.storeApplication(rmContext.getRMApps().get(applicationId));
- } catch (Exception e) {
- // For HA this exception needs to be handled by giving up
- // master status if we got fenced
- LOG.error("Failed to store application:" + applicationId, e);
- ExitUtil.terminate(1, e);
- }
LOG.info("Application with id " + applicationId.getId() +
" submitted by user " + user);
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java?rev=1469669&r1=1469668&r2=1469669&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java Fri Apr 19 02:42:16 2013
@@ -232,7 +232,8 @@ public class RMAppManager implements Eve
@SuppressWarnings("unchecked")
protected void submitApplication(
- ApplicationSubmissionContext submissionContext, long submitTime) {
+ ApplicationSubmissionContext submissionContext, long submitTime,
+ boolean isRecovered) {
ApplicationId applicationId = submissionContext.getApplicationId();
RMApp application = null;
try {
@@ -278,7 +279,8 @@ public class RMAppManager implements Eve
// All done, start the RMApp
this.rmContext.getDispatcher().getEventHandler().handle(
- new RMAppEvent(applicationId, RMAppEventType.START));
+ new RMAppEvent(applicationId, isRecovered ? RMAppEventType.RECOVER:
+ RMAppEventType.START));
} catch (IOException ie) {
LOG.info("RMAppManager submit application exception", ie);
if (application != null) {
@@ -347,7 +349,7 @@ public class RMAppManager implements Eve
if(shouldRecover) {
LOG.info("Recovering application " + appState.getAppId());
submitApplication(appState.getApplicationSubmissionContext(),
- appState.getSubmitTime());
+ appState.getSubmitTime(), true);
// re-populate attempt information in application
RMAppImpl appImpl = (RMAppImpl) rmContext.getRMApps().get(
appState.getAppId());
@@ -378,7 +380,7 @@ public class RMAppManager implements Eve
ApplicationSubmissionContext submissionContext =
((RMAppManagerSubmitEvent)event).getSubmissionContext();
long submitTime = ((RMAppManagerSubmitEvent)event).getSubmitTime();
- submitApplication(submissionContext, submitTime);
+ submitApplication(submissionContext, submitTime, false);
}
break;
default:
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java?rev=1469669&r1=1469668&r2=1469669&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java Fri Apr 19 02:42:16 2013
@@ -18,6 +18,8 @@
package org.apache.hadoop.yarn.server.resourcemanager.recovery;
+import java.io.IOException;
+
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
@@ -63,6 +65,11 @@ public class MemoryRMStateStore extends
ApplicationState appState = new ApplicationState(
appStateData.getSubmitTime(),
appStateData.getApplicationSubmissionContext());
+ if (state.appState.containsKey(appState.getAppId())) {
+ Exception e = new IOException("App: " + appId + " is already stored.");
+ LOG.info("Error storing info for app: " + appId, e);
+ throw e;
+ }
state.appState.put(appState.getAppId(), appState);
}
@@ -79,6 +86,13 @@ public class MemoryRMStateStore extends
attemptState.getAttemptId().getApplicationId());
assert appState != null;
+ if (appState.attempts.containsKey(attemptState.getAttemptId())) {
+ Exception e = new IOException("Attempt: " +
+ attemptState.getAttemptId() + " is already stored.");
+ LOG.info("Error storing info for attempt: " +
+ attemptState.getAttemptId(), e);
+ throw e;
+ }
appState.attempts.put(attemptState.getAttemptId(), attemptState);
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java?rev=1469669&r1=1469668&r2=1469669&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java Fri Apr 19 02:42:16 2013
@@ -37,6 +37,7 @@ import org.apache.hadoop.yarn.event.Asyn
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppStoredEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptStoredEvent;
@@ -166,21 +167,19 @@ public abstract class RMStateStore {
public abstract RMState loadState() throws Exception;
/**
- * Blocking API
+ * Non-Blocking API
* ResourceManager services use this to store the application's state
- * This must not be called on the dispatcher thread
+ * This does not block the dispatcher threads
+ * RMAppStoredEvent will be sent on completion to notify the RMApp
*/
- public synchronized void storeApplication(RMApp app) throws Exception {
+ @SuppressWarnings("unchecked")
+ public synchronized void storeApplication(RMApp app) {
ApplicationSubmissionContext context = app
.getApplicationSubmissionContext();
assert context instanceof ApplicationSubmissionContextPBImpl;
-
- ApplicationStateDataPBImpl appStateData = new ApplicationStateDataPBImpl();
- appStateData.setSubmitTime(app.getSubmitTime());
- appStateData.setApplicationSubmissionContext(context);
-
- LOG.info("Storing info for app: " + context.getApplicationId());
- storeApplicationState(app.getApplicationId().toString(), appStateData);
+ ApplicationState appState = new ApplicationState(
+ app.getSubmitTime(), context);
+ dispatcher.getEventHandler().handle(new RMStateStoreAppEvent(appState));
}
/**
@@ -255,6 +254,30 @@ public abstract class RMStateStore {
private synchronized void handleStoreEvent(RMStateStoreEvent event) {
switch(event.getType()) {
+ case STORE_APP:
+ {
+ ApplicationState apptState =
+ ((RMStateStoreAppEvent) event).getAppState();
+ Exception storedException = null;
+ ApplicationStateDataPBImpl appStateData =
+ new ApplicationStateDataPBImpl();
+ appStateData.setSubmitTime(apptState.getSubmitTime());
+ appStateData.setApplicationSubmissionContext(
+ apptState.getApplicationSubmissionContext());
+ ApplicationId appId =
+ apptState.getApplicationSubmissionContext().getApplicationId();
+
+ LOG.info("Storing info for app: " + appId);
+ try {
+ storeApplicationState(appId.toString(), appStateData);
+ } catch (Exception e) {
+ LOG.error("Error storing app: " + appId, e);
+ storedException = e;
+ } finally {
+ notifyDoneStoringApplication(appId, storedException);
+ }
+ }
+ break;
case STORE_APP_ATTEMPT:
{
ApplicationAttemptState attemptState =
@@ -297,11 +320,25 @@ public abstract class RMStateStore {
LOG.error("Unknown RMStateStoreEvent type: " + event.getType());
}
}
+
+ @SuppressWarnings("unchecked")
+ /**
+ * In (@link handleStoreEvent}, this method is called to notify the
+ * application about operation completion
+ * @param appId id of the application that has been saved
+ * @param storedException the exception that is thrown when storing the
+ * application
+ */
+ private void notifyDoneStoringApplication(ApplicationId appId,
+ Exception storedException) {
+ rmDispatcher.getEventHandler().handle(
+ new RMAppStoredEvent(appId, storedException));
+ }
@SuppressWarnings("unchecked")
/**
- * In (@link storeApplicationAttempt}, derived class can call this method to
- * notify the application attempt about operation completion
+ * In (@link handleStoreEvent}, this method is called to notify the
+ * application attempt about operation completion
* @param appAttempt attempt that has been saved
*/
private void notifyDoneStoringApplicationAttempt(ApplicationAttemptId attemptId,
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreEventType.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreEventType.java?rev=1469669&r1=1469668&r2=1469669&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreEventType.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreEventType.java Fri Apr 19 02:42:16 2013
@@ -20,5 +20,6 @@ package org.apache.hadoop.yarn.server.re
public enum RMStateStoreEventType {
STORE_APP_ATTEMPT,
+ STORE_APP,
REMOVE_APP
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java?rev=1469669&r1=1469668&r2=1469669&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java Fri Apr 19 02:42:16 2013
@@ -21,11 +21,13 @@ package org.apache.hadoop.yarn.server.re
public enum RMAppEventType {
// Source: ClientRMService
START,
+ RECOVER,
KILL,
// Source: RMAppAttempt
APP_REJECTED,
APP_ACCEPTED,
+ APP_SAVED,
ATTEMPT_REGISTERED,
ATTEMPT_FINISHING,
ATTEMPT_FINISHED, // Will send the final state
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java?rev=1469669&r1=1469668&r2=1469669&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java Fri Apr 19 02:42:16 2013
@@ -32,6 +32,7 @@ import java.util.concurrent.locks.Reentr
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.yarn.YarnException;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -118,13 +119,25 @@ public class RMAppImpl implements RMApp,
// Transitions from NEW state
.addTransition(RMAppState.NEW, RMAppState.NEW,
RMAppEventType.NODE_UPDATE, new RMAppNodeUpdateTransition())
+ .addTransition(RMAppState.NEW, RMAppState.NEW_SAVING,
+ RMAppEventType.START, new RMAppSavingTransition())
.addTransition(RMAppState.NEW, RMAppState.SUBMITTED,
- RMAppEventType.START, new StartAppAttemptTransition())
+ RMAppEventType.RECOVER, new StartAppAttemptTransition())
.addTransition(RMAppState.NEW, RMAppState.KILLED, RMAppEventType.KILL,
new AppKilledTransition())
.addTransition(RMAppState.NEW, RMAppState.FAILED,
RMAppEventType.APP_REJECTED, new AppRejectedTransition())
+ // Transitions from NEW_SAVING state
+ .addTransition(RMAppState.NEW_SAVING, RMAppState.NEW_SAVING,
+ RMAppEventType.NODE_UPDATE, new RMAppNodeUpdateTransition())
+ .addTransition(RMAppState.NEW_SAVING, RMAppState.SUBMITTED,
+ RMAppEventType.APP_SAVED, new StartAppAttemptTransition())
+ .addTransition(RMAppState.NEW_SAVING, RMAppState.KILLED,
+ RMAppEventType.KILL, new AppKilledTransition())
+ .addTransition(RMAppState.NEW_SAVING, RMAppState.FAILED,
+ RMAppEventType.APP_REJECTED, new AppRejectedTransition())
+
// Transitions from SUBMITTED state
.addTransition(RMAppState.SUBMITTED, RMAppState.SUBMITTED,
RMAppEventType.NODE_UPDATE, new RMAppNodeUpdateTransition())
@@ -182,7 +195,7 @@ public class RMAppImpl implements RMApp,
// Transitions from FAILED state
.addTransition(RMAppState.FAILED, RMAppState.FAILED,
- RMAppEventType.KILL)
+ EnumSet.of(RMAppEventType.KILL, RMAppEventType.APP_SAVED))
// ignorable transitions
.addTransition(RMAppState.FAILED, RMAppState.FAILED,
RMAppEventType.NODE_UPDATE)
@@ -194,7 +207,7 @@ public class RMAppImpl implements RMApp,
EnumSet.of(RMAppEventType.APP_ACCEPTED,
RMAppEventType.APP_REJECTED, RMAppEventType.KILL,
RMAppEventType.ATTEMPT_FINISHED, RMAppEventType.ATTEMPT_FAILED,
- RMAppEventType.ATTEMPT_KILLED))
+ RMAppEventType.ATTEMPT_KILLED, RMAppEventType.APP_SAVED))
// ignorable transitions
.addTransition(RMAppState.KILLED, RMAppState.KILLED,
RMAppEventType.NODE_UPDATE)
@@ -358,6 +371,8 @@ public class RMAppImpl implements RMApp,
switch(rmAppState) {
case NEW:
return YarnApplicationState.NEW;
+ case NEW_SAVING:
+ return YarnApplicationState.NEW_SAVING;
case SUBMITTED:
return YarnApplicationState.SUBMITTED;
case ACCEPTED:
@@ -378,6 +393,7 @@ public class RMAppImpl implements RMApp,
private FinalApplicationStatus createFinalApplicationStatus(RMAppState state) {
switch(state) {
case NEW:
+ case NEW_SAVING:
case SUBMITTED:
case ACCEPTED:
case RUNNING:
@@ -591,6 +607,19 @@ public class RMAppImpl implements RMApp,
private static final class StartAppAttemptTransition extends RMAppTransition {
public void transition(RMAppImpl app, RMAppEvent event) {
+ if (event.getType().equals(RMAppEventType.APP_SAVED)) {
+ assert app.getState().equals(RMAppState.NEW_SAVING);
+ RMAppStoredEvent storeEvent = (RMAppStoredEvent) event;
+ if(storeEvent.getStoredException() != null) {
+ // For HA this exception needs to be handled by giving up
+ // master status if we got fenced
+ LOG.error("Failed to store application: "
+ + storeEvent.getApplicationId(),
+ storeEvent.getStoredException());
+ ExitUtil.terminate(1, storeEvent.getStoredException());
+ }
+ }
+
app.createNewAttempt(true);
};
}
@@ -603,6 +632,18 @@ public class RMAppImpl implements RMApp,
}
}
+ private static final class RMAppSavingTransition extends RMAppTransition {
+ @Override
+ public void transition(RMAppImpl app, RMAppEvent event) {
+ // If recovery is enabled then store the application information in a
+ // non-blocking call so make sure that RM has stored the information
+ // needed to restart the AM after RM restart without further client
+ // communication
+ LOG.info("Storing application with id " + app.applicationId);
+ app.rmContext.getStateStore().storeApplication(app);
+ }
+ }
+
private static class AppFinishedTransition extends FinalTransition {
public void transition(RMAppImpl app, RMAppEvent event) {
RMAppFinishedAttemptEvent finishedEvent =
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppState.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppState.java?rev=1469669&r1=1469668&r2=1469669&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppState.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppState.java Fri Apr 19 02:42:16 2013
@@ -19,5 +19,13 @@
package org.apache.hadoop.yarn.server.resourcemanager.rmapp;
public enum RMAppState {
- NEW, SUBMITTED, ACCEPTED, RUNNING, FINISHING, FINISHED, FAILED, KILLED
+ NEW,
+ NEW_SAVING,
+ SUBMITTED,
+ ACCEPTED,
+ RUNNING,
+ FINISHING,
+ FINISHED,
+ FAILED,
+ KILLED
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java?rev=1469669&r1=1469668&r2=1469669&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java Fri Apr 19 02:42:16 2013
@@ -278,9 +278,7 @@ public class AppSchedulable extends Sche
}
}
-
- @Override
- public Resource assignContainer(FSSchedulerNode node, boolean reserved) {
+ private Resource assignContainer(FSSchedulerNode node, boolean reserved) {
LOG.info("Node offered to app: " + getName() + " reserved: " + reserved);
if (reserved) {
@@ -345,4 +343,13 @@ public class AppSchedulable extends Sche
}
return Resources.none();
}
+
+ public Resource assignReservedContainer(FSSchedulerNode node) {
+ return assignContainer(node, true);
+ }
+
+ @Override
+ public Resource assignContainer(FSSchedulerNode node) {
+ return assignContainer(node, false);
+ }
}
\ No newline at end of file
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java?rev=1469669&r1=1469668&r2=1469669&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java Fri Apr 19 02:42:16 2013
@@ -40,9 +40,6 @@ public class FSLeafQueue extends FSQueue
private final List<AppSchedulable> appScheds =
new ArrayList<AppSchedulable>();
-
- /** Scheduling mode for jobs inside the queue (fair or FIFO) */
- private SchedulingMode schedulingMode;
private final FairScheduler scheduler;
private final QueueManager queueMgr;
@@ -86,13 +83,18 @@ public class FSLeafQueue extends FSQueue
return appScheds;
}
- public void setSchedulingMode(SchedulingMode mode) {
- this.schedulingMode = mode;
+ @Override
+ public void setPolicy(SchedulingPolicy policy)
+ throws AllocationConfigurationException {
+ if (!SchedulingPolicy.isApplicableTo(policy, SchedulingPolicy.DEPTH_LEAF)) {
+ throwPolicyDoesnotApplyException(policy);
+ }
+ super.policy = policy;
}
@Override
- public void recomputeFairShares() {
- schedulingMode.computeShares(getAppSchedulables(), getFairShare());
+ public void recomputeShares() {
+ policy.computeShares(getAppSchedulables(), getFairShare());
}
@Override
@@ -136,42 +138,27 @@ public class FSLeafQueue extends FSQueue
}
@Override
- public Resource assignContainer(FSSchedulerNode node, boolean reserved) {
- LOG.debug("Node offered to queue: " + getName() + " reserved: " + reserved);
- // If this queue is over its limit, reject
- if (Resources.greaterThan(getResourceUsage(),
- queueMgr.getMaxResources(getName()))) {
- return Resources.none();
+ public Resource assignContainer(FSSchedulerNode node) {
+ Resource assigned = Resources.none();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Node offered to queue: " + getName());
}
- // If this node already has reserved resources for an app, first try to
- // finish allocating resources for that app.
- if (reserved) {
- for (AppSchedulable sched : appScheds) {
- if (sched.getApp().getApplicationAttemptId() ==
- node.getReservedContainer().getApplicationAttemptId()) {
- return sched.assignContainer(node, reserved);
- }
- }
- return Resources.none(); // We should never get here
+ if (!assignContainerPreCheck(node)) {
+ return assigned;
}
- // Otherwise, chose app to schedule based on given policy.
- else {
- Comparator<Schedulable> comparator = schedulingMode.getComparator();
-
- Collections.sort(appScheds, comparator);
- for (AppSchedulable sched: appScheds) {
- if (sched.getRunnable()) {
- Resource assignedResource = sched.assignContainer(node, reserved);
- if (!assignedResource.equals(Resources.none())) {
- return assignedResource;
- }
+ Comparator<Schedulable> comparator = policy.getComparator();
+ Collections.sort(appScheds, comparator);
+ for (AppSchedulable sched : appScheds) {
+ if (sched.getRunnable()) {
+ assigned = sched.assignContainer(node);
+ if (Resources.greaterThan(assigned, Resources.none())) {
+ break;
}
}
-
- return Resources.none();
}
+ return assigned;
}
@Override
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java?rev=1469669&r1=1469668&r2=1469669&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java Fri Apr 19 02:42:16 2013
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.re
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
import java.util.List;
import org.apache.commons.logging.Log;
@@ -33,7 +34,6 @@ public class FSParentQueue extends FSQue
private static final Log LOG = LogFactory.getLog(
FSParentQueue.class.getName());
-
private final List<FSQueue> childQueues =
new ArrayList<FSQueue>();
private final QueueManager queueMgr;
@@ -50,11 +50,11 @@ public class FSParentQueue extends FSQue
}
@Override
- public void recomputeFairShares() {
- SchedulingMode.getDefault().computeShares(childQueues, getFairShare());
+ public void recomputeShares() {
+ policy.computeShares(childQueues, getFairShare());
for (FSQueue childQueue : childQueues) {
childQueue.getMetrics().setAvailableResourcesToQueue(childQueue.getFairShare());
- childQueue.recomputeFairShares();
+ childQueue.recomputeShares();
}
}
@@ -131,13 +131,41 @@ public class FSParentQueue extends FSQue
}
@Override
- public Resource assignContainer(FSSchedulerNode node, boolean reserved) {
- throw new IllegalStateException(
- "Parent queue should not be assigned container");
+ public Resource assignContainer(FSSchedulerNode node) {
+ Resource assigned = Resources.none();
+
+ // If this queue is over its limit, reject
+ if (Resources.greaterThan(getResourceUsage(),
+ queueMgr.getMaxResources(getName()))) {
+ return assigned;
+ }
+
+ Collections.sort(childQueues, policy.getComparator());
+ for (FSQueue child : childQueues) {
+ assigned = child.assignContainer(node);
+ if (node.getReservedContainer() != null
+ || Resources.greaterThan(assigned, Resources.none())) {
+ break;
+ }
+ }
+ return assigned;
}
@Override
public Collection<FSQueue> getChildQueues() {
return childQueues;
}
+
+ @Override
+ public void setPolicy(SchedulingPolicy policy)
+ throws AllocationConfigurationException {
+ boolean allowed =
+ SchedulingPolicy.isApplicableTo(policy, (this == queueMgr
+ .getRootQueue()) ? SchedulingPolicy.DEPTH_ROOT
+ : SchedulingPolicy.DEPTH_INTERMEDIATE);
+ if (!allowed) {
+ throwPolicyDoesnotApplyException(policy);
+ }
+ super.policy = policy;
+ }
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java?rev=1469669&r1=1469668&r2=1469669&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java Fri Apr 19 02:42:16 2013
@@ -45,6 +45,8 @@ public abstract class FSQueue extends Sc
protected final RecordFactory recordFactory =
RecordFactoryProvider.getRecordFactory(null);
+ protected SchedulingPolicy policy = SchedulingPolicy.getDefault();
+
public FSQueue(String name, QueueManager queueMgr,
FairScheduler scheduler, FSParentQueue parent) {
this.name = name;
@@ -63,6 +65,19 @@ public abstract class FSQueue extends Sc
return name;
}
+ public SchedulingPolicy getPolicy() {
+ return policy;
+ }
+
+ protected void throwPolicyDoesnotApplyException(SchedulingPolicy policy)
+ throws AllocationConfigurationException {
+ throw new AllocationConfigurationException("SchedulingPolicy " + policy
+ + " does not apply to queue " + getName());
+ }
+
+ public abstract void setPolicy(SchedulingPolicy policy)
+ throws AllocationConfigurationException;
+
@Override
public double getWeight() {
return queueMgr.getQueueWeight(getName());
@@ -130,13 +145,27 @@ public abstract class FSQueue extends Sc
}
/**
- * Recomputes the fair shares for all queues and applications
- * under this queue.
+ * Recomputes the shares for all child queues and applications based on this
+ * queue's current share
*/
- public abstract void recomputeFairShares();
+ public abstract void recomputeShares();
/**
* Gets the children of this queue, if any.
*/
public abstract Collection<FSQueue> getChildQueues();
+
+ /**
+ * Helper method to check if the queue should attempt assigning resources
+ *
+ * @return true if check passes (can assign) or false otherwise
+ */
+ protected boolean assignContainerPreCheck(FSSchedulerNode node) {
+ if (Resources.greaterThan(getResourceUsage(),
+ queueMgr.getMaxResources(getName()))
+ || node.getReservedContainer() != null) {
+ return false;
+ }
+ return true;
+ }
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java?rev=1469669&r1=1469668&r2=1469669&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java Fri Apr 19 02:42:16 2013
@@ -52,6 +52,7 @@ public class FSSchedulerNode extends Sch
private volatile int numContainers;
private RMContainer reservedContainer;
+ private AppSchedulable reservedAppSchedulable;
/* set of containers that are allocated containers */
private final Map<ContainerId, RMContainer> launchedContainers =
@@ -221,6 +222,7 @@ public class FSSchedulerNode extends Sch
" on node " + this + " for application " + application);
}
this.reservedContainer = reservedContainer;
+ this.reservedAppSchedulable = application.getAppSchedulable();
}
public synchronized void unreserveResource(
@@ -237,11 +239,15 @@ public class FSSchedulerNode extends Sch
" on node " + this);
}
- reservedContainer = null;
+ this.reservedContainer = null;
+ this.reservedAppSchedulable = null;
}
public synchronized RMContainer getReservedContainer() {
return reservedContainer;
}
+ public synchronized AppSchedulable getReservedAppSchedulable() {
+ return reservedAppSchedulable;
+ }
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java?rev=1469669&r1=1469668&r2=1469669&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java Fri Apr 19 02:42:16 2013
@@ -161,7 +161,7 @@ public class FairScheduler implements Re
protected boolean assignMultiple; // Allocate multiple containers per
// heartbeat
protected int maxAssign; // Max containers to assign per heartbeat
-
+
public FairScheduler() {
clock = new SystemClock();
queueMgr = new QueueManager(this);
@@ -217,7 +217,7 @@ public class FairScheduler implements Re
rootQueue.setFairShare(clusterCapacity);
// Recursively compute fair shares for all queues
// and update metrics
- rootQueue.recomputeFairShares();
+ rootQueue.recomputeShares();
// Update recorded capacity of root queue (child queues are updated
// when fair share is calculated).
@@ -786,39 +786,24 @@ public class FairScheduler implements Re
// 1. Check for reserved applications
// 2. Schedule if there are no reservations
- // If we have have an application that has reserved a resource on this node
- // already, we try to complete the reservation.
- RMContainer reservedContainer = node.getReservedContainer();
- if (reservedContainer != null) {
- FSSchedulerApp reservedApplication =
- applications.get(reservedContainer.getApplicationAttemptId());
+ AppSchedulable reservedAppSchedulable = node.getReservedAppSchedulable();
+ if (reservedAppSchedulable != null) {
+ // Reservation exists; try to fulfill the reservation
+ LOG.info("Trying to fulfill reservation for application "
+ + reservedAppSchedulable.getApp().getApplicationAttemptId()
+ + " on node: " + nm);
- // Try to fulfill the reservation
- LOG.info("Trying to fulfill reservation for application " +
- reservedApplication.getApplicationId() + " on node: " + nm);
-
- FSLeafQueue queue = queueMgr.getLeafQueue(reservedApplication.getQueueName());
- queue.assignContainer(node, true);
+ node.getReservedAppSchedulable().assignReservedContainer(node);
}
-
- // Otherwise, schedule at queue which is furthest below fair share
else {
+ // No reservation, schedule at queue which is farthest below fair share
int assignedContainers = 0;
while (node.getReservedContainer() == null) {
- // At most one task is scheduled each iteration of this loop
- List<FSLeafQueue> scheds = new ArrayList<FSLeafQueue>(
- queueMgr.getLeafQueues());
- Collections.sort(scheds, SchedulingMode.getDefault().getComparator());
boolean assignedContainer = false;
- for (FSLeafQueue sched : scheds) {
- Resource assigned = sched.assignContainer(node, false);
- if (Resources.greaterThan(assigned, Resources.none()) ||
- node.getReservedContainer() != null) {
- eventLog.log("ASSIGN", nm.getHostName(), assigned);
- assignedContainers++;
- assignedContainer = true;
- break;
- }
+ if (Resources.greaterThan(
+ queueMgr.getRootQueue().assignContainer(node),
+ Resources.none())) {
+ assignedContainer = true;
}
if (!assignedContainer) { break; }
if (!assignMultiple) { break; }
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java?rev=1469669&r1=1469668&r2=1469669&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java Fri Apr 19 02:42:16 2013
@@ -143,7 +143,6 @@ public class QueueManager {
if (leafQueue == null) {
return null;
}
- leafQueue.setSchedulingMode(info.defaultSchedulingMode);
queue = leafQueue;
} else if (queue instanceof FSParentQueue) {
return null;
@@ -302,7 +301,7 @@ public class QueueManager {
Map<String, Integer> queueMaxApps = new HashMap<String, Integer>();
Map<String, Integer> userMaxApps = new HashMap<String, Integer>();
Map<String, Double> queueWeights = new HashMap<String, Double>();
- Map<String, SchedulingMode> queueModes = new HashMap<String, SchedulingMode>();
+ Map<String, SchedulingPolicy> queuePolicies = new HashMap<String, SchedulingPolicy>();
Map<String, Long> minSharePreemptionTimeouts = new HashMap<String, Long>();
Map<String, Map<QueueACL, AccessControlList>> queueAcls =
new HashMap<String, Map<QueueACL, AccessControlList>>();
@@ -310,7 +309,7 @@ public class QueueManager {
int queueMaxAppsDefault = Integer.MAX_VALUE;
long fairSharePreemptionTimeout = Long.MAX_VALUE;
long defaultMinSharePreemptionTimeout = Long.MAX_VALUE;
- SchedulingMode defaultSchedulingMode = SchedulingMode.getDefault();
+ SchedulingPolicy defaultSchedPolicy = SchedulingPolicy.getDefault();
// Remember all queue names so we can display them on web UI, etc.
List<String> queueNamesInAllocFile = new ArrayList<String>();
@@ -339,7 +338,7 @@ public class QueueManager {
if ("queue".equals(element.getTagName()) ||
"pool".equals(element.getTagName())) {
loadQueue("root", element, minQueueResources, maxQueueResources, queueMaxApps,
- userMaxApps, queueWeights, queueModes, minSharePreemptionTimeouts,
+ userMaxApps, queueWeights, queuePolicies, minSharePreemptionTimeouts,
queueAcls, queueNamesInAllocFile);
} else if ("user".equals(element.getTagName())) {
String userName = element.getAttribute("name");
@@ -370,11 +369,12 @@ public class QueueManager {
} else if ("queueMaxAppsDefault".equals(element.getTagName())) {
String text = ((Text)element.getFirstChild()).getData().trim();
int val = Integer.parseInt(text);
- queueMaxAppsDefault = val;}
- else if ("defaultQueueSchedulingMode".equals(element.getTagName())) {
+ queueMaxAppsDefault = val;
+ } else if ("defaultQueueSchedulingPolicy".equals(element.getTagName())
+ || "defaultQueueSchedulingMode".equals(element.getTagName())) {
String text = ((Text)element.getFirstChild()).getData().trim();
- SchedulingMode.setDefault(text);
- defaultSchedulingMode = SchedulingMode.getDefault();
+ SchedulingPolicy.setDefault(text);
+ defaultSchedPolicy = SchedulingPolicy.getDefault();
} else {
LOG.warn("Bad element in allocations file: " + element.getTagName());
}
@@ -385,7 +385,7 @@ public class QueueManager {
synchronized (this) {
info = new QueueManagerInfo(minQueueResources, maxQueueResources,
queueMaxApps, userMaxApps, queueWeights, userMaxAppsDefault,
- queueMaxAppsDefault, defaultSchedulingMode, minSharePreemptionTimeouts,
+ queueMaxAppsDefault, defaultSchedPolicy, minSharePreemptionTimeouts,
queueAcls, fairSharePreemptionTimeout, defaultMinSharePreemptionTimeout);
// Root queue should have empty ACLs. As a queue's ACL is the union of
@@ -396,14 +396,15 @@ public class QueueManager {
rootAcls.put(QueueACL.SUBMIT_APPLICATIONS, new AccessControlList(" "));
rootAcls.put(QueueACL.ADMINISTER_QUEUE, new AccessControlList(" "));
queueAcls.put(ROOT_QUEUE, rootAcls);
-
+
+ // Create all queus
for (String name: queueNamesInAllocFile) {
- FSLeafQueue queue = getLeafQueue(name);
- if (queueModes.containsKey(name)) {
- queue.setSchedulingMode(queueModes.get(name));
- } else {
- queue.setSchedulingMode(defaultSchedulingMode);
- }
+ getLeafQueue(name);
+ }
+
+ // Set custom policies as specified
+ for (Map.Entry<String, SchedulingPolicy> entry : queuePolicies.entrySet()) {
+ queues.get(entry.getKey()).setPolicy(entry.getValue());
}
}
}
@@ -414,7 +415,8 @@ public class QueueManager {
private void loadQueue(String parentName, Element element, Map<String, Resource> minQueueResources,
Map<String, Resource> maxQueueResources, Map<String, Integer> queueMaxApps,
Map<String, Integer> userMaxApps, Map<String, Double> queueWeights,
- Map<String, SchedulingMode> queueModes, Map<String, Long> minSharePreemptionTimeouts,
+ Map<String, SchedulingPolicy> queuePolicies,
+ Map<String, Long> minSharePreemptionTimeouts,
Map<String, Map<QueueACL, AccessControlList>> queueAcls, List<String> queueNamesInAllocFile)
throws AllocationConfigurationException {
String queueName = parentName + "." + element.getAttribute("name");
@@ -448,9 +450,10 @@ public class QueueManager {
String text = ((Text)field.getFirstChild()).getData().trim();
long val = Long.parseLong(text) * 1000L;
minSharePreemptionTimeouts.put(queueName, val);
- } else if ("schedulingMode".equals(field.getTagName())) {
+ } else if ("schedulingPolicy".equals(field.getTagName())
+ || "schedulingMode".equals(field.getTagName())) {
String text = ((Text)field.getFirstChild()).getData().trim();
- queueModes.put(queueName, SchedulingMode.parse(text));
+ queuePolicies.put(queueName, SchedulingPolicy.parse(text));
} else if ("aclSubmitApps".equals(field.getTagName())) {
String text = ((Text)field.getFirstChild()).getData().trim();
acls.put(QueueACL.SUBMIT_APPLICATIONS, new AccessControlList(text));
@@ -459,8 +462,9 @@ public class QueueManager {
acls.put(QueueACL.ADMINISTER_QUEUE, new AccessControlList(text));
} else if ("queue".endsWith(field.getTagName()) ||
"pool".equals(field.getTagName())) {
- loadQueue(queueName, field, minQueueResources, maxQueueResources, queueMaxApps,
- userMaxApps, queueWeights, queueModes, minSharePreemptionTimeouts,
+ loadQueue(queueName, field, minQueueResources, maxQueueResources,
+ queueMaxApps, userMaxApps, queueWeights, queuePolicies,
+ minSharePreemptionTimeouts,
queueAcls, queueNamesInAllocFile);
isLeaf = false;
}
@@ -615,13 +619,13 @@ public class QueueManager {
// below half its fair share for this long, it is allowed to preempt tasks.
public final long fairSharePreemptionTimeout;
- public final SchedulingMode defaultSchedulingMode;
+ public final SchedulingPolicy defaultSchedulingPolicy;
public QueueManagerInfo(Map<String, Resource> minQueueResources,
Map<String, Resource> maxQueueResources,
Map<String, Integer> queueMaxApps, Map<String, Integer> userMaxApps,
Map<String, Double> queueWeights, int userMaxAppsDefault,
- int queueMaxAppsDefault, SchedulingMode defaultSchedulingMode,
+ int queueMaxAppsDefault, SchedulingPolicy defaultSchedulingPolicy,
Map<String, Long> minSharePreemptionTimeouts,
Map<String, Map<QueueACL, AccessControlList>> queueAcls,
long fairSharePreemptionTimeout, long defaultMinSharePreemptionTimeout) {
@@ -632,7 +636,7 @@ public class QueueManager {
this.queueWeights = queueWeights;
this.userMaxAppsDefault = userMaxAppsDefault;
this.queueMaxAppsDefault = queueMaxAppsDefault;
- this.defaultSchedulingMode = defaultSchedulingMode;
+ this.defaultSchedulingPolicy = defaultSchedulingPolicy;
this.minSharePreemptionTimeouts = minSharePreemptionTimeouts;
this.queueAcls = queueAcls;
this.fairSharePreemptionTimeout = fairSharePreemptionTimeout;
@@ -651,7 +655,7 @@ public class QueueManager {
minSharePreemptionTimeouts = new HashMap<String, Long>();
defaultMinSharePreemptionTimeout = Long.MAX_VALUE;
fairSharePreemptionTimeout = Long.MAX_VALUE;
- defaultSchedulingMode = SchedulingMode.getDefault();
+ defaultSchedulingPolicy = SchedulingPolicy.getDefault();
}
}
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/Schedulable.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/Schedulable.java?rev=1469669&r1=1469668&r2=1469669&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/Schedulable.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/Schedulable.java Fri Apr 19 02:42:16 2013
@@ -93,11 +93,9 @@ public abstract class Schedulable {
/**
* Assign a container on this node if possible, and return the amount of
- * resources assigned. If {@code reserved} is true, it means a reservation
- * already exists on this node, and the schedulable should fulfill that
- * reservation if possible.
+ * resources assigned.
*/
- public abstract Resource assignContainer(FSSchedulerNode node, boolean reserved);
+ public abstract Resource assignContainer(FSSchedulerNode node);
/** Assign a fair share to this Schedulable. */
public void setFairShare(Resource fairShare) {
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java?rev=1469669&r1=1469668&r2=1469669&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java Fri Apr 19 02:42:16 2013
@@ -63,6 +63,7 @@ public class RmController extends Contro
// limit applications to those in states relevant to scheduling
set(YarnWebParams.APP_STATE, StringHelper.cjoin(
RMAppState.NEW.toString(),
+ RMAppState.NEW_SAVING.toString(),
RMAppState.SUBMITTED.toString(),
RMAppState.ACCEPTED.toString(),
RMAppState.RUNNING.toString(),
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java?rev=1469669&r1=1469668&r2=1469669&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java Fri Apr 19 02:42:16 2013
@@ -83,7 +83,9 @@ public class AppInfo {
String trackingUrl = app.getTrackingUrl();
this.state = app.getState();
this.trackingUrlIsNotReady = trackingUrl == null || trackingUrl.isEmpty()
- || RMAppState.NEW == this.state || RMAppState.SUBMITTED == this.state
+ || RMAppState.NEW == this.state
+ || RMAppState.NEW_SAVING == this.state
+ || RMAppState.SUBMITTED == this.state
|| RMAppState.ACCEPTED == this.state;
this.trackingUI = this.trackingUrlIsNotReady ? "UNASSIGNED" : (app
.getFinishTime() == 0 ? "ApplicationMaster" : "History");
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java?rev=1469669&r1=1469668&r2=1469669&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java Fri Apr 19 02:42:16 2013
@@ -164,7 +164,8 @@ public class TestAppManager{
}
public void submitApplication(
ApplicationSubmissionContext submissionContext) {
- super.submitApplication(submissionContext, System.currentTimeMillis());
+ super.submitApplication(
+ submissionContext, System.currentTimeMillis(), false);
}
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java?rev=1469669&r1=1469668&r2=1469669&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java Fri Apr 19 02:42:16 2013
@@ -41,6 +41,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.RMAppManagerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl;
+import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AMLivelinessMonitor;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent;
@@ -138,8 +139,9 @@ public class TestRMAppTransitions {
mock(ContainerAllocationExpirer.class);
AMLivelinessMonitor amLivelinessMonitor = mock(AMLivelinessMonitor.class);
AMLivelinessMonitor amFinishingMonitor = mock(AMLivelinessMonitor.class);
+ RMStateStore store = mock(RMStateStore.class);
this.rmContext =
- new RMContextImpl(rmDispatcher,
+ new RMContextImpl(rmDispatcher, store,
containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor,
null, new ApplicationTokenSecretManager(conf),
new RMContainerTokenSecretManager(conf),
@@ -176,6 +178,9 @@ public class TestRMAppTransitions {
if(submissionContext == null) {
submissionContext = new ApplicationSubmissionContextPBImpl();
}
+ // applicationId will not be used because RMStateStore is mocked,
+ // but applicationId is still set for safety
+ submissionContext.setApplicationId(applicationId);
RMApp application =
new RMAppImpl(applicationId, rmContext, conf, name, user, queue,
@@ -264,21 +269,45 @@ public class TestRMAppTransitions {
diag.toString().matches(regex));
}
- protected RMApp testCreateAppSubmitted(
+ protected RMApp testCreateAppNewSaving(
ApplicationSubmissionContext submissionContext) throws IOException {
RMApp application = createNewTestApp(submissionContext);
- // NEW => SUBMITTED event RMAppEventType.START
+ // NEW => NEW_SAVING event RMAppEventType.START
RMAppEvent event =
new RMAppEvent(application.getApplicationId(), RMAppEventType.START);
application.handle(event);
assertStartTimeSet(application);
+ assertAppState(RMAppState.NEW_SAVING, application);
+ return application;
+ }
+
+ protected RMApp testCreateAppSubmittedNoRecovery(
+ ApplicationSubmissionContext submissionContext) throws IOException {
+ RMApp application = testCreateAppNewSaving(submissionContext);
+ // NEW_SAVING => SUBMITTED event RMAppEventType.APP_SAVED
+ RMAppEvent event =
+ new RMAppStoredEvent(application.getApplicationId(), null);
+ application.handle(event);
+ assertStartTimeSet(application);
+ assertAppState(RMAppState.SUBMITTED, application);
+ return application;
+ }
+
+ protected RMApp testCreateAppSubmittedRecovery(
+ ApplicationSubmissionContext submissionContext) throws IOException {
+ RMApp application = createNewTestApp(submissionContext);
+ // NEW => SUBMITTED event RMAppEventType.RECOVER
+ RMAppEvent event =
+ new RMAppEvent(application.getApplicationId(), RMAppEventType.RECOVER);
+ application.handle(event);
+ assertStartTimeSet(application);
assertAppState(RMAppState.SUBMITTED, application);
return application;
}
protected RMApp testCreateAppAccepted(
ApplicationSubmissionContext submissionContext) throws IOException {
- RMApp application = testCreateAppSubmitted(submissionContext);
+ RMApp application = testCreateAppSubmittedNoRecovery(submissionContext);
// SUBMITTED => ACCEPTED event RMAppEventType.APP_ACCEPTED
RMAppEvent event =
new RMAppEvent(application.getApplicationId(),
@@ -375,7 +404,13 @@ public class TestRMAppTransitions {
application.getDiagnostics().indexOf(diagMsg) != -1);
}
- @Test
+ @Test (timeout = 30000)
+ public void testAppRecoverPath() throws IOException {
+ LOG.info("--- START: testAppRecoverPath ---");
+ testCreateAppSubmittedRecovery(null);
+ }
+
+ @Test (timeout = 30000)
public void testAppNewKill() throws IOException {
LOG.info("--- START: testAppNewKill ---");
@@ -402,11 +437,38 @@ public class TestRMAppTransitions {
assertFailed(application, rejectedText);
}
- @Test
+ @Test (timeout = 30000)
+ public void testAppNewSavingKill() throws IOException {
+ LOG.info("--- START: testAppNewSavingKill ---");
+
+ RMApp application = testCreateAppNewSaving(null);
+ // NEW_SAVING => KILLED event RMAppEventType.KILL
+ RMAppEvent event =
+ new RMAppEvent(application.getApplicationId(), RMAppEventType.KILL);
+ application.handle(event);
+ rmDispatcher.await();
+ assertKilled(application);
+ }
+
+ @Test (timeout = 30000)
+ public void testAppNewSavingReject() throws IOException {
+ LOG.info("--- START: testAppNewSavingReject ---");
+
+ RMApp application = testCreateAppNewSaving(null);
+ // NEW_SAVING => FAILED event RMAppEventType.APP_REJECTED
+ String rejectedText = "Test Application Rejected";
+ RMAppEvent event =
+ new RMAppRejectedEvent(application.getApplicationId(), rejectedText);
+ application.handle(event);
+ rmDispatcher.await();
+ assertFailed(application, rejectedText);
+ }
+
+ @Test (timeout = 30000)
public void testAppSubmittedRejected() throws IOException {
LOG.info("--- START: testAppSubmittedRejected ---");
- RMApp application = testCreateAppSubmitted(null);
+ RMApp application = testCreateAppSubmittedNoRecovery(null);
// SUBMITTED => FAILED event RMAppEventType.APP_REJECTED
String rejectedText = "app rejected";
RMAppEvent event =
@@ -419,7 +481,7 @@ public class TestRMAppTransitions {
@Test
public void testAppSubmittedKill() throws IOException, InterruptedException {
LOG.info("--- START: testAppSubmittedKill---");
- RMApp application = testCreateAppSubmitted(null);
+ RMApp application = testCreateAppSubmittedNoRecovery(null);
// SUBMITTED => KILLED event RMAppEventType.KILL
RMAppEvent event = new RMAppEvent(application.getApplicationId(),
RMAppEventType.KILL);
@@ -570,7 +632,37 @@ public class TestRMAppTransitions {
"", diag.toString());
}
- @Test
+ @Test (timeout = 30000)
+ public void testAppFailedFailed() throws IOException {
+ LOG.info("--- START: testAppFailedFailed ---");
+
+ RMApp application = testCreateAppNewSaving(null);
+
+ // NEW_SAVING => FAILED event RMAppEventType.APP_REJECTED
+ RMAppEvent event =
+ new RMAppRejectedEvent(application.getApplicationId(), "");
+ application.handle(event);
+ rmDispatcher.await();
+ assertTimesAtFinish(application);
+ assertAppState(RMAppState.FAILED, application);
+
+ // FAILED => FAILED event RMAppEventType.KILL
+ event =
+ new RMAppEvent(application.getApplicationId(), RMAppEventType.KILL);
+ application.handle(event);
+ rmDispatcher.await();
+ assertTimesAtFinish(application);
+ assertAppState(RMAppState.FAILED, application);
+
+ // FAILED => FAILED event RMAppEventType.APP_SAVED
+ event = new RMAppStoredEvent(application.getApplicationId(), null);
+ application.handle(event);
+ rmDispatcher.await();
+ assertTimesAtFinish(application);
+ assertAppState(RMAppState.FAILED, application);
+ }
+
+ @Test (timeout = 30000)
public void testAppKilledKilled() throws IOException {
LOG.info("--- START: testAppKilledKilled ---");
@@ -616,6 +708,13 @@ public class TestRMAppTransitions {
rmDispatcher.await();
assertTimesAtFinish(application);
assertAppState(RMAppState.KILLED, application);
+
+ // KILLED => KILLED event RMAppEventType.APP_SAVED
+ event = new RMAppStoredEvent(application.getApplicationId(), null);
+ application.handle(event);
+ rmDispatcher.await();
+ assertTimesAtFinish(application);
+ assertAppState(RMAppState.KILLED, application);
}
@Test
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java?rev=1469669&r1=1469668&r2=1469669&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java Fri Apr 19 02:42:16 2013
@@ -68,7 +68,7 @@ public class FakeSchedulable extends Sch
}
@Override
- public Resource assignContainer(FSSchedulerNode node, boolean reserved) {
+ public Resource assignContainer(FSSchedulerNode node) {
return null;
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestComputeFairShares.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestComputeFairShares.java?rev=1469669&r1=1469668&r2=1469669&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestComputeFairShares.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestComputeFairShares.java Fri Apr 19 02:42:16 2013
@@ -24,7 +24,7 @@ import java.util.List;
import junit.framework.Assert;
import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.modes.FairSchedulingMode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FairSharePolicy;
import org.junit.Before;
import org.junit.Test;
@@ -33,12 +33,12 @@ import org.junit.Test;
*/
public class TestComputeFairShares {
private List<Schedulable> scheds;
- private SchedulingMode schedulingMode;
+ private SchedulingPolicy schedulingMode;
@Before
public void setUp() throws Exception {
scheds = new ArrayList<Schedulable>();
- schedulingMode = new FairSchedulingMode();
+ schedulingMode = new FairSharePolicy();
}
/**