You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by su...@apache.org on 2012/11/30 20:58:44 UTC
svn commit: r1415815 [2/5] - in
/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project: ./
hadoop-yarn/dev-support/
hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/
hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-a...
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java Fri Nov 30 19:58:09 2012
@@ -56,6 +56,8 @@ import org.apache.hadoop.yarn.event.Disp
import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.AuxServicesEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.AuxServicesEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncherEvent;
@@ -65,6 +67,8 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ContainerLocalizationRequestEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizationEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizationEventType;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorEventType;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
@@ -208,6 +212,32 @@ public class TestContainer {
}
}
}
+
+ @Test
+ @SuppressWarnings("unchecked") // mocked generic
+ public void testInitWhileDone() throws Exception {
+ WrappedContainer wc = null;
+ try {
+ wc = new WrappedContainer(6, 314159265358979L, 4344, "yak");
+ wc.initContainer();
+ wc.localizeResources();
+ wc.launchContainer();
+ reset(wc.localizerBus);
+ wc.containerSuccessful();
+ wc.containerResourcesCleanup();
+ assertEquals(ContainerState.DONE, wc.c.getContainerState());
+ // Now in DONE, issue INIT
+ wc.initContainer();
+ // Verify still in DONE
+ assertEquals(ContainerState.DONE, wc.c.getContainerState());
+ verifyCleanupCall(wc);
+ }
+ finally {
+ if (wc != null) {
+ wc.finished();
+ }
+ }
+ }
@Test
@SuppressWarnings("unchecked") // mocked generic
@@ -506,6 +536,8 @@ public class TestContainer {
final EventHandler<ContainersLauncherEvent> launcherBus;
final EventHandler<ContainersMonitorEvent> monitorBus;
final EventHandler<AuxServicesEvent> auxBus;
+ final EventHandler<ApplicationEvent> appBus;
+ final EventHandler<LogHandlerEvent> LogBus;
final ContainerLaunchContext ctxt;
final ContainerId cId;
@@ -527,10 +559,14 @@ public class TestContainer {
launcherBus = mock(EventHandler.class);
monitorBus = mock(EventHandler.class);
auxBus = mock(EventHandler.class);
+ appBus = mock(EventHandler.class);
+ LogBus = mock(EventHandler.class);
dispatcher.register(LocalizationEventType.class, localizerBus);
dispatcher.register(ContainersLauncherEventType.class, launcherBus);
dispatcher.register(ContainersMonitorEventType.class, monitorBus);
dispatcher.register(AuxServicesEventType.class, auxBus);
+ dispatcher.register(ApplicationEventType.class, appBus);
+ dispatcher.register(LogHandlerEventType.class, LogBus);
this.user = user;
ctxt = mock(ContainerLaunchContext.class);
@@ -654,6 +690,11 @@ public class TestContainer {
ContainerEventType.CONTAINER_EXITED_WITH_SUCCESS));
drainDispatcherEvents();
}
+ public void containerResourcesCleanup() {
+ c.handle(new ContainerEvent(cId,
+ ContainerEventType.CONTAINER_RESOURCES_CLEANEDUP));
+ drainDispatcherEvents();
+ }
public void containerFailed(int exitCode) {
c.handle(new ContainerExitEvent(cId,
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java Fri Nov 30 19:58:09 2012
@@ -319,6 +319,7 @@ public class TestLogAggregationService e
this.user, null,
ContainerLogsRetentionPolicy.AM_AND_FAILED_CONTAINERS_ONLY, this.acls));
+ dispatcher.await();
ApplicationEvent expectedInitEvents[] = new ApplicationEvent[]{
new ApplicationEvent(
application1,
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java Fri Nov 30 19:58:09 2012
@@ -36,7 +36,6 @@ import org.apache.hadoop.yarn.event.Even
import org.apache.hadoop.yarn.ipc.RPCUtil;
import org.apache.hadoop.yarn.security.client.ClientTokenIdentifier;
import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants;
-import org.apache.hadoop.yarn.server.resourcemanager.recovery.ApplicationsStore.ApplicationStore;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType;
@@ -251,17 +250,12 @@ public class RMAppManager implements Eve
YarnConfiguration.DEFAULT_APPLICATION_NAME);
}
- // Store application for recovery
- ApplicationStore appStore = rmContext.getApplicationsStore()
- .createApplicationStore(submissionContext.getApplicationId(),
- submissionContext);
-
// Create RMApp
application =
new RMAppImpl(applicationId, rmContext, this.conf,
submissionContext.getApplicationName(),
submissionContext.getUser(), submissionContext.getQueue(),
- submissionContext, clientTokenStr, appStore, this.scheduler,
+ submissionContext, clientTokenStr, this.scheduler,
this.masterService, submitTime);
// Sanity check - duplicate?
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java Fri Nov 30 19:58:09 2012
@@ -23,8 +23,6 @@ import java.util.concurrent.ConcurrentMa
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.event.Dispatcher;
-import org.apache.hadoop.yarn.server.resourcemanager.recovery.ApplicationsStore;
-import org.apache.hadoop.yarn.server.resourcemanager.recovery.NodeStore;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AMLivelinessMonitor;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer;
@@ -41,10 +39,6 @@ public interface RMContext {
Dispatcher getDispatcher();
- NodeStore getNodeStore();
-
- ApplicationsStore getApplicationsStore();
-
ConcurrentMap<ApplicationId, RMApp> getRMApps();
ConcurrentMap<String, RMNode> getInactiveRMNodes();
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java Fri Nov 30 19:58:09 2012
@@ -24,9 +24,6 @@ import java.util.concurrent.ConcurrentMa
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.event.Dispatcher;
-import org.apache.hadoop.yarn.server.resourcemanager.recovery.ApplicationsStore;
-import org.apache.hadoop.yarn.server.resourcemanager.recovery.NodeStore;
-import org.apache.hadoop.yarn.server.resourcemanager.recovery.Store;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AMLivelinessMonitor;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer;
@@ -39,7 +36,6 @@ import org.apache.hadoop.yarn.server.res
public class RMContextImpl implements RMContext {
private final Dispatcher rmDispatcher;
- private final Store store;
private final ConcurrentMap<ApplicationId, RMApp> applications
= new ConcurrentHashMap<ApplicationId, RMApp>();
@@ -58,7 +54,7 @@ public class RMContextImpl implements RM
private final RMContainerTokenSecretManager containerTokenSecretManager;
private final ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager;
- public RMContextImpl(Store store, Dispatcher rmDispatcher,
+ public RMContextImpl(Dispatcher rmDispatcher,
ContainerAllocationExpirer containerAllocationExpirer,
AMLivelinessMonitor amLivelinessMonitor,
AMLivelinessMonitor amFinishingMonitor,
@@ -66,7 +62,6 @@ public class RMContextImpl implements RM
ApplicationTokenSecretManager appTokenSecretManager,
RMContainerTokenSecretManager containerTokenSecretManager,
ClientToAMTokenSecretManagerInRM clientTokenSecretManager) {
- this.store = store;
this.rmDispatcher = rmDispatcher;
this.containerAllocationExpirer = containerAllocationExpirer;
this.amLivelinessMonitor = amLivelinessMonitor;
@@ -83,16 +78,6 @@ public class RMContextImpl implements RM
}
@Override
- public NodeStore getNodeStore() {
- return store;
- }
-
- @Override
- public ApplicationsStore getApplicationsStore() {
- return store;
- }
-
- @Override
public ConcurrentMap<ApplicationId, RMApp> getRMApps() {
return this.applications;
}
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java Fri Nov 30 19:58:09 2012
@@ -46,8 +46,8 @@ import org.apache.hadoop.yarn.server.RMD
import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEventType;
import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.Recoverable;
-import org.apache.hadoop.yarn.server.resourcemanager.recovery.Store;
-import org.apache.hadoop.yarn.server.resourcemanager.recovery.Store.RMState;
+import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
+import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.StoreFactory;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent;
@@ -119,12 +119,12 @@ public class ResourceManager extends Com
protected RMDelegationTokenSecretManager rmDTSecretManager;
private WebApp webApp;
protected RMContext rmContext;
- private final Store store;
+ private final RMStateStore store;
protected ResourceTrackerService resourceTracker;
private Configuration conf;
- public ResourceManager(Store store) {
+ public ResourceManager(RMStateStore store) {
super("ResourceManager");
this.store = store;
}
@@ -161,7 +161,7 @@ public class ResourceManager extends Com
this.containerTokenSecretManager = createContainerTokenSecretManager(conf);
this.rmContext =
- new RMContextImpl(this.store, this.rmDispatcher,
+ new RMContextImpl(this.rmDispatcher,
this.containerAllocationExpirer, amLivelinessMonitor,
amFinishingMonitor, tokenRenewer, this.appTokenSecretManager,
this.containerTokenSecretManager, this.clientToAMSecretManager);
@@ -643,8 +643,6 @@ public class ResourceManager extends Com
@Override
public void recover(RMState state) throws Exception {
- resourceTracker.recover(state);
- scheduler.recover(state);
}
public static void main(String argv[]) {
@@ -652,14 +650,13 @@ public class ResourceManager extends Com
StringUtils.startupShutdownMessage(ResourceManager.class, argv, LOG);
try {
Configuration conf = new YarnConfiguration();
- Store store = StoreFactory.getStore(conf);
+ RMStateStore store = StoreFactory.getStore(conf);
ResourceManager resourceManager = new ResourceManager(store);
ShutdownHookManager.get().addShutdownHook(
new CompositeServiceShutdownHook(resourceManager),
SHUTDOWN_HOOK_PRIORITY);
resourceManager.init(conf);
//resourceManager.recover(store.restore());
- //store.doneWithRecovery();
resourceManager.start();
} catch (Throwable t) {
LOG.fatal("Error starting ResourceManager", t);
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java Fri Nov 30 19:58:09 2012
@@ -44,7 +44,7 @@ import org.apache.hadoop.yarn.server.api
import org.apache.hadoop.yarn.server.api.records.NodeAction;
import org.apache.hadoop.yarn.server.api.records.NodeStatus;
import org.apache.hadoop.yarn.server.api.records.RegistrationResponse;
-import org.apache.hadoop.yarn.server.resourcemanager.recovery.Store.RMState;
+import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEventType;
@@ -297,28 +297,6 @@ public class ResourceTrackerService exte
return nodeHeartBeatResponse;
}
- public void recover(RMState state) {
-//
-// List<RMNode> nodeManagers = state.getStoredNodeManagers();
-// for (RMNode nm : nodeManagers) {
-// createNewNode(nm.getNodeID(), nm.getNodeHostName(), nm
-// .getCommandPort(), nm.getHttpPort(), nm.getNode(), nm
-// .getTotalCapability());
-// }
-// for (Map.Entry<ApplicationId, ApplicationInfo> entry : state
-// .getStoredApplications().entrySet()) {
-// List<Container> containers = entry.getValue().getContainers();
-// List<Container> containersToAdd = new ArrayList<Container>();
-// for (Container c : containers) {
-// RMNode containerNode = this.rmContext.getNodesCollection()
-// .getNodeInfo(c.getNodeId());
-// containersToAdd.add(c);
-// containerNode.allocateContainer(entry.getKey(), containersToAdd);
-// containersToAdd.clear();
-// }
-// }
- }
-
/**
* resolving the network topology.
* @param hostName the hostname of this node.
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/Recoverable.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/Recoverable.java?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/Recoverable.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/Recoverable.java Fri Nov 30 19:58:09 2012
@@ -17,7 +17,7 @@
*/
package org.apache.hadoop.yarn.server.resourcemanager.recovery;
-import org.apache.hadoop.yarn.server.resourcemanager.recovery.Store.RMState;
+import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState;
public interface Recoverable {
public void recover(RMState state) throws Exception;
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/StoreFactory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/StoreFactory.java?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/StoreFactory.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/StoreFactory.java Fri Nov 30 19:58:09 2012
@@ -17,53 +17,17 @@
*/
package org.apache.hadoop.yarn.server.resourcemanager.recovery;
-import java.io.IOException;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.hadoop.yarn.api.records.ApplicationMaster;
-import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.server.resourcemanager.recovery.ApplicationsStore.ApplicationStore;
public class StoreFactory {
- public static Store getStore(Configuration conf) {
- Store store = ReflectionUtils.newInstance(
+ public static RMStateStore getStore(Configuration conf) {
+ RMStateStore store = ReflectionUtils.newInstance(
conf.getClass(YarnConfiguration.RM_STORE,
- MemStore.class, Store.class),
+ FileRMStateStore.class, RMStateStore.class),
conf);
return store;
}
-
- public static ApplicationStore createVoidAppStore() {
- return new VoidApplicationStore();
- }
-
- private static class VoidApplicationStore implements ApplicationStore {
-
- public VoidApplicationStore() {}
-
- @Override
- public void storeContainer(Container container) throws IOException {
- }
-
- @Override
- public void removeContainer(Container container) throws IOException {
- }
-
- @Override
- public void storeMasterContainer(Container container) throws IOException {
- }
-
- @Override
- public void updateApplicationState(ApplicationMaster master)
- throws IOException {
- }
-
- @Override
- public boolean isLoggable() {
- return false;
- }
- }
}
\ No newline at end of file
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java Fri Nov 30 19:58:09 2012
@@ -29,7 +29,6 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.hadoop.yarn.server.resourcemanager.recovery.ApplicationsStore.ApplicationStore;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
@@ -132,13 +131,6 @@ public interface RMApp extends EventHand
int pullRMNodeUpdates(Collection<RMNode> updatedNodes);
/**
- * Application level metadata is stored in {@link ApplicationStore} which
- * can persist the information.
- * @return the {@link ApplicationStore} for this {@link RMApp}.
- */
- ApplicationStore getApplicationStore();
-
- /**
* The finish time of the {@link RMApp}
* @return the finish time of the application.,
*/
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java Fri Nov 30 19:58:09 2012
@@ -49,7 +49,6 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.RMAppManagerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.RMAppManagerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
-import org.apache.hadoop.yarn.server.resourcemanager.recovery.ApplicationsStore.ApplicationStore;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppNodeUpdateEvent.RMAppNodeUpdateType;
import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
@@ -81,7 +80,6 @@ public class RMAppImpl implements RMApp
private final String name;
private final ApplicationSubmissionContext submissionContext;
private final String clientTokenStr;
- private final ApplicationStore appStore;
private final Dispatcher dispatcher;
private final YarnScheduler scheduler;
private final ApplicationMasterService masterService;
@@ -213,7 +211,6 @@ public class RMAppImpl implements RMApp
public RMAppImpl(ApplicationId applicationId, RMContext rmContext,
Configuration config, String name, String user, String queue,
ApplicationSubmissionContext submissionContext, String clientTokenStr,
- ApplicationStore appStore,
YarnScheduler scheduler, ApplicationMasterService masterService,
long submitTime) {
@@ -227,7 +224,6 @@ public class RMAppImpl implements RMApp
this.queue = queue;
this.submissionContext = submissionContext;
this.clientTokenStr = clientTokenStr;
- this.appStore = appStore;
this.scheduler = scheduler;
this.masterService = masterService;
this.submitTime = submitTime;
@@ -340,11 +336,6 @@ public class RMAppImpl implements RMApp
}
}
- @Override
- public ApplicationStore getApplicationStore() {
- return this.appStore;
- }
-
private YarnApplicationState createApplicationState(RMAppState rmAppState) {
switch(rmAppState) {
case NEW:
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java Fri Nov 30 19:58:09 2012
@@ -33,6 +33,7 @@ import java.util.concurrent.locks.Reentr
import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
+import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -399,7 +400,7 @@ public class RMAppAttemptImpl implements
final String trackingUriWithoutScheme) {
this.readLock.lock();
try {
- URI trackingUri = trackingUriWithoutScheme == null ? null :
+ URI trackingUri = StringUtils.isEmpty(trackingUriWithoutScheme) ? null :
ProxyUriUtils.getUriFromAMUrl(trackingUriWithoutScheme);
String proxy = YarnConfiguration.getProxyHostAndPort(conf);
URI proxyUri = ProxyUriUtils.getUriFromAMUrl(proxy);
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java Fri Nov 30 19:58:09 2012
@@ -98,10 +98,13 @@ public class RMContainerImpl implements
RMContainerEventType.KILL, new KillTransition())
.addTransition(RMContainerState.RUNNING, RMContainerState.RELEASED,
RMContainerEventType.RELEASED, new KillTransition())
+ .addTransition(RMContainerState.RUNNING, RMContainerState.RUNNING,
+ RMContainerEventType.EXPIRE)
// Transitions from COMPLETED state
.addTransition(RMContainerState.COMPLETED, RMContainerState.COMPLETED,
- EnumSet.of(RMContainerEventType.RELEASED, RMContainerEventType.KILL))
+ EnumSet.of(RMContainerEventType.EXPIRE, RMContainerEventType.RELEASED,
+ RMContainerEventType.KILL))
// Transitions from EXPIRED state
.addTransition(RMContainerState.EXPIRED, RMContainerState.EXPIRED,
@@ -109,13 +112,13 @@ public class RMContainerImpl implements
// Transitions from RELEASED state
.addTransition(RMContainerState.RELEASED, RMContainerState.RELEASED,
- EnumSet.of(RMContainerEventType.RELEASED, RMContainerEventType.KILL,
- RMContainerEventType.FINISHED))
+ EnumSet.of(RMContainerEventType.EXPIRE, RMContainerEventType.RELEASED,
+ RMContainerEventType.KILL, RMContainerEventType.FINISHED))
// Transitions from KILLED state
.addTransition(RMContainerState.KILLED, RMContainerState.KILLED,
- EnumSet.of(RMContainerEventType.RELEASED, RMContainerEventType.KILL,
- RMContainerEventType.FINISHED))
+ EnumSet.of(RMContainerEventType.EXPIRE, RMContainerEventType.RELEASED,
+ RMContainerEventType.KILL, RMContainerEventType.FINISHED))
// create the topology tables
.installTopology();
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java Fri Nov 30 19:58:09 2012
@@ -36,7 +36,6 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
-import org.apache.hadoop.yarn.server.resourcemanager.recovery.ApplicationsStore.ApplicationStore;
import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
@@ -70,14 +69,12 @@ public class AppSchedulingInfo {
boolean pending = true; // for app metrics
public AppSchedulingInfo(ApplicationAttemptId appAttemptId,
- String user, Queue queue, ActiveUsersManager activeUsersManager,
- ApplicationStore store) {
+ String user, Queue queue, ActiveUsersManager activeUsersManager) {
this.applicationAttemptId = appAttemptId;
this.applicationId = appAttemptId.getApplicationId();
this.queue = queue;
this.queueName = queue.getQueueName();
this.user = user;
- //this.store = store;
this.activeUsersManager = activeUsersManager;
}
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java Fri Nov 30 19:58:09 2012
@@ -49,7 +49,7 @@ import org.apache.hadoop.yarn.factory.pr
import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger;
import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
-import org.apache.hadoop.yarn.server.resourcemanager.recovery.Store.RMState;
+import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState;
import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType;
@@ -365,7 +365,7 @@ implements ResourceScheduler, CapacitySc
// TODO: Fix store
FiCaSchedulerApp SchedulerApp =
new FiCaSchedulerApp(applicationAttemptId, user, queue,
- queue.getActiveUsersManager(), rmContext, null);
+ queue.getActiveUsersManager(), rmContext);
// Submit to the queue
try {
@@ -767,18 +767,7 @@ implements ResourceScheduler, CapacitySc
@Override
@Lock(Lock.NoLock.class)
public void recover(RMState state) throws Exception {
- // TODO: VINDOKVFIXME recovery
-// applications.clear();
-// for (Map.Entry<ApplicationId, ApplicationInfo> entry : state.getStoredApplications().entrySet()) {
-// ApplicationId appId = entry.getKey();
-// ApplicationInfo appInfo = entry.getValue();
-// SchedulerApp app = applications.get(appId);
-// app.allocate(appInfo.getContainers());
-// for (Container c: entry.getValue().getContainers()) {
-// Queue queue = queues.get(appInfo.getApplicationSubmissionContext().getQueue());
-// queue.recoverContainer(clusterResource, applications.get(appId), c);
-// }
-// }
+ // NOT IMPLEMENTED
}
@Override
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java Fri Nov 30 19:58:09 2012
@@ -43,7 +43,6 @@ import org.apache.hadoop.yarn.factory.pr
import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger;
import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
-import org.apache.hadoop.yarn.server.resourcemanager.recovery.ApplicationsStore.ApplicationStore;
import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
@@ -111,11 +110,11 @@ public class FiCaSchedulerApp extends Sc
private final RMContext rmContext;
public FiCaSchedulerApp(ApplicationAttemptId applicationAttemptId,
String user, Queue queue, ActiveUsersManager activeUsersManager,
- RMContext rmContext, ApplicationStore store) {
+ RMContext rmContext) {
this.rmContext = rmContext;
this.appSchedulingInfo =
new AppSchedulingInfo(applicationAttemptId, user, queue,
- activeUsersManager, store);
+ activeUsersManager);
this.queue = queue;
}
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java Fri Nov 30 19:58:09 2012
@@ -50,10 +50,10 @@ public class AppSchedulable extends Sche
private long startTime;
private static RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
private static final Log LOG = LogFactory.getLog(AppSchedulable.class);
- private FSQueue queue;
+ private FSLeafQueue queue;
private RMContainerTokenSecretManager containerTokenSecretManager;
- public AppSchedulable(FairScheduler scheduler, FSSchedulerApp app, FSQueue queue) {
+ public AppSchedulable(FairScheduler scheduler, FSSchedulerApp app, FSLeafQueue queue) {
this.scheduler = scheduler;
this.app = app;
this.startTime = System.currentTimeMillis();
@@ -97,11 +97,8 @@ public class AppSchedulable extends Sche
}
@Override
- public void redistributeShare() {}
-
- @Override
public Resource getResourceUsage() {
- return this.app.getCurrentConsumption();
+ return app.getCurrentConsumption();
}
@@ -114,7 +111,7 @@ public class AppSchedulable extends Sche
* Get metrics reference from containing queue.
*/
public QueueMetrics getMetrics() {
- return this.queue.getQueueSchedulable().getMetrics();
+ return queue.getMetrics();
}
@Override
@@ -190,9 +187,9 @@ public class AppSchedulable extends Sche
RMContainer rmContainer = application.reserve(node, priority, null,
container);
node.reserveResource(application, priority, rmContainer);
- getMetrics().reserveResource(this.app.getUser(),
+ getMetrics().reserveResource(app.getUser(),
container.getResource());
- scheduler.getRootQueueMetrics().reserveResource(this.app.getUser(),
+ scheduler.getRootQueueMetrics().reserveResource(app.getUser(),
container.getResource());
}
@@ -257,13 +254,13 @@ public class AppSchedulable extends Sche
// TODO this should subtract resource just assigned
// TEMPROARY
getMetrics().setAvailableResourcesToQueue(
- this.scheduler.getClusterCapacity());
+ scheduler.getClusterCapacity());
}
// If we had previously made a reservation, delete it
if (reserved) {
- this.unreserve(application, priority, node);
+ unreserve(application, priority, node);
}
// Inform the node
@@ -290,7 +287,7 @@ public class AppSchedulable extends Sche
// Make sure the application still needs requests at this priority
if (app.getTotalRequiredResources(priority) == 0) {
- this.unreserve(app, priority, node);
+ unreserve(app, priority, node);
return Resources.none();
}
} else {
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java Fri Nov 30 19:58:09 2012
@@ -20,65 +20,112 @@ package org.apache.hadoop.yarn.server.re
import java.util.ArrayList;
import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-
-/**
- * A queue containing several applications.
- */
-@Private
-@Unstable
-public class FSQueue {
- /** Queue name. */
- private String name;
-
- /** Applications in this specific queue; does not include children queues' jobs. */
- private Collection<FSSchedulerApp> applications =
- new ArrayList<FSSchedulerApp>();
-
- /** Scheduling mode for jobs inside the queue (fair or FIFO) */
- private SchedulingMode schedulingMode;
-
- private FairScheduler scheduler;
-
- private FSQueueSchedulable queueSchedulable;
-
- public FSQueue(FairScheduler scheduler, String name) {
+import org.apache.hadoop.security.authorize.AccessControlList;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.QueueACL;
+import org.apache.hadoop.yarn.api.records.QueueInfo;
+import org.apache.hadoop.yarn.api.records.QueueState;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.factories.RecordFactory;
+import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
+
+public abstract class FSQueue extends Schedulable implements Queue {
+ private final String name;
+ private final QueueManager queueMgr;
+ private final FairScheduler scheduler;
+ private final QueueMetrics metrics;
+
+ protected final FSParentQueue parent;
+ protected final RecordFactory recordFactory =
+ RecordFactoryProvider.getRecordFactory(null);
+
+ public FSQueue(String name, QueueManager queueMgr,
+ FairScheduler scheduler, FSParentQueue parent) {
this.name = name;
- this.queueSchedulable = new FSQueueSchedulable(scheduler, this);
+ this.queueMgr = queueMgr;
this.scheduler = scheduler;
+ this.metrics = QueueMetrics.forQueue(getName(), parent, true, scheduler.getConf());
+ this.parent = parent;
}
-
- public Collection<FSSchedulerApp> getApplications() {
- return applications;
- }
-
- public void addApp(FSSchedulerApp app) {
- applications.add(app);
- AppSchedulable appSchedulable = new AppSchedulable(scheduler, app, this);
- app.setAppSchedulable(appSchedulable);
- queueSchedulable.addApp(appSchedulable);
- }
-
- public void removeJob(FSSchedulerApp app) {
- applications.remove(app);
- queueSchedulable.removeApp(app);
- }
-
+
public String getName() {
return name;
}
-
- public SchedulingMode getSchedulingMode() {
- return schedulingMode;
- }
-
- public void setSchedulingMode(SchedulingMode schedulingMode) {
- this.schedulingMode = schedulingMode;
- }
-
- public FSQueueSchedulable getQueueSchedulable() {
- return queueSchedulable;
+
+ @Override
+ public String getQueueName() {
+ return name;
}
+
+ @Override
+ public double getWeight() {
+ return queueMgr.getQueueWeight(getName());
+ }
+
+ @Override
+ public Resource getMinShare() {
+ return queueMgr.getMinResources(getName());
+ }
+
+ @Override
+ public long getStartTime() {
+ return 0;
+ }
+
+ @Override
+ public Priority getPriority() {
+ Priority p = recordFactory.newRecordInstance(Priority.class);
+ p.setPriority(1);
+ return p;
+ }
+
+ @Override
+ public QueueInfo getQueueInfo(boolean includeChildQueues, boolean recursive) {
+ QueueInfo queueInfo = recordFactory.newRecordInstance(QueueInfo.class);
+ queueInfo.setQueueName(getQueueName());
+ // TODO: we might change these queue metrics around a little bit
+ // to match the semantics of the fair scheduler.
+ queueInfo.setCapacity((float) getFairShare().getMemory() /
+ scheduler.getClusterCapacity().getMemory());
+ queueInfo.setCapacity((float) getResourceUsage().getMemory() /
+ scheduler.getClusterCapacity().getMemory());
+
+ ArrayList<QueueInfo> childQueueInfos = new ArrayList<QueueInfo>();
+ if (includeChildQueues) {
+ Collection<FSQueue> childQueues = getChildQueues();
+ for (FSQueue child : childQueues) {
+ childQueueInfos.add(child.getQueueInfo(recursive, recursive));
+ }
+ }
+ queueInfo.setChildQueues(childQueueInfos);
+ queueInfo.setQueueState(QueueState.RUNNING);
+ return queueInfo;
+ }
+
+ @Override
+ public Map<QueueACL, AccessControlList> getQueueAcls() {
+ Map<QueueACL, AccessControlList> acls = queueMgr.getQueueAcls(getName());
+ return new HashMap<QueueACL, AccessControlList>(acls);
+ }
+
+ @Override
+ public QueueMetrics getMetrics() {
+ return metrics;
+ }
+
+ /**
+ * Recomputes the fair shares for all queues and applications
+ * under this queue.
+ */
+ public abstract void recomputeFairShares();
+
+ /**
+ * Gets the children of this queue, if any.
+ */
+ public abstract Collection<FSQueue> getChildQueues();
}
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java Fri Nov 30 19:58:09 2012
@@ -42,7 +42,6 @@ import org.apache.hadoop.yarn.factory.pr
import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants;
-import org.apache.hadoop.yarn.server.resourcemanager.recovery.ApplicationsStore.ApplicationStore;
import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
@@ -103,21 +102,21 @@ public class FSSchedulerApp extends Sche
private final RMContext rmContext;
public FSSchedulerApp(ApplicationAttemptId applicationAttemptId,
String user, Queue queue, ActiveUsersManager activeUsersManager,
- RMContext rmContext, ApplicationStore store) {
+ RMContext rmContext) {
this.rmContext = rmContext;
this.appSchedulingInfo =
new AppSchedulingInfo(applicationAttemptId, user, queue,
- activeUsersManager, store);
+ activeUsersManager);
this.queue = queue;
}
public ApplicationId getApplicationId() {
- return this.appSchedulingInfo.getApplicationId();
+ return appSchedulingInfo.getApplicationId();
}
@Override
public ApplicationAttemptId getApplicationAttemptId() {
- return this.appSchedulingInfo.getApplicationAttemptId();
+ return appSchedulingInfo.getApplicationAttemptId();
}
public void setAppSchedulable(AppSchedulable appSchedulable) {
@@ -129,7 +128,7 @@ public class FSSchedulerApp extends Sche
}
public String getUser() {
- return this.appSchedulingInfo.getUser();
+ return appSchedulingInfo.getUser();
}
public synchronized void updateResourceRequests(
@@ -138,19 +137,19 @@ public class FSSchedulerApp extends Sche
}
public Map<String, ResourceRequest> getResourceRequests(Priority priority) {
- return this.appSchedulingInfo.getResourceRequests(priority);
+ return appSchedulingInfo.getResourceRequests(priority);
}
public int getNewContainerId() {
- return this.appSchedulingInfo.getNewContainerId();
+ return appSchedulingInfo.getNewContainerId();
}
public Collection<Priority> getPriorities() {
- return this.appSchedulingInfo.getPriorities();
+ return appSchedulingInfo.getPriorities();
}
public ResourceRequest getResourceRequest(Priority priority, String nodeAddress) {
- return this.appSchedulingInfo.getResourceRequest(priority, nodeAddress);
+ return appSchedulingInfo.getResourceRequest(priority, nodeAddress);
}
public synchronized int getTotalRequiredResources(Priority priority) {
@@ -158,7 +157,7 @@ public class FSSchedulerApp extends Sche
}
public Resource getResource(Priority priority) {
- return this.appSchedulingInfo.getResource(priority);
+ return appSchedulingInfo.getResource(priority);
}
/**
@@ -167,11 +166,11 @@ public class FSSchedulerApp extends Sche
*/
@Override
public boolean isPending() {
- return this.appSchedulingInfo.isPending();
+ return appSchedulingInfo.isPending();
}
public String getQueueName() {
- return this.appSchedulingInfo.getQueueName();
+ return appSchedulingInfo.getQueueName();
}
/**
@@ -185,7 +184,7 @@ public class FSSchedulerApp extends Sche
public synchronized void stop(RMAppAttemptState rmAppAttemptFinalState) {
// Cleanup all scheduling information
- this.appSchedulingInfo.stop(rmAppAttemptFinalState);
+ appSchedulingInfo.stop(rmAppAttemptFinalState);
}
@SuppressWarnings("unchecked")
@@ -196,7 +195,7 @@ public class FSSchedulerApp extends Sche
getRMContainer(containerId);
if (rmContainer == null) {
// Some unknown container sneaked into the system. Kill it.
- this.rmContext.getDispatcher().getEventHandler()
+ rmContext.getDispatcher().getEventHandler()
.handle(new RMNodeCleanContainerEvent(nodeId, containerId));
return;
}
@@ -272,7 +271,7 @@ public class FSSchedulerApp extends Sche
}
synchronized public void addSchedulingOpportunity(Priority priority) {
- this.schedulingOpportunities.setCount(priority,
+ schedulingOpportunities.setCount(priority,
schedulingOpportunities.count(priority) + 1);
}
@@ -282,19 +281,19 @@ public class FSSchedulerApp extends Sche
* successfully did so.
*/
synchronized public int getSchedulingOpportunities(Priority priority) {
- return this.schedulingOpportunities.count(priority);
+ return schedulingOpportunities.count(priority);
}
synchronized void resetReReservations(Priority priority) {
- this.reReservations.setCount(priority, 0);
+ reReservations.setCount(priority, 0);
}
synchronized void addReReservation(Priority priority) {
- this.reReservations.add(priority);
+ reReservations.add(priority);
}
synchronized public int getReReservations(Priority priority) {
- return this.reReservations.count(priority);
+ return reReservations.count(priority);
}
public synchronized int getNumReservedContainers(Priority priority) {
@@ -458,8 +457,8 @@ public class FSSchedulerApp extends Sche
* @param priority The priority of the container scheduled.
*/
synchronized public void resetSchedulingOpportunities(Priority priority) {
- this.lastScheduledContainer.put(priority, System.currentTimeMillis());
- this.schedulingOpportunities.setCount(priority, 0);
+ lastScheduledContainer.put(priority, System.currentTimeMillis());
+ schedulingOpportunities.setCount(priority, 0);
}
/**
@@ -494,14 +493,14 @@ public class FSSchedulerApp extends Sche
rackLocalityThreshold;
// Relax locality constraints once we've surpassed threshold.
- if (this.getSchedulingOpportunities(priority) > (numNodes * threshold)) {
+ if (getSchedulingOpportunities(priority) > (numNodes * threshold)) {
if (allowed.equals(NodeType.NODE_LOCAL)) {
allowedLocalityLevel.put(priority, NodeType.RACK_LOCAL);
- this.resetSchedulingOpportunities(priority);
+ resetSchedulingOpportunities(priority);
}
else if (allowed.equals(NodeType.RACK_LOCAL)) {
allowedLocalityLevel.put(priority, NodeType.OFF_SWITCH);
- this.resetSchedulingOpportunities(priority);
+ resetSchedulingOpportunities(priority);
}
}
return allowedLocalityLevel.get(priority);
@@ -512,7 +511,7 @@ public class FSSchedulerApp extends Sche
Priority priority, ResourceRequest request,
Container container) {
// Update allowed locality level
- NodeType allowed = this.allowedLocalityLevel.get(priority);
+ NodeType allowed = allowedLocalityLevel.get(priority);
if (allowed != null) {
if (allowed.equals(NodeType.OFF_SWITCH) &&
(type.equals(NodeType.NODE_LOCAL) ||
@@ -532,9 +531,9 @@ public class FSSchedulerApp extends Sche
}
// Create RMContainer
- RMContainer rmContainer = new RMContainerImpl(container, this
- .getApplicationAttemptId(), node.getNodeID(), this.rmContext
- .getDispatcher().getEventHandler(), this.rmContext
+ RMContainer rmContainer = new RMContainerImpl(container,
+ getApplicationAttemptId(), node.getNodeID(), rmContext
+ .getDispatcher().getEventHandler(), rmContext
.getContainerAllocationExpirer());
// Add it to allContainers list.
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java?rev=1415815&r1=1415814&r2=1415815&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java Fri Nov 30 19:58:09 2012
@@ -67,25 +67,25 @@ public class FSSchedulerNode extends Sch
}
public RMNode getRMNode() {
- return this.rmNode;
+ return rmNode;
}
public NodeId getNodeID() {
- return this.rmNode.getNodeID();
+ return rmNode.getNodeID();
}
public String getHttpAddress() {
- return this.rmNode.getHttpAddress();
+ return rmNode.getHttpAddress();
}
@Override
public String getHostName() {
- return this.rmNode.getHostName();
+ return rmNode.getHostName();
}
@Override
public String getRackName() {
- return this.rmNode.getRackName();
+ return rmNode.getRackName();
}
/**
@@ -112,17 +112,18 @@ public class FSSchedulerNode extends Sch
@Override
public synchronized Resource getAvailableResource() {
- return this.availableResource;
+ return availableResource;
}
@Override
public synchronized Resource getUsedResource() {
- return this.usedResource;
+ return usedResource;
}
private synchronized boolean isValidContainer(Container c) {
- if (launchedContainers.containsKey(c.getId()))
+ if (launchedContainers.containsKey(c.getId())) {
return true;
+ }
return false;
}