You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by vi...@apache.org on 2011/10/19 08:40:03 UTC
svn commit: r1185988 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/
hadoop-yarn/hadoop-yarn-server/hadoop-ya...
Author: vinodkv
Date: Wed Oct 19 06:40:03 2011
New Revision: 1185988
URL: http://svn.apache.org/viewvc?rev=1185988&view=rev
Log:
MAPREDUCE-3162. Separated application-init and container-init event types in NodeManager's Application state machine. Contributed by Todd Lipcon.
Added:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationContainerInitEvent.java
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationEventType.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationInitEvent.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java
Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1185988&r1=1185987&r2=1185988&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Wed Oct 19 06:40:03 2011
@@ -1680,6 +1680,9 @@ Release 0.23.0 - Unreleased
MAPREDUCE-3199. Fixed pom files to include correct log4j configuration for
tests. (vinodkv)
+ MAPREDUCE-3162. Separated application-init and container-init event types
+ in NodeManager's Application state machine. (Todd Lipcon via vinodkv)
+
Release 0.22.0 - Unreleased
INCOMPATIBLE CHANGES
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java?rev=1185988&r1=1185987&r2=1185988&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java Wed Oct 19 06:40:03 2011
@@ -69,6 +69,7 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger.AuditConstants;
import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdater;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationContainerInitEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationImpl;
@@ -225,6 +226,7 @@ public class ContainerManagerImpl extend
/**
* Start a container on this NodeManager.
*/
+ @SuppressWarnings("unchecked")
@Override
public StartContainerResponse startContainer(StartContainerRequest request)
throws YarnRemoteException {
@@ -274,10 +276,13 @@ public class ContainerManagerImpl extend
context.getApplications().putIfAbsent(applicationID, application)) {
LOG.info("Creating a new application reference for app "
+ applicationID);
+ dispatcher.getEventHandler().handle(
+ new ApplicationInitEvent(applicationID));
}
// TODO: Validate the request
- dispatcher.getEventHandler().handle(new ApplicationInitEvent(container));
+ dispatcher.getEventHandler().handle(
+ new ApplicationContainerInitEvent(container));
NMAuditLogger.logSuccess(launchContext.getUser(),
AuditConstants.START_CONTAINER, "ContainerManageImpl",
Added: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationContainerInitEvent.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationContainerInitEvent.java?rev=1185988&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationContainerInitEvent.java (added)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationContainerInitEvent.java Wed Oct 19 06:40:03 2011
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.application;
+
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerInitEvent;
+
+/**
+ * Event sent from {@link ContainerManagerImpl} to {@link ApplicationImpl} to
+ * request the initialization of a container. This is funneled through
+ * the Application so that the application life-cycle can be checked, and container
+ * launches can be delayed until the application is fully initialized.
+ *
+ * Once the application is initialized,
+ * {@link ApplicationImpl.InitContainerTransition} simply passes this event on as a
+ * {@link ContainerInitEvent}.
+ *
+ */
+public class ApplicationContainerInitEvent extends ApplicationEvent {
+ final Container container;
+
+ public ApplicationContainerInitEvent(Container container) {
+ super(container.getContainerID().getApplicationAttemptId().getApplicationId(),
+ ApplicationEventType.INIT_CONTAINER);
+ this.container = container;
+ }
+
+ Container getContainer() {
+ return container;
+ }
+}
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationEventType.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationEventType.java?rev=1185988&r1=1185987&r2=1185988&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationEventType.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationEventType.java Wed Oct 19 06:40:03 2011
@@ -22,6 +22,7 @@ public enum ApplicationEventType {
// Source: ContainerManager
INIT_APPLICATION,
+ INIT_CONTAINER,
FINISH_APPLICATION,
// Source: ResourceLocalizationService
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java?rev=1185988&r1=1185987&r2=1185988&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java Wed Oct 19 06:40:03 2011
@@ -104,11 +104,14 @@ public class ApplicationImpl implements
// Transitions from NEW state
.addTransition(ApplicationState.NEW, ApplicationState.INITING,
ApplicationEventType.INIT_APPLICATION, new AppInitTransition())
+ .addTransition(ApplicationState.NEW, ApplicationState.NEW,
+ ApplicationEventType.INIT_CONTAINER,
+ new InitContainerTransition())
// Transitions from INITING state
.addTransition(ApplicationState.INITING, ApplicationState.INITING,
- ApplicationEventType.INIT_APPLICATION,
- new AppIsInitingTransition())
+ ApplicationEventType.INIT_CONTAINER,
+ new InitContainerTransition())
.addTransition(ApplicationState.INITING,
EnumSet.of(ApplicationState.FINISHING_CONTAINERS_WAIT,
ApplicationState.APPLICATION_RESOURCES_CLEANINGUP),
@@ -121,8 +124,8 @@ public class ApplicationImpl implements
// Transitions from RUNNING state
.addTransition(ApplicationState.RUNNING,
ApplicationState.RUNNING,
- ApplicationEventType.INIT_APPLICATION,
- new DuplicateAppInitTransition())
+ ApplicationEventType.INIT_CONTAINER,
+ new InitContainerTransition())
.addTransition(ApplicationState.RUNNING,
ApplicationState.RUNNING,
ApplicationEventType.APPLICATION_CONTAINER_FINISHED,
@@ -167,9 +170,6 @@ public class ApplicationImpl implements
SingleArcTransition<ApplicationImpl, ApplicationEvent> {
@Override
public void transition(ApplicationImpl app, ApplicationEvent event) {
- ApplicationInitEvent initEvent = (ApplicationInitEvent) event;
- Container container = initEvent.getContainer();
- app.containers.put(container.getContainerID(), container);
app.dispatcher.getEventHandler().handle(
new ApplicationLocalizationEvent(
LocalizationEventType.INIT_APPLICATION_RESOURCES, app));
@@ -177,17 +177,36 @@ public class ApplicationImpl implements
}
/**
- * Absorb initialization events while the application initializes.
+ * Handles INIT_CONTAINER events which request that we launch a new
+ * container. When we're still in the INITTING state, we simply
+ * queue these up. When we're in the RUNNING state, we pass along
+ * an ContainerInitEvent to the appropriate ContainerImpl.
*/
- static class AppIsInitingTransition implements
+ @SuppressWarnings("unchecked")
+ static class InitContainerTransition implements
SingleArcTransition<ApplicationImpl, ApplicationEvent> {
@Override
public void transition(ApplicationImpl app, ApplicationEvent event) {
- ApplicationInitEvent initEvent = (ApplicationInitEvent) event;
+ ApplicationContainerInitEvent initEvent =
+ (ApplicationContainerInitEvent) event;
Container container = initEvent.getContainer();
app.containers.put(container.getContainerID(), container);
LOG.info("Adding " + container.getContainerID()
+ " to application " + app.toString());
+
+ switch (app.getApplicationState()) {
+ case RUNNING:
+ app.dispatcher.getEventHandler().handle(new ContainerInitEvent(
+ container.getContainerID()));
+ break;
+ case INITING:
+ case NEW:
+ // these get queued up and sent out in AppInitDoneTransition
+ break;
+ default:
+ assert false : "Invalid state for InitContainerTransition: " +
+ app.getApplicationState();
+ }
}
}
@@ -211,20 +230,6 @@ public class ApplicationImpl implements
}
}
- @SuppressWarnings("unchecked")
- static class DuplicateAppInitTransition implements
- SingleArcTransition<ApplicationImpl, ApplicationEvent> {
- @Override
- public void transition(ApplicationImpl app, ApplicationEvent event) {
- ApplicationInitEvent initEvent = (ApplicationInitEvent) event;
- Container container = initEvent.getContainer();
- app.containers.put(container.getContainerID(), container);
- LOG.info("Adding " + container.getContainerID()
- + " to application " + app.toString());
- app.dispatcher.getEventHandler().handle(new ContainerInitEvent(
- container.getContainerID()));
- }
- }
static final class ContainerDoneTransition implements
SingleArcTransition<ApplicationImpl, ApplicationEvent> {
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationInitEvent.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationInitEvent.java?rev=1185988&r1=1185987&r2=1185988&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationInitEvent.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationInitEvent.java Wed Oct 19 06:40:03 2011
@@ -18,20 +18,11 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.application;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
public class ApplicationInitEvent extends ApplicationEvent {
- private final Container container;
-
- public ApplicationInitEvent(Container container) {
- super(container.getContainerID().getApplicationAttemptId().getApplicationId(),
- ApplicationEventType.INIT_APPLICATION);
- this.container = container;
- }
-
- public Container getContainer() {
- return this.container;
+ public ApplicationInitEvent(ApplicationId appId) {
+ super(appId, ApplicationEventType.INIT_APPLICATION);
}
-
}
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java?rev=1185988&r1=1185987&r2=1185988&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java Wed Oct 19 06:40:03 2011
@@ -41,11 +41,12 @@ public class TestApplication {
WrappedApplication wa = null;
try {
wa = new WrappedApplication(1, 314159265358979L, "yak", 3);
- wa.initApplication(1);
+ wa.initApplication();
+ wa.initContainer(1);
assertEquals(ApplicationState.INITING, wa.app.getApplicationState());
assertEquals(1, wa.app.getContainers().size());
- wa.initApplication(0);
- wa.initApplication(2);
+ wa.initContainer(0);
+ wa.initContainer(2);
assertEquals(ApplicationState.INITING, wa.app.getApplicationState());
assertEquals(3, wa.app.getContainers().size());
wa.applicationInited();
@@ -70,7 +71,8 @@ public class TestApplication {
WrappedApplication wa = null;
try {
wa = new WrappedApplication(2, 314159265358979L, "yak", 3);
- wa.initApplication(0);
+ wa.initApplication();
+ wa.initContainer(0);
assertEquals(ApplicationState.INITING, wa.app.getApplicationState());
assertEquals(1, wa.app.getContainers().size());
@@ -80,8 +82,8 @@ public class TestApplication {
argThat(new ContainerInitMatcher(wa.containers.get(0)
.getContainerID())));
- wa.initApplication(1);
- wa.initApplication(2);
+ wa.initContainer(1);
+ wa.initContainer(2);
assertEquals(ApplicationState.RUNNING, wa.app.getApplicationState());
assertEquals(3, wa.app.getContainers().size());
@@ -105,7 +107,8 @@ public class TestApplication {
WrappedApplication wa = null;
try {
wa = new WrappedApplication(3, 314159265358979L, "yak", 3);
- wa.initApplication(-1);
+ wa.initApplication();
+ wa.initContainer(-1);
assertEquals(ApplicationState.INITING, wa.app.getApplicationState());
wa.applicationInited();
assertEquals(ApplicationState.RUNNING, wa.app.getApplicationState());
@@ -130,7 +133,8 @@ public class TestApplication {
WrappedApplication wa = null;
try {
wa = new WrappedApplication(4, 314159265358979L, "yak", 3);
- wa.initApplication(-1);
+ wa.initApplication();
+ wa.initContainer(-1);
assertEquals(ApplicationState.INITING, wa.app.getApplicationState());
wa.applicationInited();
assertEquals(ApplicationState.RUNNING, wa.app.getApplicationState());
@@ -185,7 +189,8 @@ public class TestApplication {
WrappedApplication wa = null;
try {
wa = new WrappedApplication(5, 314159265358979L, "yak", 3);
- wa.initApplication(-1);
+ wa.initApplication();
+ wa.initContainer(-1);
assertEquals(ApplicationState.INITING, wa.app.getApplicationState());
wa.applicationInited();
assertEquals(ApplicationState.RUNNING, wa.app.getApplicationState());
@@ -220,7 +225,8 @@ public class TestApplication {
WrappedApplication wa = null;
try {
wa = new WrappedApplication(5, 314159265358979L, "yak", 3);
- wa.initApplication(-1);
+ wa.initApplication();
+ wa.initContainer(-1);
assertEquals(ApplicationState.INITING, wa.app.getApplicationState());
wa.applicationInited();
assertEquals(ApplicationState.RUNNING, wa.app.getApplicationState());
@@ -256,7 +262,8 @@ public class TestApplication {
WrappedApplication wa = null;
try {
wa = new WrappedApplication(1, 314159265358979L, "yak", 3);
- wa.initApplication(0);
+ wa.initApplication();
+ wa.initContainer(0);
assertEquals(ApplicationState.INITING, wa.app.getApplicationState());
assertEquals(1, wa.app.getContainers().size());
@@ -276,7 +283,7 @@ public class TestApplication {
refEq(new ApplicationLocalizationEvent(
LocalizationEventType.DESTROY_APPLICATION_RESOURCES, wa.app)));
- wa.initApplication(1);
+ wa.initContainer(1);
assertEquals(ApplicationState.APPLICATION_RESOURCES_CLEANINGUP,
wa.app.getApplicationState());
assertEquals(0, wa.app.getContainers().size());
@@ -376,13 +383,18 @@ public class TestApplication {
dispatcher.stop();
}
- public void initApplication(int containerNum) {
+ public void initApplication() {
+ app.handle(new ApplicationInitEvent(appId));
+ }
+
+
+ public void initContainer(int containerNum) {
if (containerNum == -1) {
for (int i = 0; i < containers.size(); i++) {
- app.handle(new ApplicationInitEvent(containers.get(i)));
+ app.handle(new ApplicationContainerInitEvent(containers.get(i)));
}
} else {
- app.handle(new ApplicationInitEvent(containers.get(containerNum)));
+ app.handle(new ApplicationContainerInitEvent(containers.get(containerNum)));
}
drainDispatcherEvents();
}