You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by ar...@apache.org on 2013/09/10 20:03:14 UTC
svn commit: r1521566 [1/2] - in
/hadoop/common/branches/HDFS-2832/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/
hadoop-...
Author: arp
Date: Tue Sep 10 18:03:08 2013
New Revision: 1521566
URL: http://svn.apache.org/r1521566
Log:
Merging r1520450 through r1521565 from trunk to branch HDFS-2832
Added:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ContainerContext.java
- copied unchanged from r1521565, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ContainerContext.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ContainerInitializationContext.java
- copied unchanged from r1521565, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ContainerInitializationContext.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ContainerTerminationContext.java
- copied unchanged from r1521565, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ContainerTerminationContext.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptImpl.java
- copied unchanged from r1521565, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptImpl.java
Modified:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerExitStatus.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/AuxiliaryService.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServicesEvent.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServicesEventType.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/CHANGES.txt?rev=1521566&r1=1521565&r2=1521566&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/CHANGES.txt Tue Sep 10 18:03:08 2013
@@ -27,14 +27,13 @@ Release 2.3.0 - UNRELEASED
IMPROVEMENTS
YARN-905. Add state filters to nodes CLI (Wei Yan via Sandy Ryza)
+ YARN-1098. Separate out RM services into Always On and Active (Karthik
+ Kambatla via bikas)
OPTIMIZATIONS
BUG FIXES
- YARN-758. Augment MockNM to use multiple cores (Karthik Kambatla via
- Sandy Ryza)
-
YARN-1060. Two tests in TestFairScheduler are missing @Test annotation
(Niranjan Singh via Sandy Ryza)
@@ -89,6 +88,16 @@ Release 2.1.1-beta - UNRELEASED
YARN-1065. NM should provide AuxillaryService data to the container (Xuan
Gong via bikas)
+ YARN-758. Augment MockNM to use multiple cores (Karthik Kambatla via
+ Sandy Ryza)
+
+ YARN-696. Changed RMWebservice apps call to take in multiple application
+ states. (Trevor Lorimer via vinodkv)
+
+ YARN-910. Augmented auxiliary services to listen for container starts and
+ completions in addition to application events. (Alejandro Abdelnur via
+ vinodkv)
+
OPTIMIZATIONS
BUG FIXES
@@ -156,6 +165,23 @@ Release 2.1.1-beta - UNRELEASED
need more than a node's total capability were incorrectly allocated on that
node causing apps to hang. (Omkar Vinit Joshi via vinodkv)
+ YARN-1107. Fixed a bug in ResourceManager because of which RM in secure mode
+ fails to restart. (Omkar Vinit Joshi via vinodkv)
+
+ YARN-1049. ContainerExistStatus should define a status for preempted
+ containers. (tucu)
+
+ YARN-1144. Unmanaged AMs registering a tracking URI should not be
+ proxy-fied. (tucu)
+
+ YARN-1152. Fixed a bug in ResourceManager that was causing clients to get
+ invalid client token key errors when an appliation is about to finish.
+ (Jason Lowe via vinodkv)
+
+ YARN-292. Fixed FifoScheduler and FairScheduler to make their applications
+ data structures thread safe to avoid RM crashing with
+ ArrayIndexOutOfBoundsException. (Zhijie Shen via vinodkv)
+
Release 2.1.0-beta - 2013-08-22
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerExitStatus.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerExitStatus.java?rev=1521566&r1=1521565&r2=1521566&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerExitStatus.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerExitStatus.java Tue Sep 10 18:03:08 2013
@@ -41,4 +41,9 @@ public class ContainerExitStatus {
* threshold number of the nodemanager-log-directories become bad.
*/
public static final int DISKS_FAILED = -101;
+
+ /**
+ * Containers preempted by the framework.
+ */
+ public static final int PREEMPTED = -102;
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/AuxiliaryService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/AuxiliaryService.java?rev=1521566&r1=1521565&r2=1521566&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/AuxiliaryService.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/AuxiliaryService.java Tue Sep 10 18:03:08 2013
@@ -79,4 +79,27 @@ public abstract class AuxiliaryService e
* applications.
*/
public abstract ByteBuffer getMetaData();
-}
\ No newline at end of file
+
+ /**
+ * A new container is started on this NodeManager. This is a signal to
+ * this {@link AuxiliaryService} about the container initialization.
+ * This method is called when the NodeManager receives the container launch
+ * command from the ApplicationMaster and before the container process is
+ * launched.
+ *
+ * @param initContainerContext context for the container's initialization
+ */
+ public void initializeContainer(ContainerInitializationContext
+ initContainerContext) {
+ }
+
+ /**
+ * A container is finishing on this NodeManager. This is a signal to this
+ * {@link AuxiliaryService} about the same.
+ *
+ * @param stopContainerContext context for the container termination
+ */
+ public void stopContainer(ContainerTerminationContext stopContainerContext) {
+ }
+
+}
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java?rev=1521566&r1=1521565&r2=1521566&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java Tue Sep 10 18:03:08 2013
@@ -37,6 +37,8 @@ import org.apache.hadoop.yarn.event.Even
import org.apache.hadoop.yarn.server.api.ApplicationTerminationContext;
import org.apache.hadoop.yarn.server.api.AuxiliaryService;
import org.apache.hadoop.yarn.server.api.ApplicationInitializationContext;
+import org.apache.hadoop.yarn.server.api.ContainerInitializationContext;
+import org.apache.hadoop.yarn.server.api.ContainerTerminationContext;
public class AuxServices extends AbstractService
implements ServiceStateChangeListener, EventHandler<AuxServicesEvent> {
@@ -178,7 +180,21 @@ public class AuxServices extends Abstrac
.getApplicationID()));
}
break;
- default:
+ case CONTAINER_INIT:
+ for (AuxiliaryService serv : serviceMap.values()) {
+ serv.initializeContainer(new ContainerInitializationContext(
+ event.getUser(), event.getContainer().getContainerId(),
+ event.getContainer().getResource()));
+ }
+ break;
+ case CONTAINER_STOP:
+ for (AuxiliaryService serv : serviceMap.values()) {
+ serv.stopContainer(new ContainerTerminationContext(
+ event.getUser(), event.getContainer().getContainerId(),
+ event.getContainer().getResource()));
+ }
+ break;
+ default:
throw new RuntimeException("Unknown type: " + event.getType());
}
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServicesEvent.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServicesEvent.java?rev=1521566&r1=1521565&r2=1521566&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServicesEvent.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServicesEvent.java Tue Sep 10 18:03:08 2013
@@ -21,7 +21,10 @@ package org.apache.hadoop.yarn.server.no
import java.nio.ByteBuffer;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.event.AbstractEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container
+ .Container;
public class AuxServicesEvent extends AbstractEvent<AuxServicesEventType> {
@@ -29,18 +32,30 @@ public class AuxServicesEvent extends Ab
private final String serviceId;
private final ByteBuffer serviceData;
private final ApplicationId appId;
+ private final Container container;
public AuxServicesEvent(AuxServicesEventType eventType, ApplicationId appId) {
this(eventType, null, appId, null, null);
}
+ public AuxServicesEvent(AuxServicesEventType eventType, Container container) {
+ this(eventType, null, container.getContainerId().getApplicationAttemptId()
+ .getApplicationId(), null, null, container);
+ }
+
public AuxServicesEvent(AuxServicesEventType eventType, String user,
ApplicationId appId, String serviceId, ByteBuffer serviceData) {
+ this(eventType, user, appId, serviceId, serviceData, null);
+ }
+ public AuxServicesEvent(AuxServicesEventType eventType, String user,
+ ApplicationId appId, String serviceId, ByteBuffer serviceData,
+ Container container) {
super(eventType);
this.user = user;
this.appId = appId;
this.serviceId = serviceId;
this.serviceData = serviceData;
+ this.container = container;
}
public String getServiceID() {
@@ -59,4 +74,8 @@ public class AuxServicesEvent extends Ab
return appId;
}
+ public Container getContainer() {
+ return container;
+ }
+
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServicesEventType.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServicesEventType.java?rev=1521566&r1=1521565&r2=1521566&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServicesEventType.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServicesEventType.java Tue Sep 10 18:03:08 2013
@@ -20,5 +20,7 @@ package org.apache.hadoop.yarn.server.no
public enum AuxServicesEventType {
APPLICATION_INIT,
- APPLICATION_STOP
+ APPLICATION_STOP,
+ CONTAINER_INIT,
+ CONTAINER_STOP
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java?rev=1521566&r1=1521565&r2=1521566&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java Tue Sep 10 18:03:08 2013
@@ -503,6 +503,9 @@ public class ContainerImpl implements Co
final ContainerLaunchContext ctxt = container.launchContext;
container.metrics.initingContainer();
+ container.dispatcher.getEventHandler().handle(new AuxServicesEvent
+ (AuxServicesEventType.CONTAINER_INIT, container));
+
// Inform the AuxServices about the opaque serviceData
Map<String,ByteBuffer> csd = ctxt.getServiceData();
if (csd != null) {
@@ -820,8 +823,16 @@ public class ContainerImpl implements Co
static class ContainerDoneTransition implements
SingleArcTransition<ContainerImpl, ContainerEvent> {
@Override
+ @SuppressWarnings("unchecked")
public void transition(ContainerImpl container, ContainerEvent event) {
container.finished();
+ //if the current state is NEW it means the CONTAINER_INIT was never
+ // sent for the event, thus no need to send the CONTAINER_STOP
+ if (container.getCurrentState()
+ != org.apache.hadoop.yarn.api.records.ContainerState.NEW) {
+ container.dispatcher.getEventHandler().handle(new AuxServicesEvent
+ (AuxServicesEventType.CONTAINER_STOP, container));
+ }
}
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java?rev=1521566&r1=1521565&r2=1521566&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java Tue Sep 10 18:03:08 2013
@@ -22,6 +22,7 @@ import static org.apache.hadoop.service.
import static org.apache.hadoop.service.Service.STATE.STARTED;
import static org.apache.hadoop.service.Service.STATE.STOPPED;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -34,11 +35,21 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.Service;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.api.ApplicationInitializationContext;
import org.apache.hadoop.yarn.server.api.ApplicationTerminationContext;
import org.apache.hadoop.yarn.server.api.AuxiliaryService;
+import org.apache.hadoop.yarn.server.api.ContainerInitializationContext;
+import org.apache.hadoop.yarn.server.api.ContainerTerminationContext;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container
+ .Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container
+ .ContainerImpl;
import org.junit.Test;
public class TestAuxServices {
@@ -52,8 +63,10 @@ public class TestAuxServices {
private int remaining_stop;
private ByteBuffer meta = null;
private ArrayList<Integer> stoppedApps;
+ private ContainerId containerId;
+ private Resource resource;
- LightService(String name, char idef, int expected_appId) {
+ LightService(String name, char idef, int expected_appId) {
this(name, idef, expected_appId, null);
}
LightService(String name, char idef, int expected_appId, ByteBuffer meta) {
@@ -95,7 +108,22 @@ public class TestAuxServices {
public ByteBuffer getMetaData() {
return meta;
}
- }
+
+ @Override
+ public void initializeContainer(
+ ContainerInitializationContext initContainerContext) {
+ containerId = initContainerContext.getContainerId();
+ resource = initContainerContext.getResource();
+ }
+
+ @Override
+ public void stopContainer(
+ ContainerTerminationContext stopContainerContext) {
+ containerId = stopContainerContext.getContainerId();
+ resource = stopContainerContext.getResource();
+ }
+
+ }
static class ServiceA extends LightService {
public ServiceA() {
@@ -142,6 +170,35 @@ public class TestAuxServices {
assertEquals("app not properly stopped", 1, appIds.size());
assertTrue("wrong app stopped", appIds.contains((Integer)66));
}
+
+ for (AuxiliaryService serv : servs) {
+ assertNull(((LightService) serv).containerId);
+ assertNull(((LightService) serv).resource);
+ }
+
+
+ ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId1, 1);
+ ContainerTokenIdentifier cti = new ContainerTokenIdentifier(
+ ContainerId.newInstance(attemptId, 1), "", "",
+ Resource.newInstance(1, 1), 0,0,0);
+ Container container = new ContainerImpl(null, null, null, null, null, cti);
+ ContainerId containerId = container.getContainerId();
+ Resource resource = container.getResource();
+ event = new AuxServicesEvent(AuxServicesEventType.CONTAINER_INIT,container);
+ aux.handle(event);
+ for (AuxiliaryService serv : servs) {
+ assertEquals(containerId, ((LightService) serv).containerId);
+ assertEquals(resource, ((LightService) serv).resource);
+ ((LightService) serv).containerId = null;
+ ((LightService) serv).resource = null;
+ }
+
+ event = new AuxServicesEvent(AuxServicesEventType.CONTAINER_STOP, container);
+ aux.handle(event);
+ for (AuxiliaryService serv : servs) {
+ assertEquals(containerId, ((LightService) serv).containerId);
+ assertEquals(resource, ((LightService) serv).resource);
+ }
}
@Test
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java?rev=1521566&r1=1521565&r2=1521566&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java Tue Sep 10 18:03:08 2013
@@ -43,10 +43,10 @@ import org.apache.hadoop.service.Abstrac
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest;
import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest;
@@ -78,7 +78,6 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.ipc.RPCUtil;
@@ -88,7 +87,6 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType;
-import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
@@ -160,9 +158,6 @@ public class ClientRMService extends Abs
this.server.start();
clientBindAddress = conf.updateConnectAddr(YarnConfiguration.RM_ADDRESS,
server.getListenerAddress());
- // enable RM to short-circuit token operations directly to itself
- RMDelegationTokenIdentifier.Renewer.setSecretManager(
- rmDTSecretManager, clientBindAddress);
super.serviceStart();
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java?rev=1521566&r1=1521565&r2=1521566&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java Tue Sep 10 18:03:08 2013
@@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
+import org.apache.hadoop.yarn.server.resourcemanager.security.RMDelegationTokenSecretManager;
/**
* Context of the ResourceManager.
@@ -64,4 +65,13 @@ public interface RMContext {
NMTokenSecretManagerInRM getNMTokenSecretManager();
ClientToAMTokenSecretManagerInRM getClientToAMTokenSecretManager();
+
+ void setClientRMService(ClientRMService clientRMService);
+
+ ClientRMService getClientRMService();
+
+ RMDelegationTokenSecretManager getRMDelegationTokenSecretManager();
+
+ void setRMDelegationTokenSecretManager(
+ RMDelegationTokenSecretManager delegationTokenSecretManager);
}
\ No newline at end of file
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java?rev=1521566&r1=1521565&r2=1521566&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java Tue Sep 10 18:03:08 2013
@@ -36,6 +36,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
+import org.apache.hadoop.yarn.server.resourcemanager.security.RMDelegationTokenSecretManager;
import com.google.common.annotations.VisibleForTesting;
@@ -61,6 +62,8 @@ public class RMContextImpl implements RM
private final RMContainerTokenSecretManager containerTokenSecretManager;
private final NMTokenSecretManagerInRM nmTokenSecretManager;
private final ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager;
+ private ClientRMService clientRMService;
+ private RMDelegationTokenSecretManager rmDelegationTokenSecretManager;
public RMContextImpl(Dispatcher rmDispatcher,
RMStateStore store,
@@ -178,4 +181,25 @@ public class RMContextImpl implements RM
public void setStateStore(RMStateStore store) {
stateStore = store;
}
+
+ @Override
+ public ClientRMService getClientRMService() {
+ return this.clientRMService;
+ }
+
+ @Override
+ public void setClientRMService(ClientRMService clientRMService) {
+ this.clientRMService = clientRMService;
+ }
+
+ @Override
+ public RMDelegationTokenSecretManager getRMDelegationTokenSecretManager() {
+ return this.rmDelegationTokenSecretManager;
+ }
+
+ @Override
+ public void setRMDelegationTokenSecretManager(
+ RMDelegationTokenSecretManager delegationTokenSecretManager) {
+ this.rmDelegationTokenSecretManager = delegationTokenSecretManager;
+ }
}
\ No newline at end of file
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java?rev=1521566&r1=1521565&r2=1521566&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java Tue Sep 10 18:03:08 2013
@@ -107,9 +107,18 @@ public class ResourceManager extends Com
private static final Log LOG = LogFactory.getLog(ResourceManager.class);
public static final long clusterTimeStamp = System.currentTimeMillis();
+ /**
+ * "Active" services. Services that need to run only on the Active RM.
+ * These services are managed (initialized, started, stopped) by the
+ * {@link CompositeService} RMActiveServices.
+ *
+ * RM is active when (1) HA is disabled, or (2) HA is enabled and the RM is
+ * in Active state.
+ */
+ protected RMActiveServices activeServices;
protected ClientToAMTokenSecretManagerInRM clientToAMSecretManager =
new ClientToAMTokenSecretManagerInRM();
-
+
protected RMContainerTokenSecretManager containerTokenSecretManager;
protected NMTokenSecretManagerInRM nmTokenSecretManager;
@@ -135,6 +144,8 @@ public class ResourceManager extends Com
protected ResourceTrackerService resourceTracker;
private boolean recoveryEnabled;
+ /** End of Active services */
+
private Configuration conf;
public ResourceManager() {
@@ -147,133 +158,11 @@ public class ResourceManager extends Com
@Override
protected void serviceInit(Configuration conf) throws Exception {
-
validateConfigs(conf);
-
this.conf = conf;
- this.conf.setBoolean(Dispatcher.DISPATCHER_EXIT_ON_ERROR_KEY, true);
-
- this.rmDispatcher = createDispatcher();
- addIfService(this.rmDispatcher);
-
- this.amRmTokenSecretManager = createAMRMTokenSecretManager(conf);
-
- this.containerAllocationExpirer = new ContainerAllocationExpirer(
- this.rmDispatcher);
- addService(this.containerAllocationExpirer);
-
- AMLivelinessMonitor amLivelinessMonitor = createAMLivelinessMonitor();
- addService(amLivelinessMonitor);
-
- AMLivelinessMonitor amFinishingMonitor = createAMLivelinessMonitor();
- addService(amFinishingMonitor);
-
- if (UserGroupInformation.isSecurityEnabled()) {
- this.delegationTokenRenewer = createDelegationTokenRenewer();
- addService(delegationTokenRenewer);
- }
-
- this.containerTokenSecretManager = createContainerTokenSecretManager(conf);
- this.nmTokenSecretManager = createNMTokenSecretManager(conf);
-
- boolean isRecoveryEnabled = conf.getBoolean(
- YarnConfiguration.RECOVERY_ENABLED,
- YarnConfiguration.DEFAULT_RM_RECOVERY_ENABLED);
-
- RMStateStore rmStore = null;
- if(isRecoveryEnabled) {
- recoveryEnabled = true;
- rmStore = RMStateStoreFactory.getStore(conf);
- } else {
- recoveryEnabled = false;
- rmStore = new NullRMStateStore();
- }
-
- try {
- rmStore.init(conf);
- rmStore.setRMDispatcher(rmDispatcher);
- } catch (Exception e) {
- // the Exception from stateStore.init() needs to be handled for
- // HA and we need to give up master status if we got fenced
- LOG.error("Failed to init state store", e);
- ExitUtil.terminate(1, e);
- }
-
- this.rmContext =
- new RMContextImpl(this.rmDispatcher, rmStore,
- this.containerAllocationExpirer, amLivelinessMonitor,
- amFinishingMonitor, delegationTokenRenewer, this.amRmTokenSecretManager,
- this.containerTokenSecretManager, this.nmTokenSecretManager,
- this.clientToAMSecretManager);
-
- // Register event handler for NodesListManager
- this.nodesListManager = new NodesListManager(this.rmContext);
- this.rmDispatcher.register(NodesListManagerEventType.class,
- this.nodesListManager);
- addService(nodesListManager);
-
- // Initialize the scheduler
- this.scheduler = createScheduler();
- this.schedulerDispatcher = createSchedulerEventDispatcher();
- addIfService(this.schedulerDispatcher);
- this.rmDispatcher.register(SchedulerEventType.class,
- this.schedulerDispatcher);
-
- // Register event handler for RmAppEvents
- this.rmDispatcher.register(RMAppEventType.class,
- new ApplicationEventDispatcher(this.rmContext));
-
- // Register event handler for RmAppAttemptEvents
- this.rmDispatcher.register(RMAppAttemptEventType.class,
- new ApplicationAttemptEventDispatcher(this.rmContext));
-
- // Register event handler for RmNodes
- this.rmDispatcher.register(RMNodeEventType.class,
- new NodeEventDispatcher(this.rmContext));
-
- this.nmLivelinessMonitor = createNMLivelinessMonitor();
- addService(this.nmLivelinessMonitor);
-
- this.resourceTracker = createResourceTrackerService();
- addService(resourceTracker);
-
- DefaultMetricsSystem.initialize("ResourceManager");
- JvmMetrics.initSingleton("ResourceManager", null);
-
- try {
- this.scheduler.reinitialize(conf, this.rmContext);
- } catch (IOException ioe) {
- throw new RuntimeException("Failed to initialize scheduler", ioe);
- }
-
- // creating monitors that handle preemption
- createPolicyMonitors();
-
- masterService = createApplicationMasterService();
- addService(masterService) ;
-
- this.applicationACLsManager = new ApplicationACLsManager(conf);
-
- this.rmAppManager = createRMAppManager();
- // Register event handler for RMAppManagerEvents
- this.rmDispatcher.register(RMAppManagerEventType.class,
- this.rmAppManager);
- this.rmDTSecretManager = createRMDelegationTokenSecretManager(this.rmContext);
- clientRM = createClientRMService();
- addService(clientRM);
-
- adminService = createAdminService(clientRM, masterService, resourceTracker);
- addService(adminService);
-
- this.applicationMasterLauncher = createAMLauncher();
- this.rmDispatcher.register(AMLauncherEventType.class,
- this.applicationMasterLauncher);
-
- addService(applicationMasterLauncher);
-
- new RMNMInfo(this.rmContext, this.scheduler);
-
+ activeServices = new RMActiveServices();
+ addService(activeServices);
super.serviceInit(conf);
}
@@ -374,6 +263,217 @@ public class ResourceManager extends Com
}
}
+ /**
+ * RMActiveServices handles all the Active services in the RM.
+ */
+ @Private
+ class RMActiveServices extends CompositeService {
+ RMActiveServices() {
+ super("RMActiveServices");
+ }
+
+ @Override
+ protected void serviceInit(Configuration configuration) throws Exception {
+ conf.setBoolean(Dispatcher.DISPATCHER_EXIT_ON_ERROR_KEY, true);
+
+ rmDispatcher = createDispatcher();
+ addIfService(rmDispatcher);
+
+ amRmTokenSecretManager = createAMRMTokenSecretManager(conf);
+
+ containerAllocationExpirer = new ContainerAllocationExpirer(rmDispatcher);
+ addService(containerAllocationExpirer);
+
+ AMLivelinessMonitor amLivelinessMonitor = createAMLivelinessMonitor();
+ addService(amLivelinessMonitor);
+
+ AMLivelinessMonitor amFinishingMonitor = createAMLivelinessMonitor();
+ addService(amFinishingMonitor);
+
+ containerTokenSecretManager = createContainerTokenSecretManager(conf);
+ nmTokenSecretManager = createNMTokenSecretManager(conf);
+
+ boolean isRecoveryEnabled = conf.getBoolean(
+ YarnConfiguration.RECOVERY_ENABLED,
+ YarnConfiguration.DEFAULT_RM_RECOVERY_ENABLED);
+
+ RMStateStore rmStore = null;
+ if(isRecoveryEnabled) {
+ recoveryEnabled = true;
+ rmStore = RMStateStoreFactory.getStore(conf);
+ } else {
+ recoveryEnabled = false;
+ rmStore = new NullRMStateStore();
+ }
+
+ try {
+ rmStore.init(conf);
+ rmStore.setRMDispatcher(rmDispatcher);
+ } catch (Exception e) {
+ // the Exception from stateStore.init() needs to be handled for
+ // HA and we need to give up master status if we got fenced
+ LOG.error("Failed to init state store", e);
+ ExitUtil.terminate(1, e);
+ }
+
+ if (UserGroupInformation.isSecurityEnabled()) {
+ delegationTokenRenewer = createDelegationTokenRenewer();
+ }
+
+ rmContext = new RMContextImpl(
+ rmDispatcher, rmStore, containerAllocationExpirer, amLivelinessMonitor,
+ amFinishingMonitor, delegationTokenRenewer, amRmTokenSecretManager,
+ containerTokenSecretManager, nmTokenSecretManager,
+ clientToAMSecretManager);
+
+ // Register event handler for NodesListManager
+ nodesListManager = new NodesListManager(rmContext);
+ rmDispatcher.register(NodesListManagerEventType.class, nodesListManager);
+ addService(nodesListManager);
+
+ // Initialize the scheduler
+ scheduler = createScheduler();
+ schedulerDispatcher = createSchedulerEventDispatcher();
+ addIfService(schedulerDispatcher);
+ rmDispatcher.register(SchedulerEventType.class, schedulerDispatcher);
+
+ // Register event handler for RmAppEvents
+ rmDispatcher.register(RMAppEventType.class,
+ new ApplicationEventDispatcher(rmContext));
+
+ // Register event handler for RmAppAttemptEvents
+ rmDispatcher.register(RMAppAttemptEventType.class,
+ new ApplicationAttemptEventDispatcher(rmContext));
+
+ // Register event handler for RmNodes
+ rmDispatcher.register(
+ RMNodeEventType.class, new NodeEventDispatcher(rmContext));
+
+ nmLivelinessMonitor = createNMLivelinessMonitor();
+ addService(nmLivelinessMonitor);
+
+ resourceTracker = createResourceTrackerService();
+ addService(resourceTracker);
+
+ DefaultMetricsSystem.initialize("ResourceManager");
+ JvmMetrics.initSingleton("ResourceManager", null);
+
+ try {
+ scheduler.reinitialize(conf, rmContext);
+ } catch (IOException ioe) {
+ throw new RuntimeException("Failed to initialize scheduler", ioe);
+ }
+
+ // creating monitors that handle preemption
+ createPolicyMonitors();
+
+ masterService = createApplicationMasterService();
+ addService(masterService) ;
+
+ applicationACLsManager = new ApplicationACLsManager(conf);
+
+ rmAppManager = createRMAppManager();
+ // Register event handler for RMAppManagerEvents
+ rmDispatcher.register(RMAppManagerEventType.class, rmAppManager);
+ rmDTSecretManager = createRMDelegationTokenSecretManager(rmContext);
+ rmContext.setRMDelegationTokenSecretManager(rmDTSecretManager);
+ clientRM = createClientRMService();
+ rmContext.setClientRMService(clientRM);
+ addService(clientRM);
+
+ adminService = createAdminService(clientRM, masterService, resourceTracker);
+ addService(adminService);
+
+ applicationMasterLauncher = createAMLauncher();
+ rmDispatcher.register(AMLauncherEventType.class,
+ applicationMasterLauncher);
+
+ addService(applicationMasterLauncher);
+ if (UserGroupInformation.isSecurityEnabled()) {
+ addService(delegationTokenRenewer);
+ delegationTokenRenewer.setRMContext(rmContext);
+ }
+
+ new RMNMInfo(rmContext, scheduler);
+
+ super.serviceInit(conf);
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+ amRmTokenSecretManager.start();
+ containerTokenSecretManager.start();
+ nmTokenSecretManager.start();
+
+ RMStateStore rmStore = rmContext.getStateStore();
+ // The state store needs to start irrespective of recoveryEnabled as apps
+ // need events to move to further states.
+ rmStore.start();
+
+ if(recoveryEnabled) {
+ try {
+ RMState state = rmStore.loadState();
+ recover(state);
+ } catch (Exception e) {
+ // the Exception from loadState() needs to be handled for
+ // HA and we need to give up master status if we got fenced
+ LOG.error("Failed to load/recover state", e);
+ ExitUtil.terminate(1, e);
+ }
+ }
+
+ startWepApp();
+ try {
+ rmDTSecretManager.startThreads();
+ } catch(IOException ie) {
+ throw new YarnRuntimeException("Failed to start secret manager threads", ie);
+ }
+
+ if (getConfig().getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) {
+ String hostname = getConfig().get(YarnConfiguration.RM_WEBAPP_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS);
+ hostname = (hostname.contains(":")) ? hostname.substring(0, hostname.indexOf(":")) : hostname;
+ int port = webApp.port();
+ String resolvedAddress = hostname + ":" + port;
+ conf.set(YarnConfiguration.RM_WEBAPP_ADDRESS, resolvedAddress);
+ }
+
+ super.serviceStart();
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ if (webApp != null) {
+ webApp.stop();
+ }
+ if (rmDTSecretManager != null) {
+ rmDTSecretManager.stopThreads();
+ }
+
+ if (amRmTokenSecretManager != null) {
+ amRmTokenSecretManager.stop();
+ }
+ if (containerTokenSecretManager != null) {
+ containerTokenSecretManager.stop();
+ }
+ if(nmTokenSecretManager != null) {
+ nmTokenSecretManager.stop();
+ }
+
+ DefaultMetricsSystem.shutdown();
+
+ if (rmContext != null) {
+ RMStateStore store = rmContext.getStateStore();
+ try {
+ store.close();
+ } catch (Exception e) {
+ LOG.error("Error closing store.", e);
+ }
+ }
+ super.serviceStop();
+ }
+ }
+
@Private
public static class SchedulerEventDispatcher extends AbstractService
implements EventHandler<SchedulerEvent> {
@@ -616,61 +716,7 @@ public class ResourceManager extends Com
throw new YarnRuntimeException("Failed to login", ie);
}
- this.amRmTokenSecretManager.start();
- this.containerTokenSecretManager.start();
- this.nmTokenSecretManager.start();
-
- // Explicitly start DTRenewer too in secure mode before kicking recovery as
- // tokens will start getting added for renewal as part of the recovery
- // process itself.
- if (UserGroupInformation.isSecurityEnabled()) {
- this.delegationTokenRenewer.start();
- }
-
- RMStateStore rmStore = rmContext.getStateStore();
- // The state store needs to start irrespective of recoveryEnabled as apps
- // need events to move to further states.
- rmStore.start();
-
- if(recoveryEnabled) {
- try {
- RMState state = rmStore.loadState();
- recover(state);
- } catch (Exception e) {
- // the Exception from loadState() needs to be handled for
- // HA and we need to give up master status if we got fenced
- LOG.error("Failed to load/recover state", e);
- ExitUtil.terminate(1, e);
- }
- }
-
- startWepApp();
- try {
- rmDTSecretManager.startThreads();
- } catch(IOException ie) {
- throw new YarnRuntimeException("Failed to start secret manager threads", ie);
- }
-
- if (getConfig().getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) {
- String hostname = getConfig().get(YarnConfiguration.RM_WEBAPP_ADDRESS,
- YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS);
- hostname = (hostname.contains(":")) ? hostname.substring(0, hostname.indexOf(":")) : hostname;
- int port = webApp.port();
- String resolvedAddress = hostname + ":" + port;
- conf.set(YarnConfiguration.RM_WEBAPP_ADDRESS, resolvedAddress);
- }
-
super.serviceStart();
-
- /*synchronized(shutdown) {
- try {
- while(!shutdown.get()) {
- shutdown.wait();
- }
- } catch(InterruptedException ie) {
- LOG.info("Interrupted while waiting", ie);
- }
- }*/
}
protected void doSecureLogin() throws IOException {
@@ -680,39 +726,6 @@ public class ResourceManager extends Com
@Override
protected void serviceStop() throws Exception {
- if (webApp != null) {
- webApp.stop();
- }
- if (rmDTSecretManager != null) {
- rmDTSecretManager.stopThreads();
- }
-
- if (amRmTokenSecretManager != null) {
- this.amRmTokenSecretManager.stop();
- }
- if (containerTokenSecretManager != null) {
- this.containerTokenSecretManager.stop();
- }
- if(nmTokenSecretManager != null) {
- nmTokenSecretManager.stop();
- }
-
- /*synchronized(shutdown) {
- shutdown.set(true);
- shutdown.notifyAll();
- }*/
-
- DefaultMetricsSystem.shutdown();
-
- if (rmContext != null) {
- RMStateStore store = rmContext.getStateStore();
- try {
- store.close();
- } catch (Exception e) {
- LOG.error("Error closing store.", e);
- }
- }
-
super.serviceStop();
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java?rev=1521566&r1=1521565&r2=1521566&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java Tue Sep 10 18:03:08 2013
@@ -432,18 +432,18 @@ public class RMAppImpl implements RMApp,
currentApplicationAttemptId = this.currentAttempt.getAppAttemptId();
trackingUrl = this.currentAttempt.getTrackingUrl();
origTrackingUrl = this.currentAttempt.getOriginalTrackingUrl();
- if (UserGroupInformation.isSecurityEnabled()
- && clientUserName != null) {
+ if (UserGroupInformation.isSecurityEnabled()) {
+ // get a token so the client can communicate with the app attempt
+ // NOTE: token may be unavailable if the attempt is not running
Token<ClientToAMTokenIdentifier> attemptClientToAMToken =
- new Token<ClientToAMTokenIdentifier>(
- new ClientToAMTokenIdentifier(
- currentApplicationAttemptId, clientUserName),
- rmContext.getClientToAMTokenSecretManager());
- clientToAMToken = BuilderUtils.newClientToAMToken(
- attemptClientToAMToken.getIdentifier(),
- attemptClientToAMToken.getKind().toString(),
- attemptClientToAMToken.getPassword(),
- attemptClientToAMToken.getService().toString());
+ this.currentAttempt.createClientToken(clientUserName);
+ if (attemptClientToAMToken != null) {
+ clientToAMToken = BuilderUtils.newClientToAMToken(
+ attemptClientToAMToken.getIdentifier(),
+ attemptClientToAMToken.getKind().toString(),
+ attemptClientToAMToken.getPassword(),
+ attemptClientToAMToken.getService().toString());
+ }
}
host = this.currentAttempt.getHost();
rpcPort = this.currentAttempt.getRpcPort();
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java?rev=1521566&r1=1521565&r2=1521566&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java Tue Sep 10 18:03:08 2013
@@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
+import org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
/**
@@ -156,6 +157,13 @@ public interface RMAppAttempt extends Ev
SecretKey getClientTokenMasterKey();
/**
+ * Create a token for authenticating a client connection to the app attempt
+ * @param clientName the name of the client requesting the token
+ * @return the token or null if the attempt is not running
+ */
+ Token<ClientToAMTokenIdentifier> createClientToken(String clientName);
+
+ /**
* Get application container and resource usage information.
* @return an ApplicationResourceUsageReport object.
*/
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java?rev=1521566&r1=1521565&r2=1521566&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java Tue Sep 10 18:03:08 2013
@@ -61,6 +61,7 @@ import org.apache.hadoop.yarn.event.Even
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
+import org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier;
import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEvent;
@@ -89,6 +90,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils;
import org.apache.hadoop.yarn.state.InvalidStateTransitonException;
@@ -440,7 +442,8 @@ public class RMAppAttemptImpl implements
public String getTrackingUrl() {
this.readLock.lock();
try {
- return this.proxiedTrackingUrl;
+ return (getSubmissionContext().getUnmanagedAM()) ?
+ this.origTrackingUrl : this.proxiedTrackingUrl;
} finally {
this.readLock.unlock();
}
@@ -508,6 +511,26 @@ public class RMAppAttemptImpl implements
}
@Override
+ public Token<ClientToAMTokenIdentifier> createClientToken(String client) {
+ this.readLock.lock();
+
+ try {
+ Token<ClientToAMTokenIdentifier> token = null;
+ ClientToAMTokenSecretManagerInRM secretMgr =
+ this.rmContext.getClientToAMTokenSecretManager();
+ if (client != null &&
+ secretMgr.getMasterKey(this.applicationAttemptId) != null) {
+ token = new Token<ClientToAMTokenIdentifier>(
+ new ClientToAMTokenIdentifier(this.applicationAttemptId, client),
+ secretMgr);
+ }
+ return token;
+ } finally {
+ this.readLock.unlock();
+ }
+ }
+
+ @Override
public String getDiagnostics() {
this.readLock.lock();
@@ -810,7 +833,11 @@ public class RMAppAttemptImpl implements
Allocation amContainerAllocation = appAttempt.scheduler.allocate(
appAttempt.applicationAttemptId, EMPTY_CONTAINER_REQUEST_LIST,
EMPTY_CONTAINER_RELEASE_LIST, null, null);
-
+ // There must be at least one container allocated, because a
+ // CONTAINER_ALLOCATED is emitted after an RMContainer is constructed,
+ // and is put in SchedulerApplication#newlyAllocatedContainers. Then,
+ // YarnScheduler#allocate will fetch it.
+ assert amContainerAllocation.getContainers().size() != 0;
// Set the masterContainer
appAttempt.setMasterContainer(amContainerAllocation.getContainers().get(
0));
@@ -961,7 +988,7 @@ public class RMAppAttemptImpl implements
}
}
- private static final class AMRegisteredTransition extends BaseTransition {
+ static final class AMRegisteredTransition extends BaseTransition {
@Override
public void transition(RMAppAttemptImpl appAttempt,
RMAppAttemptEvent event) {
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java?rev=1521566&r1=1521565&r2=1521566&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java Tue Sep 10 18:03:08 2013
@@ -63,11 +63,11 @@ public class SchedulerUtils {
public static final String UNRESERVED_CONTAINER =
"Container reservation no longer required.";
-
+
/**
* Utility to create a {@link ContainerStatus} during exceptional
* circumstances.
- *
+ *
* @param containerId {@link ContainerId} of returned/released/lost container.
* @param diagnostics diagnostic message
* @return <code>ContainerStatus</code> for an returned/released/lost
@@ -75,12 +75,41 @@ public class SchedulerUtils {
*/
public static ContainerStatus createAbnormalContainerStatus(
ContainerId containerId, String diagnostics) {
+ return createAbnormalContainerStatus(containerId,
+ ContainerExitStatus.ABORTED, diagnostics);
+ }
+
+ /**
+ * Utility to create a {@link ContainerStatus} during exceptional
+ * circumstances.
+ *
+ * @param containerId {@link ContainerId} of returned/released/lost container.
+ * @param diagnostics diagnostic message
+ * @return <code>ContainerStatus</code> for an returned/released/lost
+ * container
+ */
+ public static ContainerStatus createPreemptedContainerStatus(
+ ContainerId containerId, String diagnostics) {
+ return createAbnormalContainerStatus(containerId,
+ ContainerExitStatus.PREEMPTED, diagnostics);
+ }
+
+ /**
+ * Utility to create a {@link ContainerStatus} during exceptional
+ * circumstances.
+ *
+ * @param containerId {@link ContainerId} of returned/released/lost container.
+ * @param diagnostics diagnostic message
+ * @return <code>ContainerStatus</code> for an returned/released/lost
+ * container
+ */
+ private static ContainerStatus createAbnormalContainerStatus(
+ ContainerId containerId, int exitStatus, String diagnostics) {
ContainerStatus containerStatus =
recordFactory.newRecordInstance(ContainerStatus.class);
containerStatus.setContainerId(containerId);
containerStatus.setDiagnostics(diagnostics);
- containerStatus.setExitStatus(
- ContainerExitStatus.ABORTED);
+ containerStatus.setExitStatus(exitStatus);
containerStatus.setState(ContainerState.COMPLETE);
return containerStatus;
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java?rev=1521566&r1=1521565&r2=1521566&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java Tue Sep 10 18:03:08 2013
@@ -80,6 +80,8 @@ import org.apache.hadoop.yarn.server.uti
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
+import com.google.common.annotations.VisibleForTesting;
+
@LimitedPrivate("yarn")
@Evolving
@SuppressWarnings("unchecked")
@@ -179,7 +181,8 @@ public class CapacityScheduler
private Resource minimumAllocation;
private Resource maximumAllocation;
- private Map<ApplicationAttemptId, FiCaSchedulerApp> applications =
+ @VisibleForTesting
+ protected Map<ApplicationAttemptId, FiCaSchedulerApp> applications =
new ConcurrentHashMap<ApplicationAttemptId, FiCaSchedulerApp>();
private boolean initialized = false;
@@ -904,7 +907,7 @@ public class CapacityScheduler
LOG.debug("KILL_CONTAINER: container" + cont.toString());
}
completedContainer(cont,
- SchedulerUtils.createAbnormalContainerStatus(
+ SchedulerUtils.createPreemptedContainerStatus(
cont.getContainerId(),"Container being forcibly preempted:"
+ cont.getContainerId()),
RMContainerEventType.KILL);
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java?rev=1521566&r1=1521565&r2=1521566&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java Tue Sep 10 18:03:08 2013
@@ -37,7 +37,6 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -50,6 +49,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger;
import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants;
@@ -154,8 +154,9 @@ public class FairScheduler implements Re
// This stores per-application scheduling information, indexed by
// attempt ID's for fast lookup.
+ @VisibleForTesting
protected Map<ApplicationAttemptId, FSSchedulerApp> applications =
- new HashMap<ApplicationAttemptId, FSSchedulerApp>();
+ new ConcurrentHashMap<ApplicationAttemptId, FSSchedulerApp>();
// Nodes in the cluster, indexed by NodeId
private Map<NodeId, FSSchedulerNode> nodes =
@@ -444,7 +445,7 @@ public class FairScheduler implements Re
// proceed with kill
if (time + waitTimeBeforeKill < clock.getTime()) {
ContainerStatus status =
- SchedulerUtils.createAbnormalContainerStatus(
+ SchedulerUtils.createPreemptedContainerStatus(
container.getContainerId(), SchedulerUtils.PREEMPTED_CONTAINER);
// TODO: Not sure if this ever actually adds this to the list of cleanup
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java?rev=1521566&r1=1521565&r2=1521566&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java Tue Sep 10 18:03:08 2013
@@ -25,8 +25,8 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.TreeMap;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentSkipListMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -36,12 +36,10 @@ import org.apache.hadoop.conf.Configurab
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AccessControlList;
-import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.QueueACL;
@@ -50,7 +48,9 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger;
@@ -90,6 +90,8 @@ import org.apache.hadoop.yarn.util.resou
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
+import com.google.common.annotations.VisibleForTesting;
+
@LimitedPrivate("yarn")
@Evolving
@SuppressWarnings("unchecked")
@@ -113,8 +115,10 @@ public class FifoScheduler implements Re
private Resource maximumAllocation;
private boolean usePortForNodeName;
- private Map<ApplicationAttemptId, FiCaSchedulerApp> applications
- = new TreeMap<ApplicationAttemptId, FiCaSchedulerApp>();
+ // Use ConcurrentSkipListMap because applications need to be ordered
+ @VisibleForTesting
+ protected Map<ApplicationAttemptId, FiCaSchedulerApp> applications
+ = new ConcurrentSkipListMap<ApplicationAttemptId, FiCaSchedulerApp>();
private ActiveUsersManager activeUsersManager;
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java?rev=1521566&r1=1521565&r2=1521566&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java Tue Sep 10 18:03:08 2013
@@ -34,6 +34,7 @@ import java.util.TimerTask;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -47,6 +48,8 @@ import org.apache.hadoop.service.Abstrac
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import com.google.common.annotations.VisibleForTesting;
@@ -64,6 +67,7 @@ public class DelegationTokenRenewer exte
// global single timer (daemon)
private Timer renewalTimer;
+ private RMContext rmContext;
// delegation token canceler thread
private DelegationTokenCancelThread dtCancelThread =
@@ -80,6 +84,9 @@ public class DelegationTokenRenewer exte
private long tokenRemovalDelayMs;
private Thread delayedRemovalThread;
+ private boolean isServiceStarted = false;
+ private List<DelegationTokenToRenew> pendingTokenForRenewal =
+ new ArrayList<DelegationTokenRenewer.DelegationTokenToRenew>();
private boolean tokenKeepAliveEnabled;
@@ -100,7 +107,6 @@ public class DelegationTokenRenewer exte
@Override
protected void serviceStart() throws Exception {
-
dtCancelThread.start();
renewalTimer = new Timer(true);
if (tokenKeepAliveEnabled) {
@@ -109,6 +115,15 @@ public class DelegationTokenRenewer exte
"DelayedTokenCanceller");
delayedRemovalThread.start();
}
+ // enable RM to short-circuit token operations directly to itself
+ RMDelegationTokenIdentifier.Renewer.setSecretManager(
+ rmContext.getRMDelegationTokenSecretManager(),
+ rmContext.getClientRMService().getBindAddress());
+ // Delegation token renewal is delayed until ClientRMService starts. As
+ // it is required to short circuit the token renewal calls.
+ isServiceStarted = true;
+ renewIfServiceIsStarted(pendingTokenForRenewal);
+ pendingTokenForRenewal.clear();
super.serviceStart();
}
@@ -275,8 +290,8 @@ public class DelegationTokenRenewer exte
* @throws IOException
*/
public void addApplication(
- ApplicationId applicationId, Credentials ts, boolean shouldCancelAtEnd)
- throws IOException {
+ ApplicationId applicationId, Credentials ts, boolean shouldCancelAtEnd)
+ throws IOException {
if (ts == null) {
return; //nothing to add
}
@@ -291,25 +306,40 @@ public class DelegationTokenRenewer exte
// find tokens for renewal, but don't add timers until we know
// all renewable tokens are valid
- Set<DelegationTokenToRenew> dtrs = new HashSet<DelegationTokenToRenew>();
+ // At RM restart it is safe to assume that all the previously added tokens
+ // are valid
+ List<DelegationTokenToRenew> tokenList =
+ new ArrayList<DelegationTokenRenewer.DelegationTokenToRenew>();
for(Token<?> token : tokens) {
- // first renew happens immediately
if (token.isManaged()) {
- DelegationTokenToRenew dtr =
- new DelegationTokenToRenew(applicationId, token, getConfig(), now,
- shouldCancelAtEnd);
- renewToken(dtr);
- dtrs.add(dtr);
+ tokenList.add(new DelegationTokenToRenew(applicationId,
+ token, getConfig(), now, shouldCancelAtEnd));
}
}
- for (DelegationTokenToRenew dtr : dtrs) {
- addTokenToList(dtr);
- setTimerForTokenRenewal(dtr);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Registering token for renewal for:" +
- " service = " + dtr.token.getService() +
- " for appId = " + applicationId);
+ if (!tokenList.isEmpty()){
+ renewIfServiceIsStarted(tokenList);
+ }
+ }
+
+ protected void renewIfServiceIsStarted(List<DelegationTokenToRenew> dtrs)
+ throws IOException {
+ if (isServiceStarted) {
+ // Renewing token and adding it to timer calls are separated purposefully
+ // If user provides incorrect token then it should not be added for
+ // renewal.
+ for (DelegationTokenToRenew dtr : dtrs) {
+ renewToken(dtr);
+ }
+ for (DelegationTokenToRenew dtr : dtrs) {
+ addTokenToList(dtr);
+ setTimerForTokenRenewal(dtr);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Registering token for renewal for:" + " service = "
+ + dtr.token.getService() + " for appId = " + dtr.applicationId);
+ }
}
+ } else {
+ pendingTokenForRenewal.addAll(dtrs);
}
}
@@ -513,4 +543,7 @@ public class DelegationTokenRenewer exte
}
}
+ public void setRMContext(RMContext rmContext) {
+ this.rmContext = rmContext;
+ }
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java?rev=1521566&r1=1521565&r2=1521566&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java Tue Sep 10 18:03:08 2013
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
import java.io.IOException;
+import java.util.Arrays;
import java.util.Collection;
import java.util.EnumSet;
import java.util.HashSet;
@@ -231,6 +232,7 @@ public class RMWebServices {
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public AppsInfo getApps(@Context HttpServletRequest hsr,
@QueryParam("state") String stateQuery,
+ @QueryParam("states") Set<String> statesQuery,
@QueryParam("finalStatus") String finalStatusQuery,
@QueryParam("user") String userQuery,
@QueryParam("queue") String queueQuery,
@@ -245,6 +247,7 @@ public class RMWebServices {
boolean checkStart = false;
boolean checkEnd = false;
boolean checkAppTypes = false;
+ boolean checkAppStates = false;
long countNum = 0;
// set values suitable in case both of begin/end not specified
@@ -321,6 +324,36 @@ public class RMWebServices {
checkAppTypes = true;
}
+ String allAppStates;
+ RMAppState[] stateArray = RMAppState.values();
+ allAppStates = Arrays.toString(stateArray);
+
+ Set<String> appStates = new HashSet<String>();
+ // stateQuery is deprecated.
+ if (stateQuery != null && !stateQuery.isEmpty()) {
+ statesQuery.add(stateQuery);
+ }
+ if (!statesQuery.isEmpty()) {
+ for (String applicationState : statesQuery) {
+ if (applicationState != null && !applicationState.isEmpty()) {
+ String[] states = applicationState.split(",");
+ for (String state : states) {
+ try {
+ RMAppState.valueOf(state.trim());
+ } catch (IllegalArgumentException iae) {
+ throw new BadRequestException(
+ "Invalid application-state " + state
+ + " specified. It should be one of " + allAppStates);
+ }
+ appStates.add(state.trim().toLowerCase());
+ }
+ }
+ }
+ }
+ if (!appStates.isEmpty()) {
+ checkAppStates = true;
+ }
+
final ConcurrentMap<ApplicationId, RMApp> apps = rm.getRMContext()
.getRMApps();
AppsInfo allApps = new AppsInfo();
@@ -329,11 +362,10 @@ public class RMWebServices {
if (checkCount && num == countNum) {
break;
}
- if (stateQuery != null && !stateQuery.isEmpty()) {
- RMAppState.valueOf(stateQuery);
- if (!rmapp.getState().toString().equalsIgnoreCase(stateQuery)) {
- continue;
- }
+
+ if (checkAppStates
+ && !appStates.contains(rmapp.getState().toString().toLowerCase())) {
+ continue;
}
if (finalStatusQuery != null && !finalStatusQuery.isEmpty()) {
FinalApplicationStatus.valueOf(finalStatusQuery);
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java?rev=1521566&r1=1521565&r2=1521566&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java Tue Sep 10 18:03:08 2013
@@ -19,7 +19,6 @@
package org.apache.hadoop.yarn.server.resourcemanager;
import java.io.IOException;
-import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.UnknownHostException;
import java.nio.ByteBuffer;
@@ -35,6 +34,7 @@ import org.apache.hadoop.fs.CommonConfig
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.SaslRpcServer.AuthMethod;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
@@ -90,7 +90,7 @@ public class TestRMRestart {
conf.set(YarnConfiguration.RECOVERY_ENABLED, "true");
conf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName());
- rmAddr = new InetSocketAddress(InetAddress.getLocalHost(), 123);
+ rmAddr = new InetSocketAddress("localhost", 8032);
}
@Test (timeout=180000)
@@ -592,7 +592,12 @@ public class TestRMRestart {
@Test
public void testRMDelegationTokenRestoredOnRMRestart() throws Exception {
conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 2);
-
+
+ conf.set(
+ CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
+ "kerberos");
+ conf.set(YarnConfiguration.RM_ADDRESS, "localhost:8032");
+ UserGroupInformation.setConfiguration(conf);
MemoryRMStateStore memStore = new MemoryRMStateStore();
memStore.init(conf);
RMState rmState = memStore.getState();
@@ -614,6 +619,8 @@ public class TestRMRestart {
// request a token and add into credential
GetDelegationTokenRequest request1 =
GetDelegationTokenRequest.newInstance("renewer1");
+ UserGroupInformation.getCurrentUser().setAuthenticationMethod(
+ AuthMethod.KERBEROS);
GetDelegationTokenResponse response1 =
rm1.getClientRMService().getDelegationToken(request1);
org.apache.hadoop.yarn.api.records.Token delegationToken1 =
@@ -644,7 +651,7 @@ public class TestRMRestart {
rm1.getRMDTSecretManager().getAllTokens();
Assert.assertEquals(tokenIdentSet, allTokensRM1.keySet());
Assert.assertEquals(allTokensRM1, rmDTState);
-
+
// assert sequence number is saved
Assert.assertEquals(
rm1.getRMDTSecretManager().getLatestDTSequenceNumber(),
@@ -682,7 +689,7 @@ public class TestRMRestart {
// assert master keys and tokens are populated back to DTSecretManager
Map<RMDelegationTokenIdentifier, Long> allTokensRM2 =
rm2.getRMDTSecretManager().getAllTokens();
- Assert.assertEquals(allTokensRM1, allTokensRM2);
+ Assert.assertEquals(allTokensRM2.keySet(), allTokensRM1.keySet());
// rm2 has its own master keys when it starts, we use containsAll here
Assert.assertTrue(rm2.getRMDTSecretManager().getAllMasterKeys()
.containsAll(allKeysRM1));
@@ -735,15 +742,24 @@ public class TestRMRestart {
}
@Override
- protected void doSecureLogin() throws IOException {
- // Do nothing.
+ protected ClientRMService createClientRMService() {
+ return new ClientRMService(getRMContext(), getResourceScheduler(),
+ rmAppManager, applicationACLsManager, rmDTSecretManager){
+ @Override
+ protected void serviceStart() throws Exception {
+ // do nothing
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ //do nothing
+ }
+ };
}
@Override
- protected void serviceInit(Configuration conf) throws Exception {
- super.serviceInit(conf);
- RMDelegationTokenIdentifier.Renewer.setSecretManager(
- this.getRMDTSecretManager(), rmAddr);
+ protected void doSecureLogin() throws IOException {
+ // Do nothing.
}
}
}