You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by cm...@apache.org on 2014/08/20 01:51:01 UTC
svn commit: r1619012 [5/26] - in
/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project: ./
hadoop-yarn/bin/ hadoop-yarn/conf/ hadoop-yarn/dev-support/
hadoop-yarn/hadoop-yarn-api/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/...
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml Tue Aug 19 23:49:39 2014
@@ -71,6 +71,17 @@
</property>
<property>
+ <description>
+ The actual address the server will bind to. If this optional address is
+ set, the RPC and webapp servers will bind to this address and the port specified in
+ yarn.resourcemanager.address and yarn.resourcemanager.webapp.address, respectively. This
+ is most useful for making RM listen to all interfaces by setting to 0.0.0.0.
+ </description>
+ <name>yarn.resourcemanager.bind-host</name>
+ <value></value>
+ </property>
+
+ <property>
<description>The number of threads used to handle applications manager requests.</description>
<name>yarn.resourcemanager.client.thread-count</name>
<value>50</value>
@@ -195,6 +206,15 @@
</property>
<property>
+ <description>Flag to enable override of the default kerberos authentication
+ filter with the RM authentication filter to allow authentication using
+ delegation tokens(fallback to kerberos if the tokens are missing). Only
+ applicable when the http authentication type is kerberos.</description>
+ <name>yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled</name>
+ <value>true</value>
+ </property>
+
+ <property>
<description>How long to wait until a node manager is considered dead.</description>
<name>yarn.nm.liveness-monitor.expiry-interval-ms</name>
<value>600000</value>
@@ -270,6 +290,14 @@
</property>
<property>
+ <description>Enable RM work preserving recovery. This configuration is private
+ to YARN for experimenting the feature.
+ </description>
+ <name>yarn.resourcemanager.work-preserving-recovery.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
<description>The class to use as the persistent store.
If org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore
@@ -309,14 +337,17 @@
<property>
<description>Number of times RM tries to connect to ZooKeeper.</description>
<name>yarn.resourcemanager.zk-num-retries</name>
- <value>500</value>
+ <value>1000</value>
</property>
<property>
<description>Retry interval in milliseconds when connecting to ZooKeeper.
+ When HA is enabled, the value here is NOT used. It is generated
+ automatically from yarn.resourcemanager.zk-timeout-ms and
+ yarn.resourcemanager.zk-num-retries.
</description>
<name>yarn.resourcemanager.zk-retry-interval-ms</name>
- <value>2000</value>
+ <value>1000</value>
</property>
<property>
@@ -370,6 +401,18 @@
</property>
<property>
+ <description>
+ Specify the auths to be used for the ACL's specified in both the
+ yarn.resourcemanager.zk-acl and
+ yarn.resourcemanager.zk-state-store.root-node.acl properties. This
+ takes a comma-separated list of authentication mechanisms, each of the
+ form 'scheme:auth' (the same syntax used for the 'addAuth' command in
+ the ZK CLI).
+ </description>
+ <name>yarn.resourcemanager.zk-auth</name>
+ </property>
+
+ <property>
<description>URI pointing to the location of the FileSystem path where
RM state will be stored. This must be supplied when using
org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore
@@ -431,7 +474,7 @@
<property>
<description>Name of the cluster. In a HA setting,
this is used to ensure the RM participates in leader
- election fo this cluster and ensures it does not affect
+ election for this cluster and ensures it does not affect
other clusters</description>
<name>yarn.resourcemanager.cluster-id</name>
<!--value>yarn-cluster</value-->
@@ -604,6 +647,17 @@
</property>
<property>
+ <description>
+ The actual address the server will bind to. If this optional address is
+ set, the RPC and webapp servers will bind to this address and the port specified in
+ yarn.nodemanager.address and yarn.nodemanager.webapp.address, respectively. This is
+ most useful for making NM listen to all interfaces by setting to 0.0.0.0.
+ </description>
+ <name>yarn.nodemanager.bind-host</name>
+ <value></value>
+ </property>
+
+ <property>
<description>Environment variables that should be forwarded from the NodeManager's environment to the container's.</description>
<name>yarn.nodemanager.admin-env</name>
<value>MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX</value>
@@ -733,7 +787,14 @@
</property>
<property>
- <description>Whether to enable log aggregation</description>
+ <description>Whether to enable log aggregation. Log aggregation collects
+ each container's logs and moves these logs onto a file-system, for e.g.
+ HDFS, after the application completes. Users can configure the
+ "yarn.nodemanager.remote-app-log-dir" and
+ "yarn.nodemanager.remote-app-log-dir-suffix" properties to determine
+ where these logs are moved to. Users can access the logs via the
+ Application Timeline Server.
+ </description>
<name>yarn.log-aggregation-enable</name>
<value>false</value>
</property>
@@ -1105,7 +1166,7 @@
If enabled, clients will put entities and events to the timeline server.
</description>
<name>yarn.timeline-service.enabled</name>
- <value>true</value>
+ <value>false</value>
</property>
<property>
@@ -1134,9 +1195,21 @@
</property>
<property>
+ <description>
+ The actual address the server will bind to. If this optional address is
+ set, the RPC and webapp servers will bind to this address and the port specified in
+ yarn.timeline-service.address and yarn.timeline-service.webapp.address, respectively.
+ This is most useful for making the service listen to all interfaces by setting to
+ 0.0.0.0.
+ </description>
+ <name>yarn.timeline-service.bind-host</name>
+ <value></value>
+ </property>
+
+ <property>
<description>Store class name for timeline store.</description>
<name>yarn.timeline-service.store-class</name>
- <value>org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.LeveldbTimelineStore</value>
+ <value>org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore</value>
</property>
<property>
@@ -1154,7 +1227,7 @@
<property>
<description>Store file name for leveldb timeline store.</description>
<name>yarn.timeline-service.leveldb-timeline-store.path</name>
- <value>${yarn.log.dir}/timeline</value>
+ <value>${hadoop.tmp.dir}/yarn/timeline</value>
</property>
<property>
@@ -1188,6 +1261,36 @@
</property>
<property>
+ <name>yarn.timeline-service.http-authentication.type</name>
+ <value>simple</value>
+ <description>
+ Defines authentication used for the timeline server HTTP endpoint.
+ Supported values are: simple | kerberos | #AUTHENTICATION_HANDLER_CLASSNAME#
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.timeline-service.http-authentication.simple.anonymous.allowed</name>
+ <value>true</value>
+ <description>
+ Indicates if anonymous requests are allowed by the timeline server when using
+ 'simple' authentication.
+ </description>
+ </property>
+
+ <property>
+ <description>The Kerberos principal for the timeline server.</description>
+ <name>yarn.timeline-service.principal</name>
+ <value></value>
+ </property>
+
+ <property>
+ <description>The Kerberos keytab for the timeline server.</description>
+ <name>yarn.timeline-service.keytab</name>
+ <value>/etc/krb5.keytab</value>
+ </property>
+
+ <property>
<description>Indicate to ResourceManager as well as clients whether
history-service is enabled or not. If enabled, ResourceManager starts
recording historical data that ApplicationHistory service can consume.
@@ -1203,7 +1306,7 @@
org.apache.hadoop.yarn.server.applicationhistoryservice.FileSystemApplicationHistoryStore
as the value for yarn.timeline-service.generic-application-history.store-class</description>
<name>yarn.timeline-service.generic-application-history.fs-history-store.uri</name>
- <value>${hadoop.log.dir}/yarn/system/history</value>
+ <value>${hadoop.tmp.dir}/yarn/timeline/generic-history</value>
</property>
<property>
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java Tue Aug 19 23:49:39 2014
@@ -24,8 +24,6 @@ import java.net.SocketTimeoutException;
import java.util.ArrayList;
import java.util.List;
-import org.junit.Assert;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -46,6 +44,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -55,6 +54,7 @@ import org.apache.hadoop.yarn.factory.pr
import org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
+import org.junit.Assert;
import org.junit.Test;
/*
@@ -102,7 +102,8 @@ public class TestContainerLaunchRPC {
Resource resource = Resource.newInstance(1234, 2);
ContainerTokenIdentifier containerTokenIdentifier =
new ContainerTokenIdentifier(containerId, "localhost", "user",
- resource, System.currentTimeMillis() + 10000, 42, 42);
+ resource, System.currentTimeMillis() + 10000, 42, 42,
+ Priority.newInstance(0), 0);
Token containerToken =
TestRPC.newContainerToken(nodeId, "password".getBytes(),
containerTokenIdentifier);
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java Tue Aug 19 23:49:39 2014
@@ -23,8 +23,6 @@ import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.List;
-import org.junit.Assert;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.ipc.ProtobufRpcEngine;
@@ -50,6 +48,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -61,6 +60,7 @@ import org.apache.hadoop.yarn.ipc.RPCUti
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.util.Records;
+import org.junit.Assert;
import org.junit.Test;
public class TestRPC {
@@ -129,7 +129,8 @@ public class TestRPC {
Resource resource = Resource.newInstance(1234, 2);
ContainerTokenIdentifier containerTokenIdentifier =
new ContainerTokenIdentifier(containerId, "localhost", "user",
- resource, System.currentTimeMillis() + 10000, 42, 42);
+ resource, System.currentTimeMillis() + 10000, 42, 42,
+ Priority.newInstance(0), 0);
Token containerToken = newContainerToken(nodeId, "password".getBytes(),
containerTokenIdentifier);
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfiguration.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfiguration.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfiguration.java Tue Aug 19 23:49:39 2014
@@ -28,6 +28,7 @@ import java.net.SocketAddress;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
public class TestYarnConfiguration {
@@ -75,4 +76,131 @@ public class TestYarnConfiguration {
YarnConfiguration.DEFAULT_NM_PORT);
assertEquals(1234, addr.getPort());
}
+
+ @Test
+ public void testGetSocketAddr() throws Exception {
+
+ YarnConfiguration conf;
+ InetSocketAddress resourceTrackerAddress;
+
+ //all default
+ conf = new YarnConfiguration();
+ resourceTrackerAddress = conf.getSocketAddr(
+ YarnConfiguration.RM_BIND_HOST,
+ YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
+ assertEquals(
+ new InetSocketAddress(
+ YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS.split(":")[0],
+ YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT),
+ resourceTrackerAddress);
+
+ //with address
+ conf.set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, "10.0.0.1");
+ resourceTrackerAddress = conf.getSocketAddr(
+ YarnConfiguration.RM_BIND_HOST,
+ YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
+ assertEquals(
+ new InetSocketAddress(
+ "10.0.0.1",
+ YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT),
+ resourceTrackerAddress);
+
+ //address and socket
+ conf.set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, "10.0.0.2:5001");
+ resourceTrackerAddress = conf.getSocketAddr(
+ YarnConfiguration.RM_BIND_HOST,
+ YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
+ assertEquals(
+ new InetSocketAddress(
+ "10.0.0.2",
+ 5001),
+ resourceTrackerAddress);
+
+ //bind host only
+ conf = new YarnConfiguration();
+ conf.set(YarnConfiguration.RM_BIND_HOST, "10.0.0.3");
+ resourceTrackerAddress = conf.getSocketAddr(
+ YarnConfiguration.RM_BIND_HOST,
+ YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
+ assertEquals(
+ new InetSocketAddress(
+ "10.0.0.3",
+ YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT),
+ resourceTrackerAddress);
+
+ //bind host and address no port
+ conf.set(YarnConfiguration.RM_BIND_HOST, "0.0.0.0");
+ conf.set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, "10.0.0.2");
+ resourceTrackerAddress = conf.getSocketAddr(
+ YarnConfiguration.RM_BIND_HOST,
+ YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
+ assertEquals(
+ new InetSocketAddress(
+ "0.0.0.0",
+ YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT),
+ resourceTrackerAddress);
+
+ //bind host and address with port
+ conf.set(YarnConfiguration.RM_BIND_HOST, "0.0.0.0");
+ conf.set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, "10.0.0.2:5003");
+ resourceTrackerAddress = conf.getSocketAddr(
+ YarnConfiguration.RM_BIND_HOST,
+ YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
+ assertEquals(
+ new InetSocketAddress(
+ "0.0.0.0",
+ 5003),
+ resourceTrackerAddress);
+
+ }
+
+ @Test
+ public void testUpdateConnectAddr() throws Exception {
+ YarnConfiguration conf;
+ InetSocketAddress resourceTrackerConnectAddress;
+ InetSocketAddress serverAddress;
+
+ //no override, old behavior. Won't work on a host named "yo.yo.yo"
+ conf = new YarnConfiguration();
+ conf.set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, "yo.yo.yo");
+ serverAddress = new InetSocketAddress(
+ YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS.split(":")[0],
+ Integer.valueOf(YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS.split(":")[1]));
+
+ resourceTrackerConnectAddress = conf.updateConnectAddr(
+ YarnConfiguration.RM_BIND_HOST,
+ YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
+ serverAddress);
+
+ assertFalse(resourceTrackerConnectAddress.toString().startsWith("yo.yo.yo"));
+
+ //cause override with address
+ conf = new YarnConfiguration();
+ conf.set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, "yo.yo.yo");
+ conf.set(YarnConfiguration.RM_BIND_HOST, "0.0.0.0");
+ serverAddress = new InetSocketAddress(
+ YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS.split(":")[0],
+ Integer.valueOf(YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS.split(":")[1]));
+
+ resourceTrackerConnectAddress = conf.updateConnectAddr(
+ YarnConfiguration.RM_BIND_HOST,
+ YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
+ serverAddress);
+
+ assertTrue(resourceTrackerConnectAddress.toString().startsWith("yo.yo.yo"));
+ }
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/DrainDispatcher.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/DrainDispatcher.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/DrainDispatcher.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/DrainDispatcher.java Tue Aug 19 23:49:39 2014
@@ -28,6 +28,7 @@ public class DrainDispatcher extends Asy
// and similar grotesqueries
private volatile boolean drained = false;
private final BlockingQueue<Event> queue;
+ final Object mutex;
public DrainDispatcher() {
this(new LinkedBlockingQueue<Event>());
@@ -36,6 +37,7 @@ public class DrainDispatcher extends Asy
private DrainDispatcher(BlockingQueue<Event> eventQueue) {
super(eventQueue);
this.queue = eventQueue;
+ this.mutex = this;
}
/**
@@ -53,8 +55,10 @@ public class DrainDispatcher extends Asy
@Override
public void run() {
while (!Thread.currentThread().isInterrupted()) {
- // !drained if dispatch queued new events on this dispatcher
- drained = queue.isEmpty();
+ synchronized (mutex) {
+ // !drained if dispatch queued new events on this dispatcher
+ drained = queue.isEmpty();
+ }
Event event;
try {
event = queue.take();
@@ -75,8 +79,10 @@ public class DrainDispatcher extends Asy
return new EventHandler() {
@Override
public void handle(Event event) {
- drained = false;
- actual.handle(event);
+ synchronized (mutex) {
+ actual.handle(event);
+ drained = false;
+ }
}
};
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java Tue Aug 19 23:49:39 2014
@@ -23,6 +23,7 @@ import static org.apache.hadoop.fs.Creat
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assume.assumeTrue;
import java.io.File;
import java.io.FileOutputStream;
@@ -66,6 +67,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalDirAllocator;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.util.Shell;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
@@ -308,6 +310,11 @@ public class TestFSDownload {
FileContext files = FileContext.getLocalFSFileContext(conf);
Path basedir = files.makeQualified(new Path("target",
TestFSDownload.class.getSimpleName()));
+
+ // if test directory doesn't have ancestor permission, skip this test
+ FileSystem f = basedir.getFileSystem(conf);
+ assumeTrue(FSDownload.ancestorsHaveExecutePermissions(f, basedir, null));
+
files.mkdir(basedir, null, true);
conf.setStrings(TestFSDownload.class.getName(), basedir.toString());
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestTimes.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestTimes.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestTimes.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestTimes.java Tue Aug 19 23:49:39 2014
@@ -50,4 +50,15 @@ public class TestTimes {
elapsed = Times.elapsed(5, 10, false);
Assert.assertEquals("Elapsed time is not 5", 5, elapsed);
}
+
+ @Test
+ public void testFinishTimesAheadOfStartTimes() {
+ long elapsed = Times.elapsed(10, 5, true);
+ Assert.assertEquals("Elapsed time is not -1", -1, elapsed);
+ elapsed = Times.elapsed(10, 5, false);
+ Assert.assertEquals("Elapsed time is not -1", -1, elapsed);
+ // use Long.MAX_VALUE to ensure started time is after the current one
+ elapsed = Times.elapsed(Long.MAX_VALUE, 0, true);
+ Assert.assertEquals("Elapsed time is not -1", -1, elapsed);
+ }
}
\ No newline at end of file
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestSubViews.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestSubViews.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestSubViews.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestSubViews.java Tue Aug 19 23:49:39 2014
@@ -70,6 +70,6 @@ public class TestSubViews {
out.flush();
verify(out).print("sub1 text");
verify(out).print("sub2 text");
- verify(out, times(15)).println(); // test inline transition across views
+ verify(out, times(16)).println(); // test inline transition across views
}
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/view/TestHtmlPage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/view/TestHtmlPage.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/view/TestHtmlPage.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/view/TestHtmlPage.java Tue Aug 19 23:49:39 2014
@@ -22,11 +22,12 @@ import com.google.inject.Injector;
import java.io.PrintWriter;
+import org.apache.hadoop.yarn.webapp.MimeType;
import org.apache.hadoop.yarn.webapp.WebAppException;
import org.apache.hadoop.yarn.webapp.test.WebAppTests;
import org.apache.hadoop.yarn.webapp.view.HtmlPage;
-
import org.junit.Test;
+
import static org.mockito.Mockito.*;
public class TestHtmlPage {
@@ -53,6 +54,12 @@ public class TestHtmlPage {
Injector injector = WebAppTests.testPage(TestView.class);
PrintWriter out = injector.getInstance(PrintWriter.class);
+ // Verify the HTML page has correct meta tags in the header
+ verify(out).print(" http-equiv=\"X-UA-Compatible\"");
+ verify(out).print(" content=\"IE=8\"");
+ verify(out).print(" http-equiv=\"Content-type\"");
+ verify(out).print(String.format(" content=\"%s\"", MimeType.HTML));
+
verify(out).print("test");
verify(out).print(" id=\"testid\"");
verify(out).print("test note");
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/view/TestInfoBlock.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/view/TestInfoBlock.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/view/TestInfoBlock.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/view/TestInfoBlock.java Tue Aug 19 23:49:39 2014
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.webapp.vi
import java.io.PrintWriter;
import java.io.StringWriter;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import org.apache.hadoop.yarn.webapp.ResponseInfo;
@@ -34,6 +35,33 @@ public class TestInfoBlock {
public static PrintWriter pw;
+ static final String JAVASCRIPT = "<script>alert('text')</script>";
+ static final String JAVASCRIPT_ESCAPED =
+ "<script>alert('text')</script>";
+
+ public static class JavaScriptInfoBlock extends InfoBlock{
+
+ static ResponseInfo resInfo;
+
+ static {
+ resInfo = new ResponseInfo();
+ resInfo._("User_Name", JAVASCRIPT);
+ }
+
+ @Override
+ public PrintWriter writer() {
+ return TestInfoBlock.pw;
+ }
+
+ JavaScriptInfoBlock(ResponseInfo info) {
+ super(resInfo);
+ }
+
+ public JavaScriptInfoBlock() {
+ super(resInfo);
+ }
+ }
+
public static class MultilineInfoBlock extends InfoBlock{
static ResponseInfo resInfo;
@@ -78,4 +106,13 @@ public class TestInfoBlock {
+ " This is second line.%n </div>%n");
assertTrue(output.contains(expectedSinglelineData) && output.contains(expectedMultilineData));
}
+
+ @Test(timeout=60000L)
+ public void testJavaScriptInfoBlock() throws Exception{
+ WebAppTests.testBlock(JavaScriptInfoBlock.class);
+ TestInfoBlock.pw.flush();
+ String output = TestInfoBlock.sw.toString();
+ assertFalse(output.contains("<script>"));
+ assertTrue(output.contains(JAVASCRIPT_ESCAPED));
+ }
}
\ No newline at end of file
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml Tue Aug 19 23:49:39 2014
@@ -45,24 +45,6 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>commons-el</groupId>
- <artifactId>commons-el</artifactId>
- </exclusion>
- <exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-runtime</artifactId>
- </exclusion>
- <exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-compiler</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jsp-2.1-jetty</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
@@ -169,15 +151,6 @@
</dependency>
<dependency>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-core-asl</artifactId>
- </dependency>
- <dependency>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-mapper-asl</artifactId>
- </dependency>
-
- <dependency>
<groupId>commons-collections</groupId>
<artifactId>commons-collections</artifactId>
</dependency>
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java Tue Aug 19 23:49:39 2014
@@ -56,6 +56,7 @@ import org.apache.hadoop.yarn.exceptions
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.ipc.RPCUtil;
import org.apache.hadoop.yarn.ipc.YarnRPC;
public class ApplicationHistoryClientService extends AbstractService {
@@ -75,10 +76,11 @@ public class ApplicationHistoryClientSer
protected void serviceStart() throws Exception {
Configuration conf = getConfig();
YarnRPC rpc = YarnRPC.create(conf);
- InetSocketAddress address =
- conf.getSocketAddr(YarnConfiguration.TIMELINE_SERVICE_ADDRESS,
- YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ADDRESS,
- YarnConfiguration.DEFAULT_TIMELINE_SERVICE_PORT);
+ InetSocketAddress address = conf.getSocketAddr(
+ YarnConfiguration.TIMELINE_SERVICE_BIND_HOST,
+ YarnConfiguration.TIMELINE_SERVICE_ADDRESS,
+ YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ADDRESS,
+ YarnConfiguration.DEFAULT_TIMELINE_SERVICE_PORT);
server =
rpc.getServer(ApplicationHistoryProtocol.class, protocolHandler,
@@ -88,8 +90,10 @@ public class ApplicationHistoryClientSer
server.start();
this.bindAddress =
- conf.updateConnectAddr(YarnConfiguration.TIMELINE_SERVICE_ADDRESS,
- server.getListenerAddress());
+ conf.updateConnectAddr(YarnConfiguration.TIMELINE_SERVICE_BIND_HOST,
+ YarnConfiguration.TIMELINE_SERVICE_ADDRESS,
+ YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ADDRESS,
+ server.getListenerAddress());
LOG.info("Instantiated ApplicationHistoryClientService at "
+ this.bindAddress);
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java Tue Aug 19 23:49:39 2014
@@ -18,12 +18,18 @@
package org.apache.hadoop.yarn.server.applicationhistoryservice;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.ArrayList;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.metrics2.source.JvmMetrics;
+import org.apache.hadoop.security.AuthenticationFilterInitializer;
+import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.service.CompositeService;
import org.apache.hadoop.service.Service;
import org.apache.hadoop.util.ExitUtil;
@@ -33,9 +39,13 @@ import org.apache.hadoop.util.StringUtil
import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineStore;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.LeveldbTimelineStore;
import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp;
+import org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore;
+import org.apache.hadoop.yarn.server.timeline.TimelineDataManager;
+import org.apache.hadoop.yarn.server.timeline.TimelineStore;
+import org.apache.hadoop.yarn.server.timeline.security.TimelineACLsManager;
+import org.apache.hadoop.yarn.server.timeline.security.TimelineAuthenticationFilterInitializer;
+import org.apache.hadoop.yarn.server.timeline.security.TimelineDelegationTokenSecretManagerService;
import org.apache.hadoop.yarn.webapp.WebApp;
import org.apache.hadoop.yarn.webapp.WebApps;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
@@ -52,9 +62,11 @@ public class ApplicationHistoryServer ex
private static final Log LOG = LogFactory
.getLog(ApplicationHistoryServer.class);
- ApplicationHistoryClientService ahsClientService;
- ApplicationHistoryManager historyManager;
- TimelineStore timelineStore;
+ private ApplicationHistoryClientService ahsClientService;
+ private ApplicationHistoryManager historyManager;
+ private TimelineStore timelineStore;
+ private TimelineDelegationTokenSecretManagerService secretManagerService;
+ private TimelineDataManager timelineDataManager;
private WebApp webApp;
public ApplicationHistoryServer() {
@@ -63,19 +75,31 @@ public class ApplicationHistoryServer ex
@Override
protected void serviceInit(Configuration conf) throws Exception {
- historyManager = createApplicationHistory();
+ // init timeline services first
+ timelineStore = createTimelineStore(conf);
+ addIfService(timelineStore);
+ secretManagerService = createTimelineDelegationTokenSecretManagerService(conf);
+ addService(secretManagerService);
+ timelineDataManager = createTimelineDataManager(conf);
+
+ // init generic history service afterwards
+ historyManager = createApplicationHistoryManager(conf);
ahsClientService = createApplicationHistoryClientService(historyManager);
addService(ahsClientService);
addService((Service) historyManager);
- timelineStore = createTimelineStore(conf);
- addIfService(timelineStore);
+
+ DefaultMetricsSystem.initialize("ApplicationHistoryServer");
+ JvmMetrics.initSingleton("ApplicationHistoryServer", null);
super.serviceInit(conf);
}
@Override
protected void serviceStart() throws Exception {
- DefaultMetricsSystem.initialize("ApplicationHistoryServer");
- JvmMetrics.initSingleton("ApplicationHistoryServer", null);
+ try {
+ doSecureLogin(getConfig());
+ } catch(IOException ie) {
+ throw new YarnRuntimeException("Failed to login", ie);
+ }
startWebApp();
super.serviceStart();
@@ -93,21 +117,22 @@ public class ApplicationHistoryServer ex
@Private
@VisibleForTesting
- public ApplicationHistoryClientService getClientService() {
+ ApplicationHistoryClientService getClientService() {
return this.ahsClientService;
}
- protected ApplicationHistoryClientService
- createApplicationHistoryClientService(
- ApplicationHistoryManager historyManager) {
- return new ApplicationHistoryClientService(historyManager);
- }
-
- protected ApplicationHistoryManager createApplicationHistory() {
- return new ApplicationHistoryManagerImpl();
+ /**
+ * @return ApplicationTimelineStore
+ */
+ @Private
+ @VisibleForTesting
+ public TimelineStore getTimelineStore() {
+ return timelineStore;
}
- protected ApplicationHistoryManager getApplicationHistory() {
+ @Private
+ @VisibleForTesting
+ ApplicationHistoryManager getApplicationHistoryManager() {
return this.historyManager;
}
@@ -136,45 +161,104 @@ public class ApplicationHistoryServer ex
launchAppHistoryServer(args);
}
- protected ApplicationHistoryManager createApplicationHistoryManager(
+ private ApplicationHistoryClientService
+ createApplicationHistoryClientService(
+ ApplicationHistoryManager historyManager) {
+ return new ApplicationHistoryClientService(historyManager);
+ }
+
+ private ApplicationHistoryManager createApplicationHistoryManager(
Configuration conf) {
return new ApplicationHistoryManagerImpl();
}
- protected TimelineStore createTimelineStore(
+ private TimelineStore createTimelineStore(
Configuration conf) {
return ReflectionUtils.newInstance(conf.getClass(
YarnConfiguration.TIMELINE_SERVICE_STORE, LeveldbTimelineStore.class,
TimelineStore.class), conf);
}
- protected void startWebApp() {
- String bindAddress = WebAppUtils.getAHSWebAppURLWithoutScheme(getConfig());
+ private TimelineDelegationTokenSecretManagerService
+ createTimelineDelegationTokenSecretManagerService(Configuration conf) {
+ return new TimelineDelegationTokenSecretManagerService();
+ }
+
+ private TimelineDataManager createTimelineDataManager(Configuration conf) {
+ return new TimelineDataManager(
+ timelineStore, new TimelineACLsManager(conf));
+ }
+
+ private void startWebApp() {
+ Configuration conf = getConfig();
+ // Always load pseudo authentication filter to parse "user.name" in an URL
+ // to identify a HTTP request's user in insecure mode.
+ // When Kerberos authentication type is set (i.e., secure mode is turned on),
+ // the customized filter will be loaded by the timeline server to do Kerberos
+ // + DT authentication.
+ String initializers = conf.get("hadoop.http.filter.initializers");
+
+ initializers =
+ initializers == null || initializers.length() == 0 ? "" : initializers;
+
+ if (!initializers.contains(TimelineAuthenticationFilterInitializer.class
+ .getName())) {
+ initializers =
+ TimelineAuthenticationFilterInitializer.class.getName() + ","
+ + initializers;
+ }
+
+ String[] parts = initializers.split(",");
+ ArrayList<String> target = new ArrayList<String>();
+ for (String filterInitializer : parts) {
+ filterInitializer = filterInitializer.trim();
+ if (filterInitializer.equals(AuthenticationFilterInitializer.class
+ .getName())) {
+ continue;
+ }
+ target.add(filterInitializer);
+ }
+ String actualInitializers =
+ org.apache.commons.lang.StringUtils.join(target, ",");
+ if (!actualInitializers.equals(initializers)) {
+ conf.set("hadoop.http.filter.initializers", actualInitializers);
+ }
+ String bindAddress = WebAppUtils.getWebAppBindURL(conf,
+ YarnConfiguration.TIMELINE_SERVICE_BIND_HOST,
+ WebAppUtils.getAHSWebAppURLWithoutScheme(conf));
LOG.info("Instantiating AHSWebApp at " + bindAddress);
try {
+ AHSWebApp ahsWebApp = AHSWebApp.getInstance();
+ ahsWebApp.setApplicationHistoryManager(historyManager);
+ ahsWebApp.setTimelineDelegationTokenSecretManagerService(secretManagerService);
+ ahsWebApp.setTimelineDataManager(timelineDataManager);
webApp =
WebApps
.$for("applicationhistory", ApplicationHistoryClientService.class,
- ahsClientService, "ws")
- .with(getConfig())
- .withHttpSpnegoPrincipalKey(
- YarnConfiguration.TIMELINE_SERVICE_WEBAPP_SPNEGO_USER_NAME_KEY)
- .withHttpSpnegoKeytabKey(
- YarnConfiguration.TIMELINE_SERVICE_WEBAPP_SPNEGO_KEYTAB_FILE_KEY)
- .at(bindAddress)
- .start(new AHSWebApp(historyManager, timelineStore));
+ ahsClientService, "ws")
+ .with(conf).at(bindAddress).start(ahsWebApp);
} catch (Exception e) {
String msg = "AHSWebApp failed to start.";
LOG.error(msg, e);
throw new YarnRuntimeException(msg, e);
}
}
+
+ private void doSecureLogin(Configuration conf) throws IOException {
+ InetSocketAddress socAddr = getBindAddress(conf);
+ SecurityUtil.login(conf, YarnConfiguration.TIMELINE_SERVICE_KEYTAB,
+ YarnConfiguration.TIMELINE_SERVICE_PRINCIPAL, socAddr.getHostName());
+ }
+
/**
- * @return ApplicationTimelineStore
+ * Retrieve the timeline server bind address from configuration
+ *
+ * @param conf
+ * @return InetSocketAddress
*/
- @Private
- @VisibleForTesting
- public TimelineStore getTimelineStore() {
- return timelineStore;
+ private static InetSocketAddress getBindAddress(Configuration conf) {
+ return conf.getSocketAddr(YarnConfiguration.TIMELINE_SERVICE_ADDRESS,
+ YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ADDRESS,
+ YarnConfiguration.DEFAULT_TIMELINE_SERVICE_PORT);
}
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java Tue Aug 19 23:49:39 2014
@@ -19,23 +19,69 @@ package org.apache.hadoop.yarn.server.ap
import static org.apache.hadoop.yarn.util.StringHelper.pajoin;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.yarn.server.api.ApplicationContext;
import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryManager;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineStore;
+import org.apache.hadoop.yarn.server.timeline.TimelineDataManager;
+import org.apache.hadoop.yarn.server.timeline.security.TimelineDelegationTokenSecretManagerService;
+import org.apache.hadoop.yarn.server.timeline.webapp.TimelineWebServices;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.WebApp;
import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
import org.apache.hadoop.yarn.webapp.YarnWebParams;
+import com.google.common.annotations.VisibleForTesting;
+
public class AHSWebApp extends WebApp implements YarnWebParams {
- private final ApplicationHistoryManager applicationHistoryManager;
- private final TimelineStore timelineStore;
+ private ApplicationHistoryManager applicationHistoryManager;
+ private TimelineDelegationTokenSecretManagerService secretManagerService;
+ private TimelineDataManager timelineDataManager;
+
+ private static AHSWebApp instance = null;
+
+ public static AHSWebApp getInstance() {
+ if (instance == null) {
+ instance = new AHSWebApp();
+ }
+ return instance;
+ }
+
+ @Private
+ @VisibleForTesting
+ public static void resetInstance() {
+ instance = null;
+ }
+
+ private AHSWebApp() {
+
+ }
+
+ public ApplicationHistoryManager getApplicationHistoryManager() {
+ return applicationHistoryManager;
+ }
- public AHSWebApp(ApplicationHistoryManager applicationHistoryManager,
- TimelineStore timelineStore) {
+ public void setApplicationHistoryManager(
+ ApplicationHistoryManager applicationHistoryManager) {
this.applicationHistoryManager = applicationHistoryManager;
- this.timelineStore = timelineStore;
+ }
+
+ public TimelineDelegationTokenSecretManagerService
+ getTimelineDelegationTokenSecretManagerService() {
+ return secretManagerService;
+ }
+
+ public void setTimelineDelegationTokenSecretManagerService(
+ TimelineDelegationTokenSecretManagerService secretManagerService) {
+ this.secretManagerService = secretManagerService;
+ }
+
+ public TimelineDataManager getTimelineDataManager() {
+ return timelineDataManager;
+ }
+
+ public void setTimelineDataManager(TimelineDataManager timelineDataManager) {
+ this.timelineDataManager = timelineDataManager;
}
@Override
@@ -45,7 +91,9 @@ public class AHSWebApp extends WebApp im
bind(TimelineWebServices.class);
bind(GenericExceptionHandler.class);
bind(ApplicationContext.class).toInstance(applicationHistoryManager);
- bind(TimelineStore.class).toInstance(timelineStore);
+ bind(TimelineDelegationTokenSecretManagerService.class).toInstance(
+ secretManagerService);
+ bind(TimelineDataManager.class).toInstance(timelineDataManager);
route("/", AHSController.class);
route(pajoin("/apps", APP_STATE), AHSController.class);
route(pajoin("/app", APPLICATION_ID), AHSController.class, "app");
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java Tue Aug 19 23:49:39 2014
@@ -44,6 +44,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ContainerReport;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import org.junit.After;
import org.junit.Before;
@@ -68,12 +69,13 @@ public class TestApplicationHistoryClien
historyServer.init(config);
historyServer.start();
store =
- ((ApplicationHistoryManagerImpl) historyServer.getApplicationHistory())
+ ((ApplicationHistoryManagerImpl) historyServer.getApplicationHistoryManager())
.getHistoryStore();
}
@After
public void tearDown() throws Exception {
+ AHSWebApp.resetInstance();
historyServer.stop();
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java Tue Aug 19 23:49:39 2014
@@ -23,10 +23,14 @@ import static org.junit.Assert.assertNot
import static org.junit.Assert.fail;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.AuthenticationFilterInitializer;
import org.apache.hadoop.service.Service.STATE;
import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp;
+import org.apache.hadoop.yarn.server.timeline.security.TimelineAuthenticationFilterInitializer;
import org.junit.After;
+import org.junit.Assert;
import org.junit.Test;
public class TestApplicationHistoryServer {
@@ -40,7 +44,7 @@ public class TestApplicationHistoryServe
Configuration config = new YarnConfiguration();
historyServer.init(config);
assertEquals(STATE.INITED, historyServer.getServiceState());
- assertEquals(3, historyServer.getServices().size());
+ assertEquals(4, historyServer.getServices().size());
ApplicationHistoryClientService historyService =
historyServer.getClientService();
assertNotNull(historyServer.getClientService());
@@ -68,10 +72,36 @@ public class TestApplicationHistoryServe
}
}
+ @Test(timeout = 50000)
+ public void testFilteOverrides() throws Exception {
+
+ String[] filterInitializers =
+ {
+ AuthenticationFilterInitializer.class.getName(),
+ TimelineAuthenticationFilterInitializer.class.getName(),
+ AuthenticationFilterInitializer.class.getName() + ","
+ + TimelineAuthenticationFilterInitializer.class.getName(),
+ AuthenticationFilterInitializer.class.getName() + ", "
+ + TimelineAuthenticationFilterInitializer.class.getName() };
+ for (String filterInitializer : filterInitializers) {
+ historyServer = new ApplicationHistoryServer();
+ Configuration config = new YarnConfiguration();
+ config.set("hadoop.http.filter.initializers", filterInitializer);
+ historyServer.init(config);
+ historyServer.start();
+ Configuration tmp = historyServer.getConfig();
+ assertEquals(TimelineAuthenticationFilterInitializer.class.getName(),
+ tmp.get("hadoop.http.filter.initializers"));
+ historyServer.stop();
+ AHSWebApp.resetInstance();
+ }
+ }
+
@After
public void stop() {
if (historyServer != null) {
historyServer.stop();
}
+ AHSWebApp.resetInstance();
}
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestMemoryApplicationHistoryStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestMemoryApplicationHistoryStore.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestMemoryApplicationHistoryStore.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestMemoryApplicationHistoryStore.java Tue Aug 19 23:49:39 2014
@@ -198,7 +198,7 @@ public class TestMemoryApplicationHistor
writeContainerFinishData(containerId);
}
long usedMemoryAfter = (runtime.totalMemory() - runtime.freeMemory()) / mb;
- Assert.assertTrue((usedMemoryAfter - usedMemoryBefore) < 200);
+ Assert.assertTrue((usedMemoryAfter - usedMemoryBefore) < 400);
}
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml Tue Aug 19 23:49:39 2014
@@ -38,24 +38,6 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>commons-el</groupId>
- <artifactId>commons-el</artifactId>
- </exclusion>
- <exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-runtime</artifactId>
- </exclusion>
- <exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-compiler</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jsp-2.1-jetty</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
<dependency>
@@ -103,6 +85,10 @@
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.fusesource.leveldbjni</groupId>
+ <artifactId>leveldbjni-all</artifactId>
+ </dependency>
</dependencies>
<build>
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerRequest.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerRequest.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerRequest.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerRequest.java Tue Aug 19 23:49:39 2014
@@ -20,16 +20,17 @@ package org.apache.hadoop.yarn.server.ap
import java.util.List;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.util.Records;
public abstract class RegisterNodeManagerRequest {
-
+
public static RegisterNodeManagerRequest newInstance(NodeId nodeId,
int httpPort, Resource resource, String nodeManagerVersionId,
- List<ContainerStatus> containerStatuses) {
+ List<NMContainerStatus> containerStatuses,
+ List<ApplicationId> runningApplications) {
RegisterNodeManagerRequest request =
Records.newRecord(RegisterNodeManagerRequest.class);
request.setHttpPort(httpPort);
@@ -37,6 +38,7 @@ public abstract class RegisterNodeManage
request.setNodeId(nodeId);
request.setNMVersion(nodeManagerVersionId);
request.setContainerStatuses(containerStatuses);
+ request.setRunningApplications(runningApplications);
return request;
}
@@ -44,11 +46,32 @@ public abstract class RegisterNodeManage
public abstract int getHttpPort();
public abstract Resource getResource();
public abstract String getNMVersion();
- public abstract List<ContainerStatus> getContainerStatuses();
+ public abstract List<NMContainerStatus> getNMContainerStatuses();
+
+ /**
+ * We introduce this here because currently YARN RM doesn't persist nodes info
+ * for application running. When RM restart happened, we cannot determinate if
+ * a node should do application cleanup (like log-aggregation, status update,
+ * etc.) or not. <p/>
+ * When we have this running application list in node manager register
+ * request, we can recover nodes info for running applications. And then we
+ * can take actions accordingly
+ *
+ * @return running application list in this node
+ */
+ public abstract List<ApplicationId> getRunningApplications();
public abstract void setNodeId(NodeId nodeId);
public abstract void setHttpPort(int port);
public abstract void setResource(Resource resource);
public abstract void setNMVersion(String version);
- public abstract void setContainerStatuses(List<ContainerStatus> containerStatuses);
+ public abstract void setContainerStatuses(
+ List<NMContainerStatus> containerStatuses);
+
+ /**
+ * Setter for {@link RegisterNodeManagerRequest#getRunningApplications()}
+ * @param runningApplications running application in this node
+ */
+ public abstract void setRunningApplications(
+ List<ApplicationId> runningApplications);
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerRequestPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerRequestPBImpl.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerRequestPBImpl.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerRequestPBImpl.java Tue Aug 19 23:49:39 2014
@@ -23,21 +23,26 @@ import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
-import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.impl.pb.NodeIdPBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerStatusPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.NodeIdPBImpl;
-import org.apache.hadoop.yarn.api.records.impl.pb.ProtoBase;
import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl;
-import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStatusProto;
import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
+import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NMContainerStatusProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.RegisterNodeManagerRequestProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.RegisterNodeManagerRequestProtoOrBuilder;
+import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerRequest;
@@ -49,7 +54,8 @@ public class RegisterNodeManagerRequestP
private Resource resource = null;
private NodeId nodeId = null;
- private List<ContainerStatus> containerStatuses = null;
+ private List<NMContainerStatus> containerStatuses = null;
+ private List<ApplicationId> runningApplications = null;
public RegisterNodeManagerRequestPBImpl() {
builder = RegisterNodeManagerRequestProto.newBuilder();
@@ -69,7 +75,10 @@ public class RegisterNodeManagerRequestP
private void mergeLocalToBuilder() {
if (this.containerStatuses != null) {
- addContainerStatusesToProto();
+ addNMContainerStatusesToProto();
+ }
+ if (this.runningApplications != null) {
+ addRunningApplicationsToProto();
}
if (this.resource != null) {
builder.setResource(convertToProtoFormat(this.resource));
@@ -80,6 +89,18 @@ public class RegisterNodeManagerRequestP
}
+ private synchronized void addNMContainerStatusesToProto() {
+ maybeInitBuilder();
+ builder.clearContainerStatuses();
+ List<NMContainerStatusProto> list =
+ new ArrayList<NMContainerStatusProto>();
+ for (NMContainerStatus status : this.containerStatuses) {
+ list.add(convertToProtoFormat(status));
+ }
+ builder.addAllContainerStatuses(list);
+ }
+
+
private void mergeLocalToProto() {
if (viaProto)
maybeInitBuilder();
@@ -152,46 +173,46 @@ public class RegisterNodeManagerRequestP
maybeInitBuilder();
builder.setHttpPort(httpPort);
}
-
+
@Override
- public List<ContainerStatus> getContainerStatuses() {
- initContainerStatuses();
- return containerStatuses;
+ public List<ApplicationId> getRunningApplications() {
+ initRunningApplications();
+ return runningApplications;
}
- private void initContainerStatuses() {
- if (this.containerStatuses != null) {
+ private void initRunningApplications() {
+ if (this.runningApplications != null) {
return;
}
RegisterNodeManagerRequestProtoOrBuilder p = viaProto ? proto : builder;
- List<ContainerStatusProto> list = p.getContainerStatusesList();
- this.containerStatuses = new ArrayList<ContainerStatus>();
- for (ContainerStatusProto c : list) {
- this.containerStatuses.add(convertFromProtoFormat(c));
+ List<ApplicationIdProto> list = p.getRunningApplicationsList();
+ this.runningApplications = new ArrayList<ApplicationId>();
+ for (ApplicationIdProto c : list) {
+ this.runningApplications.add(convertFromProtoFormat(c));
}
}
@Override
- public void setContainerStatuses(List<ContainerStatus> containers) {
- if (containers == null) {
+ public void setRunningApplications(List<ApplicationId> apps) {
+ if (apps == null) {
return;
}
- initContainerStatuses();
- this.containerStatuses.addAll(containers);
+ initRunningApplications();
+ this.runningApplications.addAll(apps);
}
- private void addContainerStatusesToProto() {
+ private void addRunningApplicationsToProto() {
maybeInitBuilder();
- builder.clearContainerStatuses();
- if (containerStatuses == null) {
+ builder.clearRunningApplications();
+ if (runningApplications == null) {
return;
}
- Iterable<ContainerStatusProto> it = new Iterable<ContainerStatusProto>() {
+ Iterable<ApplicationIdProto> it = new Iterable<ApplicationIdProto>() {
@Override
- public Iterator<ContainerStatusProto> iterator() {
- return new Iterator<ContainerStatusProto>() {
- Iterator<ContainerStatus> iter = containerStatuses.iterator();
+ public Iterator<ApplicationIdProto> iterator() {
+ return new Iterator<ApplicationIdProto>() {
+ Iterator<ApplicationId> iter = runningApplications.iterator();
@Override
public boolean hasNext() {
@@ -199,7 +220,7 @@ public class RegisterNodeManagerRequestP
}
@Override
- public ContainerStatusProto next() {
+ public ApplicationIdProto next() {
return convertToProtoFormat(iter.next());
}
@@ -210,7 +231,35 @@ public class RegisterNodeManagerRequestP
};
}
};
- builder.addAllContainerStatuses(it);
+ builder.addAllRunningApplications(it);
+ }
+
+ @Override
+ public List<NMContainerStatus> getNMContainerStatuses() {
+ initContainerRecoveryReports();
+ return containerStatuses;
+ }
+
+ private void initContainerRecoveryReports() {
+ if (this.containerStatuses != null) {
+ return;
+ }
+ RegisterNodeManagerRequestProtoOrBuilder p = viaProto ? proto : builder;
+ List<NMContainerStatusProto> list = p.getContainerStatusesList();
+ this.containerStatuses = new ArrayList<NMContainerStatus>();
+ for (NMContainerStatusProto c : list) {
+ this.containerStatuses.add(convertFromProtoFormat(c));
+ }
+ }
+
+ @Override
+ public void setContainerStatuses(
+ List<NMContainerStatus> containerReports) {
+ if (containerReports == null) {
+ return;
+ }
+ initContainerRecoveryReports();
+ this.containerStatuses.addAll(containerReports);
}
@Override
@@ -242,6 +291,14 @@ public class RegisterNodeManagerRequestP
maybeInitBuilder();
builder.setNmVersion(version);
}
+
+ private ApplicationIdPBImpl convertFromProtoFormat(ApplicationIdProto p) {
+ return new ApplicationIdPBImpl(p);
+ }
+
+ private ApplicationIdProto convertToProtoFormat(ApplicationId t) {
+ return ((ApplicationIdPBImpl)t).getProto();
+ }
private NodeIdPBImpl convertFromProtoFormat(NodeIdProto p) {
return new NodeIdPBImpl(p);
@@ -259,11 +316,11 @@ public class RegisterNodeManagerRequestP
return ((ResourcePBImpl)t).getProto();
}
- private ContainerStatusPBImpl convertFromProtoFormat(ContainerStatusProto c) {
- return new ContainerStatusPBImpl(c);
+ private NMContainerStatusPBImpl convertFromProtoFormat(NMContainerStatusProto c) {
+ return new NMContainerStatusPBImpl(c);
}
- private ContainerStatusProto convertToProtoFormat(ContainerStatus c) {
- return ((ContainerStatusPBImpl)c).getProto();
+ private NMContainerStatusProto convertToProtoFormat(NMContainerStatus c) {
+ return ((NMContainerStatusPBImpl)c).getProto();
}
}
\ No newline at end of file
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseContainerTokenSecretManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseContainerTokenSecretManager.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseContainerTokenSecretManager.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseContainerTokenSecretManager.java Tue Aug 19 23:49:39 2014
@@ -43,7 +43,7 @@ public class BaseContainerTokenSecretMan
private static Log LOG = LogFactory
.getLog(BaseContainerTokenSecretManager.class);
- private int serialNo = new SecureRandom().nextInt();
+ protected int serialNo = new SecureRandom().nextInt();
protected final ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
protected final Lock readLock = readWriteLock.readLock();
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseNMTokenSecretManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseNMTokenSecretManager.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseNMTokenSecretManager.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseNMTokenSecretManager.java Tue Aug 19 23:49:39 2014
@@ -42,7 +42,7 @@ public class BaseNMTokenSecretManager ex
private static Log LOG = LogFactory
.getLog(BaseNMTokenSecretManager.class);
- private int serialNo = new SecureRandom().nextInt();
+ protected int serialNo = new SecureRandom().nextInt();
protected final ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
protected final Lock readLock = readWriteLock.readLock();
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java Tue Aug 19 23:49:39 2014
@@ -156,8 +156,7 @@ public class BuilderUtils {
byte[] password, long rmIdentifier) throws IOException {
ContainerTokenIdentifier identifier =
new ContainerTokenIdentifier(cId, host + ":" + port, user, r,
- expiryTime,
- masterKeyId, rmIdentifier);
+ expiryTime, masterKeyId, rmIdentifier, Priority.newInstance(0), 0);
return newContainerToken(BuilderUtils.newNodeId(host, port), password,
identifier);
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainerInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainerInfo.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainerInfo.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainerInfo.java Tue Aug 19 23:49:39 2014
@@ -57,7 +57,7 @@ public class ContainerInfo {
assignedNodeId = container.getAssignedNode().toString();
}
priority = container.getPriority().getPriority();
- startedTime = container.getStartTime();
+ startedTime = container.getCreationTime();
finishedTime = container.getFinishTime();
elapsedTime = Times.elapsed(startedTime, finishedTime);
diagnosticsInfo = container.getDiagnosticsInfo();
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto Tue Aug 19 23:49:39 2014
@@ -47,4 +47,10 @@ message NodeHealthStatusProto {
optional bool is_node_healthy = 1;
optional string health_report = 2;
optional int64 last_health_report_time = 3;
-}
\ No newline at end of file
+}
+
+message VersionProto {
+ optional int32 major_version = 1;
+ optional int32 minor_version = 2;
+}
+
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto Tue Aug 19 23:49:39 2014
@@ -30,7 +30,8 @@ message RegisterNodeManagerRequestProto
optional int32 http_port = 3;
optional ResourceProto resource = 4;
optional string nm_version = 5;
- repeated ContainerStatusProto containerStatuses = 6;
+ repeated NMContainerStatusProto container_statuses = 6;
+ repeated ApplicationIdProto runningApplications = 7;
}
message RegisterNodeManagerResponseProto {
@@ -58,3 +59,13 @@ message NodeHeartbeatResponseProto {
optional int64 nextHeartBeatInterval = 7;
optional string diagnostics_message = 8;
}
+
+message NMContainerStatusProto {
+ optional ContainerIdProto container_id = 1;
+ optional ContainerStateProto container_state = 2;
+ optional ResourceProto resource = 3;
+ optional PriorityProto priority = 4;
+ optional string diagnostics = 5 [default = "N/A"];
+ optional int32 container_exit_status = 6;
+ optional int64 creation_time = 7;
+}
\ No newline at end of file
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml Tue Aug 19 23:49:39 2014
@@ -40,24 +40,6 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>commons-el</groupId>
- <artifactId>commons-el</artifactId>
- </exclusion>
- <exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-runtime</artifactId>
- </exclusion>
- <exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-compiler</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jsp-2.1-jetty</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
@@ -174,6 +156,10 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-common</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.fusesource.leveldbjni</groupId>
+ <artifactId>leveldbjni-all</artifactId>
+ </dependency>
</dependencies>
<profiles>
@@ -310,6 +296,7 @@
<source>
<directory>${basedir}/src/main/proto</directory>
<includes>
+ <include>yarn_server_nodemanager_recovery.proto</include>
<include>yarn_server_nodemanager_service_protos.proto</include>
<include>LocalizationProtocol.proto</include>
</includes>
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java Tue Aug 19 23:49:39 2014
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.nodemanager;
+import java.io.File;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.util.ArrayList;
@@ -29,17 +30,18 @@ import java.util.concurrent.locks.Reentr
import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
+import org.apache.commons.io.FileUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.util.Shell.ShellCommandExecutor;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerDiagnosticsUpdateEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch;
import org.apache.hadoop.yarn.server.nodemanager.util.ProcessIdFileReader;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.StringUtils;
@@ -126,9 +128,76 @@ public abstract class ContainerExecutor
public abstract void deleteAsUser(String user, Path subDir, Path... basedirs)
throws IOException, InterruptedException;
+ public abstract boolean isContainerProcessAlive(String user, String pid)
+ throws IOException;
+
+ /**
+ * Recover an already existing container. This is a blocking call and returns
+ * only when the container exits. Note that the container must have been
+ * activated prior to this call.
+ * @param user the user of the container
+ * @param containerId The ID of the container to reacquire
+ * @return The exit code of the pre-existing container
+ * @throws IOException
+ */
+ public int reacquireContainer(String user, ContainerId containerId)
+ throws IOException {
+ Path pidPath = getPidFilePath(containerId);
+ if (pidPath == null) {
+ LOG.warn(containerId + " is not active, returning terminated error");
+ return ExitCode.TERMINATED.getExitCode();
+ }
+
+ String pid = null;
+ pid = ProcessIdFileReader.getProcessId(pidPath);
+ if (pid == null) {
+ throw new IOException("Unable to determine pid for " + containerId);
+ }
+
+ LOG.info("Reacquiring " + containerId + " with pid " + pid);
+ try {
+ while(isContainerProcessAlive(user, pid)) {
+ Thread.sleep(1000);
+ }
+ } catch (InterruptedException e) {
+ throw new IOException("Interrupted while waiting for process " + pid
+ + " to exit", e);
+ }
+
+ // wait for exit code file to appear
+ String exitCodeFile = ContainerLaunch.getExitCodeFile(pidPath.toString());
+ File file = new File(exitCodeFile);
+ final int sleepMsec = 100;
+ int msecLeft = 2000;
+ while (!file.exists() && msecLeft >= 0) {
+ if (!isContainerActive(containerId)) {
+ LOG.info(containerId + " was deactivated");
+ return ExitCode.TERMINATED.getExitCode();
+ }
+ try {
+ Thread.sleep(sleepMsec);
+ } catch (InterruptedException e) {
+ throw new IOException(
+ "Interrupted while waiting for exit code from " + containerId, e);
+ }
+ msecLeft -= sleepMsec;
+ }
+ if (msecLeft < 0) {
+ throw new IOException("Timeout while waiting for exit code from "
+ + containerId);
+ }
+
+ try {
+ return Integer.parseInt(FileUtils.readFileToString(file).trim());
+ } catch (NumberFormatException e) {
+ throw new IOException("Error parsing exit code from pid " + pid, e);
+ }
+ }
+
public enum ExitCode {
FORCE_KILLED(137),
- TERMINATED(143);
+ TERMINATED(143),
+ LOST(154);
private final int code;
private ExitCode(int exitCode) {