You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by sz...@apache.org on 2014/01/26 17:34:39 UTC
svn commit: r1561516 [2/3] - in
/hadoop/common/branches/HDFS-5535/hadoop-yarn-project: ./ hadoop-yarn/
hadoop-yarn/bin/ hadoop-yarn/conf/ hadoop-yarn/hadoop-yarn-api/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/
hadoop-yarn/had...
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java?rev=1561516&r1=1561515&r2=1561516&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java Sun Jan 26 16:34:30 2014
@@ -43,19 +43,26 @@ import junit.framework.Assert;
import org.apache.commons.lang.time.DateFormatUtils;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerReport;
+import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.util.Records;
import org.junit.Before;
import org.junit.Test;
+import org.mortbay.log.Log;
import org.apache.commons.cli.Options;
@@ -114,19 +121,180 @@ public class TestYarnCLI {
}
@Test
+ public void testGetApplicationAttemptReport() throws Exception {
+ ApplicationCLI cli = createAndGetAppCLI();
+ ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
+ ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
+ applicationId, 1);
+ ApplicationAttemptReport attemptReport = ApplicationAttemptReport
+ .newInstance(attemptId, "host", 124, "url", "diagnostics",
+ YarnApplicationAttemptState.FINISHED, ContainerId.newInstance(
+ attemptId, 1));
+ when(
+ client
+ .getApplicationAttemptReport(any(ApplicationAttemptId.class)))
+ .thenReturn(attemptReport);
+ int result = cli.run(new String[] { "applicationattempt", "-status",
+ attemptId.toString() });
+ assertEquals(0, result);
+ verify(client).getApplicationAttemptReport(attemptId);
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ PrintWriter pw = new PrintWriter(baos);
+ pw.println("Application Attempt Report : ");
+ pw.println("\tApplicationAttempt-Id : appattempt_1234_0005_000001");
+ pw.println("\tState : FINISHED");
+ pw.println("\tAMContainer : container_1234_0005_01_000001");
+ pw.println("\tTracking-URL : url");
+ pw.println("\tRPC Port : 124");
+ pw.println("\tAM Host : host");
+ pw.println("\tDiagnostics : diagnostics");
+ pw.close();
+ String appReportStr = baos.toString("UTF-8");
+ Assert.assertEquals(appReportStr, sysOutStream.toString());
+ verify(sysOut, times(1)).println(isA(String.class));
+ }
+
+ @Test
+ public void testGetApplicationAttempts() throws Exception {
+ ApplicationCLI cli = createAndGetAppCLI();
+ ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
+ ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
+ applicationId, 1);
+ ApplicationAttemptId attemptId1 = ApplicationAttemptId.newInstance(
+ applicationId, 2);
+ ApplicationAttemptReport attemptReport = ApplicationAttemptReport
+ .newInstance(attemptId, "host", 124, "url", "diagnostics",
+ YarnApplicationAttemptState.FINISHED, ContainerId.newInstance(
+ attemptId, 1));
+ ApplicationAttemptReport attemptReport1 = ApplicationAttemptReport
+ .newInstance(attemptId1, "host", 124, "url", "diagnostics",
+ YarnApplicationAttemptState.FINISHED, ContainerId.newInstance(
+ attemptId1, 1));
+ List<ApplicationAttemptReport> reports = new ArrayList<ApplicationAttemptReport>();
+ reports.add(attemptReport);
+ reports.add(attemptReport1);
+ when(client.getApplicationAttempts(any(ApplicationId.class)))
+ .thenReturn(reports);
+ int result = cli.run(new String[] { "applicationattempt", "-list",
+ applicationId.toString() });
+ assertEquals(0, result);
+ verify(client).getApplicationAttempts(applicationId);
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ PrintWriter pw = new PrintWriter(baos);
+ pw.println("Total number of application attempts :2");
+ pw.print(" ApplicationAttempt-Id");
+ pw.print("\t State");
+ pw.print("\t AM-Container-Id");
+ pw.println("\t Tracking-URL");
+ pw.print(" appattempt_1234_0005_000001");
+ pw.print("\t FINISHED");
+ pw.print("\t container_1234_0005_01_000001");
+ pw.println("\t url");
+ pw.print(" appattempt_1234_0005_000002");
+ pw.print("\t FINISHED");
+ pw.print("\t container_1234_0005_02_000001");
+ pw.println("\t url");
+ pw.close();
+ String appReportStr = baos.toString("UTF-8");
+ Assert.assertEquals(appReportStr, sysOutStream.toString());
+ }
+
+ @Test
+ public void testGetContainerReport() throws Exception {
+ ApplicationCLI cli = createAndGetAppCLI();
+ ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
+ ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
+ applicationId, 1);
+ ContainerId containerId = ContainerId.newInstance(attemptId, 1);
+ ContainerReport container = ContainerReport.newInstance(containerId, null,
+ NodeId.newInstance("host", 1234), Priority.UNDEFINED, 1234, 5678,
+ "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE);
+ when(client.getContainerReport(any(ContainerId.class))).thenReturn(
+ container);
+ int result = cli.run(new String[] { "container", "-status",
+ containerId.toString() });
+ assertEquals(0, result);
+ verify(client).getContainerReport(containerId);
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ PrintWriter pw = new PrintWriter(baos);
+ pw.println("Container Report : ");
+ pw.println("\tContainer-Id : container_1234_0005_01_000001");
+ pw.println("\tStart-Time : 1234");
+ pw.println("\tFinish-Time : 5678");
+ pw.println("\tState : COMPLETE");
+ pw.println("\tLOG-URL : logURL");
+ pw.println("\tHost : host:1234");
+ pw.println("\tDiagnostics : diagnosticInfo");
+ pw.close();
+ String appReportStr = baos.toString("UTF-8");
+ Assert.assertEquals(appReportStr, sysOutStream.toString());
+ verify(sysOut, times(1)).println(isA(String.class));
+ }
+
+ @Test
+ public void testGetContainers() throws Exception {
+ ApplicationCLI cli = createAndGetAppCLI();
+ ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
+ ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
+ applicationId, 1);
+ ContainerId containerId = ContainerId.newInstance(attemptId, 1);
+ ContainerId containerId1 = ContainerId.newInstance(attemptId, 2);
+ ContainerReport container = ContainerReport.newInstance(containerId, null,
+ NodeId.newInstance("host", 1234), Priority.UNDEFINED, 1234, 5678,
+ "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE);
+ ContainerReport container1 = ContainerReport.newInstance(containerId1, null,
+ NodeId.newInstance("host", 1234), Priority.UNDEFINED, 1234, 5678,
+ "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE);
+ List<ContainerReport> reports = new ArrayList<ContainerReport>();
+ reports.add(container);
+ reports.add(container1);
+ when(client.getContainers(any(ApplicationAttemptId.class))).thenReturn(
+ reports);
+ int result = cli.run(new String[] { "container", "-list",
+ attemptId.toString() });
+ assertEquals(0, result);
+ verify(client).getContainers(attemptId);
+ Log.info(sysOutStream.toString());
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ PrintWriter pw = new PrintWriter(baos);
+ pw.println("Total number of containers :2");
+ pw.print(" Container-Id");
+ pw.print("\t Start Time");
+ pw.print("\t Finish Time");
+ pw.print("\t State");
+ pw.print("\t Host");
+ pw.println("\t LOG-URL");
+ pw.print(" container_1234_0005_01_000001");
+ pw.print("\t 1234");
+ pw.print("\t 5678");
+ pw.print("\t COMPLETE");
+ pw.print("\t host:1234");
+ pw.println("\t logURL");
+ pw.print(" container_1234_0005_01_000002");
+ pw.print("\t 1234");
+ pw.print("\t 5678");
+ pw.print("\t COMPLETE");
+ pw.print("\t host:1234");
+ pw.println("\t logURL");
+ pw.close();
+ String appReportStr = baos.toString("UTF-8");
+ Assert.assertEquals(appReportStr, sysOutStream.toString());
+ }
+
+ @Test
public void testGetApplicationReportException() throws Exception {
ApplicationCLI cli = createAndGetAppCLI();
ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
when(client.getApplicationReport(any(ApplicationId.class))).thenThrow(
- new ApplicationNotFoundException("Application with id '"
- + applicationId + "' doesn't exist in RM."));
+ new ApplicationNotFoundException("History file for application"
+ + applicationId + " is not found"));
try {
cli.run(new String[] { "-status", applicationId.toString() });
Assert.fail();
} catch (Exception ex) {
Assert.assertTrue(ex instanceof ApplicationNotFoundException);
- Assert.assertEquals("Application with id '" + applicationId
- + "' doesn't exist in RM.", ex.getMessage());
+ Assert.assertEquals("History file for application"
+ + applicationId + " is not found", ex.getMessage());
}
}
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml?rev=1561516&r1=1561515&r2=1561516&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml Sun Jan 26 16:34:30 2014
@@ -199,6 +199,7 @@
<exclude>src/main/resources/webapps/mapreduce/.keep</exclude>
<exclude>src/main/resources/webapps/jobhistory/.keep</exclude>
<exclude>src/main/resources/webapps/yarn/.keep</exclude>
+ <exclude>src/main/resources/webapps/applicationhistory/.keep</exclude>
<exclude>src/main/resources/webapps/cluster/.keep</exclude>
<exclude>src/main/resources/webapps/test/.keep</exclude>
<exclude>src/main/resources/webapps/proxy/.keep</exclude>
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerLogAppender.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerLogAppender.java?rev=1561516&r1=1561515&r2=1561516&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerLogAppender.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ContainerLogAppender.java Sun Jan 26 16:34:30 2014
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn;
import java.io.File;
+import java.io.Flushable;
import java.util.LinkedList;
import java.util.Queue;
@@ -33,7 +34,9 @@ import org.apache.log4j.spi.LoggingEvent
*/
@Public
@Unstable
-public class ContainerLogAppender extends FileAppender {
+public class ContainerLogAppender extends FileAppender
+ implements Flushable
+{
private String containerLogDir;
//so that log4j can configure it from the configuration(log4j.properties).
private int maxEvents;
@@ -65,6 +68,7 @@ public class ContainerLogAppender extend
}
}
+ @Override
public void flush() {
if (qw != null) {
qw.flush();
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java?rev=1561516&r1=1561515&r2=1561516&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java Sun Jan 26 16:34:30 2014
@@ -252,6 +252,25 @@ public class RegisterApplicationMasterRe
this.containersFromPreviousAttempt = new ArrayList<Container>();
this.containersFromPreviousAttempt.addAll(containers);
}
+
+ @Override
+ public String getQueue() {
+ RegisterApplicationMasterResponseProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasQueue()) {
+ return null;
+ }
+ return p.getQueue();
+ }
+
+ @Override
+ public void setQueue(String queue) {
+ maybeInitBuilder();
+ if (queue == null) {
+ builder.clearQueue();
+ } else {
+ builder.setQueue(queue);
+ }
+ }
private void initRunningContainersList() {
RegisterApplicationMasterResponseProtoOrBuilder p = viaProto ? proto : builder;
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java?rev=1561516&r1=1561515&r2=1561516&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java Sun Jan 26 16:34:30 2014
@@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.QueueState;
+import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.proto.YarnProtos.AMCommandProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAccessTypeProto;
@@ -45,6 +46,7 @@ import org.apache.hadoop.yarn.proto.Yarn
import org.apache.hadoop.yarn.proto.YarnProtos.NodeStateProto;
import org.apache.hadoop.yarn.proto.YarnProtos.QueueACLProto;
import org.apache.hadoop.yarn.proto.YarnProtos.QueueStateProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationAttemptStateProto;
import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationStateProto;
import com.google.protobuf.ByteString;
@@ -97,6 +99,21 @@ public class ProtoUtils {
}
/*
+ * YarnApplicationAttemptState
+ */
+ private static String YARN_APPLICATION_ATTEMPT_STATE_PREFIX = "APP_ATTEMPT_";
+ public static YarnApplicationAttemptStateProto convertToProtoFormat(
+ YarnApplicationAttemptState e) {
+ return YarnApplicationAttemptStateProto
+ .valueOf(YARN_APPLICATION_ATTEMPT_STATE_PREFIX + e.name());
+ }
+ public static YarnApplicationAttemptState convertFromProtoFormat(
+ YarnApplicationAttemptStateProto e) {
+ return YarnApplicationAttemptState.valueOf(e.name().replace(
+ YARN_APPLICATION_ATTEMPT_STATE_PREFIX, ""));
+ }
+
+ /*
* ApplicationResourceUsageReport
*/
public static ApplicationResourceUsageReportProto convertToProtoFormat(ApplicationResourceUsageReport e) {
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/StringHelper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/StringHelper.java?rev=1561516&r1=1561515&r2=1561516&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/StringHelper.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/StringHelper.java Sun Jan 26 16:34:30 2014
@@ -178,4 +178,8 @@ public final class StringHelper {
public static String percent(double value) {
return String.format("%.2f", value * 100);
}
+
+ public static String getPartUrl(String url, String part) {
+ return url.substring(url.indexOf(part));
+ }
}
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java?rev=1561516&r1=1561515&r2=1561516&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java Sun Jan 26 16:34:30 2014
@@ -24,6 +24,7 @@ import org.apache.hadoop.classification.
public interface YarnWebParams {
String NM_NODENAME = "nm.id";
String APPLICATION_ID = "app.id";
+ String APPLICATION_ATTEMPT_ID = "appattempt.id";
String CONTAINER_ID = "container.id";
String CONTAINER_LOG_TYPE= "log.type";
String ENTITY_STRING = "entity.string";
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java?rev=1561516&r1=1561515&r2=1561516&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java Sun Jan 26 16:34:30 2014
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.yarn.webapp.util;
+import static org.apache.hadoop.yarn.util.StringHelper.join;
+
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.UnknownHostException;
@@ -27,7 +29,9 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.http.HttpConfig.Policy;
import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.util.ConverterUtils;
import com.google.common.base.Joiner;
@@ -144,6 +148,16 @@ public class WebAppUtils {
YarnConfiguration.DEFAULT_NM_WEBAPP_ADDRESS);
}
}
+
+ public static String getAHSWebAppURLWithoutScheme(Configuration conf) {
+ if (HttpConfig.isSecure()) {
+ return conf.get(YarnConfiguration.AHS_WEBAPP_HTTPS_ADDRESS,
+ YarnConfiguration.DEFAULT_AHS_WEBAPP_HTTPS_ADDRESS);
+ } else {
+ return conf.get(YarnConfiguration.AHS_WEBAPP_ADDRESS,
+ YarnConfiguration.DEFAULT_AHS_WEBAPP_ADDRESS);
+ }
+ }
/**
* if url has scheme then it will be returned as it is else it will return
@@ -160,4 +174,11 @@ public class WebAppUtils {
return schemePrefix + url;
}
}
+
+ public static String getLogUrl(String nodeHttpAddress, String allocatedNode,
+ ContainerId containerId, String user) {
+ return join(HttpConfig.getSchemePrefix(), nodeHttpAddress, "/logs", "/",
+ allocatedNode, "/", ConverterUtils.toString(containerId), "/",
+ ConverterUtils.toString(containerId), "/", user);
+ }
}
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml?rev=1561516&r1=1561515&r2=1561516&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml Sun Jan 26 16:34:30 2014
@@ -357,7 +357,7 @@
By default, when this property is not set, we use the ACLs from
yarn.resourcemanager.zk-acl for shared admin access and
- rm-address:cluster-timestamp for username-based exclusive create-delete
+ rm-address:random-number for username-based exclusive create-delete
access.
This property allows users to set ACLs of their choice instead of using
@@ -564,6 +564,30 @@
<value>org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy</value>
</property>
+ <property>
+ <description>Indicate to ResourceManager as well as clients whether
+ history-service is enabled or not. If enabled, ResourceManager starts
+ recording historical data that ApplicationHistory service can consume.
+ Similarly, clients can redirect to the history service when applications
+ finish if this is enabled.</description>
+ <name>yarn.ahs.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <description>Number of worker threads that write the history data.</description>
+ <name>yarn.resourcemanager.history-writer.multi-threaded-dispatcher.pool-size</name>
+ <value>10</value>
+ </property>
+
+ <property>
+ <description>The implementation class of ApplicationHistoryStore, which is
+ to be used by RMApplicationHistoryWriter.
+ </description>
+ <name>yarn.resourcemanager.history-writer.class</name>
+ <value>org.apache.hadoop.yarn.server.applicationhistoryservice.NullApplicationHistoryStore</value>
+ </property>
+
<!-- Node Manager Configs -->
<property>
<description>The hostname of the NM.</description>
@@ -1041,6 +1065,61 @@
<value></value>
</property>
+ <!-- Application History Service's Configuration-->
+
+ <property>
+ <description>The hostname of the AHS.</description>
+ <name>yarn.ahs.hostname</name>
+ <value>0.0.0.0</value>
+ </property>
+
+ <property>
+ <description>The http address of the AHS web application.</description>
+ <name>yarn.ahs.webapp.address</name>
+ <value>${yarn.ahs.hostname}:8188</value>
+ </property>
+
+ <property>
+ <description>The https adddress of the AHS web application.</description>
+ <name>yarn.ahs.webapp.https.address</name>
+ <value>${yarn.ahs.hostname}:8190</value>
+ </property>
+
+ <property>
+ <description>URI pointing to the location of the FileSystem path where
+ the history will be persisted. This must be supplied when using
+ org.apache.hadoop.yarn.server.applicationhistoryservice.FileSystemApplicationHistoryStore
+ as the value for yarn.resourcemanager.history-writer.store.class</description>
+ <name>yarn.ahs.fs-history-store.uri</name>
+ <value>${hadoop.log.dir}/yarn/system/ahstore</value>
+ </property>
+
+ <property>
+ <description>This is default address for the Application History server
+ to start the RPC server.</description>
+ <name>yarn.ahs.address</name>
+ <value>0.0.0.0:10200</value>
+ </property>
+
+ <property>
+ <description>CLient thread count to serve the client requests.</description>
+ <name>yarn.ahs.client.thread-count</name>
+ <value>10</value>
+ </property>
+
+ <property>
+ <description>T-file compression types used to compress history data.</description>
+ <name>yarn.ahs.fs-history-store.compression-type</name>
+ <value>none</value>
+ </property>
+
+ <property>
+ <description> Store class name for history store, defaulting to file
+ system store </description>
+ <name>yarn.ahs.store.class</name>
+ <value>org.apache.hadoop.yarn.server.applicationhistoryservice.FileSystemApplicationHistoryStore</value>
+ </property>
+
<!-- Other configuration -->
<property>
<description>The interval that the yarn client library uses to poll the
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java?rev=1561516&r1=1561515&r2=1561516&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java Sun Jan 26 16:34:30 2014
@@ -638,8 +638,8 @@ public class ResourceLocalizationService
super("Public Localizer");
this.lfs = getLocalFileContext(conf);
this.conf = conf;
- this.pending =
- new ConcurrentHashMap<Future<Path>, LocalizerResourceRequestEvent>();
+ this.pending = Collections.synchronizedMap(
+ new HashMap<Future<Path>, LocalizerResourceRequestEvent>());
this.threadPool = createLocalizerExecutor(conf);
this.queue = new ExecutorCompletionService<Path>(threadPool);
}
@@ -675,8 +675,12 @@ public class ResourceLocalizationService
publicDirDestPath =
new Path(publicDirDestPath, Long.toString(publicRsrc
.nextUniqueNumber()));
- pending.put(queue.submit(new FSDownload(lfs, null, conf,
- publicDirDestPath, resource)), request);
+ // explicitly synchronize pending here to avoid future task
+ // completing and being dequeued before pending updated
+ synchronized (pending) {
+ pending.put(queue.submit(new FSDownload(lfs, null, conf,
+ publicDirDestPath, resource)), request);
+ }
} catch (IOException e) {
rsrc.unlock();
// TODO Need to Fix IO Exceptions - Notifying resource
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml?rev=1561516&r1=1561515&r2=1561516&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml Sun Jan 26 16:34:30 2014
@@ -185,6 +185,11 @@
<!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-applicationhistoryservice</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-web-proxy</artifactId>
</dependency>
<dependency>
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java?rev=1561516&r1=1561515&r2=1561516&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java Sun Jan 26 16:34:30 2014
@@ -267,6 +267,7 @@ public class ApplicationMasterService ex
.getMaximumResourceCapability());
response.setApplicationACLs(app.getRMAppAttempt(applicationAttemptId)
.getSubmissionContext().getAMContainerSpec().getApplicationACLs());
+ response.setQueue(app.getQueue());
if (UserGroupInformation.isSecurityEnabled()) {
LOG.info("Setting client token master key");
response.setClientToAMTokenMasterKey(java.nio.ByteBuffer.wrap(rmContext
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java?rev=1561516&r1=1561515&r2=1561516&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java Sun Jan 26 16:34:30 2014
@@ -24,6 +24,7 @@ import org.apache.hadoop.ha.HAServicePro
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.event.Dispatcher;
+import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AMLivelinessMonitor;
@@ -33,8 +34,8 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer;
-import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
+import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMDelegationTokenSecretManager;
/**
@@ -90,4 +91,10 @@ public interface RMContext {
void setRMDelegationTokenSecretManager(
RMDelegationTokenSecretManager delegationTokenSecretManager);
+
+ RMApplicationHistoryWriter getRMApplicationHistoryWriter();
+
+ void setRMApplicationHistoryWriter(
+ RMApplicationHistoryWriter rmApplicationHistoryWriter);
+
}
\ No newline at end of file
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java?rev=1561516&r1=1561515&r2=1561516&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java Sun Jan 26 16:34:30 2014
@@ -27,6 +27,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
+import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.NullRMStateStore;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
@@ -76,6 +77,7 @@ public class RMContextImpl implements RM
private NodesListManager nodesListManager;
private ResourceTrackerService resourceTrackerService;
private ApplicationMasterService applicationMasterService;
+ private RMApplicationHistoryWriter rmApplicationHistoryWriter;
/**
* Default constructor. To be used in conjunction with setter methods for
@@ -95,7 +97,8 @@ public class RMContextImpl implements RM
AMRMTokenSecretManager appTokenSecretManager,
RMContainerTokenSecretManager containerTokenSecretManager,
NMTokenSecretManagerInRM nmTokenSecretManager,
- ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager) {
+ ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager,
+ RMApplicationHistoryWriter rmApplicationHistoryWriter) {
this();
this.setDispatcher(rmDispatcher);
this.setContainerAllocationExpirer(containerAllocationExpirer);
@@ -106,6 +109,7 @@ public class RMContextImpl implements RM
this.setContainerTokenSecretManager(containerTokenSecretManager);
this.setNMTokenSecretManager(nmTokenSecretManager);
this.setClientToAMTokenSecretManager(clientToAMTokenSecretManager);
+ this.setRMApplicationHistoryWriter(rmApplicationHistoryWriter);
RMStateStore nullStore = new NullRMStateStore();
nullStore.setRMDispatcher(rmDispatcher);
@@ -318,4 +322,16 @@ public class RMContextImpl implements RM
return haServiceState;
}
}
+
+ @Override
+ public RMApplicationHistoryWriter getRMApplicationHistoryWriter() {
+ return rmApplicationHistoryWriter;
+ }
+
+ @Override
+ public void setRMApplicationHistoryWriter(
+ RMApplicationHistoryWriter rmApplicationHistoryWriter) {
+ this.rmApplicationHistoryWriter = rmApplicationHistoryWriter;
+ }
+
}
\ No newline at end of file
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java?rev=1561516&r1=1561515&r2=1561516&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java Sun Jan 26 16:34:30 2014
@@ -33,9 +33,14 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.exceptions.InvalidContainerReleaseException;
import org.apache.hadoop.yarn.exceptions.InvalidResourceBlacklistRequestException;
import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
@@ -43,6 +48,7 @@ import org.apache.hadoop.yarn.server.res
* Utility methods to aid serving RM data through the REST and RPC APIs
*/
public class RMServerUtils {
+
public static List<RMNode> queryRMNodes(RMContext context,
EnumSet<NodeState> acceptedStates) {
// nodes contains nodes that are NEW, RUNNING OR UNHEALTHY
@@ -56,7 +62,7 @@ public class RMServerUtils {
}
}
}
-
+
// inactiveNodes contains nodes that are DECOMMISSIONED, LOST, OR REBOOTED
if (acceptedStates.contains(NodeState.DECOMMISSIONED) ||
acceptedStates.contains(NodeState.LOST) ||
@@ -69,7 +75,7 @@ public class RMServerUtils {
}
return results;
}
-
+
/**
* Utility method to validate a list resource requests, by insuring that the
* requested memory/vcore is non-negative and not greater than max
@@ -85,8 +91,9 @@ public class RMServerUtils {
* @throw <code>InvalidResourceBlacklistRequestException </code> if the
* resource is not able to be added to the blacklist.
*/
- public static void validateBlacklistRequest(ResourceBlacklistRequest blacklistRequest)
- throws InvalidResourceBlacklistRequestException {
+ public static void validateBlacklistRequest(
+ ResourceBlacklistRequest blacklistRequest)
+ throws InvalidResourceBlacklistRequestException {
if (blacklistRequest != null) {
List<String> plus = blacklistRequest.getBlacklistAdditions();
if (plus != null && plus.contains(ResourceRequest.ANY)) {
@@ -100,10 +107,12 @@ public class RMServerUtils {
* It will validate to make sure all the containers belong to correct
* application attempt id. If not then it will throw
* {@link InvalidContainerReleaseException}
- * @param containerReleaseList containers to be released as requested by
- * application master.
- * @param appAttemptId Application attempt Id
- * @throws InvalidContainerReleaseException
+ *
+ * @param containerReleaseList
+ * containers to be released as requested by application master.
+ * @param appAttemptId
+ * Application attempt Id
+ * @throws InvalidContainerReleaseException
*/
public static void
validateContainerReleaseRequest(List<ContainerId> containerReleaseList,
@@ -111,9 +120,11 @@ public class RMServerUtils {
throws InvalidContainerReleaseException {
for (ContainerId cId : containerReleaseList) {
if (!appAttemptId.equals(cId.getApplicationAttemptId())) {
- throw new InvalidContainerReleaseException("Cannot release container : "
- + cId.toString() + " not belonging to this application attempt : "
- + appAttemptId);
+ throw new InvalidContainerReleaseException(
+ "Cannot release container : "
+ + cId.toString()
+ + " not belonging to this application attempt : "
+ + appAttemptId);
}
}
}
@@ -157,4 +168,63 @@ public class RMServerUtils {
}
return user;
}
+
+ public static YarnApplicationState createApplicationState(
+ RMAppState rmAppState) {
+ switch (rmAppState) {
+ case NEW:
+ return YarnApplicationState.NEW;
+ case NEW_SAVING:
+ return YarnApplicationState.NEW_SAVING;
+ case SUBMITTED:
+ return YarnApplicationState.SUBMITTED;
+ case ACCEPTED:
+ return YarnApplicationState.ACCEPTED;
+ case RUNNING:
+ return YarnApplicationState.RUNNING;
+ case FINISHING:
+ case FINISHED:
+ return YarnApplicationState.FINISHED;
+ case KILLED:
+ return YarnApplicationState.KILLED;
+ case FAILED:
+ return YarnApplicationState.FAILED;
+ default:
+ throw new YarnRuntimeException("Unknown state passed!");
+ }
+ }
+
+ public static YarnApplicationAttemptState createApplicationAttemptState(
+ RMAppAttemptState rmAppAttemptState) {
+ switch (rmAppAttemptState) {
+ case NEW:
+ return YarnApplicationAttemptState.NEW;
+ case SUBMITTED:
+ return YarnApplicationAttemptState.SUBMITTED;
+ case SCHEDULED:
+ return YarnApplicationAttemptState.SCHEDULED;
+ case ALLOCATED:
+ return YarnApplicationAttemptState.ALLOCATED;
+ case LAUNCHED:
+ return YarnApplicationAttemptState.LAUNCHED;
+ case ALLOCATED_SAVING:
+ case LAUNCHED_UNMANAGED_SAVING:
+ return YarnApplicationAttemptState.ALLOCATED_SAVING;
+ case RUNNING:
+ return YarnApplicationAttemptState.RUNNING;
+ case FINISHING:
+ return YarnApplicationAttemptState.FINISHING;
+ case FINAL_SAVING:
+ return YarnApplicationAttemptState.FINAL_SAVING;
+ case FINISHED:
+ return YarnApplicationAttemptState.FINISHED;
+ case KILLED:
+ return YarnApplicationAttemptState.KILLED;
+ case FAILED:
+ return YarnApplicationAttemptState.FAILED;
+ default:
+ throw new YarnRuntimeException("Unknown state passed!");
+ }
+ }
+
}
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java?rev=1561516&r1=1561515&r2=1561516&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java Sun Jan 26 16:34:30 2014
@@ -52,6 +52,7 @@ import org.apache.hadoop.yarn.event.Asyn
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter;
import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEventType;
import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher;
import org.apache.hadoop.yarn.server.resourcemanager.monitor.SchedulingEditPolicy;
@@ -261,6 +262,10 @@ public class ResourceManager extends Com
this.applicationACLsManager, this.conf);
}
+ protected RMApplicationHistoryWriter createRMApplicationHistoryWriter() {
+ return new RMApplicationHistoryWriter();
+ }
+
// sanity check for configurations
protected static void validateConfigs(Configuration conf) {
// validate max-attempts
@@ -345,6 +350,11 @@ public class ResourceManager extends Com
rmContext.setDelegationTokenRenewer(delegationTokenRenewer);
}
+ RMApplicationHistoryWriter rmApplicationHistoryWriter =
+ createRMApplicationHistoryWriter();
+ addService(rmApplicationHistoryWriter);
+ rmContext.setRMApplicationHistoryWriter(rmApplicationHistoryWriter);
+
// Register event handler for NodesListManager
nodesListManager = new NodesListManager(rmContext);
rmDispatcher.register(NodesListManagerEventType.class, nodesListManager);
@@ -607,7 +617,7 @@ public class ResourceManager extends Com
@Override
public void handle(RMFatalEvent event) {
LOG.fatal("Received a " + RMFatalEvent.class.getName() + " of type " +
- event.getType().name());
+ event.getType().name() + ". Cause:\n" + event.getCause());
if (event.getType() == RMFatalEventType.STATE_STORE_FENCED) {
LOG.info("RMStateStore has been fenced");
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java?rev=1561516&r1=1561515&r2=1561516&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java Sun Jan 26 16:34:30 2014
@@ -24,6 +24,7 @@ import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.security.NoSuchAlgorithmException;
+import java.security.SecureRandom;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@@ -47,7 +48,6 @@ import org.apache.hadoop.yarn.proto.Yarn
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.ApplicationStateDataProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RMStateVersionProto;
import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
-import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.RMStateVersion;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationAttemptStateDataPBImpl;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationStateDataPBImpl;
@@ -74,6 +74,7 @@ import com.google.common.annotations.Vis
public class ZKRMStateStore extends RMStateStore {
public static final Log LOG = LogFactory.getLog(ZKRMStateStore.class);
+ private final SecureRandom random = new SecureRandom();
protected static final String ROOT_ZNODE_NAME = "ZKRMStateRoot";
protected static final RMStateVersion CURRENT_VERSION_INFO = RMStateVersion
@@ -136,6 +137,8 @@ public class ZKRMStateStore extends RMSt
private String fencingNodePath;
private Op createFencingNodePathOp;
private Op deleteFencingNodePathOp;
+ private String zkRootNodeUsername;
+ private final String zkRootNodePassword = Long.toString(random.nextLong());
@VisibleForTesting
List<ACL> zkRootNodeAcl;
@@ -145,9 +148,6 @@ public class ZKRMStateStore extends RMSt
private final String zkRootNodeAuthScheme =
new DigestAuthenticationProvider().getScheme();
- private String zkRootNodeUsername;
- private String zkRootNodePassword;
-
/**
* Given the {@link Configuration} and {@link ACL}s used (zkAcl) for
* ZooKeeper access, construct the {@link ACL}s for the store's root node.
@@ -172,7 +172,6 @@ public class ZKRMStateStore extends RMSt
zkRootNodeUsername = HAUtil.getConfValueForRMInstance(
YarnConfiguration.RM_ADDRESS,
YarnConfiguration.DEFAULT_RM_ADDRESS, conf);
- zkRootNodePassword = Long.toString(ResourceManager.getClusterTimeStamp());
Id rmId = new Id(zkRootNodeAuthScheme,
DigestAuthenticationProvider.generateDigest(
zkRootNodeUsername + ":" + zkRootNodePassword));
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java?rev=1561516&r1=1561515&r2=1561516&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java Sun Jan 26 16:34:30 2014
@@ -54,6 +54,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.RMAppManagerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.RMAppManagerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.ApplicationState;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.Recoverable;
@@ -335,6 +336,8 @@ public class RMAppImpl implements RMApp,
this.writeLock = lock.writeLock();
this.stateMachine = stateMachineFactory.make(this);
+
+ rmContext.getRMApplicationHistoryWriter().applicationStarted(this);
}
@Override
@@ -1002,6 +1005,11 @@ public class RMAppImpl implements RMApp,
app.handler.handle(
new RMAppManagerEvent(app.applicationId,
RMAppManagerEventType.APP_COMPLETED));
+
+ // TODO: We need to fix for the problem that RMApp enters the final state
+ // after RMAppAttempt in the killing case
+ app.rmContext.getRMApplicationHistoryWriter()
+ .applicationFinished(app);
};
}
@@ -1069,27 +1077,7 @@ public class RMAppImpl implements RMApp,
if (rmAppState.equals(RMAppState.KILLING)) {
rmAppState = stateBeforeKilling;
}
- switch (rmAppState) {
- case NEW:
- return YarnApplicationState.NEW;
- case NEW_SAVING:
- return YarnApplicationState.NEW_SAVING;
- case SUBMITTED:
- return YarnApplicationState.SUBMITTED;
- case ACCEPTED:
- return YarnApplicationState.ACCEPTED;
- case RUNNING:
- return YarnApplicationState.RUNNING;
- case FINISHING:
- case FINISHED:
- return YarnApplicationState.FINISHED;
- case KILLED:
- return YarnApplicationState.KILLED;
- case FAILED:
- return YarnApplicationState.FAILED;
- default:
- throw new YarnRuntimeException("Unknown state passed!");
- }
+ return RMServerUtils.createApplicationState(rmAppState);
}
public static boolean isAppInFinalState(RMApp rmApp) {
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java?rev=1561516&r1=1561515&r2=1561516&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java Sun Jan 26 16:34:30 2014
@@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
@@ -178,4 +179,21 @@ public interface RMAppAttempt extends Ev
* @return the start time of the application.
*/
long getStartTime();
+
+ /**
+ * The current state of the {@link RMAppAttempt}.
+ *
+ * @return the current state {@link RMAppAttemptState} for this application
+ * attempt.
+ */
+ RMAppAttemptState getState();
+
+ /**
+ * Create the external user-facing state of the attempt of ApplicationMaster
+ * from the current state of the {@link RMAppAttempt}.
+ *
+ * @return the external user-facing state of the attempt ApplicationMaster.
+ */
+ YarnApplicationAttemptState createApplicationAttemptState();
+
}
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java?rev=1561516&r1=1561515&r2=1561516&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java Sun Jan 26 16:34:30 2014
@@ -55,6 +55,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
@@ -62,6 +63,7 @@ import org.apache.hadoop.yarn.security.A
import org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier;
import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEvent;
import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEventType;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
@@ -1046,6 +1048,9 @@ public class RMAppAttemptImpl implements
appAttempt.eventHandler.handle(new AppAttemptRemovedSchedulerEvent(
appAttemptId, finalAttemptState, keepContainersAcrossAppAttempts));
appAttempt.removeCredentials(appAttempt);
+
+ appAttempt.rmContext.getRMApplicationHistoryWriter()
+ .applicationAttemptFinished(appAttempt);
}
}
@@ -1143,6 +1148,9 @@ public class RMAppAttemptImpl implements
// write at AM launch time, so we don't save the AM's tracking URL anywhere
// as that would mean an extra state-store write. For now, we hope that in
// work-preserving restart, AMs are forced to reregister.
+
+ appAttempt.rmContext.getRMApplicationHistoryWriter()
+ .applicationAttemptStarted(appAttempt);
}
}
@@ -1514,6 +1522,23 @@ public class RMAppAttemptImpl implements
}
}
+ @Override
+ public RMAppAttemptState getState() {
+ this.readLock.lock();
+
+ try {
+ return this.stateMachine.getCurrentState();
+ } finally {
+ this.readLock.unlock();
+ }
+ }
+
+ @Override
+ public YarnApplicationAttemptState createApplicationAttemptState() {
+ RMAppAttemptState state = getState();
+ return RMServerUtils.createApplicationAttemptState(state);
+ }
+
private void launchAttempt(){
// Send event to launch the AM Container
eventHandler.handle(new AMLauncherEvent(AMLauncherEventType.LAUNCH, this));
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java?rev=1561516&r1=1561515&r2=1561516&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java Sun Jan 26 16:34:30 2014
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.re
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -50,4 +51,22 @@ public interface RMContainer extends Eve
Priority getReservedPriority();
+ Resource getAllocatedResource();
+
+ NodeId getAllocatedNode();
+
+ Priority getAllocatedPriority();
+
+ long getStartTime();
+
+ long getFinishTime();
+
+ String getDiagnosticsInfo();
+
+ String getLogURL();
+
+ int getContainerExitStatus();
+
+ ContainerState getContainerState();
+
}
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java?rev=1561516&r1=1561515&r2=1561516&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java Sun Jan 26 16:34:30 2014
@@ -28,10 +28,13 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptContainerAcquiredEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptContainerAllocatedEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptContainerFinishedEvent;
@@ -40,6 +43,7 @@ import org.apache.hadoop.yarn.state.Inva
import org.apache.hadoop.yarn.state.SingleArcTransition;
import org.apache.hadoop.yarn.state.StateMachine;
import org.apache.hadoop.yarn.state.StateMachineFactory;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
@SuppressWarnings({"unchecked", "rawtypes"})
public class RMContainerImpl implements RMContainer {
@@ -133,28 +137,39 @@ public class RMContainerImpl implements
private final ApplicationAttemptId appAttemptId;
private final NodeId nodeId;
private final Container container;
+ private final RMContext rmContext;
private final EventHandler eventHandler;
private final ContainerAllocationExpirer containerAllocationExpirer;
+ private final String user;
private Resource reservedResource;
private NodeId reservedNode;
private Priority reservedPriority;
+ private long startTime;
+ private long finishTime;
+ private String logURL;
+ private ContainerStatus finishedStatus;
+
public RMContainerImpl(Container container,
ApplicationAttemptId appAttemptId, NodeId nodeId,
- EventHandler handler,
- ContainerAllocationExpirer containerAllocationExpirer) {
+ String user, RMContext rmContext) {
this.stateMachine = stateMachineFactory.make(this);
this.containerId = container.getId();
this.nodeId = nodeId;
this.container = container;
this.appAttemptId = appAttemptId;
- this.eventHandler = handler;
- this.containerAllocationExpirer = containerAllocationExpirer;
+ this.user = user;
+ this.startTime = System.currentTimeMillis();
+ this.rmContext = rmContext;
+ this.eventHandler = rmContext.getDispatcher().getEventHandler();
+ this.containerAllocationExpirer = rmContext.getContainerAllocationExpirer();
ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
this.readLock = lock.readLock();
this.writeLock = lock.writeLock();
+
+ rmContext.getRMApplicationHistoryWriter().containerStarted(this);
}
@Override
@@ -197,7 +212,77 @@ public class RMContainerImpl implements
public Priority getReservedPriority() {
return reservedPriority;
}
-
+
+ @Override
+ public Resource getAllocatedResource() {
+ return container.getResource();
+ }
+
+ @Override
+ public NodeId getAllocatedNode() {
+ return container.getNodeId();
+ }
+
+ @Override
+ public Priority getAllocatedPriority() {
+ return container.getPriority();
+ }
+
+ @Override
+ public long getStartTime() {
+ return startTime;
+ }
+
+ @Override
+ public long getFinishTime() {
+ try {
+ readLock.lock();
+ return finishTime;
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ @Override
+ public String getDiagnosticsInfo() {
+ try {
+ readLock.lock();
+ return finishedStatus.getDiagnostics();
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ @Override
+ public String getLogURL() {
+ try {
+ readLock.lock();
+ return logURL;
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ @Override
+ public int getContainerExitStatus() {
+ try {
+ readLock.lock();
+ return finishedStatus.getExitStatus();
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ @Override
+ public ContainerState getContainerState() {
+ try {
+ readLock.lock();
+ return finishedStatus.getState();
+ } finally {
+ readLock.unlock();
+ }
+ }
+
@Override
public String toString() {
return containerId.toString();
@@ -276,6 +361,11 @@ public class RMContainerImpl implements
@Override
public void transition(RMContainerImpl container, RMContainerEvent event) {
+ // The logs of running containers should be found on NM webUI
+ // The logs should be accessible after the container is launched
+ container.logURL = WebAppUtils.getLogUrl(container.container
+ .getNodeHttpAddress(), container.getAllocatedNode().toString(),
+ container.containerId, container.user);
// Unregister from containerAllocationExpirer.
container.containerAllocationExpirer.unregister(container
.getContainerId());
@@ -288,9 +378,17 @@ public class RMContainerImpl implements
public void transition(RMContainerImpl container, RMContainerEvent event) {
RMContainerFinishedEvent finishedEvent = (RMContainerFinishedEvent) event;
+ container.finishTime = System.currentTimeMillis();
+ container.finishedStatus = finishedEvent.getRemoteContainerStatus();
+ // TODO: when AHS webUI is ready, logURL should be updated to point to
+ // the web page that will show the aggregated logs
+
// Inform AppAttempt
container.eventHandler.handle(new RMAppAttemptContainerFinishedEvent(
container.appAttemptId, finishedEvent.getRemoteContainerStatus()));
+
+ container.rmContext.getRMApplicationHistoryWriter()
+ .containerFinished(container);
}
}
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java?rev=1561516&r1=1561515&r2=1561516&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java Sun Jan 26 16:34:30 2014
@@ -233,8 +233,7 @@ public abstract class SchedulerApplicati
if (rmContainer == null) {
rmContainer =
new RMContainerImpl(container, getApplicationAttemptId(),
- node.getNodeID(), rmContext.getDispatcher().getEventHandler(),
- rmContext.getContainerAllocationExpirer());
+ node.getNodeID(), appSchedulingInfo.getUser(), rmContext);
Resources.addTo(currentReservation, container.getResource());
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java?rev=1561516&r1=1561515&r2=1561516&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java Sun Jan 26 16:34:30 2014
@@ -121,9 +121,8 @@ public class FiCaSchedulerApp extends Sc
// Create RMContainer
RMContainer rmContainer = new RMContainerImpl(container, this
- .getApplicationAttemptId(), node.getNodeID(), this.rmContext
- .getDispatcher().getEventHandler(), this.rmContext
- .getContainerAllocationExpirer());
+ .getApplicationAttemptId(), node.getNodeID(),
+ appSchedulingInfo.getUser(), this.rmContext);
// Add it to allContainers list.
newlyAllocatedContainers.add(rmContainer);
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java?rev=1561516&r1=1561515&r2=1561516&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java Sun Jan 26 16:34:30 2014
@@ -271,9 +271,8 @@ public class FSSchedulerApp extends Sche
// Create RMContainer
RMContainer rmContainer = new RMContainerImpl(container,
- getApplicationAttemptId(), node.getNodeID(), rmContext
- .getDispatcher().getEventHandler(), rmContext
- .getContainerAllocationExpirer());
+ getApplicationAttemptId(), node.getNodeID(),
+ appSchedulingInfo.getUser(), rmContext);
// Add it to allContainers list.
newlyAllocatedContainers.add(rmContainer);
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementPolicy.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementPolicy.java?rev=1561516&r1=1561515&r2=1561516&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementPolicy.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementPolicy.java Sun Jan 26 16:34:30 2014
@@ -80,7 +80,13 @@ public class QueuePlacementPolicy {
Node node = elements.item(i);
if (node instanceof Element) {
Element element = (Element)node;
- String ruleName = element.getTagName();
+
+ String ruleName = element.getAttribute("name");
+ if ("".equals(ruleName)) {
+ throw new AllocationConfigurationException("No name provided for a " +
+ "rule element");
+ }
+
Class<? extends QueuePlacementRule> clazz = ruleClasses.get(ruleName);
if (clazz == null) {
throw new AllocationConfigurationException("No rule class found for "
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java?rev=1561516&r1=1561515&r2=1561516&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java Sun Jan 26 16:34:30 2014
@@ -22,9 +22,9 @@ package org.apache.hadoop.yarn.server.re
import static org.mockito.Matchers.isA;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
-import static org.mockito.Mockito.times;
import java.util.HashMap;
import java.util.List;
@@ -47,6 +47,7 @@ import org.apache.hadoop.yarn.event.Even
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.MockRMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
@@ -104,9 +105,10 @@ public class TestAppManager{
rmDispatcher);
AMLivelinessMonitor amFinishingMonitor = new AMLivelinessMonitor(
rmDispatcher);
+ RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
RMContext context = new RMContextImpl(rmDispatcher,
containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor,
- null, null, null, null, null) {
+ null, null, null, null, null, writer) {
@Override
public ConcurrentMap<ApplicationId, RMApp> getRMApps() {
return map;
Modified: hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java?rev=1561516&r1=1561515&r2=1561516&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java (original)
+++ hadoop/common/branches/HDFS-5535/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java Sun Jan 26 16:34:30 2014
@@ -19,21 +19,21 @@
package org.apache.hadoop.yarn.server.resourcemanager;
import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyBoolean;
import static org.mockito.Matchers.anyString;
-import static org.mockito.Matchers.any;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.HashSet;
import java.util.EnumSet;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -51,9 +51,9 @@ import org.apache.hadoop.security.UserGr
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.yarn.MockApps;
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
+import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoResponse;
@@ -81,6 +81,7 @@ import org.apache.hadoop.yarn.factories.
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
+import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.NullRMStateStore;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
@@ -599,6 +600,8 @@ public class TestClientRMService {
.thenReturn(queInfo);
when(yarnScheduler.getQueueInfo(eq("nonexistentqueue"), anyBoolean(), anyBoolean()))
.thenThrow(new IOException("queue does not exist"));
+ RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
+ when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer);
ConcurrentHashMap<ApplicationId, RMApp> apps = getRMApps(rmContext,
yarnScheduler);
when(rmContext.getRMApps()).thenReturn(apps);