You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by cn...@apache.org on 2014/01/27 19:09:30 UTC
svn commit: r1561771 [2/3] - in
/hadoop/common/branches/HDFS-4685/hadoop-yarn-project: ./ hadoop-yarn/
hadoop-yarn/bin/ hadoop-yarn/conf/ hadoop-yarn/hadoop-yarn-api/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/
hadoop-yarn/had...
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java?rev=1561771&r1=1561770&r2=1561771&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java Mon Jan 27 18:09:22 2014
@@ -29,9 +29,13 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerReport;
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.QueueInfo;
@@ -40,6 +44,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.api.records.YarnClusterMetrics;
import org.apache.hadoop.yarn.client.api.impl.YarnClientImpl;
+import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
@@ -360,4 +365,75 @@ public abstract class YarnClient extends
*/
public abstract List<QueueUserACLInfo> getQueueAclsInfo() throws YarnException,
IOException;
+
+ /**
+ * <p>
+ * Get a report of the given ApplicationAttempt.
+ * </p>
+ *
+ * <p>
+ * In secure mode, <code>YARN</code> verifies access to the application, queue
+ * etc. before accepting the request.
+ * </p>
+ *
+ * @param applicationAttemptId
+ * {@link ApplicationAttemptId} of the application attempt that needs
+ * a report
+ * @return application attempt report
+ * @throws YarnException
+ * @throws {@link ApplicationAttemptNotFoundException} if application attempt
+ * not found
+ * @throws IOException
+ */
+ public abstract ApplicationAttemptReport getApplicationAttemptReport(
+ ApplicationAttemptId applicationAttemptId) throws YarnException, IOException;
+
+ /**
+ * <p>
+ * Get a report of all (ApplicationAttempts) of Application in the cluster.
+ * </p>
+ *
+ * @param applicationId
+ * @return a list of reports for all application attempts for specified
+ * application.
+ * @throws YarnException
+ * @throws IOException
+ */
+ public abstract List<ApplicationAttemptReport> getApplicationAttempts(
+ ApplicationId applicationId) throws YarnException, IOException;
+
+ /**
+ * <p>
+ * Get a report of the given Container.
+ * </p>
+ *
+ * <p>
+ * In secure mode, <code>YARN</code> verifies access to the application, queue
+ * etc. before accepting the request.
+ * </p>
+ *
+ * @param containerId
+ * {@link ContainerId} of the container that needs a report
+ * @return container report
+ * @throws YarnException
+ * @throws {@link ContainerNotFoundException} if container not found.
+ * @throws IOException
+ */
+ public abstract ContainerReport getContainerReport(ContainerId containerId)
+ throws YarnException, IOException;
+
+ /**
+ * <p>
+ * Get a report of all (Containers) of ApplicationAttempt in the cluster.
+ * </p>
+ *
+ * @param applicationAttemptId
+ * @return a list of reports of all containers for specified application
+ * attempts
+ * @throws YarnException
+ * @throws IOException
+ */
+ public abstract List<ContainerReport> getContainers(
+ ApplicationAttemptId applicationAttemptId) throws YarnException,
+ IOException;
}
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java?rev=1561771&r1=1561770&r2=1561771&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java Mon Jan 27 18:09:22 2014
@@ -49,9 +49,13 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest;
import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerReport;
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.QueueInfo;
@@ -60,9 +64,11 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.api.records.YarnClusterMetrics;
import org.apache.hadoop.yarn.client.ClientRMProxy;
+import org.apache.hadoop.yarn.client.api.AHSClient;
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.client.api.YarnClientApplication;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
@@ -80,6 +86,8 @@ public class YarnClientImpl extends Yarn
protected ApplicationClientProtocol rmClient;
protected long submitPollIntervalMillis;
private long asyncApiPollIntervalMillis;
+ protected AHSClient historyClient;
+ private boolean historyServiceEnabled;
private static final String ROOT = "root";
@@ -100,6 +108,14 @@ public class YarnClientImpl extends Yarn
YarnConfiguration.YARN_CLIENT_APP_SUBMISSION_POLL_INTERVAL_MS,
YarnConfiguration.DEFAULT_YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_INTERVAL_MS);
}
+
+ if (conf.getBoolean(YarnConfiguration.YARN_HISTORY_SERVICE_ENABLED,
+ YarnConfiguration.DEFAULT_YARN_HISTORY_SERVICE_ENABLED)) {
+ historyServiceEnabled = true;
+ historyClient = AHSClientImpl.createAHSClient();
+ historyClient.init(getConfig());
+ }
+
super.serviceInit(conf);
}
@@ -107,7 +123,10 @@ public class YarnClientImpl extends Yarn
protected void serviceStart() throws Exception {
try {
rmClient = ClientRMProxy.createRMProxy(getConfig(),
- ApplicationClientProtocol.class);
+ ApplicationClientProtocol.class);
+ if (historyServiceEnabled) {
+ historyClient.start();
+ }
} catch (IOException e) {
throw new YarnRuntimeException(e);
}
@@ -119,6 +138,9 @@ public class YarnClientImpl extends Yarn
if (this.rmClient != null) {
RPC.stopProxy(this.rmClient);
}
+ if (historyServiceEnabled) {
+ historyClient.stop();
+ }
super.serviceStop();
}
@@ -207,11 +229,27 @@ public class YarnClientImpl extends Yarn
@Override
public ApplicationReport getApplicationReport(ApplicationId appId)
throws YarnException, IOException {
- GetApplicationReportRequest request =
- Records.newRecord(GetApplicationReportRequest.class);
- request.setApplicationId(appId);
- GetApplicationReportResponse response =
- rmClient.getApplicationReport(request);
+ GetApplicationReportResponse response = null;
+ try {
+ GetApplicationReportRequest request = Records
+ .newRecord(GetApplicationReportRequest.class);
+ request.setApplicationId(appId);
+ response = rmClient.getApplicationReport(request);
+ } catch (YarnException e) {
+
+ if (!historyServiceEnabled) {
+ // Just throw it as usual if historyService is not enabled.
+ throw e;
+ }
+
+ // Even if history-service is enabled, treat all exceptions still the same
+ // except the following
+ if (!(e.getClass() == ApplicationNotFoundException.class)) {
+ throw e;
+ }
+
+ return historyClient.getApplicationReport(appId);
+ }
return response.getApplicationReport();
}
@@ -373,4 +411,41 @@ public class YarnClientImpl extends Yarn
public void setRMClient(ApplicationClientProtocol rmClient) {
this.rmClient = rmClient;
}
+
+ @Override
+ public ApplicationAttemptReport getApplicationAttemptReport(
+ ApplicationAttemptId appAttemptId) throws YarnException, IOException {
+ if (historyServiceEnabled) {
+ return historyClient.getApplicationAttemptReport(appAttemptId);
+ }
+ throw new YarnException("History service is not enabled.");
+ }
+
+ @Override
+ public List<ApplicationAttemptReport> getApplicationAttempts(
+ ApplicationId appId) throws YarnException, IOException {
+ if (historyServiceEnabled) {
+ return historyClient.getApplicationAttempts(appId);
+ }
+ throw new YarnException("History service is not enabled.");
+ }
+
+ @Override
+ public ContainerReport getContainerReport(ContainerId containerId)
+ throws YarnException, IOException {
+ if (historyServiceEnabled) {
+ return historyClient.getContainerReport(containerId);
+ }
+ throw new YarnException("History service is not enabled.");
+ }
+
+ @Override
+ public List<ContainerReport> getContainers(
+ ApplicationAttemptId applicationAttemptId) throws YarnException,
+ IOException {
+ if (historyServiceEnabled) {
+ return historyClient.getContainers(applicationAttemptId);
+ }
+ throw new YarnException("History service is not enabled.");
+ }
}
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java?rev=1561771&r1=1561770&r2=1561771&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java Mon Jan 27 18:09:22 2014
@@ -35,8 +35,10 @@ import org.apache.commons.cli.Options;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.util.ToolRunner;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.ContainerReport;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.util.ConverterUtils;
@@ -46,13 +48,22 @@ import com.google.common.annotations.Vis
@Private
@Unstable
public class ApplicationCLI extends YarnCLI {
- private static final String APPLICATIONS_PATTERN =
- "%30s\t%20s\t%20s\t%10s\t%10s\t%18s\t%18s\t%15s\t%35s" +
- System.getProperty("line.separator");
+ private static final String APPLICATIONS_PATTERN =
+ "%30s\t%20s\t%20s\t%10s\t%10s\t%18s\t%18s\t%15s\t%35s"
+ + System.getProperty("line.separator");
+ private static final String APPLICATION_ATTEMPTS_PATTERN =
+ "%30s\t%20s\t%35s\t%35s"
+ + System.getProperty("line.separator");
+ private static final String CONTAINER_PATTERN =
+ "%30s\t%20s\t%20s\t%20s\t%20s\t%35s"
+ + System.getProperty("line.separator");
private static final String APP_TYPE_CMD = "appTypes";
- private static final String APP_STATE_CMD ="appStates";
+ private static final String APP_STATE_CMD = "appStates";
private static final String ALLSTATES_OPTION = "ALL";
+ public static final String APPLICATION = "application";
+ public static final String APPLICATION_ATTEMPT = "applicationattempt";
+ public static final String CONTAINER = "container";
private boolean allAppStates;
@@ -69,23 +80,33 @@ public class ApplicationCLI extends Yarn
public int run(String[] args) throws Exception {
Options opts = new Options();
- opts.addOption(STATUS_CMD, true, "Prints the status of the application.");
- opts.addOption(LIST_CMD, false, "List applications from the RM. " +
- "Supports optional use of -appTypes to filter applications " +
- "based on application type, " +
- "and -appStates to filter applications based on application state");
+ opts.addOption(STATUS_CMD, true,
+ "Prints the status of the application.");
+ if (args.length > 0
+ && args[0].compareToIgnoreCase(APPLICATION_ATTEMPT) == 0) {
+ opts.addOption(LIST_CMD, true,
+ "List application attempts for aplication from AHS. ");
+ } else if (args.length > 0 && args[0].compareToIgnoreCase("container") == 0) {
+ opts.addOption(LIST_CMD, true,
+ "List containers for application attempts from AHS. ");
+ } else {
+ opts.addOption(LIST_CMD, false, "List applications from the RM. "
+ + "Supports optional use of -appTypes to filter applications "
+ + "based on application type, "
+ + "and -appStates to filter applications based on application state");
+ }
opts.addOption(KILL_CMD, true, "Kills the application.");
opts.addOption(HELP_CMD, false, "Displays help for all commands.");
- Option appTypeOpt = new Option(APP_TYPE_CMD, true, "Works with -list to " +
- "filter applications based on " +
- "input comma-separated list of application types.");
+ Option appTypeOpt = new Option(APP_TYPE_CMD, true, "Works with -list to "
+ + "filter applications based on "
+ + "input comma-separated list of application types.");
appTypeOpt.setValueSeparator(',');
appTypeOpt.setArgs(Option.UNLIMITED_VALUES);
appTypeOpt.setArgName("Types");
opts.addOption(appTypeOpt);
- Option appStateOpt = new Option(APP_STATE_CMD, true, "Works with -list " +
- "to filter applications based on input comma-separated list of " +
- "application states. " + getAllValidApplicationStates());
+ Option appStateOpt = new Option(APP_STATE_CMD, true, "Works with -list "
+ + "to filter applications based on input comma-separated list of "
+ + "application states. " + getAllValidApplicationStates());
appStateOpt.setValueSeparator(',');
appStateOpt.setArgs(Option.UNLIMITED_VALUES);
appStateOpt.setArgName("States");
@@ -104,50 +125,77 @@ public class ApplicationCLI extends Yarn
}
if (cliParser.hasOption(STATUS_CMD)) {
- if (args.length != 2) {
+ if ((args[0].compareToIgnoreCase(APPLICATION) == 0)
+ || (args[0].compareToIgnoreCase(APPLICATION_ATTEMPT) == 0)
+ || (args[0].compareToIgnoreCase(CONTAINER) == 0)) {
+ if (args.length != 3) {
+ printUsage(opts);
+ return exitCode;
+ }
+ } else if (args.length != 2) {
printUsage(opts);
return exitCode;
}
- printApplicationReport(cliParser.getOptionValue(STATUS_CMD));
+ if (args[0].compareToIgnoreCase(APPLICATION_ATTEMPT) == 0) {
+ printApplicationAttemptReport(cliParser.getOptionValue(STATUS_CMD));
+ } else if (args[0].compareToIgnoreCase(CONTAINER) == 0) {
+ printContainerReport(cliParser.getOptionValue(STATUS_CMD));
+ } else {
+ printApplicationReport(cliParser.getOptionValue(STATUS_CMD));
+ }
} else if (cliParser.hasOption(LIST_CMD)) {
- allAppStates = false;
- Set<String> appTypes = new HashSet<String>();
- if(cliParser.hasOption(APP_TYPE_CMD)) {
- String[] types = cliParser.getOptionValues(APP_TYPE_CMD);
- if (types != null) {
- for (String type : types) {
- if (!type.trim().isEmpty()) {
- appTypes.add(type.toUpperCase().trim());
+ if (args[0].compareToIgnoreCase(APPLICATION_ATTEMPT) == 0) {
+ if (args.length != 3) {
+ printUsage(opts);
+ return exitCode;
+ }
+ listApplicationAttempts(cliParser.getOptionValue(LIST_CMD));
+ } else if (args[0].compareToIgnoreCase(CONTAINER) == 0) {
+ if (args.length != 3) {
+ printUsage(opts);
+ return exitCode;
+ }
+ listContainers(cliParser.getOptionValue(LIST_CMD));
+ } else {
+ allAppStates = false;
+ Set<String> appTypes = new HashSet<String>();
+ if (cliParser.hasOption(APP_TYPE_CMD)) {
+ String[] types = cliParser.getOptionValues(APP_TYPE_CMD);
+ if (types != null) {
+ for (String type : types) {
+ if (!type.trim().isEmpty()) {
+ appTypes.add(type.toUpperCase().trim());
+ }
}
}
}
- }
- EnumSet<YarnApplicationState> appStates =
- EnumSet.noneOf(YarnApplicationState.class);
- if (cliParser.hasOption(APP_STATE_CMD)) {
- String[] states = cliParser.getOptionValues(APP_STATE_CMD);
- if (states != null) {
- for (String state : states) {
- if (!state.trim().isEmpty()) {
- if (state.trim().equalsIgnoreCase(ALLSTATES_OPTION)) {
- allAppStates = true;
- break;
- }
- try {
- appStates.add(YarnApplicationState.valueOf(state.toUpperCase()
- .trim()));
- } catch (IllegalArgumentException ex) {
- sysout.println("The application state " + state
- + " is invalid.");
- sysout.println(getAllValidApplicationStates());
- return exitCode;
+ EnumSet<YarnApplicationState> appStates = EnumSet
+ .noneOf(YarnApplicationState.class);
+ if (cliParser.hasOption(APP_STATE_CMD)) {
+ String[] states = cliParser.getOptionValues(APP_STATE_CMD);
+ if (states != null) {
+ for (String state : states) {
+ if (!state.trim().isEmpty()) {
+ if (state.trim().equalsIgnoreCase(ALLSTATES_OPTION)) {
+ allAppStates = true;
+ break;
+ }
+ try {
+ appStates.add(YarnApplicationState.valueOf(state
+ .toUpperCase().trim()));
+ } catch (IllegalArgumentException ex) {
+ sysout.println("The application state " + state
+ + " is invalid.");
+ sysout.println(getAllValidApplicationStates());
+ return exitCode;
+ }
}
}
}
}
+ listApplications(appTypes, appStates);
}
- listApplications(appTypes, appStates);
} else if (cliParser.hasOption(KILL_CMD)) {
if (args.length != 2) {
printUsage(opts);
@@ -175,8 +223,85 @@ public class ApplicationCLI extends Yarn
}
/**
- * Lists the applications matching the given application Types
- * And application States present in the Resource Manager
+ * Prints the application attempt report for an application attempt id.
+ *
+ * @param applicationAttemptId
+ * @throws YarnException
+ */
+ private void printApplicationAttemptReport(String applicationAttemptId)
+ throws YarnException, IOException {
+ ApplicationAttemptReport appAttemptReport = client
+ .getApplicationAttemptReport(ConverterUtils
+ .toApplicationAttemptId(applicationAttemptId));
+ // Use PrintWriter.println, which uses correct platform line ending.
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ PrintWriter appAttemptReportStr = new PrintWriter(baos);
+ if (appAttemptReport != null) {
+ appAttemptReportStr.println("Application Attempt Report : ");
+ appAttemptReportStr.print("\tApplicationAttempt-Id : ");
+ appAttemptReportStr.println(appAttemptReport.getApplicationAttemptId());
+ appAttemptReportStr.print("\tState : ");
+ appAttemptReportStr.println(appAttemptReport
+ .getYarnApplicationAttemptState());
+ appAttemptReportStr.print("\tAMContainer : ");
+ appAttemptReportStr.println(appAttemptReport.getAMContainerId()
+ .toString());
+ appAttemptReportStr.print("\tTracking-URL : ");
+ appAttemptReportStr.println(appAttemptReport.getTrackingUrl());
+ appAttemptReportStr.print("\tRPC Port : ");
+ appAttemptReportStr.println(appAttemptReport.getRpcPort());
+ appAttemptReportStr.print("\tAM Host : ");
+ appAttemptReportStr.println(appAttemptReport.getHost());
+ appAttemptReportStr.print("\tDiagnostics : ");
+ appAttemptReportStr.print(appAttemptReport.getDiagnostics());
+ } else {
+ appAttemptReportStr.print("Application Attempt with id '"
+ + applicationAttemptId + "' doesn't exist in History Server.");
+ }
+ appAttemptReportStr.close();
+ sysout.println(baos.toString("UTF-8"));
+ }
+
+ /**
+ * Prints the container report for an container id.
+ *
+ * @param containerId
+ * @throws YarnException
+ */
+ private void printContainerReport(String containerId) throws YarnException,
+ IOException {
+ ContainerReport containerReport = client.getContainerReport((ConverterUtils
+ .toContainerId(containerId)));
+ // Use PrintWriter.println, which uses correct platform line ending.
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ PrintWriter containerReportStr = new PrintWriter(baos);
+ if (containerReport != null) {
+ containerReportStr.println("Container Report : ");
+ containerReportStr.print("\tContainer-Id : ");
+ containerReportStr.println(containerReport.getContainerId());
+ containerReportStr.print("\tStart-Time : ");
+ containerReportStr.println(containerReport.getStartTime());
+ containerReportStr.print("\tFinish-Time : ");
+ containerReportStr.println(containerReport.getFinishTime());
+ containerReportStr.print("\tState : ");
+ containerReportStr.println(containerReport.getContainerState());
+ containerReportStr.print("\tLOG-URL : ");
+ containerReportStr.println(containerReport.getLogUrl());
+ containerReportStr.print("\tHost : ");
+ containerReportStr.println(containerReport.getAssignedNode());
+ containerReportStr.print("\tDiagnostics : ");
+ containerReportStr.print(containerReport.getDiagnosticsInfo());
+ } else {
+ containerReportStr.print("Container with id '" + containerId
+ + "' doesn't exist in Hostory Server.");
+ }
+ containerReportStr.close();
+ sysout.println(baos.toString("UTF-8"));
+ }
+
+ /**
+ * Lists the applications matching the given application Types And application
+ * States present in the Resource Manager
*
* @param appTypes
* @param appStates
@@ -188,7 +313,7 @@ public class ApplicationCLI extends Yarn
IOException {
PrintWriter writer = new PrintWriter(sysout);
if (allAppStates) {
- for(YarnApplicationState appState : YarnApplicationState.values()) {
+ for (YarnApplicationState appState : YarnApplicationState.values()) {
appStates.add(appState);
}
} else {
@@ -199,23 +324,24 @@ public class ApplicationCLI extends Yarn
}
}
- List<ApplicationReport> appsReport =
- client.getApplications(appTypes, appStates);
+ List<ApplicationReport> appsReport = client.getApplications(appTypes,
+ appStates);
- writer
- .println("Total number of applications (application-types: " + appTypes
- + " and states: " + appStates + ")" + ":" + appsReport.size());
- writer.printf(APPLICATIONS_PATTERN, "Application-Id",
- "Application-Name","Application-Type", "User", "Queue",
- "State", "Final-State","Progress", "Tracking-URL");
+ writer.println("Total number of applications (application-types: "
+ + appTypes + " and states: " + appStates + ")" + ":"
+ + appsReport.size());
+ writer.printf(APPLICATIONS_PATTERN, "Application-Id", "Application-Name",
+ "Application-Type", "User", "Queue", "State", "Final-State",
+ "Progress", "Tracking-URL");
for (ApplicationReport appReport : appsReport) {
DecimalFormat formatter = new DecimalFormat("###.##%");
String progress = formatter.format(appReport.getProgress());
writer.printf(APPLICATIONS_PATTERN, appReport.getApplicationId(),
- appReport.getName(),appReport.getApplicationType(), appReport.getUser(),
- appReport.getQueue(),appReport.getYarnApplicationState(),
- appReport.getFinalApplicationStatus(),progress,
- appReport.getOriginalTrackingUrl());
+ appReport.getName(), appReport.getApplicationType(), appReport
+ .getUser(), appReport.getQueue(), appReport
+ .getYarnApplicationState(),
+ appReport.getFinalApplicationStatus(), progress, appReport
+ .getOriginalTrackingUrl());
}
writer.flush();
}
@@ -227,8 +353,8 @@ public class ApplicationCLI extends Yarn
* @throws YarnException
* @throws IOException
*/
- private void killApplication(String applicationId)
- throws YarnException, IOException {
+ private void killApplication(String applicationId) throws YarnException,
+ IOException {
ApplicationId appId = ConverterUtils.toApplicationId(applicationId);
ApplicationReport appReport = client.getApplicationReport(appId);
if (appReport.getYarnApplicationState() == YarnApplicationState.FINISHED
@@ -296,14 +422,63 @@ public class ApplicationCLI extends Yarn
private String getAllValidApplicationStates() {
StringBuilder sb = new StringBuilder();
- sb.append("The valid application state can be"
- + " one of the following: ");
+ sb.append("The valid application state can be" + " one of the following: ");
sb.append(ALLSTATES_OPTION + ",");
- for (YarnApplicationState appState : YarnApplicationState
- .values()) {
- sb.append(appState+",");
+ for (YarnApplicationState appState : YarnApplicationState.values()) {
+ sb.append(appState + ",");
}
String output = sb.toString();
- return output.substring(0, output.length()-1);
+ return output.substring(0, output.length() - 1);
+ }
+
+ /**
+ * Lists the application attempts matching the given applicationid
+ *
+ * @param applicationId
+ * @throws YarnException
+ * @throws IOException
+ */
+ private void listApplicationAttempts(String appId) throws YarnException,
+ IOException {
+ PrintWriter writer = new PrintWriter(sysout);
+
+ List<ApplicationAttemptReport> appAttemptsReport = client
+ .getApplicationAttempts(ConverterUtils.toApplicationId(appId));
+ writer.println("Total number of application attempts " + ":"
+ + appAttemptsReport.size());
+ writer.printf(APPLICATION_ATTEMPTS_PATTERN, "ApplicationAttempt-Id",
+ "State", "AM-Container-Id", "Tracking-URL");
+ for (ApplicationAttemptReport appAttemptReport : appAttemptsReport) {
+ writer.printf(APPLICATION_ATTEMPTS_PATTERN, appAttemptReport
+ .getApplicationAttemptId(), appAttemptReport
+ .getYarnApplicationAttemptState(), appAttemptReport
+ .getAMContainerId().toString(), appAttemptReport.getTrackingUrl());
+ }
+ writer.flush();
+ }
+
+ /**
+ * Lists the containers matching the given application attempts
+ *
+ * @param appAttemptId
+ * @throws YarnException
+ * @throws IOException
+ */
+ private void listContainers(String appAttemptId) throws YarnException,
+ IOException {
+ PrintWriter writer = new PrintWriter(sysout);
+
+ List<ContainerReport> appsReport = client
+ .getContainers(ConverterUtils.toApplicationAttemptId(appAttemptId));
+ writer.println("Total number of containers " + ":" + appsReport.size());
+ writer.printf(CONTAINER_PATTERN, "Container-Id", "Start Time",
+ "Finish Time", "State", "Host", "LOG-URL");
+ for (ContainerReport containerReport : appsReport) {
+ writer.printf(CONTAINER_PATTERN, containerReport.getContainerId(),
+ containerReport.getStartTime(), containerReport.getFinishTime(),
+ containerReport.getContainerState(), containerReport
+ .getAssignedNode(), containerReport.getLogUrl());
+ }
+ writer.flush();
}
}
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java?rev=1561771&r1=1561770&r2=1561771&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java Mon Jan 27 18:09:22 2014
@@ -58,12 +58,10 @@ import org.apache.hadoop.yarn.client.api
import org.apache.hadoop.yarn.client.api.YarnClientApplication;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.server.MiniYARNCluster;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
-import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.util.Records;
import org.apache.log4j.Level;
import org.apache.log4j.LogManager;
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java?rev=1561771&r1=1561770&r2=1561771&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java Mon Jan 27 18:09:22 2014
@@ -43,19 +43,26 @@ import junit.framework.Assert;
import org.apache.commons.lang.time.DateFormatUtils;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerReport;
+import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.util.Records;
import org.junit.Before;
import org.junit.Test;
+import org.mortbay.log.Log;
import org.apache.commons.cli.Options;
@@ -114,19 +121,180 @@ public class TestYarnCLI {
}
@Test
+ public void testGetApplicationAttemptReport() throws Exception {
+ ApplicationCLI cli = createAndGetAppCLI();
+ ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
+ ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
+ applicationId, 1);
+ ApplicationAttemptReport attemptReport = ApplicationAttemptReport
+ .newInstance(attemptId, "host", 124, "url", "diagnostics",
+ YarnApplicationAttemptState.FINISHED, ContainerId.newInstance(
+ attemptId, 1));
+ when(
+ client
+ .getApplicationAttemptReport(any(ApplicationAttemptId.class)))
+ .thenReturn(attemptReport);
+ int result = cli.run(new String[] { "applicationattempt", "-status",
+ attemptId.toString() });
+ assertEquals(0, result);
+ verify(client).getApplicationAttemptReport(attemptId);
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ PrintWriter pw = new PrintWriter(baos);
+ pw.println("Application Attempt Report : ");
+ pw.println("\tApplicationAttempt-Id : appattempt_1234_0005_000001");
+ pw.println("\tState : FINISHED");
+ pw.println("\tAMContainer : container_1234_0005_01_000001");
+ pw.println("\tTracking-URL : url");
+ pw.println("\tRPC Port : 124");
+ pw.println("\tAM Host : host");
+ pw.println("\tDiagnostics : diagnostics");
+ pw.close();
+ String appReportStr = baos.toString("UTF-8");
+ Assert.assertEquals(appReportStr, sysOutStream.toString());
+ verify(sysOut, times(1)).println(isA(String.class));
+ }
+
+ @Test
+ public void testGetApplicationAttempts() throws Exception {
+ ApplicationCLI cli = createAndGetAppCLI();
+ ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
+ ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
+ applicationId, 1);
+ ApplicationAttemptId attemptId1 = ApplicationAttemptId.newInstance(
+ applicationId, 2);
+ ApplicationAttemptReport attemptReport = ApplicationAttemptReport
+ .newInstance(attemptId, "host", 124, "url", "diagnostics",
+ YarnApplicationAttemptState.FINISHED, ContainerId.newInstance(
+ attemptId, 1));
+ ApplicationAttemptReport attemptReport1 = ApplicationAttemptReport
+ .newInstance(attemptId1, "host", 124, "url", "diagnostics",
+ YarnApplicationAttemptState.FINISHED, ContainerId.newInstance(
+ attemptId1, 1));
+ List<ApplicationAttemptReport> reports = new ArrayList<ApplicationAttemptReport>();
+ reports.add(attemptReport);
+ reports.add(attemptReport1);
+ when(client.getApplicationAttempts(any(ApplicationId.class)))
+ .thenReturn(reports);
+ int result = cli.run(new String[] { "applicationattempt", "-list",
+ applicationId.toString() });
+ assertEquals(0, result);
+ verify(client).getApplicationAttempts(applicationId);
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ PrintWriter pw = new PrintWriter(baos);
+ pw.println("Total number of application attempts :2");
+ pw.print(" ApplicationAttempt-Id");
+ pw.print("\t State");
+ pw.print("\t AM-Container-Id");
+ pw.println("\t Tracking-URL");
+ pw.print(" appattempt_1234_0005_000001");
+ pw.print("\t FINISHED");
+ pw.print("\t container_1234_0005_01_000001");
+ pw.println("\t url");
+ pw.print(" appattempt_1234_0005_000002");
+ pw.print("\t FINISHED");
+ pw.print("\t container_1234_0005_02_000001");
+ pw.println("\t url");
+ pw.close();
+ String appReportStr = baos.toString("UTF-8");
+ Assert.assertEquals(appReportStr, sysOutStream.toString());
+ }
+
+ @Test
+ public void testGetContainerReport() throws Exception {
+ ApplicationCLI cli = createAndGetAppCLI();
+ ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
+ ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
+ applicationId, 1);
+ ContainerId containerId = ContainerId.newInstance(attemptId, 1);
+ ContainerReport container = ContainerReport.newInstance(containerId, null,
+ NodeId.newInstance("host", 1234), Priority.UNDEFINED, 1234, 5678,
+ "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE);
+ when(client.getContainerReport(any(ContainerId.class))).thenReturn(
+ container);
+ int result = cli.run(new String[] { "container", "-status",
+ containerId.toString() });
+ assertEquals(0, result);
+ verify(client).getContainerReport(containerId);
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ PrintWriter pw = new PrintWriter(baos);
+ pw.println("Container Report : ");
+ pw.println("\tContainer-Id : container_1234_0005_01_000001");
+ pw.println("\tStart-Time : 1234");
+ pw.println("\tFinish-Time : 5678");
+ pw.println("\tState : COMPLETE");
+ pw.println("\tLOG-URL : logURL");
+ pw.println("\tHost : host:1234");
+ pw.println("\tDiagnostics : diagnosticInfo");
+ pw.close();
+ String appReportStr = baos.toString("UTF-8");
+ Assert.assertEquals(appReportStr, sysOutStream.toString());
+ verify(sysOut, times(1)).println(isA(String.class));
+ }
+
+ @Test
+ public void testGetContainers() throws Exception {
+ ApplicationCLI cli = createAndGetAppCLI();
+ ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
+ ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
+ applicationId, 1);
+ ContainerId containerId = ContainerId.newInstance(attemptId, 1);
+ ContainerId containerId1 = ContainerId.newInstance(attemptId, 2);
+ ContainerReport container = ContainerReport.newInstance(containerId, null,
+ NodeId.newInstance("host", 1234), Priority.UNDEFINED, 1234, 5678,
+ "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE);
+ ContainerReport container1 = ContainerReport.newInstance(containerId1, null,
+ NodeId.newInstance("host", 1234), Priority.UNDEFINED, 1234, 5678,
+ "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE);
+ List<ContainerReport> reports = new ArrayList<ContainerReport>();
+ reports.add(container);
+ reports.add(container1);
+ when(client.getContainers(any(ApplicationAttemptId.class))).thenReturn(
+ reports);
+ int result = cli.run(new String[] { "container", "-list",
+ attemptId.toString() });
+ assertEquals(0, result);
+ verify(client).getContainers(attemptId);
+ Log.info(sysOutStream.toString());
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ PrintWriter pw = new PrintWriter(baos);
+ pw.println("Total number of containers :2");
+ pw.print(" Container-Id");
+ pw.print("\t Start Time");
+ pw.print("\t Finish Time");
+ pw.print("\t State");
+ pw.print("\t Host");
+ pw.println("\t LOG-URL");
+ pw.print(" container_1234_0005_01_000001");
+ pw.print("\t 1234");
+ pw.print("\t 5678");
+ pw.print("\t COMPLETE");
+ pw.print("\t host:1234");
+ pw.println("\t logURL");
+ pw.print(" container_1234_0005_01_000002");
+ pw.print("\t 1234");
+ pw.print("\t 5678");
+ pw.print("\t COMPLETE");
+ pw.print("\t host:1234");
+ pw.println("\t logURL");
+ pw.close();
+ String appReportStr = baos.toString("UTF-8");
+ Assert.assertEquals(appReportStr, sysOutStream.toString());
+ }
+
+ @Test
public void testGetApplicationReportException() throws Exception {
ApplicationCLI cli = createAndGetAppCLI();
ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
when(client.getApplicationReport(any(ApplicationId.class))).thenThrow(
- new ApplicationNotFoundException("Application with id '"
- + applicationId + "' doesn't exist in RM."));
+ new ApplicationNotFoundException("History file for application"
+ + applicationId + " is not found"));
try {
cli.run(new String[] { "-status", applicationId.toString() });
Assert.fail();
} catch (Exception ex) {
Assert.assertTrue(ex instanceof ApplicationNotFoundException);
- Assert.assertEquals("Application with id '" + applicationId
- + "' doesn't exist in RM.", ex.getMessage());
+ Assert.assertEquals("History file for application"
+ + applicationId + " is not found", ex.getMessage());
}
}
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml?rev=1561771&r1=1561770&r2=1561771&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml Mon Jan 27 18:09:22 2014
@@ -199,6 +199,7 @@
<exclude>src/main/resources/webapps/mapreduce/.keep</exclude>
<exclude>src/main/resources/webapps/jobhistory/.keep</exclude>
<exclude>src/main/resources/webapps/yarn/.keep</exclude>
+ <exclude>src/main/resources/webapps/applicationhistory/.keep</exclude>
<exclude>src/main/resources/webapps/cluster/.keep</exclude>
<exclude>src/main/resources/webapps/test/.keep</exclude>
<exclude>src/main/resources/webapps/proxy/.keep</exclude>
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java?rev=1561771&r1=1561770&r2=1561771&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java Mon Jan 27 18:09:22 2014
@@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.QueueState;
+import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.proto.YarnProtos.AMCommandProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAccessTypeProto;
@@ -45,6 +46,7 @@ import org.apache.hadoop.yarn.proto.Yarn
import org.apache.hadoop.yarn.proto.YarnProtos.NodeStateProto;
import org.apache.hadoop.yarn.proto.YarnProtos.QueueACLProto;
import org.apache.hadoop.yarn.proto.YarnProtos.QueueStateProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationAttemptStateProto;
import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationStateProto;
import com.google.protobuf.ByteString;
@@ -97,6 +99,21 @@ public class ProtoUtils {
}
/*
+ * YarnApplicationAttemptState
+ */
+ private static String YARN_APPLICATION_ATTEMPT_STATE_PREFIX = "APP_ATTEMPT_";
+ public static YarnApplicationAttemptStateProto convertToProtoFormat(
+ YarnApplicationAttemptState e) {
+ return YarnApplicationAttemptStateProto
+ .valueOf(YARN_APPLICATION_ATTEMPT_STATE_PREFIX + e.name());
+ }
+ public static YarnApplicationAttemptState convertFromProtoFormat(
+ YarnApplicationAttemptStateProto e) {
+ return YarnApplicationAttemptState.valueOf(e.name().replace(
+ YARN_APPLICATION_ATTEMPT_STATE_PREFIX, ""));
+ }
+
+ /*
* ApplicationResourceUsageReport
*/
public static ApplicationResourceUsageReportProto convertToProtoFormat(ApplicationResourceUsageReport e) {
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/StringHelper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/StringHelper.java?rev=1561771&r1=1561770&r2=1561771&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/StringHelper.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/StringHelper.java Mon Jan 27 18:09:22 2014
@@ -178,4 +178,8 @@ public final class StringHelper {
public static String percent(double value) {
return String.format("%.2f", value * 100);
}
+
+ public static String getPartUrl(String url, String part) {
+ return url.substring(url.indexOf(part));
+ }
}
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java?rev=1561771&r1=1561770&r2=1561771&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java Mon Jan 27 18:09:22 2014
@@ -24,6 +24,7 @@ import org.apache.hadoop.classification.
public interface YarnWebParams {
String NM_NODENAME = "nm.id";
String APPLICATION_ID = "app.id";
+ String APPLICATION_ATTEMPT_ID = "appattempt.id";
String CONTAINER_ID = "container.id";
String CONTAINER_LOG_TYPE= "log.type";
String ENTITY_STRING = "entity.string";
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java?rev=1561771&r1=1561770&r2=1561771&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java Mon Jan 27 18:09:22 2014
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.yarn.webapp.util;
+import static org.apache.hadoop.yarn.util.StringHelper.join;
+
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.UnknownHostException;
@@ -27,7 +29,9 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.http.HttpConfig.Policy;
import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.util.ConverterUtils;
import com.google.common.base.Joiner;
@@ -144,6 +148,16 @@ public class WebAppUtils {
YarnConfiguration.DEFAULT_NM_WEBAPP_ADDRESS);
}
}
+
+ public static String getAHSWebAppURLWithoutScheme(Configuration conf) {
+ if (HttpConfig.isSecure()) {
+ return conf.get(YarnConfiguration.AHS_WEBAPP_HTTPS_ADDRESS,
+ YarnConfiguration.DEFAULT_AHS_WEBAPP_HTTPS_ADDRESS);
+ } else {
+ return conf.get(YarnConfiguration.AHS_WEBAPP_ADDRESS,
+ YarnConfiguration.DEFAULT_AHS_WEBAPP_ADDRESS);
+ }
+ }
/**
* if url has scheme then it will be returned as it is else it will return
@@ -160,4 +174,11 @@ public class WebAppUtils {
return schemePrefix + url;
}
}
+
+ public static String getLogUrl(String nodeHttpAddress, String allocatedNode,
+ ContainerId containerId, String user) {
+ return join(HttpConfig.getSchemePrefix(), nodeHttpAddress, "/logs", "/",
+ allocatedNode, "/", ConverterUtils.toString(containerId), "/",
+ ConverterUtils.toString(containerId), "/", user);
+ }
}
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml?rev=1561771&r1=1561770&r2=1561771&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml Mon Jan 27 18:09:22 2014
@@ -564,6 +564,30 @@
<value>org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.ProportionalCapacityPreemptionPolicy</value>
</property>
+ <property>
+ <description>Indicate to ResourceManager as well as clients whether
+ history-service is enabled or not. If enabled, ResourceManager starts
+ recording historical data that ApplicationHistory service can consume.
+ Similarly, clients can redirect to the history service when applications
+ finish if this is enabled.</description>
+ <name>yarn.ahs.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <description>Number of worker threads that write the history data.</description>
+ <name>yarn.resourcemanager.history-writer.multi-threaded-dispatcher.pool-size</name>
+ <value>10</value>
+ </property>
+
+ <property>
+ <description>The implementation class of ApplicationHistoryStore, which is
+ to be used by RMApplicationHistoryWriter.
+ </description>
+ <name>yarn.resourcemanager.history-writer.class</name>
+ <value>org.apache.hadoop.yarn.server.applicationhistoryservice.NullApplicationHistoryStore</value>
+ </property>
+
<!-- Node Manager Configs -->
<property>
<description>The hostname of the NM.</description>
@@ -1041,6 +1065,61 @@
<value></value>
</property>
+ <!-- Application History Service's Configuration-->
+
+ <property>
+ <description>The hostname of the AHS.</description>
+ <name>yarn.ahs.hostname</name>
+ <value>0.0.0.0</value>
+ </property>
+
+ <property>
+ <description>The http address of the AHS web application.</description>
+ <name>yarn.ahs.webapp.address</name>
+ <value>${yarn.ahs.hostname}:8188</value>
+ </property>
+
+ <property>
+ <description>The https adddress of the AHS web application.</description>
+ <name>yarn.ahs.webapp.https.address</name>
+ <value>${yarn.ahs.hostname}:8190</value>
+ </property>
+
+ <property>
+ <description>URI pointing to the location of the FileSystem path where
+ the history will be persisted. This must be supplied when using
+ org.apache.hadoop.yarn.server.applicationhistoryservice.FileSystemApplicationHistoryStore
+ as the value for yarn.resourcemanager.history-writer.store.class</description>
+ <name>yarn.ahs.fs-history-store.uri</name>
+ <value>${hadoop.log.dir}/yarn/system/ahstore</value>
+ </property>
+
+ <property>
+ <description>This is default address for the Application History server
+ to start the RPC server.</description>
+ <name>yarn.ahs.address</name>
+ <value>0.0.0.0:10200</value>
+ </property>
+
+ <property>
+ <description>CLient thread count to serve the client requests.</description>
+ <name>yarn.ahs.client.thread-count</name>
+ <value>10</value>
+ </property>
+
+ <property>
+ <description>T-file compression types used to compress history data.</description>
+ <name>yarn.ahs.fs-history-store.compression-type</name>
+ <value>none</value>
+ </property>
+
+ <property>
+ <description> Store class name for history store, defaulting to file
+ system store </description>
+ <name>yarn.ahs.store.class</name>
+ <value>org.apache.hadoop.yarn.server.applicationhistoryservice.FileSystemApplicationHistoryStore</value>
+ </property>
+
<!-- Other configuration -->
<property>
<description>The interval that the yarn client library uses to poll the
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java?rev=1561771&r1=1561770&r2=1561771&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java Mon Jan 27 18:09:22 2014
@@ -638,8 +638,8 @@ public class ResourceLocalizationService
super("Public Localizer");
this.lfs = getLocalFileContext(conf);
this.conf = conf;
- this.pending =
- new ConcurrentHashMap<Future<Path>, LocalizerResourceRequestEvent>();
+ this.pending = Collections.synchronizedMap(
+ new HashMap<Future<Path>, LocalizerResourceRequestEvent>());
this.threadPool = createLocalizerExecutor(conf);
this.queue = new ExecutorCompletionService<Path>(threadPool);
}
@@ -675,8 +675,12 @@ public class ResourceLocalizationService
publicDirDestPath =
new Path(publicDirDestPath, Long.toString(publicRsrc
.nextUniqueNumber()));
- pending.put(queue.submit(new FSDownload(lfs, null, conf,
- publicDirDestPath, resource)), request);
+ // explicitly synchronize pending here to avoid future task
+ // completing and being dequeued before pending updated
+ synchronized (pending) {
+ pending.put(queue.submit(new FSDownload(lfs, null, conf,
+ publicDirDestPath, resource)), request);
+ }
} catch (IOException e) {
rsrc.unlock();
// TODO Need to Fix IO Exceptions - Notifying resource
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml?rev=1561771&r1=1561770&r2=1561771&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml Mon Jan 27 18:09:22 2014
@@ -185,6 +185,11 @@
<!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-applicationhistoryservice</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-web-proxy</artifactId>
</dependency>
<dependency>
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java?rev=1561771&r1=1561770&r2=1561771&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java Mon Jan 27 18:09:22 2014
@@ -24,6 +24,7 @@ import org.apache.hadoop.ha.HAServicePro
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.event.Dispatcher;
+import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AMLivelinessMonitor;
@@ -33,8 +34,8 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer;
-import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
+import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMDelegationTokenSecretManager;
/**
@@ -90,4 +91,10 @@ public interface RMContext {
void setRMDelegationTokenSecretManager(
RMDelegationTokenSecretManager delegationTokenSecretManager);
+
+ RMApplicationHistoryWriter getRMApplicationHistoryWriter();
+
+ void setRMApplicationHistoryWriter(
+ RMApplicationHistoryWriter rmApplicationHistoryWriter);
+
}
\ No newline at end of file
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java?rev=1561771&r1=1561770&r2=1561771&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java Mon Jan 27 18:09:22 2014
@@ -27,6 +27,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
+import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.NullRMStateStore;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
@@ -76,6 +77,7 @@ public class RMContextImpl implements RM
private NodesListManager nodesListManager;
private ResourceTrackerService resourceTrackerService;
private ApplicationMasterService applicationMasterService;
+ private RMApplicationHistoryWriter rmApplicationHistoryWriter;
/**
* Default constructor. To be used in conjunction with setter methods for
@@ -95,7 +97,8 @@ public class RMContextImpl implements RM
AMRMTokenSecretManager appTokenSecretManager,
RMContainerTokenSecretManager containerTokenSecretManager,
NMTokenSecretManagerInRM nmTokenSecretManager,
- ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager) {
+ ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager,
+ RMApplicationHistoryWriter rmApplicationHistoryWriter) {
this();
this.setDispatcher(rmDispatcher);
this.setContainerAllocationExpirer(containerAllocationExpirer);
@@ -106,6 +109,7 @@ public class RMContextImpl implements RM
this.setContainerTokenSecretManager(containerTokenSecretManager);
this.setNMTokenSecretManager(nmTokenSecretManager);
this.setClientToAMTokenSecretManager(clientToAMTokenSecretManager);
+ this.setRMApplicationHistoryWriter(rmApplicationHistoryWriter);
RMStateStore nullStore = new NullRMStateStore();
nullStore.setRMDispatcher(rmDispatcher);
@@ -318,4 +322,16 @@ public class RMContextImpl implements RM
return haServiceState;
}
}
+
+ @Override
+ public RMApplicationHistoryWriter getRMApplicationHistoryWriter() {
+ return rmApplicationHistoryWriter;
+ }
+
+ @Override
+ public void setRMApplicationHistoryWriter(
+ RMApplicationHistoryWriter rmApplicationHistoryWriter) {
+ this.rmApplicationHistoryWriter = rmApplicationHistoryWriter;
+ }
+
}
\ No newline at end of file
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java?rev=1561771&r1=1561770&r2=1561771&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java Mon Jan 27 18:09:22 2014
@@ -33,9 +33,14 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.exceptions.InvalidContainerReleaseException;
import org.apache.hadoop.yarn.exceptions.InvalidResourceBlacklistRequestException;
import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
@@ -43,6 +48,7 @@ import org.apache.hadoop.yarn.server.res
* Utility methods to aid serving RM data through the REST and RPC APIs
*/
public class RMServerUtils {
+
public static List<RMNode> queryRMNodes(RMContext context,
EnumSet<NodeState> acceptedStates) {
// nodes contains nodes that are NEW, RUNNING OR UNHEALTHY
@@ -56,7 +62,7 @@ public class RMServerUtils {
}
}
}
-
+
// inactiveNodes contains nodes that are DECOMMISSIONED, LOST, OR REBOOTED
if (acceptedStates.contains(NodeState.DECOMMISSIONED) ||
acceptedStates.contains(NodeState.LOST) ||
@@ -69,7 +75,7 @@ public class RMServerUtils {
}
return results;
}
-
+
/**
* Utility method to validate a list resource requests, by insuring that the
* requested memory/vcore is non-negative and not greater than max
@@ -85,8 +91,9 @@ public class RMServerUtils {
* @throw <code>InvalidResourceBlacklistRequestException </code> if the
* resource is not able to be added to the blacklist.
*/
- public static void validateBlacklistRequest(ResourceBlacklistRequest blacklistRequest)
- throws InvalidResourceBlacklistRequestException {
+ public static void validateBlacklistRequest(
+ ResourceBlacklistRequest blacklistRequest)
+ throws InvalidResourceBlacklistRequestException {
if (blacklistRequest != null) {
List<String> plus = blacklistRequest.getBlacklistAdditions();
if (plus != null && plus.contains(ResourceRequest.ANY)) {
@@ -100,10 +107,12 @@ public class RMServerUtils {
* It will validate to make sure all the containers belong to correct
* application attempt id. If not then it will throw
* {@link InvalidContainerReleaseException}
- * @param containerReleaseList containers to be released as requested by
- * application master.
- * @param appAttemptId Application attempt Id
- * @throws InvalidContainerReleaseException
+ *
+ * @param containerReleaseList
+ * containers to be released as requested by application master.
+ * @param appAttemptId
+ * Application attempt Id
+ * @throws InvalidContainerReleaseException
*/
public static void
validateContainerReleaseRequest(List<ContainerId> containerReleaseList,
@@ -111,9 +120,11 @@ public class RMServerUtils {
throws InvalidContainerReleaseException {
for (ContainerId cId : containerReleaseList) {
if (!appAttemptId.equals(cId.getApplicationAttemptId())) {
- throw new InvalidContainerReleaseException("Cannot release container : "
- + cId.toString() + " not belonging to this application attempt : "
- + appAttemptId);
+ throw new InvalidContainerReleaseException(
+ "Cannot release container : "
+ + cId.toString()
+ + " not belonging to this application attempt : "
+ + appAttemptId);
}
}
}
@@ -157,4 +168,63 @@ public class RMServerUtils {
}
return user;
}
+
+ public static YarnApplicationState createApplicationState(
+ RMAppState rmAppState) {
+ switch (rmAppState) {
+ case NEW:
+ return YarnApplicationState.NEW;
+ case NEW_SAVING:
+ return YarnApplicationState.NEW_SAVING;
+ case SUBMITTED:
+ return YarnApplicationState.SUBMITTED;
+ case ACCEPTED:
+ return YarnApplicationState.ACCEPTED;
+ case RUNNING:
+ return YarnApplicationState.RUNNING;
+ case FINISHING:
+ case FINISHED:
+ return YarnApplicationState.FINISHED;
+ case KILLED:
+ return YarnApplicationState.KILLED;
+ case FAILED:
+ return YarnApplicationState.FAILED;
+ default:
+ throw new YarnRuntimeException("Unknown state passed!");
+ }
+ }
+
+ public static YarnApplicationAttemptState createApplicationAttemptState(
+ RMAppAttemptState rmAppAttemptState) {
+ switch (rmAppAttemptState) {
+ case NEW:
+ return YarnApplicationAttemptState.NEW;
+ case SUBMITTED:
+ return YarnApplicationAttemptState.SUBMITTED;
+ case SCHEDULED:
+ return YarnApplicationAttemptState.SCHEDULED;
+ case ALLOCATED:
+ return YarnApplicationAttemptState.ALLOCATED;
+ case LAUNCHED:
+ return YarnApplicationAttemptState.LAUNCHED;
+ case ALLOCATED_SAVING:
+ case LAUNCHED_UNMANAGED_SAVING:
+ return YarnApplicationAttemptState.ALLOCATED_SAVING;
+ case RUNNING:
+ return YarnApplicationAttemptState.RUNNING;
+ case FINISHING:
+ return YarnApplicationAttemptState.FINISHING;
+ case FINAL_SAVING:
+ return YarnApplicationAttemptState.FINAL_SAVING;
+ case FINISHED:
+ return YarnApplicationAttemptState.FINISHED;
+ case KILLED:
+ return YarnApplicationAttemptState.KILLED;
+ case FAILED:
+ return YarnApplicationAttemptState.FAILED;
+ default:
+ throw new YarnRuntimeException("Unknown state passed!");
+ }
+ }
+
}
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java?rev=1561771&r1=1561770&r2=1561771&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java Mon Jan 27 18:09:22 2014
@@ -52,6 +52,7 @@ import org.apache.hadoop.yarn.event.Asyn
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter;
import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEventType;
import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher;
import org.apache.hadoop.yarn.server.resourcemanager.monitor.SchedulingEditPolicy;
@@ -261,6 +262,10 @@ public class ResourceManager extends Com
this.applicationACLsManager, this.conf);
}
+ protected RMApplicationHistoryWriter createRMApplicationHistoryWriter() {
+ return new RMApplicationHistoryWriter();
+ }
+
// sanity check for configurations
protected static void validateConfigs(Configuration conf) {
// validate max-attempts
@@ -345,6 +350,11 @@ public class ResourceManager extends Com
rmContext.setDelegationTokenRenewer(delegationTokenRenewer);
}
+ RMApplicationHistoryWriter rmApplicationHistoryWriter =
+ createRMApplicationHistoryWriter();
+ addService(rmApplicationHistoryWriter);
+ rmContext.setRMApplicationHistoryWriter(rmApplicationHistoryWriter);
+
// Register event handler for NodesListManager
nodesListManager = new NodesListManager(rmContext);
rmDispatcher.register(NodesListManagerEventType.class, nodesListManager);
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java?rev=1561771&r1=1561770&r2=1561771&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java Mon Jan 27 18:09:22 2014
@@ -54,6 +54,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.RMAppManagerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.RMAppManagerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.ApplicationState;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.Recoverable;
@@ -335,6 +336,8 @@ public class RMAppImpl implements RMApp,
this.writeLock = lock.writeLock();
this.stateMachine = stateMachineFactory.make(this);
+
+ rmContext.getRMApplicationHistoryWriter().applicationStarted(this);
}
@Override
@@ -1002,6 +1005,11 @@ public class RMAppImpl implements RMApp,
app.handler.handle(
new RMAppManagerEvent(app.applicationId,
RMAppManagerEventType.APP_COMPLETED));
+
+ // TODO: We need to fix for the problem that RMApp enters the final state
+ // after RMAppAttempt in the killing case
+ app.rmContext.getRMApplicationHistoryWriter()
+ .applicationFinished(app);
};
}
@@ -1069,27 +1077,7 @@ public class RMAppImpl implements RMApp,
if (rmAppState.equals(RMAppState.KILLING)) {
rmAppState = stateBeforeKilling;
}
- switch (rmAppState) {
- case NEW:
- return YarnApplicationState.NEW;
- case NEW_SAVING:
- return YarnApplicationState.NEW_SAVING;
- case SUBMITTED:
- return YarnApplicationState.SUBMITTED;
- case ACCEPTED:
- return YarnApplicationState.ACCEPTED;
- case RUNNING:
- return YarnApplicationState.RUNNING;
- case FINISHING:
- case FINISHED:
- return YarnApplicationState.FINISHED;
- case KILLED:
- return YarnApplicationState.KILLED;
- case FAILED:
- return YarnApplicationState.FAILED;
- default:
- throw new YarnRuntimeException("Unknown state passed!");
- }
+ return RMServerUtils.createApplicationState(rmAppState);
}
public static boolean isAppInFinalState(RMApp rmApp) {
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java?rev=1561771&r1=1561770&r2=1561771&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java Mon Jan 27 18:09:22 2014
@@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
@@ -178,4 +179,21 @@ public interface RMAppAttempt extends Ev
* @return the start time of the application.
*/
long getStartTime();
+
+ /**
+ * The current state of the {@link RMAppAttempt}.
+ *
+ * @return the current state {@link RMAppAttemptState} for this application
+ * attempt.
+ */
+ RMAppAttemptState getState();
+
+ /**
+ * Create the external user-facing state of the attempt of ApplicationMaster
+ * from the current state of the {@link RMAppAttempt}.
+ *
+ * @return the external user-facing state of the attempt ApplicationMaster.
+ */
+ YarnApplicationAttemptState createApplicationAttemptState();
+
}
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java?rev=1561771&r1=1561770&r2=1561771&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java Mon Jan 27 18:09:22 2014
@@ -55,6 +55,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
@@ -62,6 +63,7 @@ import org.apache.hadoop.yarn.security.A
import org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier;
import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEvent;
import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEventType;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
@@ -1046,6 +1048,9 @@ public class RMAppAttemptImpl implements
appAttempt.eventHandler.handle(new AppAttemptRemovedSchedulerEvent(
appAttemptId, finalAttemptState, keepContainersAcrossAppAttempts));
appAttempt.removeCredentials(appAttempt);
+
+ appAttempt.rmContext.getRMApplicationHistoryWriter()
+ .applicationAttemptFinished(appAttempt);
}
}
@@ -1143,6 +1148,9 @@ public class RMAppAttemptImpl implements
// write at AM launch time, so we don't save the AM's tracking URL anywhere
// as that would mean an extra state-store write. For now, we hope that in
// work-preserving restart, AMs are forced to reregister.
+
+ appAttempt.rmContext.getRMApplicationHistoryWriter()
+ .applicationAttemptStarted(appAttempt);
}
}
@@ -1514,6 +1522,23 @@ public class RMAppAttemptImpl implements
}
}
+ @Override
+ public RMAppAttemptState getState() {
+ this.readLock.lock();
+
+ try {
+ return this.stateMachine.getCurrentState();
+ } finally {
+ this.readLock.unlock();
+ }
+ }
+
+ @Override
+ public YarnApplicationAttemptState createApplicationAttemptState() {
+ RMAppAttemptState state = getState();
+ return RMServerUtils.createApplicationAttemptState(state);
+ }
+
private void launchAttempt(){
// Send event to launch the AM Container
eventHandler.handle(new AMLauncherEvent(AMLauncherEventType.LAUNCH, this));
Modified: hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java?rev=1561771&r1=1561770&r2=1561771&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java (original)
+++ hadoop/common/branches/HDFS-4685/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java Mon Jan 27 18:09:22 2014
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.re
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -50,4 +51,22 @@ public interface RMContainer extends Eve
Priority getReservedPriority();
+ Resource getAllocatedResource();
+
+ NodeId getAllocatedNode();
+
+ Priority getAllocatedPriority();
+
+ long getStartTime();
+
+ long getFinishTime();
+
+ String getDiagnosticsInfo();
+
+ String getLogURL();
+
+ int getContainerExitStatus();
+
+ ContainerState getContainerState();
+
}