You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vi...@apache.org on 2015/09/16 05:25:19 UTC
[1/8] hadoop git commit: YARN-1884. Added nodeHttpAddress into
ContainerReport and fixed the link to NM web page. Contributed by Xuan Gong.
Repository: hadoop
Updated Branches:
refs/heads/branch-2.6.1 075d899d1 -> dba2b60fd
YARN-1884. Added nodeHttpAddress into ContainerReport and fixed the link to NM web page. Contributed by Xuan Gong.
(cherry picked from commit 85f6d67fa78511f255fcfa810afc9a156a7b483b)
(cherry picked from commit 426535007bcdc67331f7a37b5d69cc20b37c26e0)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f4154bde
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f4154bde
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f4154bde
Branch: refs/heads/branch-2.6.1
Commit: f4154bdee8d52d655f5f35ba020c3f63f80b69be
Parents: 075d899
Author: Zhijie Shen <zj...@apache.org>
Authored: Wed Mar 11 19:35:19 2015 -0700
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Fri Sep 11 11:45:29 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../yarn/api/records/ContainerReport.java | 17 ++++++++++++++-
.../src/main/proto/yarn_protos.proto | 1 +
.../hadoop/yarn/client/cli/ApplicationCLI.java | 11 +++++++---
.../hadoop/yarn/client/ProtocolHATestBase.java | 4 ++--
.../yarn/client/api/impl/TestAHSClient.java | 6 ++++--
.../yarn/client/api/impl/TestYarnClient.java | 7 +++++--
.../hadoop/yarn/client/cli/TestYarnCLI.java | 18 ++++++++++++----
.../records/impl/pb/ContainerReportPBImpl.java | 19 +++++++++++++++++
.../ApplicationHistoryManagerImpl.java | 2 +-
...pplicationHistoryManagerOnTimelineStore.java | 10 ++++++++-
.../metrics/ContainerMetricsConstants.java | 2 ++
.../yarn/server/webapp/AppAttemptBlock.java | 12 ++++++-----
.../hadoop/yarn/server/webapp/AppBlock.java | 11 ++++------
.../yarn/server/webapp/ContainerBlock.java | 7 ++++++-
.../yarn/server/webapp/dao/ContainerInfo.java | 5 +++++
.../metrics/ContainerCreatedEvent.java | 8 ++++++-
.../metrics/SystemMetricsPublisher.java | 5 ++++-
.../rmcontainer/RMContainer.java | 1 +
.../rmcontainer/RMContainerImpl.java | 22 ++++++++++++++++++--
.../metrics/TestSystemMetricsPublisher.java | 5 ++++-
21 files changed, 142 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4154bde/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index b6bafb8..c82c892 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -205,6 +205,9 @@ Release 2.6.1 - 2015-09-09
YARN-4047. ClientRMService getApplications has high scheduler lock contention.
(Jason Lowe via jianhe)
+ YARN-1884. Added nodeHttpAddress into ContainerReport and fixed the link to NM
+ web page. (Xuan Gong via zjshen)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4154bde/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerReport.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerReport.java
index 4cce77f..72b8edf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerReport.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerReport.java
@@ -41,6 +41,7 @@ import org.apache.hadoop.yarn.util.Records;
* <li>{@link ContainerState} of the container.</li>
* <li>Diagnostic information in case of errors.</li>
* <li>Log URL.</li>
+ * <li>nodeHttpAddress</li>
* </ul>
* </p>
*
@@ -54,7 +55,8 @@ public abstract class ContainerReport {
public static ContainerReport newInstance(ContainerId containerId,
Resource allocatedResource, NodeId assignedNode, Priority priority,
long creationTime, long finishTime, String diagnosticInfo, String logUrl,
- int containerExitStatus, ContainerState containerState) {
+ int containerExitStatus, ContainerState containerState,
+ String nodeHttpAddress) {
ContainerReport report = Records.newRecord(ContainerReport.class);
report.setContainerId(containerId);
report.setAllocatedResource(allocatedResource);
@@ -66,6 +68,7 @@ public abstract class ContainerReport {
report.setLogUrl(logUrl);
report.setContainerExitStatus(containerExitStatus);
report.setContainerState(containerState);
+ report.setNodeHttpAddress(nodeHttpAddress);
return report;
}
@@ -199,4 +202,16 @@ public abstract class ContainerReport {
@Unstable
public abstract void setContainerExitStatus(int containerExitStatus);
+ /**
+ * Get the Node Http address of the container
+ *
+ * @return the node http address of the container
+ */
+ @Public
+ @Unstable
+ public abstract String getNodeHttpAddress();
+
+ @Private
+ @Unstable
+ public abstract void setNodeHttpAddress(String nodeHttpAddress);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4154bde/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index 5c86c2d..db01770 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -98,6 +98,7 @@ message ContainerReportProto {
optional string log_url = 8;
optional int32 container_exit_status = 9;
optional ContainerStateProto container_state = 10;
+ optional string node_http_address = 11;
}
enum YarnApplicationStateProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4154bde/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
index 83d212d..259f66b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
@@ -58,7 +58,7 @@ public class ApplicationCLI extends YarnCLI {
"%30s\t%20s\t%35s\t%35s"
+ System.getProperty("line.separator");
private static final String CONTAINER_PATTERN =
- "%30s\t%20s\t%20s\t%20s\t%20s\t%35s"
+ "%30s\t%20s\t%20s\t%20s\t%20s\t%20s\t%35s"
+ System.getProperty("line.separator");
private static final String APP_TYPE_CMD = "appTypes";
@@ -315,6 +315,9 @@ public class ApplicationCLI extends YarnCLI {
containerReportStr.println(containerReport.getLogUrl());
containerReportStr.print("\tHost : ");
containerReportStr.println(containerReport.getAssignedNode());
+ containerReportStr.print("\tNodeHttpAddress : ");
+ containerReportStr.println(containerReport.getNodeHttpAddress() == null
+ ? "N/A" : containerReport.getNodeHttpAddress());
containerReportStr.print("\tDiagnostics : ");
containerReportStr.print(containerReport.getDiagnosticsInfo());
} else {
@@ -535,7 +538,7 @@ public class ApplicationCLI extends YarnCLI {
.getContainers(ConverterUtils.toApplicationAttemptId(appAttemptId));
writer.println("Total number of containers " + ":" + appsReport.size());
writer.printf(CONTAINER_PATTERN, "Container-Id", "Start Time",
- "Finish Time", "State", "Host", "LOG-URL");
+ "Finish Time", "State", "Host", "Node Http Address", "LOG-URL");
for (ContainerReport containerReport : appsReport) {
writer.printf(
CONTAINER_PATTERN,
@@ -543,7 +546,9 @@ public class ApplicationCLI extends YarnCLI {
Times.format(containerReport.getCreationTime()),
Times.format(containerReport.getFinishTime()),
containerReport.getContainerState(), containerReport
- .getAssignedNode(), containerReport.getLogUrl());
+ .getAssignedNode(), containerReport.getNodeHttpAddress() == null
+ ? "N/A" : containerReport.getNodeHttpAddress(),
+ containerReport.getLogUrl());
}
writer.flush();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4154bde/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
index 4b2b4bc..ea9e361 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
@@ -77,7 +77,6 @@ import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenRequest;
import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenResponse;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse;
-import org.apache.hadoop.yarn.api.records.AMCommand;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -715,7 +714,8 @@ public abstract class ProtocolHATestBase extends ClientBaseWithFixes {
public ContainerReport createFakeContainerReport() {
return ContainerReport.newInstance(createFakeContainerId(), null,
NodeId.newInstance("localhost", 0), null, 1000l, 1200l, "", "", 0,
- ContainerState.COMPLETE);
+ ContainerState.COMPLETE,
+ "http://" + NodeId.newInstance("localhost", 0).toString());
}
public List<ContainerReport> createFakeContainerReports() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4154bde/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java
index a88189e..c3e3c41 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java
@@ -371,14 +371,16 @@ public class TestAHSClient {
ContainerReport.newInstance(
ContainerId.newContainerId(attempt.getApplicationAttemptId(), 1),
null, NodeId.newInstance("host", 1234), Priority.UNDEFINED, 1234,
- 5678, "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE);
+ 5678, "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE,
+ "http://" + NodeId.newInstance("host", 2345).toString());
containerReports.add(container);
ContainerReport container1 =
ContainerReport.newInstance(
ContainerId.newContainerId(attempt.getApplicationAttemptId(), 2),
null, NodeId.newInstance("host", 1234), Priority.UNDEFINED, 1234,
- 5678, "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE);
+ 5678, "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE,
+ "http://" + NodeId.newInstance("host", 2345).toString());
containerReports.add(container1);
containers.put(attempt.getApplicationAttemptId(), containerReports);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4154bde/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
index 02f2882..3471343 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
@@ -501,13 +501,16 @@ public class TestYarnClient {
ContainerReport container = ContainerReport.newInstance(
ContainerId.newContainerId(attempt.getApplicationAttemptId(), 1), null,
NodeId.newInstance("host", 1234), Priority.UNDEFINED, 1234, 5678,
- "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE);
+ "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE,
+ "http://" + NodeId.newInstance("host", 2345).toString());
containerReports.add(container);
ContainerReport container1 = ContainerReport.newInstance(
ContainerId.newContainerId(attempt.getApplicationAttemptId(), 2), null,
NodeId.newInstance("host", 1234), Priority.UNDEFINED, 1234, 5678,
- "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE);
+ "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE,
+ "http://" + NodeId.newInstance("host", 2345).toString());
+
containerReports.add(container1);
containers.put(attempt.getApplicationAttemptId(), containerReports);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4154bde/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
index b4df551..62ed913 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
@@ -223,7 +223,8 @@ public class TestYarnCLI {
ContainerId containerId = ContainerId.newContainerId(attemptId, 1);
ContainerReport container = ContainerReport.newInstance(containerId, null,
NodeId.newInstance("host", 1234), Priority.UNDEFINED, 1234, 5678,
- "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE);
+ "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE,
+ "http://" + NodeId.newInstance("host", 2345).toString());
when(client.getContainerReport(any(ContainerId.class))).thenReturn(
container);
int result = cli.run(new String[] { "container", "-status",
@@ -239,6 +240,7 @@ public class TestYarnCLI {
pw.println("\tState : COMPLETE");
pw.println("\tLOG-URL : logURL");
pw.println("\tHost : host:1234");
+ pw.println("\tNodeHttpAddress : http://host:2345");
pw.println("\tDiagnostics : diagnosticInfo");
pw.close();
String appReportStr = baos.toString("UTF-8");
@@ -258,19 +260,23 @@ public class TestYarnCLI {
long time1=1234,time2=5678;
ContainerReport container = ContainerReport.newInstance(containerId, null,
NodeId.newInstance("host", 1234), Priority.UNDEFINED, time1, time2,
- "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE);
+ "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE,
+ "http://" + NodeId.newInstance("host", 2345).toString());
ContainerReport container1 = ContainerReport.newInstance(containerId1, null,
NodeId.newInstance("host", 1234), Priority.UNDEFINED, time1, time2,
- "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE);
+ "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE,
+ "http://" + NodeId.newInstance("host", 2345).toString());
ContainerReport container2 = ContainerReport.newInstance(containerId2, null,
NodeId.newInstance("host", 1234), Priority.UNDEFINED, time1,0,
- "diagnosticInfo", "", 0, ContainerState.RUNNING);
+ "diagnosticInfo", "", 0, ContainerState.RUNNING,
+ "http://" + NodeId.newInstance("host", 2345).toString());
List<ContainerReport> reports = new ArrayList<ContainerReport>();
reports.add(container);
reports.add(container1);
reports.add(container2);
when(client.getContainers(any(ApplicationAttemptId.class))).thenReturn(
reports);
+ sysOutStream.reset();
int result = cli.run(new String[] { "container", "-list",
attemptId.toString() });
assertEquals(0, result);
@@ -283,24 +289,28 @@ public class TestYarnCLI {
pw.print("\t Finish Time");
pw.print("\t State");
pw.print("\t Host");
+ pw.print("\t Node Http Address");
pw.println("\t LOG-URL");
pw.print(" container_1234_0005_01_000001");
pw.print("\t"+Times.format(time1));
pw.print("\t"+Times.format(time2));
pw.print("\t COMPLETE");
pw.print("\t host:1234");
+ pw.print("\t http://host:2345");
pw.println("\t logURL");
pw.print(" container_1234_0005_01_000002");
pw.print("\t"+Times.format(time1));
pw.print("\t"+Times.format(time2));
pw.print("\t COMPLETE");
pw.print("\t host:1234");
+ pw.print("\t http://host:2345");
pw.println("\t logURL");
pw.print(" container_1234_0005_01_000003");
pw.print("\t"+Times.format(time1));
pw.print("\t N/A");
pw.print("\t RUNNING");
pw.print("\t host:1234");
+ pw.print("\t http://host:2345");
pw.println("\t ");
pw.close();
String appReportStr = baos.toString("UTF-8");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4154bde/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerReportPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerReportPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerReportPBImpl.java
index 18c452f..1f0405f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerReportPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerReportPBImpl.java
@@ -338,4 +338,23 @@ public class ContainerReportPBImpl extends ContainerReport {
ContainerStateProto containerState) {
return ProtoUtils.convertFromProtoFormat(containerState);
}
+
+ @Override
+ public String getNodeHttpAddress() {
+ ContainerReportProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasNodeHttpAddress()) {
+ return null;
+ }
+ return (p.getNodeHttpAddress());
+ }
+
+ @Override
+ public void setNodeHttpAddress(String nodeHttpAddress) {
+ maybeInitBuilder();
+ if (nodeHttpAddress == null) {
+ builder.clearNodeHttpAddress();
+ return;
+ }
+ builder.setNodeHttpAddress(nodeHttpAddress);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4154bde/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
index 779b182..479858f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
@@ -215,7 +215,7 @@ public class ApplicationHistoryManagerImpl extends AbstractService implements
containerHistory.getStartTime(), containerHistory.getFinishTime(),
containerHistory.getDiagnosticsInfo(), logUrl,
containerHistory.getContainerExitStatus(),
- containerHistory.getContainerState());
+ containerHistory.getContainerState(), null);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4154bde/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
index 56ad469..4a407c3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
@@ -410,6 +410,7 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
String diagnosticsInfo = null;
int exitStatus = ContainerExitStatus.INVALID;
ContainerState state = null;
+ String nodeHttpAddress = null;
Map<String, Object> entityInfo = entity.getOtherInfo();
if (entityInfo != null) {
if (entityInfo
@@ -439,6 +440,12 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
allocatedPriority = (Integer) entityInfo.get(
ContainerMetricsConstants.ALLOCATED_PRIORITY_ENTITY_INFO);
}
+ if (entityInfo.containsKey(
+ ContainerMetricsConstants.ALLOCATED_HOST_HTTP_ADDRESS_ENTITY_INFO)) {
+ nodeHttpAddress =
+ (String) entityInfo
+ .get(ContainerMetricsConstants.ALLOCATED_HOST_HTTP_ADDRESS_ENTITY_INFO);
+ }
}
List<TimelineEvent> events = entity.getEvents();
if (events != null) {
@@ -488,7 +495,8 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
Resource.newInstance(allocatedMem, allocatedVcore),
NodeId.newInstance(allocatedHost, allocatedPort),
Priority.newInstance(allocatedPriority),
- createdTime, finishedTime, diagnosticsInfo, logUrl, exitStatus, state);
+ createdTime, finishedTime, diagnosticsInfo, logUrl, exitStatus, state,
+ nodeHttpAddress);
}
private ApplicationReportExt generateApplicationReport(TimelineEntity entity,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4154bde/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ContainerMetricsConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ContainerMetricsConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ContainerMetricsConstants.java
index 8791da4..0d5540d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ContainerMetricsConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ContainerMetricsConstants.java
@@ -57,4 +57,6 @@ public class ContainerMetricsConstants {
public static final String STATE_EVENT_INFO =
"YARN_CONTAINER_STATE";
+ public static final String ALLOCATED_HOST_HTTP_ADDRESS_ENTITY_INFO =
+ "YARN_CONTAINER_ALLOCATED_HOST_HTTP_ADDRESS";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4154bde/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java
index ea33f4f..ae5f092 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java
@@ -191,12 +191,14 @@ public class AppAttemptBlock extends HtmlBlock {
.append(url("container", container.getContainerId()))
.append("'>")
.append(container.getContainerId())
- .append("</a>\",\"<a href='")
- .append("#") // TODO: replace with node http address (YARN-1884)
+ .append("</a>\",\"<a ")
+ .append(
+ container.getNodeHttpAddress() == null ? "#" : "href='"
+ + container.getNodeHttpAddress())
.append("'>")
- .append(container.getAssignedNodeId() == null ? "N/A" :
+ .append(container.getNodeHttpAddress() == null ? "N/A" :
StringEscapeUtils.escapeJavaScript(StringEscapeUtils
- .escapeHtml(container.getAssignedNodeId())))
+ .escapeHtml(container.getNodeHttpAddress())))
.append("</a>\",\"")
.append(container.getContainerExitStatus()).append("\",\"<a href='")
.append(container.getLogUrl() == null ?
@@ -224,4 +226,4 @@ public class AppAttemptBlock extends HtmlBlock {
}
return false;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4154bde/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
index d52975b..e62c783 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
@@ -291,15 +291,12 @@ public class AppBlock extends HtmlBlock {
}
long startTime = 0L;
String logsLink = null;
+ String nodeLink = null;
if (containerReport != null) {
ContainerInfo container = new ContainerInfo(containerReport);
startTime = container.getStartedTime();
logsLink = containerReport.getLogUrl();
- }
- String nodeLink = null;
- if (appAttempt.getHost() != null && appAttempt.getRpcPort() >= 0
- && appAttempt.getRpcPort() < 65536) {
- nodeLink = appAttempt.getHost() + ":" + appAttempt.getRpcPort();
+ nodeLink = containerReport.getNodeHttpAddress();
}
// AppAttemptID numerical value parsed by parseHadoopID in
// yarn.dt.plugins.js
@@ -310,8 +307,8 @@ public class AppBlock extends HtmlBlock {
.append(appAttempt.getAppAttemptId())
.append("</a>\",\"")
.append(startTime)
- .append("\",\"<a href='")
- .append("#") // TODO: replace with node http address (YARN-1884)
+ .append("\",\"<a ")
+ .append(nodeLink == null ? "#" : "href='" + nodeLink)
.append("'>")
.append(nodeLink == null ? "N/A" : StringEscapeUtils
.escapeJavaScript(StringEscapeUtils.escapeHtml(nodeLink)))
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4154bde/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ContainerBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ContainerBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ContainerBlock.java
index ed50c7a..cae8d2e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ContainerBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ContainerBlock.java
@@ -104,7 +104,12 @@ public class ContainerBlock extends HtmlBlock {
container.getContainerState() == null ? UNAVAILABLE : container
.getContainerState())
._("Exit Status:", container.getContainerExitStatus())
- ._("Node:", container.getAssignedNodeId())
+ ._(
+ "Node:",
+ container.getNodeHttpAddress() == null ? "#" : container
+ .getNodeHttpAddress(),
+ container.getNodeHttpAddress() == null ? "N/A" : container
+ .getNodeHttpAddress())
._("Priority:", container.getPriority())
._("Started:", Times.format(container.getStartedTime()))
._(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4154bde/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainerInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainerInfo.java
index bdcc7b2..0d18e7a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainerInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainerInfo.java
@@ -42,6 +42,7 @@ public class ContainerInfo {
protected String logUrl;
protected int containerExitStatus;
protected ContainerState containerState;
+ protected String nodeHttpAddress;
public ContainerInfo() {
// JAXB needs this
@@ -64,6 +65,7 @@ public class ContainerInfo {
logUrl = container.getLogUrl();
containerExitStatus = container.getContainerExitStatus();
containerState = container.getContainerState();
+ nodeHttpAddress = container.getNodeHttpAddress();
}
public String getContainerId() {
@@ -114,4 +116,7 @@ public class ContainerInfo {
return containerState;
}
+ public String getNodeHttpAddress() {
+ return nodeHttpAddress;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4154bde/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ContainerCreatedEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ContainerCreatedEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ContainerCreatedEvent.java
index eeda181..05b6781 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ContainerCreatedEvent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ContainerCreatedEvent.java
@@ -29,18 +29,21 @@ public class ContainerCreatedEvent extends SystemMetricsEvent {
private Resource allocatedResource;
private NodeId allocatedNode;
private Priority allocatedPriority;
+ private String nodeHttpAddress;
public ContainerCreatedEvent(
ContainerId containerId,
Resource allocatedResource,
NodeId allocatedNode,
Priority allocatedPriority,
- long createdTime) {
+ long createdTime,
+ String nodeHttpAddress) {
super(SystemMetricsEventType.CONTAINER_CREATED, createdTime);
this.containerId = containerId;
this.allocatedResource = allocatedResource;
this.allocatedNode = allocatedNode;
this.allocatedPriority = allocatedPriority;
+ this.nodeHttpAddress = nodeHttpAddress;
}
@Override
@@ -64,4 +67,7 @@ public class ContainerCreatedEvent extends SystemMetricsEvent {
return allocatedPriority;
}
+ public String getNodeHttpAddress() {
+ return nodeHttpAddress;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4154bde/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
index d8e7ef6..fb1edc62 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
@@ -179,7 +179,7 @@ public class SystemMetricsPublisher extends CompositeService {
container.getAllocatedResource(),
container.getAllocatedNode(),
container.getAllocatedPriority(),
- createdTime));
+ createdTime, container.getNodeHttpAddress()));
}
}
@@ -380,6 +380,9 @@ public class SystemMetricsPublisher extends CompositeService {
event.getAllocatedNode().getPort());
entityInfo.put(ContainerMetricsConstants.ALLOCATED_PRIORITY_ENTITY_INFO,
event.getAllocatedPriority().getPriority());
+ entityInfo.put(
+ ContainerMetricsConstants.ALLOCATED_HOST_HTTP_ADDRESS_ENTITY_INFO,
+ event.getNodeHttpAddress());
entity.setOtherInfo(entityInfo);
TimelineEvent tEvent = new TimelineEvent();
tEvent.setEventType(ContainerMetricsConstants.CREATED_EVENT_TYPE);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4154bde/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java
index 9e9dcb9..20087f5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java
@@ -79,4 +79,5 @@ public interface RMContainer extends EventHandler<RMContainerEvent> {
List<ResourceRequest> getResourceRequests();
+ String getNodeHttpAddress();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4154bde/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
index b69968f..7ee5467 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
@@ -42,7 +42,6 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
-import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppRunningOnNodeEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptContainerAllocatedEvent;
@@ -608,10 +607,29 @@ public class RMContainerImpl implements RMContainer {
this.getAllocatedResource(), this.getAllocatedNode(),
this.getAllocatedPriority(), this.getCreationTime(),
this.getFinishTime(), this.getDiagnosticsInfo(), this.getLogURL(),
- this.getContainerExitStatus(), this.getContainerState());
+ this.getContainerExitStatus(), this.getContainerState(),
+ this.getNodeHttpAddress());
} finally {
this.readLock.unlock();
}
return containerReport;
}
+
+ @Override
+ public String getNodeHttpAddress() {
+ try {
+ readLock.lock();
+ if (container.getNodeHttpAddress() != null) {
+ StringBuilder httpAddress = new StringBuilder();
+ httpAddress.append(WebAppUtils.getHttpSchemePrefix(rmContext
+ .getYarnConfiguration()));
+ httpAddress.append(container.getNodeHttpAddress());
+ return httpAddress.toString();
+ } else {
+ return null;
+ }
+ } finally {
+ readLock.unlock();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4154bde/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
index 65c8547..2fa60dd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
@@ -38,7 +38,6 @@ import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp;
import org.apache.hadoop.yarn.server.metrics.AppAttemptMetricsConstants;
import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
import org.apache.hadoop.yarn.server.metrics.ContainerMetricsConstants;
@@ -369,6 +368,10 @@ public class TestSystemMetricsPublisher {
when(container.getDiagnosticsInfo()).thenReturn("test diagnostics info");
when(container.getContainerExitStatus()).thenReturn(-1);
when(container.getContainerState()).thenReturn(ContainerState.COMPLETE);
+ Container mockContainer = mock(Container.class);
+ when(container.getContainer()).thenReturn(mockContainer);
+ when(mockContainer.getNodeHttpAddress())
+ .thenReturn("http://localhost:1234");
return container;
}
[3/8] hadoop git commit: YARN-3740. Fixed the typo in the
configuration name: APPLICATION_HISTORY_PREFIX_MAX_APPS. Contributed by Xuan
Gong.
Posted by vi...@apache.org.
YARN-3740. Fixed the typo in the configuration name: APPLICATION_HISTORY_PREFIX_MAX_APPS. Contributed by Xuan Gong.
(cherry picked from commit eb6bf91eeacf97afb4cefe590f75ba94f3187d2b)
(cherry picked from commit 68cddb894a95a1ee63291af162849a23cb4eeb5f)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3ab820e6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3ab820e6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3ab820e6
Branch: refs/heads/branch-2.6.1
Commit: 3ab820e696ef9493bf8c7e2c302e4cb8bf3bf72d
Parents: 85ec6eb
Author: Zhijie Shen <zj...@apache.org>
Authored: Fri May 29 18:16:48 2015 -0700
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Fri Sep 11 11:53:51 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java | 4 ++--
.../ApplicationHistoryManagerOnTimelineStore.java | 4 ++--
.../TestApplicationHistoryClientService.java | 2 +-
4 files changed, 8 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ab820e6/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index cab1f0c..6641bb2 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -211,6 +211,9 @@ Release 2.6.1 - 2015-09-09
YARN-3171. Sort by Application id, AppAttempt and ContainerID doesn't work
in ATS / RM web ui. (Naganarasimha G R via xgong)
+ YARN-3740. Fixed the typo in the configuration name:
+ APPLICATION_HISTORY_PREFIX_MAX_APPS. (Xuan Gong via zjshen)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ab820e6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 84322c3..881ecb5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1304,9 +1304,9 @@ public class YarnConfiguration extends Configuration {
* REST API or application history protocol and shown in timeline
* server web ui.
*/
- public static final String APPLICATION_HISTORY_PREFIX_MAX_APPS =
+ public static final String APPLICATION_HISTORY_MAX_APPS =
APPLICATION_HISTORY_PREFIX + "max-applications";
- public static final long DEFAULT_APPLICATION_HISTORY_PREFIX_MAX_APPS = 10000;
+ public static final long DEFAULT_APPLICATION_HISTORY_MAX_APPS = 10000;
/** Timeline service store class */
public static final String TIMELINE_SERVICE_STORE =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ab820e6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
index 4a407c3..b8f98c5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
@@ -89,8 +89,8 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
serverHttpAddress = WebAppUtils.getHttpSchemePrefix(conf) +
WebAppUtils.getAHSWebAppURLWithoutScheme(conf);
maxLoadedApplications =
- conf.getLong(YarnConfiguration.APPLICATION_HISTORY_PREFIX_MAX_APPS,
- YarnConfiguration.DEFAULT_APPLICATION_HISTORY_PREFIX_MAX_APPS);
+ conf.getLong(YarnConfiguration.APPLICATION_HISTORY_MAX_APPS,
+ YarnConfiguration.DEFAULT_APPLICATION_HISTORY_MAX_APPS);
super.serviceInit(conf);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ab820e6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
index feab8c1..ae49a2d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
@@ -172,7 +172,7 @@ public class TestApplicationHistoryClientService {
// Create a historyManager, and set the max_apps can be loaded
// as 1.
Configuration conf = new YarnConfiguration();
- conf.setLong(YarnConfiguration.APPLICATION_HISTORY_PREFIX_MAX_APPS, 1);
+ conf.setLong(YarnConfiguration.APPLICATION_HISTORY_MAX_APPS, 1);
ApplicationHistoryManagerOnTimelineStore historyManager2 =
new ApplicationHistoryManagerOnTimelineStore(dataManager,
new ApplicationACLsManager(conf));
[4/8] hadoop git commit: YARN-3379. Fixed missing data in
localityTable and ResourceRequests table in RM WebUI. Contributed by Xuan
Gong (cherry picked from commit 4e886eb9cbd2dcb128bbfd17309c734083093a4c)
Posted by vi...@apache.org.
YARN-3379. Fixed missing data in localityTable and ResourceRequests table in RM WebUI. Contributed by Xuan Gong
(cherry picked from commit 4e886eb9cbd2dcb128bbfd17309c734083093a4c)
(cherry picked from commit 3f0c9e5fe36d201de021d989b23ebaeb2d9a027b)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e914220a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e914220a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e914220a
Branch: refs/heads/branch-2.6.1
Commit: e914220ab9d090b709d445bb415e45eab0c4dea5
Parents: 3ab820e
Author: Jian He <ji...@apache.org>
Authored: Thu Mar 19 22:27:21 2015 -0700
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Mon Sep 14 12:54:01 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../yarn/server/webapp/AppAttemptBlock.java | 3 +-
.../hadoop/yarn/server/webapp/AppBlock.java | 47 ++--------
.../resourcemanager/webapp/AppAttemptPage.java | 3 +-
.../server/resourcemanager/webapp/AppPage.java | 3 +-
.../webapp/RMAppAttemptBlock.java | 56 ++++++++++++
.../resourcemanager/webapp/RMAppBlock.java | 94 ++++++++++++++++++++
7 files changed, 165 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e914220a/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 6641bb2..e7d847f 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -214,6 +214,9 @@ Release 2.6.1 - 2015-09-09
YARN-3740. Fixed the typo in the configuration name:
APPLICATION_HISTORY_PREFIX_MAX_APPS. (Xuan Gong via zjshen)
+ YARN-3379. Fixed missing data in localityTable and ResourceRequests table
+ in RM WebUI. (Xuan Gong via jianhe)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e914220a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java
index ae5f092..abf7c86 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java
@@ -43,13 +43,13 @@ import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY;
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
import org.apache.hadoop.yarn.webapp.view.InfoBlock;
-
import com.google.inject.Inject;
public class AppAttemptBlock extends HtmlBlock {
private static final Log LOG = LogFactory.getLog(AppAttemptBlock.class);
protected ApplicationBaseProtocol appBaseProt;
+ protected ApplicationAttemptId appAttemptId = null;
@Inject
public AppAttemptBlock(ApplicationBaseProtocol appBaseProt, ViewContext ctx) {
@@ -65,7 +65,6 @@ public class AppAttemptBlock extends HtmlBlock {
return;
}
- ApplicationAttemptId appAttemptId = null;
try {
appAttemptId = ConverterUtils.toApplicationAttemptId(attemptid);
} catch (IllegalArgumentException e) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e914220a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
index e62c783..93b24a3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
@@ -21,15 +21,9 @@ package org.apache.hadoop.yarn.server.webapp;
import static org.apache.hadoop.yarn.util.StringHelper.join;
import static org.apache.hadoop.yarn.webapp.YarnWebParams.APPLICATION_ID;
import static org.apache.hadoop.yarn.webapp.YarnWebParams.WEB_UI_TYPE;
-import static org.apache.hadoop.yarn.webapp.view.JQueryUI._EVEN;
-import static org.apache.hadoop.yarn.webapp.view.JQueryUI._INFO_WRAP;
-import static org.apache.hadoop.yarn.webapp.view.JQueryUI._ODD;
-import static org.apache.hadoop.yarn.webapp.view.JQueryUI._TH;
import java.security.PrivilegedExceptionAction;
import java.util.Collection;
-import java.util.List;
-
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -43,12 +37,9 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerReport;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException;
@@ -57,10 +48,8 @@ import org.apache.hadoop.yarn.server.webapp.dao.AppInfo;
import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo;
import org.apache.hadoop.yarn.util.Apps;
import org.apache.hadoop.yarn.util.Times;
-import org.apache.hadoop.yarn.util.resource.Resources;
import org.apache.hadoop.yarn.webapp.YarnWebParams;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
-import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY;
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
@@ -73,9 +62,11 @@ public class AppBlock extends HtmlBlock {
private static final Log LOG = LogFactory.getLog(AppBlock.class);
protected ApplicationBaseProtocol appBaseProt;
protected Configuration conf;
+ protected ApplicationId appID = null;
@Inject
- AppBlock(ApplicationBaseProtocol appBaseProt, ViewContext ctx, Configuration conf) {
+ protected AppBlock(ApplicationBaseProtocol appBaseProt, ViewContext ctx,
+ Configuration conf) {
super(ctx);
this.appBaseProt = appBaseProt;
this.conf = conf;
@@ -90,7 +81,6 @@ public class AppBlock extends HtmlBlock {
return;
}
- ApplicationId appID = null;
try {
appID = Apps.toAppID(aid);
} catch (Exception e) {
@@ -217,31 +207,7 @@ public class AppBlock extends HtmlBlock {
return;
}
- //TODO:YARN-3284
- //The preemption metrics will be exposed from ApplicationReport
- // and ApplicationAttemptReport
- ApplicationResourceUsageReport usageReport =
- appReport.getApplicationResourceUsageReport();
- DIV<Hamlet> pdiv = html.
- _(InfoBlock.class).
- div(_INFO_WRAP);
- info("Application Overview").clear();
- info("Application Metrics")
- ._("Total Resource Preempted:",
- Resources.none()) // TODO: YARN-3284
- ._("Total Number of Non-AM Containers Preempted:",
- String.valueOf(0)) // TODO: YARN-3284
- ._("Total Number of AM Containers Preempted:",
- String.valueOf(0)) // TODO: YARN-3284
- ._("Resource Preempted from Current Attempt:",
- Resources.none()) // TODO: YARN-3284
- ._("Number of Non-AM Containers Preempted from Current Attempt:",
- 0) // TODO: YARN-3284
- ._("Aggregate Resource Allocation:",
- String.format("%d MB-seconds, %d vcore-seconds", usageReport == null
- ? 0 : usageReport.getMemorySeconds(), usageReport == null ? 0
- : usageReport.getVcoreSeconds()));
- pdiv._();
+ createApplicationMetricsTable(html);
html._(InfoBlock.class);
@@ -352,4 +318,9 @@ public class AppBlock extends HtmlBlock {
}
return status.toString();
}
+
+ // The preemption metrics only need to be shown in RM WebUI
+ protected void createApplicationMetricsTable(Block html) {
+
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e914220a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppAttemptPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppAttemptPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppAttemptPage.java
index 92eae48..2bbe4ff 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppAttemptPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppAttemptPage.java
@@ -23,7 +23,6 @@ import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;
-import org.apache.hadoop.yarn.server.webapp.AppAttemptBlock;
import org.apache.hadoop.yarn.server.webapp.WebPageUtils;
import org.apache.hadoop.yarn.webapp.SubView;
import org.apache.hadoop.yarn.webapp.YarnWebParams;
@@ -49,7 +48,7 @@ public class AppAttemptPage extends RmView {
@Override
protected Class<? extends SubView> content() {
- return AppAttemptBlock.class;
+ return RMAppAttemptBlock.class;
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e914220a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppPage.java
index 77ae44c..b0698af 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppPage.java
@@ -23,7 +23,6 @@ import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;
-import org.apache.hadoop.yarn.server.webapp.AppBlock;
import org.apache.hadoop.yarn.server.webapp.WebPageUtils;
import org.apache.hadoop.yarn.webapp.SubView;
import org.apache.hadoop.yarn.webapp.YarnWebParams;
@@ -50,6 +49,6 @@ public class AppPage extends RmView {
@Override
protected Class<? extends SubView> content() {
- return AppBlock.class;
+ return RMAppBlock.class;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e914220a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java
new file mode 100644
index 0000000..7dc7030
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
+import org.apache.hadoop.yarn.server.webapp.AppAttemptBlock;
+
+import com.google.inject.Inject;
+
+public class RMAppAttemptBlock extends AppAttemptBlock{
+
+ private final ResourceManager rm;
+ protected Configuration conf;
+
+ @Inject
+ RMAppAttemptBlock(ViewContext ctx, ResourceManager rm, Configuration conf) {
+ super(rm.getClientRMService(), ctx);
+ this.rm = rm;
+ this.conf = conf;
+ }
+
+ @Override
+ protected void render(Block html) {
+ super.render(html);
+ }
+
+ private RMAppAttempt getRMAppAttempt() {
+ ApplicationId appId = this.appAttemptId.getApplicationId();
+ RMAppAttempt attempt = null;
+ RMApp rmApp = rm.getRMContext().getRMApps().get(appId);
+ if (rmApp != null) {
+ attempt = rmApp.getAppAttempts().get(appAttemptId);
+ }
+ return attempt;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e914220a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java
new file mode 100644
index 0000000..64c5747
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java
@@ -0,0 +1,94 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp;
+
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI._INFO_WRAP;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptMetrics;
+import org.apache.hadoop.yarn.server.webapp.AppBlock;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
+import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV;
+import org.apache.hadoop.yarn.webapp.view.InfoBlock;
+
+import com.google.inject.Inject;
+
+public class RMAppBlock extends AppBlock{
+
+ private final ResourceManager rm;
+
+ @Inject
+ RMAppBlock(ViewContext ctx, Configuration conf, ResourceManager rm) {
+ super(rm.getClientRMService(), ctx, conf);
+ this.rm = rm;
+ }
+
+ @Override
+ protected void render(Block html) {
+ super.render(html);
+ }
+
+ @Override
+ protected void createApplicationMetricsTable(Block html){
+ RMApp rmApp = this.rm.getRMContext().getRMApps().get(appID);
+ RMAppMetrics appMetrics = rmApp == null ? null : rmApp.getRMAppMetrics();
+ // Get attempt metrics and fields, it is possible currentAttempt of RMApp is
+ // null. In that case, we will assume resource preempted and number of Non
+ // AM container preempted on that attempt is 0
+ RMAppAttemptMetrics attemptMetrics;
+ if (rmApp == null || null == rmApp.getCurrentAppAttempt()) {
+ attemptMetrics = null;
+ } else {
+ attemptMetrics = rmApp.getCurrentAppAttempt().getRMAppAttemptMetrics();
+ }
+ Resource attemptResourcePreempted =
+ attemptMetrics == null ? Resources.none() : attemptMetrics
+ .getResourcePreempted();
+ int attemptNumNonAMContainerPreempted =
+ attemptMetrics == null ? 0 : attemptMetrics
+ .getNumNonAMContainersPreempted();
+ DIV<Hamlet> pdiv = html.
+ _(InfoBlock.class).
+ div(_INFO_WRAP);
+ info("Application Overview").clear();
+ info("Application Metrics")
+ ._("Total Resource Preempted:",
+ appMetrics == null ? "N/A" : appMetrics.getResourcePreempted())
+ ._("Total Number of Non-AM Containers Preempted:",
+ appMetrics == null ? "N/A"
+ : appMetrics.getNumNonAMContainersPreempted())
+ ._("Total Number of AM Containers Preempted:",
+ appMetrics == null ? "N/A"
+ : appMetrics.getNumAMContainersPreempted())
+ ._("Resource Preempted from Current Attempt:",
+ attemptResourcePreempted)
+ ._("Number of Non-AM Containers Preempted from Current Attempt:",
+ attemptNumNonAMContainerPreempted)
+ ._("Aggregate Resource Allocation:",
+ String.format("%d MB-seconds, %d vcore-seconds",
+ appMetrics == null ? "N/A" : appMetrics.getMemorySeconds(),
+ appMetrics == null ? "N/A" : appMetrics.getVcoreSeconds()));
+ pdiv._();
+ }
+}
[5/8] hadoop git commit: YARN-3248. Display count of nodes
blacklisted by apps in the web UI. Contributed by Varun Vasudev
Posted by vi...@apache.org.
YARN-3248. Display count of nodes blacklisted by apps in the web UI.
Contributed by Varun Vasudev
(cherry picked from commit 4728bdfa15809db4b8b235faa286c65de4a48cf6)
(cherry picked from commit e26b6e55e96b763063dfbd39977096367eafc1e3)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7af5d6b4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7af5d6b4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7af5d6b4
Branch: refs/heads/branch-2.6.1
Commit: 7af5d6b4bae762e3c7fd0bb16f1f04d45e42ea25
Parents: e914220
Author: Xuan <xg...@apache.org>
Authored: Wed Apr 1 04:19:18 2015 -0700
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Tue Sep 15 17:30:06 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../yarn/server/webapp/AppAttemptBlock.java | 66 +++++----
.../hadoop/yarn/server/webapp/AppBlock.java | 38 +++--
.../hadoop/yarn/server/webapp/AppsBlock.java | 95 +++++++-----
.../scheduler/AppSchedulingInfo.java | 4 +
.../scheduler/SchedulerApplicationAttempt.java | 4 +
.../webapp/AppsBlockWithMetrics.java | 2 +-
.../webapp/CapacitySchedulerPage.java | 2 +-
.../webapp/RMAppAttemptBlock.java | 68 +++++++++
.../resourcemanager/webapp/RMAppBlock.java | 110 ++++++++++++++
.../resourcemanager/webapp/RMAppsBlock.java | 146 +++++++++++++++++++
.../resourcemanager/webapp/RMWebServices.java | 3 +-
.../webapp/dao/AppAttemptInfo.java | 19 ++-
.../webapp/TestRMWebServicesApps.java | 2 +-
14 files changed, 476 insertions(+), 86 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7af5d6b4/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index e7d847f..c2c2bfa 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -217,6 +217,9 @@ Release 2.6.1 - 2015-09-09
YARN-3379. Fixed missing data in localityTable and ResourceRequests table
in RM WebUI. (Xuan Gong via jianhe)
+ YARN-3248. Display count of nodes blacklisted by apps in the web UI.
+ (Varun Vasudev via xgong)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7af5d6b4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java
index abf7c86..50d4f72 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY;
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
import org.apache.hadoop.yarn.webapp.view.InfoBlock;
+
import com.google.inject.Inject;
public class AppAttemptBlock extends HtmlBlock {
@@ -73,7 +74,7 @@ public class AppAttemptBlock extends HtmlBlock {
}
UserGroupInformation callerUGI = getCallerUGI();
- ApplicationAttemptReport appAttemptReport = null;
+ ApplicationAttemptReport appAttemptReport;
try {
final GetApplicationAttemptReportRequest request =
GetApplicationAttemptReportRequest.newInstance(appAttemptId);
@@ -136,33 +137,7 @@ public class AppAttemptBlock extends HtmlBlock {
&& appAttempt.getRpcPort() < 65536) {
node = appAttempt.getHost() + ":" + appAttempt.getRpcPort();
}
- info("Application Attempt Overview")
- ._(
- "Application Attempt State:",
- appAttempt.getAppAttemptState() == null ? UNAVAILABLE : appAttempt
- .getAppAttemptState())
- ._(
- "AM Container:",
- appAttempt.getAmContainerId() == null || containers == null
- || !hasAMContainer(appAttemptReport.getAMContainerId(), containers)
- ? null : root_url("container", appAttempt.getAmContainerId()),
- String.valueOf(appAttempt.getAmContainerId()))
- ._("Node:", node)
- ._(
- "Tracking URL:",
- appAttempt.getTrackingUrl() == null
- || appAttempt.getTrackingUrl() == UNAVAILABLE ? null
- : root_url(appAttempt.getTrackingUrl()),
- appAttempt.getTrackingUrl() == null
- || appAttempt.getTrackingUrl() == UNAVAILABLE
- ? "Unassigned"
- : appAttempt.getAppAttemptState() == YarnApplicationAttemptState.FINISHED
- || appAttempt.getAppAttemptState() == YarnApplicationAttemptState.FAILED
- || appAttempt.getAppAttemptState() == YarnApplicationAttemptState.KILLED
- ? "History" : "ApplicationMaster")
- ._("Diagnostics Info:", appAttempt.getDiagnosticsInfo() == null ?
- "" : appAttempt.getDiagnosticsInfo());
-
+ generateOverview(appAttemptReport, containers, appAttempt, node);
html._(InfoBlock.class);
if (exceptionWhenGetContainerReports) {
@@ -216,7 +191,40 @@ public class AppAttemptBlock extends HtmlBlock {
tbody._()._();
}
- private boolean hasAMContainer(ContainerId containerId,
+ protected void generateOverview(ApplicationAttemptReport appAttemptReport,
+ Collection<ContainerReport> containers, AppAttemptInfo appAttempt,
+ String node) {
+ info("Application Attempt Overview")
+ ._(
+ "Application Attempt State:",
+ appAttempt.getAppAttemptState() == null ? UNAVAILABLE : appAttempt
+ .getAppAttemptState())
+ ._(
+ "AM Container:",
+ appAttempt.getAmContainerId() == null || containers == null
+ || !hasAMContainer(appAttemptReport.getAMContainerId(), containers)
+ ? null : root_url("container", appAttempt.getAmContainerId()),
+ String.valueOf(appAttempt.getAmContainerId()))
+ ._("Node:", node)
+ ._(
+ "Tracking URL:",
+ appAttempt.getTrackingUrl() == null
+ || appAttempt.getTrackingUrl().equals(UNAVAILABLE) ? null
+ : root_url(appAttempt.getTrackingUrl()),
+ appAttempt.getTrackingUrl() == null
+ || appAttempt.getTrackingUrl().equals(UNAVAILABLE)
+ ? "Unassigned"
+ : appAttempt.getAppAttemptState() == YarnApplicationAttemptState.FINISHED
+ || appAttempt.getAppAttemptState() == YarnApplicationAttemptState.FAILED
+ || appAttempt.getAppAttemptState() == YarnApplicationAttemptState.KILLED
+ ? "History" : "ApplicationMaster")
+ ._(
+ "Diagnostics Info:",
+ appAttempt.getDiagnosticsInfo() == null ? "" : appAttempt
+ .getDiagnosticsInfo());
+ }
+
+ protected boolean hasAMContainer(ContainerId containerId,
Collection<ContainerReport> containers) {
for (ContainerReport container : containers) {
if (containerId.equals(container.getContainerId())) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7af5d6b4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
index 93b24a3..c310f92 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
@@ -24,6 +24,7 @@ import static org.apache.hadoop.yarn.webapp.YarnWebParams.WEB_UI_TYPE;
import java.security.PrivilegedExceptionAction;
import java.util.Collection;
+
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -89,7 +90,7 @@ public class AppBlock extends HtmlBlock {
}
UserGroupInformation callerUGI = getCallerUGI();
- ApplicationReport appReport = null;
+ ApplicationReport appReport;
try {
final GetApplicationReportRequest request =
GetApplicationReportRequest.newInstance(appID);
@@ -161,7 +162,8 @@ public class AppBlock extends HtmlBlock {
._("Application Type:", app.getType())
._("Application Tags:",
app.getApplicationTags() == null ? "" : app.getApplicationTags())
- ._("YarnApplicationState:",
+ ._(
+ "YarnApplicationState:",
app.getAppState() == null ? UNAVAILABLE : clarifyAppState(app
.getAppState()))
._("FinalStatus Reported by AM:",
@@ -171,16 +173,19 @@ public class AppBlock extends HtmlBlock {
"Elapsed:",
StringUtils.formatTime(Times.elapsed(app.getStartedTime(),
app.getFinishedTime())))
- ._("Tracking URL:",
- app.getTrackingUrl() == null || app.getTrackingUrl() == UNAVAILABLE
- ? null : root_url(app.getTrackingUrl()),
- app.getTrackingUrl() == null || app.getTrackingUrl() == UNAVAILABLE
- ? "Unassigned" : app.getAppState() == YarnApplicationState.FINISHED
- || app.getAppState() == YarnApplicationState.FAILED
- || app.getAppState() == YarnApplicationState.KILLED ? "History"
- : "ApplicationMaster")
+ ._(
+ "Tracking URL:",
+ app.getTrackingUrl() == null
+ || app.getTrackingUrl().equals(UNAVAILABLE) ? null : root_url(app
+ .getTrackingUrl()),
+ app.getTrackingUrl() == null
+ || app.getTrackingUrl().equals(UNAVAILABLE) ? "Unassigned" : app
+ .getAppState() == YarnApplicationState.FINISHED
+ || app.getAppState() == YarnApplicationState.FAILED
+ || app.getAppState() == YarnApplicationState.KILLED ? "History"
+ : "ApplicationMaster")
._("Diagnostics:",
- app.getDiagnosticsInfo() == null ? "" : app.getDiagnosticsInfo());
+ app.getDiagnosticsInfo() == null ? "" : app.getDiagnosticsInfo());
Collection<ApplicationAttemptReport> attempts;
try {
@@ -211,6 +216,13 @@ public class AppBlock extends HtmlBlock {
html._(InfoBlock.class);
+ generateApplicationTable(html, callerUGI, attempts);
+
+ }
+
+ protected void generateApplicationTable(Block html,
+ UserGroupInformation callerUGI,
+ Collection<ApplicationAttemptReport> attempts) {
// Application Attempt Table
TBODY<TABLE<Hamlet>> tbody =
html.table("#attempts").thead().tr().th(".id", "Attempt ID")
@@ -220,7 +232,7 @@ public class AppBlock extends HtmlBlock {
StringBuilder attemptsTableData = new StringBuilder("[\n");
for (final ApplicationAttemptReport appAttemptReport : attempts) {
AppAttemptInfo appAttempt = new AppAttemptInfo(appAttemptReport);
- ContainerReport containerReport = null;
+ ContainerReport containerReport;
try {
// AM container is always the first container of the attempt
final GetContainerReportRequest request =
@@ -231,7 +243,7 @@ public class AppBlock extends HtmlBlock {
appBaseProt.getContainerReport(request).getContainerReport();
} else {
containerReport = callerUGI.doAs(
- new PrivilegedExceptionAction<ContainerReport> () {
+ new PrivilegedExceptionAction<ContainerReport>() {
@Override
public ContainerReport run() throws Exception {
ContainerReport report = null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7af5d6b4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java
index 583a0ae..6f8e10b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java
@@ -24,6 +24,7 @@ import static org.apache.hadoop.yarn.webapp.YarnWebParams.APPS_NUM;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR_VALUE;
+import java.io.IOException;
import java.security.PrivilegedExceptionAction;
import java.util.Collection;
import java.util.EnumSet;
@@ -36,6 +37,7 @@ import org.apache.hadoop.yarn.api.ApplicationBaseProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.webapp.dao.AppInfo;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE;
@@ -48,26 +50,19 @@ public class AppsBlock extends HtmlBlock {
private static final Log LOG = LogFactory.getLog(AppsBlock.class);
protected ApplicationBaseProtocol appBaseProt;
+ protected EnumSet<YarnApplicationState> reqAppStates;
+ protected UserGroupInformation callerUGI;
+ protected Collection<ApplicationReport> appReports;
@Inject
- AppsBlock(ApplicationBaseProtocol appBaseProt, ViewContext ctx) {
+ protected AppsBlock(ApplicationBaseProtocol appBaseProt, ViewContext ctx) {
super(ctx);
this.appBaseProt = appBaseProt;
}
- @Override
- public void render(Block html) {
- setTitle("Applications");
-
- TBODY<TABLE<Hamlet>> tbody =
- html.table("#apps").thead().tr().th(".id", "ID").th(".user", "User")
- .th(".name", "Name").th(".type", "Application Type")
- .th(".queue", "Queue").th(".starttime", "StartTime")
- .th(".finishtime", "FinishTime").th(".state", "State")
- .th(".finalstatus", "FinalStatus").th(".progress", "Progress")
- .th(".ui", "Tracking UI")._()._().tbody();
- EnumSet<YarnApplicationState> reqAppStates =
- EnumSet.noneOf(YarnApplicationState.class);
+ protected void fetchData() throws YarnException, IOException,
+ InterruptedException {
+ reqAppStates = EnumSet.noneOf(YarnApplicationState.class);
String reqStateString = $(APP_STATE);
if (reqStateString != null && !reqStateString.isEmpty()) {
String[] appStateStrings = reqStateString.split(",");
@@ -76,33 +71,54 @@ public class AppsBlock extends HtmlBlock {
}
}
- UserGroupInformation callerUGI = getCallerUGI();
- Collection<ApplicationReport> appReports = null;
+ callerUGI = getCallerUGI();
+ final GetApplicationsRequest request =
+ GetApplicationsRequest.newInstance(reqAppStates);
+ String appsNumStr = $(APPS_NUM);
+ if (appsNumStr != null && !appsNumStr.isEmpty()) {
+ long appsNum = Long.parseLong(appsNumStr);
+ request.setLimit(appsNum);
+ }
+ if (callerUGI == null) {
+ appReports = appBaseProt.getApplications(request).getApplicationList();
+ } else {
+ appReports =
+ callerUGI
+ .doAs(new PrivilegedExceptionAction<Collection<ApplicationReport>>() {
+ @Override
+ public Collection<ApplicationReport> run() throws Exception {
+ return appBaseProt.getApplications(request)
+ .getApplicationList();
+ }
+ });
+ }
+ }
+
+ @Override
+ public void render(Block html) {
+ setTitle("Applications");
+
try {
- final GetApplicationsRequest request =
- GetApplicationsRequest.newInstance(reqAppStates);
- String appsNumStr = $(APPS_NUM);
- if (appsNumStr != null && !appsNumStr.isEmpty()) {
- long appsNum = Long.parseLong(appsNumStr);
- request.setLimit(appsNum);
- }
- if (callerUGI == null) {
- appReports = appBaseProt.getApplications(request).getApplicationList();
- } else {
- appReports = callerUGI.doAs(
- new PrivilegedExceptionAction<Collection<ApplicationReport>> () {
- @Override
- public Collection<ApplicationReport> run() throws Exception {
- return appBaseProt.getApplications(request).getApplicationList();
- }
- });
- }
- } catch (Exception e) {
+ fetchData();
+ }
+ catch( Exception e) {
String message = "Failed to read the applications.";
LOG.error(message, e);
html.p()._(message)._();
return;
}
+ renderData(html);
+ }
+
+ protected void renderData(Block html) {
+ TBODY<TABLE<Hamlet>> tbody =
+ html.table("#apps").thead().tr().th(".id", "ID").th(".user", "User")
+ .th(".name", "Name").th(".type", "Application Type")
+ .th(".queue", "Queue").th(".starttime", "StartTime")
+ .th(".finishtime", "FinishTime").th(".state", "State")
+ .th(".finalstatus", "FinalStatus").th(".progress", "Progress")
+ .th(".ui", "Tracking UI")._()._().tbody();
+
StringBuilder appsTableData = new StringBuilder("[\n");
for (ApplicationReport appReport : appReports) {
// TODO: remove the following condition. It is still here because
@@ -123,7 +139,7 @@ public class AppsBlock extends HtmlBlock {
.append("</a>\",\"")
.append(
StringEscapeUtils.escapeJavaScript(StringEscapeUtils.escapeHtml(app
- .getUser())))
+ .getUser())))
.append("\",\"")
.append(
StringEscapeUtils.escapeJavaScript(StringEscapeUtils.escapeHtml(app
@@ -150,11 +166,12 @@ public class AppsBlock extends HtmlBlock {
.append("'> </div> </div>").append("\",\"<a ");
String trackingURL =
- app.getTrackingUrl() == null || app.getTrackingUrl() == UNAVAILABLE
- ? null : app.getTrackingUrl();
+ app.getTrackingUrl() == null
+ || app.getTrackingUrl().equals(UNAVAILABLE) ? null : app
+ .getTrackingUrl();
String trackingUI =
- app.getTrackingUrl() == null || app.getTrackingUrl() == UNAVAILABLE
+ app.getTrackingUrl() == null || app.getTrackingUrl().equals(UNAVAILABLE)
? "Unassigned"
: app.getAppState() == YarnApplicationState.FINISHED
|| app.getAppState() == YarnApplicationState.FAILED
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7af5d6b4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
index 331cee9..b8d552b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
@@ -437,6 +437,10 @@ public class AppSchedulingInfo {
return this.blacklist;
}
+ public synchronized Set<String> getBlackListCopy() {
+ return new HashSet<String>(this.blacklist);
+ }
+
public synchronized void transferStateFromPreviousAppSchedulingInfo(
AppSchedulingInfo appInfo) {
// this.priorities = appInfo.getPriorities();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7af5d6b4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
index 84975b6..510276c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
@@ -615,4 +615,8 @@ public class SchedulerApplicationAttempt {
// schedulingOpportunities
// lastScheduledContainer
}
+
+ public Set<String> getBlacklistedNodes() {
+ return this.appSchedulingInfo.getBlackListCopy();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7af5d6b4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlockWithMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlockWithMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlockWithMetrics.java
index cb0836a..29889ec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlockWithMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlockWithMetrics.java
@@ -27,6 +27,6 @@ import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
class AppsBlockWithMetrics extends HtmlBlock {
@Override public void render(Block html) {
html._(MetricsOverviewTable.class);
- html._(AppsBlock.class);
+ html._(RMAppsBlock.class);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7af5d6b4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
index 0be361a..b1d0eba 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
@@ -230,7 +230,7 @@ class CapacitySchedulerPage extends RmView {
ul._()._().
script().$type("text/javascript").
_("$('#cs').hide();")._()._().
- _(AppsBlock.class);
+ _(RMAppsBlock.class);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7af5d6b4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java
index 7dc7030..c94d59a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java
@@ -18,12 +18,23 @@
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
+import java.util.Collection;
+import java.util.Set;
+
+import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerReport;
+import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
import org.apache.hadoop.yarn.server.webapp.AppAttemptBlock;
+import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo;
import com.google.inject.Inject;
@@ -53,4 +64,61 @@ public class RMAppAttemptBlock extends AppAttemptBlock{
}
return attempt;
}
+
+ protected void generateOverview(ApplicationAttemptReport appAttemptReport,
+ Collection<ContainerReport> containers, AppAttemptInfo appAttempt,
+ String node) {
+
+ String blacklistedNodes = "-";
+ Set<String> nodes =
+ getBlacklistedNodes(rm, getRMAppAttempt().getAppAttemptId());
+ if (nodes != null) {
+ if (!nodes.isEmpty()) {
+ blacklistedNodes = StringUtils.join(nodes, ", ");
+ }
+ }
+
+ info("Application Attempt Overview")
+ ._(
+ "Application Attempt State:",
+ appAttempt.getAppAttemptState() == null ? UNAVAILABLE : appAttempt
+ .getAppAttemptState())
+ ._(
+ "AM Container:",
+ appAttempt.getAmContainerId() == null || containers == null
+ || !hasAMContainer(appAttemptReport.getAMContainerId(), containers)
+ ? null : root_url("container", appAttempt.getAmContainerId()),
+ String.valueOf(appAttempt.getAmContainerId()))
+ ._("Node:", node)
+ ._(
+ "Tracking URL:",
+ appAttempt.getTrackingUrl() == null
+ || appAttempt.getTrackingUrl().equals(UNAVAILABLE) ? null
+ : root_url(appAttempt.getTrackingUrl()),
+ appAttempt.getTrackingUrl() == null
+ || appAttempt.getTrackingUrl().equals(UNAVAILABLE)
+ ? "Unassigned"
+ : appAttempt.getAppAttemptState() == YarnApplicationAttemptState.FINISHED
+ || appAttempt.getAppAttemptState() == YarnApplicationAttemptState.FAILED
+ || appAttempt.getAppAttemptState() == YarnApplicationAttemptState.KILLED
+ ? "History" : "ApplicationMaster")
+ ._(
+ "Diagnostics Info:",
+ appAttempt.getDiagnosticsInfo() == null ? "" : appAttempt
+ .getDiagnosticsInfo())._("Blacklisted Nodes:", blacklistedNodes);
+ }
+
+ public static Set<String> getBlacklistedNodes(ResourceManager rm,
+ ApplicationAttemptId appid) {
+ if (rm.getResourceScheduler() instanceof AbstractYarnScheduler) {
+ AbstractYarnScheduler ayScheduler =
+ (AbstractYarnScheduler) rm.getResourceScheduler();
+ SchedulerApplicationAttempt attempt =
+ ayScheduler.getApplicationAttempt(appid);
+ if (attempt != null) {
+ return attempt.getBlacklistedNodes();
+ }
+ }
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7af5d6b4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java
index 64c5747..171c6c9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java
@@ -20,13 +20,29 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI._INFO_WRAP;
+import java.security.PrivilegedExceptionAction;
+import java.util.Collection;
+import java.util.Set;
+
+import org.apache.commons.lang.StringEscapeUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportRequest;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerReport;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptMetrics;
import org.apache.hadoop.yarn.server.webapp.AppBlock;
+import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo;
+import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo;
+import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV;
@@ -36,8 +52,10 @@ import com.google.inject.Inject;
public class RMAppBlock extends AppBlock{
+ private static final Log LOG = LogFactory.getLog(RMAppBlock.class);
private final ResourceManager rm;
+
@Inject
RMAppBlock(ViewContext ctx, Configuration conf, ResourceManager rm) {
super(rm.getClientRMService(), ctx, conf);
@@ -91,4 +109,96 @@ public class RMAppBlock extends AppBlock{
appMetrics == null ? "N/A" : appMetrics.getVcoreSeconds()));
pdiv._();
}
+
+ @Override
+ protected void generateApplicationTable(Block html,
+ UserGroupInformation callerUGI,
+ Collection<ApplicationAttemptReport> attempts) {
+ // Application Attempt Table
+ Hamlet.TBODY<Hamlet.TABLE<Hamlet>> tbody =
+ html.table("#attempts").thead().tr().th(".id", "Attempt ID")
+ .th(".started", "Started").th(".node", "Node").th(".logs", "Logs")
+ .th(".blacklistednodes", "Blacklisted Nodes")._()._().tbody();
+
+ StringBuilder attemptsTableData = new StringBuilder("[\n");
+ for (final ApplicationAttemptReport appAttemptReport : attempts) {
+ AppAttemptInfo appAttempt = new AppAttemptInfo(appAttemptReport);
+ ContainerReport containerReport = null;
+ try {
+ // AM container is always the first container of the attempt
+ final GetContainerReportRequest request =
+ GetContainerReportRequest.newInstance(ContainerId.newContainerId(
+ appAttemptReport.getApplicationAttemptId(), 1));
+ if (callerUGI == null) {
+ containerReport =
+ appBaseProt.getContainerReport(request).getContainerReport();
+ } else {
+ containerReport = callerUGI.doAs(
+ new PrivilegedExceptionAction<ContainerReport>() {
+ @Override
+ public ContainerReport run() throws Exception {
+ ContainerReport report = null;
+ try {
+ report = appBaseProt.getContainerReport(request)
+ .getContainerReport();
+ } catch (ContainerNotFoundException ex) {
+ LOG.warn(ex.getMessage());
+ }
+ return report;
+ }
+ });
+ }
+ } catch (Exception e) {
+ String message =
+ "Failed to read the AM container of the application attempt "
+ + appAttemptReport.getApplicationAttemptId() + ".";
+ LOG.error(message, e);
+ html.p()._(message)._();
+ return;
+ }
+ long startTime = 0L;
+ String logsLink = null;
+ String nodeLink = null;
+ if (containerReport != null) {
+ ContainerInfo container = new ContainerInfo(containerReport);
+ startTime = container.getStartedTime();
+ logsLink = containerReport.getLogUrl();
+ nodeLink = containerReport.getNodeHttpAddress();
+ }
+ String blacklistedNodesCount = "N/A";
+ Set<String> nodes = RMAppAttemptBlock.getBlacklistedNodes(rm,
+ ConverterUtils.toApplicationAttemptId(appAttempt.getAppAttemptId()));
+ if(nodes != null) {
+ blacklistedNodesCount = String.valueOf(nodes.size());
+ }
+
+ // AppAttemptID numerical value parsed by parseHadoopID in
+ // yarn.dt.plugins.js
+ attemptsTableData
+ .append("[\"<a href='")
+ .append(url("appattempt", appAttempt.getAppAttemptId()))
+ .append("'>")
+ .append(appAttempt.getAppAttemptId())
+ .append("</a>\",\"")
+ .append(startTime)
+ .append("\",\"<a ")
+ .append(nodeLink == null ? "#" : "href='" + nodeLink)
+ .append("'>")
+ .append(nodeLink == null ? "N/A" : StringEscapeUtils
+ .escapeJavaScript(StringEscapeUtils.escapeHtml(nodeLink)))
+ .append("</a>\",\"<a ")
+ .append(logsLink == null ? "#" : "href='" + logsLink).append("'>")
+ .append(logsLink == null ? "N/A" : "Logs").append("</a>\",").append(
+ "\"").append(blacklistedNodesCount).append("\"],\n");
+ }
+ if (attemptsTableData.charAt(attemptsTableData.length() - 2) == ',') {
+ attemptsTableData.delete(attemptsTableData.length() - 2,
+ attemptsTableData.length() - 1);
+ }
+ attemptsTableData.append("]");
+ html.script().$type("text/javascript")
+ ._("var attemptsTableData=" + attemptsTableData)._();
+
+ tbody._()._();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7af5d6b4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java
new file mode 100644
index 0000000..94f3afa
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java
@@ -0,0 +1,146 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp;
+
+import static org.apache.hadoop.yarn.util.StringHelper.join;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR_VALUE;
+
+import java.util.Set;
+
+import org.apache.commons.lang.StringEscapeUtils;
+import org.apache.hadoop.yarn.api.ApplicationBaseProtocol;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.webapp.AppsBlock;
+import org.apache.hadoop.yarn.server.webapp.dao.AppInfo;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.hadoop.yarn.webapp.View;
+import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
+import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE;
+import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY;
+
+import com.google.inject.Inject;
+
+public class RMAppsBlock extends AppsBlock {
+
+ private ResourceManager rm;
+
+ @Inject
+ RMAppsBlock(ResourceManager rm, ApplicationBaseProtocol appBaseProt,
+ View.ViewContext ctx) {
+ super(appBaseProt, ctx);
+ this.rm = rm;
+ }
+
+ @Override
+ protected void renderData(Block html) {
+ TBODY<TABLE<Hamlet>> tbody =
+ html.table("#apps").thead().tr().th(".id", "ID").th(".user", "User")
+ .th(".name", "Name").th(".type", "Application Type")
+ .th(".queue", "Queue").th(".starttime", "StartTime")
+ .th(".finishtime", "FinishTime").th(".state", "State")
+ .th(".finalstatus", "FinalStatus").th(".progress", "Progress")
+ .th(".ui", "Tracking UI").th(".blacklisted", "Blacklisted Nodes")._()
+ ._().tbody();
+
+ StringBuilder appsTableData = new StringBuilder("[\n");
+ for (ApplicationReport appReport : appReports) {
+ // TODO: remove the following condition. It is still here because
+ // the history side implementation of ApplicationBaseProtocol
+ // hasn't filtering capability (YARN-1819).
+ if (!reqAppStates.isEmpty()
+ && !reqAppStates.contains(appReport.getYarnApplicationState())) {
+ continue;
+ }
+
+ AppInfo app = new AppInfo(appReport);
+ String blacklistedNodesCount = "N/A";
+ Set<String> nodes =
+ RMAppAttemptBlock
+ .getBlacklistedNodes(rm, ConverterUtils.toApplicationAttemptId(app
+ .getCurrentAppAttemptId()));
+ if (nodes != null) {
+ blacklistedNodesCount = String.valueOf(nodes.size());
+ }
+ String percent = String.format("%.1f", app.getProgress());
+ // AppID numerical value parsed by parseHadoopID in yarn.dt.plugins.js
+ appsTableData
+ .append("[\"<a href='")
+ .append(url("app", app.getAppId()))
+ .append("'>")
+ .append(app.getAppId())
+ .append("</a>\",\"")
+ .append(
+ StringEscapeUtils.escapeJavaScript(StringEscapeUtils.escapeHtml(app
+ .getUser())))
+ .append("\",\"")
+ .append(
+ StringEscapeUtils.escapeJavaScript(StringEscapeUtils.escapeHtml(app
+ .getName())))
+ .append("\",\"")
+ .append(
+ StringEscapeUtils.escapeJavaScript(StringEscapeUtils.escapeHtml(app
+ .getType())))
+ .append("\",\"")
+ .append(
+ StringEscapeUtils.escapeJavaScript(StringEscapeUtils.escapeHtml(app
+ .getQueue()))).append("\",\"").append(app.getStartedTime())
+ .append("\",\"").append(app.getFinishedTime())
+ .append("\",\"")
+ .append(app.getAppState() == null ? UNAVAILABLE : app.getAppState())
+ .append("\",\"")
+ .append(app.getFinalAppStatus())
+ .append("\",\"")
+ // Progress bar
+ .append("<br title='").append(percent).append("'> <div class='")
+ .append(C_PROGRESSBAR).append("' title='").append(join(percent, '%'))
+ .append("'> ").append("<div class='").append(C_PROGRESSBAR_VALUE)
+ .append("' style='").append(join("width:", percent, '%'))
+ .append("'> </div> </div>").append("\",\"<a ");
+
+ String trackingURL =
+ app.getTrackingUrl() == null
+ || app.getTrackingUrl().equals(UNAVAILABLE) ? null : app
+ .getTrackingUrl();
+
+ String trackingUI =
+ app.getTrackingUrl() == null
+ || app.getTrackingUrl().equals(UNAVAILABLE) ? "Unassigned" : app
+ .getAppState() == YarnApplicationState.FINISHED
+ || app.getAppState() == YarnApplicationState.FAILED
+ || app.getAppState() == YarnApplicationState.KILLED ? "History"
+ : "ApplicationMaster";
+ appsTableData.append(trackingURL == null ? "#" : "href='" + trackingURL)
+ .append("'>").append(trackingUI).append("</a>\",").append("\"")
+ .append(blacklistedNodesCount).append("\"],\n");
+
+ }
+ if (appsTableData.charAt(appsTableData.length() - 2) == ',') {
+ appsTableData.delete(appsTableData.length() - 2,
+ appsTableData.length() - 1);
+ }
+ appsTableData.append("]");
+ html.script().$type("text/javascript")
+ ._("var appsTableData=" + appsTableData)._();
+
+ tbody._()._();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7af5d6b4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
index 27417a9..7746922 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
@@ -631,7 +631,8 @@ public class RMWebServices {
AppAttemptsInfo appAttemptsInfo = new AppAttemptsInfo();
for (RMAppAttempt attempt : app.getAppAttempts().values()) {
- AppAttemptInfo attemptInfo = new AppAttemptInfo(attempt, app.getUser());
+ AppAttemptInfo attemptInfo =
+ new AppAttemptInfo(rm, attempt, app.getUser());
appAttemptsInfo.add(attemptInfo);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7af5d6b4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java
index 0bb9cf7..2c7bbd0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java
@@ -21,8 +21,13 @@ import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
+import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMAppAttemptBlock;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
@@ -36,16 +41,18 @@ public class AppAttemptInfo {
protected String nodeHttpAddress;
protected String nodeId;
protected String logsLink;
+ protected String blacklistedNodes;
public AppAttemptInfo() {
}
- public AppAttemptInfo(RMAppAttempt attempt, String user) {
+ public AppAttemptInfo(ResourceManager rm, RMAppAttempt attempt, String user) {
this.startTime = 0;
this.containerId = "";
this.nodeHttpAddress = "";
this.nodeId = "";
this.logsLink = "";
+ this.blacklistedNodes = "";
if (attempt != null) {
this.id = attempt.getAppAttemptId().getAttemptId();
this.startTime = attempt.getStartTime();
@@ -57,6 +64,16 @@ public class AppAttemptInfo {
this.logsLink =
WebAppUtils.getRunningLogURL("//" + masterContainer.getNodeHttpAddress(),
ConverterUtils.toString(masterContainer.getId()), user);
+ if (rm.getResourceScheduler() instanceof AbstractYarnScheduler) {
+ AbstractYarnScheduler ayScheduler =
+ (AbstractYarnScheduler) rm.getResourceScheduler();
+ SchedulerApplicationAttempt sattempt =
+ ayScheduler.getApplicationAttempt(attempt.getAppAttemptId());
+ if (sattempt != null) {
+ blacklistedNodes =
+ StringUtils.join(sattempt.getBlacklistedNodes(), ", ");
+ }
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7af5d6b4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
index 23ea22e..3258d7e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
@@ -1599,7 +1599,7 @@ public class TestRMWebServicesApps extends JerseyTest {
String user)
throws JSONException, Exception {
- assertEquals("incorrect number of elements", 6, info.length());
+ assertEquals("incorrect number of elements", 7, info.length());
verifyAppAttemptInfoGeneric(appAttempt, info.getInt("id"),
info.getLong("startTime"), info.getString("containerId"),
[6/8] hadoop git commit: YARN-3544. Got back AM logs link on the RM
web UI for a completed app. Contributed by Xuan Gong.
Posted by vi...@apache.org.
YARN-3544. Got back AM logs link on the RM web UI for a completed app. Contributed by Xuan Gong.
(cherry picked from commit 21bf2cdcb77f69abc906e6cd401a8fb221f250e9)
(cherry picked from commit c9ee316045b83b18cb068aa4de739a1f4b50f02a)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d57c3f0a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d57c3f0a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d57c3f0a
Branch: refs/heads/branch-2.6.1
Commit: d57c3f0a2628f6f540a128e163ffdad4cccb1301
Parents: 7af5d6b
Author: Zhijie Shen <zj...@apache.org>
Authored: Wed Apr 29 17:12:52 2015 -0700
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Tue Sep 15 17:30:15 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../resourcemanager/webapp/RMAppBlock.java | 82 +++++++-------------
2 files changed, 30 insertions(+), 55 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d57c3f0a/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index c2c2bfa..17d888d 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -220,6 +220,9 @@ Release 2.6.1 - 2015-09-09
YARN-3248. Display count of nodes blacklisted by apps in the web UI.
(Varun Vasudev via xgong)
+ YARN-3544. Got back AM logs link on the RM web UI for a completed app.
+ (Xuan Gong via zjshen)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d57c3f0a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java
index 171c6c9..e7f2d1a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI._INFO_WRAP;
-import java.security.PrivilegedExceptionAction;
import java.util.Collection;
import java.util.Set;
@@ -29,36 +28,35 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerReport;
import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptMetrics;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptInfo;
import org.apache.hadoop.yarn.server.webapp.AppBlock;
-import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo;
-import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo;
-import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import org.apache.hadoop.yarn.webapp.view.InfoBlock;
import com.google.inject.Inject;
+//import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo;
public class RMAppBlock extends AppBlock{
private static final Log LOG = LogFactory.getLog(RMAppBlock.class);
private final ResourceManager rm;
+ private final Configuration conf;
@Inject
RMAppBlock(ViewContext ctx, Configuration conf, ResourceManager rm) {
super(rm.getClientRMService(), ctx, conf);
+ this.conf = conf;
this.rm = rm;
}
@@ -120,67 +118,41 @@ public class RMAppBlock extends AppBlock{
.th(".started", "Started").th(".node", "Node").th(".logs", "Logs")
.th(".blacklistednodes", "Blacklisted Nodes")._()._().tbody();
+ RMApp rmApp = this.rm.getRMContext().getRMApps().get(this.appID);
+ if (rmApp == null) {
+ return;
+ }
StringBuilder attemptsTableData = new StringBuilder("[\n");
for (final ApplicationAttemptReport appAttemptReport : attempts) {
- AppAttemptInfo appAttempt = new AppAttemptInfo(appAttemptReport);
- ContainerReport containerReport = null;
- try {
- // AM container is always the first container of the attempt
- final GetContainerReportRequest request =
- GetContainerReportRequest.newInstance(ContainerId.newContainerId(
- appAttemptReport.getApplicationAttemptId(), 1));
- if (callerUGI == null) {
- containerReport =
- appBaseProt.getContainerReport(request).getContainerReport();
- } else {
- containerReport = callerUGI.doAs(
- new PrivilegedExceptionAction<ContainerReport>() {
- @Override
- public ContainerReport run() throws Exception {
- ContainerReport report = null;
- try {
- report = appBaseProt.getContainerReport(request)
- .getContainerReport();
- } catch (ContainerNotFoundException ex) {
- LOG.warn(ex.getMessage());
- }
- return report;
- }
- });
- }
- } catch (Exception e) {
- String message =
- "Failed to read the AM container of the application attempt "
- + appAttemptReport.getApplicationAttemptId() + ".";
- LOG.error(message, e);
- html.p()._(message)._();
- return;
- }
- long startTime = 0L;
- String logsLink = null;
- String nodeLink = null;
- if (containerReport != null) {
- ContainerInfo container = new ContainerInfo(containerReport);
- startTime = container.getStartedTime();
- logsLink = containerReport.getLogUrl();
- nodeLink = containerReport.getNodeHttpAddress();
+ RMAppAttempt rmAppAttempt =
+ rmApp.getRMAppAttempt(appAttemptReport.getApplicationAttemptId());
+ if (rmAppAttempt == null) {
+ continue;
}
+ AppAttemptInfo attemptInfo =
+ new AppAttemptInfo(this.rm, rmAppAttempt, rmApp.getUser());
String blacklistedNodesCount = "N/A";
- Set<String> nodes = RMAppAttemptBlock.getBlacklistedNodes(rm,
- ConverterUtils.toApplicationAttemptId(appAttempt.getAppAttemptId()));
+ Set<String> nodes =
+ RMAppAttemptBlock.getBlacklistedNodes(rm,
+ rmAppAttempt.getAppAttemptId());
if(nodes != null) {
blacklistedNodesCount = String.valueOf(nodes.size());
}
// AppAttemptID numerical value parsed by parseHadoopID in
// yarn.dt.plugins.js
+ String nodeLink = attemptInfo.getNodeHttpAddress();
+ if (nodeLink != null) {
+ nodeLink = WebAppUtils.getHttpSchemePrefix(conf) + nodeLink;
+ }
+ String logsLink = attemptInfo.getLogsLink();
attemptsTableData
.append("[\"<a href='")
- .append(url("appattempt", appAttempt.getAppAttemptId()))
+ .append(url("appattempt", rmAppAttempt.getAppAttemptId().toString()))
.append("'>")
- .append(appAttempt.getAppAttemptId())
+ .append(String.valueOf(rmAppAttempt.getAppAttemptId()))
.append("</a>\",\"")
- .append(startTime)
+ .append(attemptInfo.getStartTime())
.append("\",\"<a ")
.append(nodeLink == null ? "#" : "href='" + nodeLink)
.append("'>")
[7/8] hadoop git commit: Revert "YARN-2890. MiniYARNCluster should
start the timeline server based on the configuration. Contributed by Mit
Desai."
Posted by vi...@apache.org.
Revert "YARN-2890. MiniYARNCluster should start the timeline server based on the configuration. Contributed by Mit Desai."
This reverts commit 8a47d1aa559c380869861c070af9d1e7a8732607.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9c4a6e12
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9c4a6e12
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9c4a6e12
Branch: refs/heads/branch-2.6.1
Commit: 9c4a6e127098bf0c337903d01d7ec902a7cc6084
Parents: d57c3f0
Author: Vinod Kumar Vavilapalli <vi...@apache.org>
Authored: Tue Sep 15 11:50:47 2015 -0700
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Tue Sep 15 17:30:15 2015 -0700
----------------------------------------------------------------------
.../jobhistory/TestJobHistoryEventHandler.java | 2 +-
.../mapred/TestMRTimelineEventHandling.java | 58 +-------------------
.../hadoop/mapreduce/v2/MiniMRYarnCluster.java | 6 +-
hadoop-yarn-project/CHANGES.txt | 3 -
.../distributedshell/TestDistributedShell.java | 2 +-
.../hadoop/yarn/client/ProtocolHATestBase.java | 6 +-
.../hadoop/yarn/server/MiniYARNCluster.java | 24 +++++++-
7 files changed, 34 insertions(+), 67 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c4a6e12/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
index de260c9..43e3dbe 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
@@ -453,7 +453,7 @@ public class TestJobHistoryEventHandler {
long currentTime = System.currentTimeMillis();
try {
yarnCluster = new MiniYARNCluster(
- TestJobHistoryEventHandler.class.getSimpleName(), 1, 1, 1, 1);
+ TestJobHistoryEventHandler.class.getSimpleName(), 1, 1, 1, 1, true);
yarnCluster.init(conf);
yarnCluster.start();
jheh.start();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c4a6e12/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
index 4ffbad7..c2ef128 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
@@ -35,58 +35,6 @@ import org.junit.Test;
public class TestMRTimelineEventHandling {
@Test
- public void testTimelineServiceStartInMiniCluster() throws Exception {
- Configuration conf = new YarnConfiguration();
-
- /*
- * Timeline service should not start if the config is set to false
- * Regardless to the value of MAPREDUCE_JOB_EMIT_TIMELINE_DATA
- */
- conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false);
- conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_EMIT_TIMELINE_DATA, true);
- MiniMRYarnCluster cluster = null;
- try {
- cluster = new MiniMRYarnCluster(
- TestJobHistoryEventHandler.class.getSimpleName(), 1);
- cluster.init(conf);
- cluster.start();
-
- //verify that the timeline service is not started.
- Assert.assertNull("Timeline Service should not have been started",
- cluster.getApplicationHistoryServer());
-
- //Run a MR job and verify it succeeds
- Path inDir = new Path("input");
- Path outDir = new Path("output");
- RunningJob job =
- UtilsForTests.runJobSucceed(new JobConf(conf), inDir, outDir);
- Assert.assertEquals(JobStatus.SUCCEEDED,
- job.getJobStatus().getState().getValue());
- }
- finally {
- if(cluster != null) {
- cluster.stop();
- }
- }
- conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false);
- conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_EMIT_TIMELINE_DATA, false);
- cluster = null;
- try {
- cluster = new MiniMRYarnCluster(
- TestJobHistoryEventHandler.class.getSimpleName(), 1);
- cluster.init(conf);
- cluster.start();
- Assert.assertNull("Timeline Service should not have been started",
- cluster.getApplicationHistoryServer());
- }
- finally {
- if(cluster != null) {
- cluster.stop();
- }
- }
- }
-
- @Test
public void testMRTimelineEventHandling() throws Exception {
Configuration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
@@ -94,7 +42,7 @@ public class TestMRTimelineEventHandling {
MiniMRYarnCluster cluster = null;
try {
cluster = new MiniMRYarnCluster(
- TestJobHistoryEventHandler.class.getSimpleName(), 1);
+ TestJobHistoryEventHandler.class.getSimpleName(), 1, true);
cluster.init(conf);
cluster.start();
TimelineStore ts = cluster.getApplicationHistoryServer()
@@ -148,7 +96,7 @@ public class TestMRTimelineEventHandling {
MiniMRYarnCluster cluster = null;
try {
cluster = new MiniMRYarnCluster(
- TestJobHistoryEventHandler.class.getSimpleName(), 1);
+ TestJobHistoryEventHandler.class.getSimpleName(), 1, true);
cluster.init(conf);
cluster.start();
TimelineStore ts = cluster.getApplicationHistoryServer()
@@ -185,7 +133,7 @@ public class TestMRTimelineEventHandling {
cluster = null;
try {
cluster = new MiniMRYarnCluster(
- TestJobHistoryEventHandler.class.getSimpleName(), 1);
+ TestJobHistoryEventHandler.class.getSimpleName(), 1, true);
cluster.init(conf);
cluster.start();
TimelineStore ts = cluster.getApplicationHistoryServer()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c4a6e12/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
index 45b736e..47b38a1 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
@@ -72,7 +72,11 @@ public class MiniMRYarnCluster extends MiniYARNCluster {
}
public MiniMRYarnCluster(String testName, int noOfNMs) {
- super(testName, 1, noOfNMs, 4, 4);
+ this(testName, noOfNMs, false);
+ }
+
+ public MiniMRYarnCluster(String testName, int noOfNMs, boolean enableAHS) {
+ super(testName, 1, noOfNMs, 4, 4, enableAHS);
historyServerWrapper = new JobHistoryServerWrapper();
addService(historyServerWrapper);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c4a6e12/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 17d888d..6ef7c62 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -49,9 +49,6 @@ Release 2.6.1 - 2015-09-09
YARN-2905. AggregatedLogsBlock page can infinitely loop if the aggregated
log file is corrupted (Varun Saxena via jlowe)
- YARN-2890. MiniYARNCluster should start the timeline server based on the
- configuration. (Mit Desai via zjshen)
-
YARN-2894. Fixed a bug regarding application view acl when RM fails over.
(Rohith Sharmaks via jianhe)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c4a6e12/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
index 904ad58..17c29ec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
@@ -84,7 +84,7 @@ public class TestDistributedShell {
if (yarnCluster == null) {
yarnCluster =
new MiniYARNCluster(TestDistributedShell.class.getSimpleName(), 1,
- numNodeManager, 1, 1);
+ numNodeManager, 1, 1, true);
yarnCluster.init(conf);
yarnCluster.start();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c4a6e12/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
index ea9e361..1cc43b0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
@@ -294,7 +294,7 @@ public abstract class ProtocolHATestBase extends ClientBaseWithFixes {
conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
cluster =
new MiniYARNClusterForHATesting(TestRMFailover.class.getName(), 2,
- numOfNMs, 1, 1, overrideClientRMService, overrideRTS,
+ numOfNMs, 1, 1, false, overrideClientRMService, overrideRTS,
overrideApplicationMasterService);
cluster.resetStartFailoverFlag(false);
cluster.init(conf);
@@ -325,10 +325,10 @@ public abstract class ProtocolHATestBase extends ClientBaseWithFixes {
public MiniYARNClusterForHATesting(String testName,
int numResourceManagers, int numNodeManagers, int numLocalDirs,
- int numLogDirs, boolean overrideClientRMService,
+ int numLogDirs, boolean enableAHS, boolean overrideClientRMService,
boolean overrideRTS, boolean overrideApplicationMasterService) {
super(testName, numResourceManagers, numNodeManagers, numLocalDirs,
- numLogDirs);
+ numLogDirs, enableAHS);
this.overrideClientRMService = overrideClientRMService;
this.overrideRTS = overrideRTS;
this.overrideApplicationMasterService = overrideApplicationMasterService;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c4a6e12/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
index 2eb5034..e83d601 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
@@ -57,6 +57,7 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerResp
import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer;
import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryStore;
import org.apache.hadoop.yarn.server.applicationhistoryservice.MemoryApplicationHistoryStore;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.NodeHealthCheckerService;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
@@ -119,6 +120,7 @@ public class MiniYARNCluster extends CompositeService {
private int numLocalDirs;
// Number of nm-log-dirs per nodemanager
private int numLogDirs;
+ private boolean enableAHS;
/**
* @param testName name of the test
@@ -126,13 +128,15 @@ public class MiniYARNCluster extends CompositeService {
* @param numNodeManagers the number of node managers in the cluster
* @param numLocalDirs the number of nm-local-dirs per nodemanager
* @param numLogDirs the number of nm-log-dirs per nodemanager
+ * @param enableAHS enable ApplicationHistoryServer or not
*/
public MiniYARNCluster(
String testName, int numResourceManagers, int numNodeManagers,
- int numLocalDirs, int numLogDirs) {
+ int numLocalDirs, int numLogDirs, boolean enableAHS) {
super(testName.replace("$", ""));
this.numLocalDirs = numLocalDirs;
this.numLogDirs = numLogDirs;
+ this.enableAHS = enableAHS;
String testSubDir = testName.replace("$", "");
File targetWorkDir = new File("target", testSubDir);
try {
@@ -184,6 +188,20 @@ public class MiniYARNCluster extends CompositeService {
/**
* @param testName name of the test
+ * @param numResourceManagers the number of resource managers in the cluster
+ * @param numNodeManagers the number of node managers in the cluster
+ * @param numLocalDirs the number of nm-local-dirs per nodemanager
+ * @param numLogDirs the number of nm-log-dirs per nodemanager
+ */
+ public MiniYARNCluster(
+ String testName, int numResourceManagers, int numNodeManagers,
+ int numLocalDirs, int numLogDirs) {
+ this(testName, numResourceManagers, numNodeManagers, numLocalDirs,
+ numLogDirs, false);
+ }
+
+ /**
+ * @param testName name of the test
* @param numNodeManagers the number of node managers in the cluster
* @param numLocalDirs the number of nm-local-dirs per nodemanager
* @param numLogDirs the number of nm-log-dirs per nodemanager
@@ -242,8 +260,8 @@ public class MiniYARNCluster extends CompositeService {
addService(new NodeManagerWrapper(index));
}
- if(conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false)) {
- addService(new ApplicationHistoryServerWrapper());
+ if (enableAHS) {
+ addService(new ApplicationHistoryServerWrapper());
}
super.serviceInit(
[8/8] hadoop git commit: YARN-2890. MiniYarnCluster should turn on
timeline service if configured to do so. Contributed by Mit Desai.
Posted by vi...@apache.org.
YARN-2890. MiniYarnCluster should turn on timeline service if configured to do so. Contributed by Mit Desai.
(cherry picked from commit 265ed1fe804743601a8b62cabc1e4dc2ec8e502f)
(cherry picked from commit 55b794e7fa205df655c19bbfe1de99091fa9dc64)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dba2b60f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dba2b60f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dba2b60f
Branch: refs/heads/branch-2.6.1
Commit: dba2b60fdd83aa501e158f42abc75d5ce9ad3cee
Parents: 9c4a6e1
Author: Hitesh Shah <hi...@apache.org>
Authored: Wed Apr 8 14:13:10 2015 -0700
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Tue Sep 15 17:30:15 2015 -0700
----------------------------------------------------------------------
.../jobhistory/TestJobHistoryEventHandler.java | 2 +-
.../mapred/TestMRTimelineEventHandling.java | 52 ++++++++-
hadoop-yarn-project/CHANGES.txt | 3 +
.../distributedshell/TestDistributedShell.java | 2 +-
.../hadoop/yarn/server/MiniYARNCluster.java | 6 +-
.../hadoop/yarn/server/TestMiniYarnCluster.java | 115 +++++++++++++++++++
6 files changed, 172 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dba2b60f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
index 43e3dbe..de260c9 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
@@ -453,7 +453,7 @@ public class TestJobHistoryEventHandler {
long currentTime = System.currentTimeMillis();
try {
yarnCluster = new MiniYARNCluster(
- TestJobHistoryEventHandler.class.getSimpleName(), 1, 1, 1, 1, true);
+ TestJobHistoryEventHandler.class.getSimpleName(), 1, 1, 1, 1);
yarnCluster.init(conf);
yarnCluster.start();
jheh.start();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dba2b60f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
index c2ef128..eab9026 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
@@ -35,6 +35,52 @@ import org.junit.Test;
public class TestMRTimelineEventHandling {
@Test
+ public void testTimelineServiceStartInMiniCluster() throws Exception {
+ Configuration conf = new YarnConfiguration();
+
+ /*
+ * Timeline service should not start if the config is set to false
+ * Regardless to the value of MAPREDUCE_JOB_EMIT_TIMELINE_DATA
+ */
+ conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false);
+ conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_EMIT_TIMELINE_DATA, true);
+ MiniMRYarnCluster cluster = null;
+ try {
+ cluster = new MiniMRYarnCluster(
+ TestJobHistoryEventHandler.class.getSimpleName(), 1);
+ cluster.init(conf);
+ cluster.start();
+
+ //verify that the timeline service is not started.
+ Assert.assertNull("Timeline Service should not have been started",
+ cluster.getApplicationHistoryServer());
+ }
+ finally {
+ if(cluster != null) {
+ cluster.stop();
+ }
+ }
+ conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false);
+ conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_EMIT_TIMELINE_DATA, false);
+ cluster = null;
+ try {
+ cluster = new MiniMRYarnCluster(
+ TestJobHistoryEventHandler.class.getSimpleName(), 1);
+ cluster.init(conf);
+ cluster.start();
+
+ //verify that the timeline service is not started.
+ Assert.assertNull("Timeline Service should not have been started",
+ cluster.getApplicationHistoryServer());
+ }
+ finally {
+ if(cluster != null) {
+ cluster.stop();
+ }
+ }
+ }
+
+ @Test
public void testMRTimelineEventHandling() throws Exception {
Configuration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
@@ -42,7 +88,7 @@ public class TestMRTimelineEventHandling {
MiniMRYarnCluster cluster = null;
try {
cluster = new MiniMRYarnCluster(
- TestJobHistoryEventHandler.class.getSimpleName(), 1, true);
+ TestJobHistoryEventHandler.class.getSimpleName(), 1);
cluster.init(conf);
cluster.start();
TimelineStore ts = cluster.getApplicationHistoryServer()
@@ -96,7 +142,7 @@ public class TestMRTimelineEventHandling {
MiniMRYarnCluster cluster = null;
try {
cluster = new MiniMRYarnCluster(
- TestJobHistoryEventHandler.class.getSimpleName(), 1, true);
+ TestJobHistoryEventHandler.class.getSimpleName(), 1);
cluster.init(conf);
cluster.start();
TimelineStore ts = cluster.getApplicationHistoryServer()
@@ -133,7 +179,7 @@ public class TestMRTimelineEventHandling {
cluster = null;
try {
cluster = new MiniMRYarnCluster(
- TestJobHistoryEventHandler.class.getSimpleName(), 1, true);
+ TestJobHistoryEventHandler.class.getSimpleName(), 1);
cluster.init(conf);
cluster.start();
TimelineStore ts = cluster.getApplicationHistoryServer()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dba2b60f/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 6ef7c62..6c72dd7 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -220,6 +220,9 @@ Release 2.6.1 - 2015-09-09
YARN-3544. Got back AM logs link on the RM web UI for a completed app.
(Xuan Gong via zjshen)
+ YARN-2890. MiniYarnCluster should turn on timeline service if
+ configured to do so. (Mit Desai via hitesh)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dba2b60f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
index 17c29ec..904ad58 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
@@ -84,7 +84,7 @@ public class TestDistributedShell {
if (yarnCluster == null) {
yarnCluster =
new MiniYARNCluster(TestDistributedShell.class.getSimpleName(), 1,
- numNodeManager, 1, 1, true);
+ numNodeManager, 1, 1);
yarnCluster.init(conf);
yarnCluster.start();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dba2b60f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
index e83d601..cdaa9f2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
@@ -57,7 +57,6 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerResp
import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer;
import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryStore;
import org.apache.hadoop.yarn.server.applicationhistoryservice.MemoryApplicationHistoryStore;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.NodeHealthCheckerService;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
@@ -260,8 +259,9 @@ public class MiniYARNCluster extends CompositeService {
addService(new NodeManagerWrapper(index));
}
- if (enableAHS) {
- addService(new ApplicationHistoryServerWrapper());
+ if(conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
+ YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED) || enableAHS) {
+ addService(new ApplicationHistoryServerWrapper());
}
super.serviceInit(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dba2b60f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnCluster.java
new file mode 100644
index 0000000..8a3c9e7
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnCluster.java
@@ -0,0 +1,115 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestMiniYarnCluster {
+ @Test
+ public void testTimelineServiceStartInMiniCluster() throws Exception {
+ Configuration conf = new YarnConfiguration();
+ int numNodeManagers = 1;
+ int numLocalDirs = 1;
+ int numLogDirs = 1;
+ boolean enableAHS;
+
+ /*
+ * Timeline service should not start if TIMELINE_SERVICE_ENABLED == false
+ * and enableAHS flag == false
+ */
+ conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false);
+ enableAHS = false;
+ MiniYARNCluster cluster = null;
+ try {
+ cluster = new MiniYARNCluster(TestMiniYarnCluster.class.getSimpleName(),
+ numNodeManagers, numLocalDirs, numLogDirs, numLogDirs, enableAHS);
+ cluster.init(conf);
+ cluster.start();
+
+ //verify that the timeline service is not started.
+ Assert.assertNull("Timeline Service should not have been started",
+ cluster.getApplicationHistoryServer());
+ }
+ finally {
+ if(cluster != null) {
+ cluster.stop();
+ }
+ }
+
+ /*
+ * Timeline service should start if TIMELINE_SERVICE_ENABLED == true
+ * and enableAHS == false
+ */
+ conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
+ enableAHS = false;
+ cluster = null;
+ try {
+ cluster = new MiniYARNCluster(TestMiniYarnCluster.class.getSimpleName(),
+ numNodeManagers, numLocalDirs, numLogDirs, numLogDirs, enableAHS);
+ cluster.init(conf);
+ cluster.start();
+
+ //Timeline service may sometime take a while to get started
+ int wait = 0;
+ while(cluster.getApplicationHistoryServer() == null && wait < 20) {
+ Thread.sleep(500);
+ wait++;
+ }
+ //verify that the timeline service is started.
+ Assert.assertNotNull("Timeline Service should have been started",
+ cluster.getApplicationHistoryServer());
+ }
+ finally {
+ if(cluster != null) {
+ cluster.stop();
+ }
+ }
+ /*
+ * Timeline service should start if TIMELINE_SERVICE_ENABLED == false
+ * and enableAHS == true
+ */
+ conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false);
+ enableAHS = true;
+ cluster = null;
+ try {
+ cluster = new MiniYARNCluster(TestMiniYarnCluster.class.getSimpleName(),
+ numNodeManagers, numLocalDirs, numLogDirs, numLogDirs, enableAHS);
+ cluster.init(conf);
+ cluster.start();
+
+ //Timeline service may sometime take a while to get started
+ int wait = 0;
+ while(cluster.getApplicationHistoryServer() == null && wait < 20) {
+ Thread.sleep(500);
+ wait++;
+ }
+ //verify that the timeline service is started.
+ Assert.assertNotNull("Timeline Service should have been started",
+ cluster.getApplicationHistoryServer());
+ }
+ finally {
+ if(cluster != null) {
+ cluster.stop();
+ }
+ }
+ }
+}
[2/8] hadoop git commit: YARN-3171. Sort by Application id,
AppAttempt and ContainerID doesn't work in ATS / RM web ui.
Contributed by Naganarasimha G R
Posted by vi...@apache.org.
YARN-3171. Sort by Application id, AppAttempt and ContainerID doesn't
work in ATS / RM web ui. Contributed by Naganarasimha G R
(cherry picked from commit 3ff1ba2a7b00fdf06270d00b2193bde4b56b06b3)
(cherry picked from commit b6eb36dbdc1c75f412616426cc7599f485e9b86e)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/85ec6eb3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/85ec6eb3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/85ec6eb3
Branch: refs/heads/branch-2.6.1
Commit: 85ec6eb37a9a62f62a5c2b7840f49ea8b0bae31c
Parents: f4154bd
Author: Xuan <xg...@apache.org>
Authored: Sun Mar 15 20:26:10 2015 -0700
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Fri Sep 11 11:49:11 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../org/apache/hadoop/yarn/server/webapp/WebPageUtils.java | 6 +++---
2 files changed, 6 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85ec6eb3/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index c82c892..cab1f0c 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -208,6 +208,9 @@ Release 2.6.1 - 2015-09-09
YARN-1884. Added nodeHttpAddress into ContainerReport and fixed the link to NM
web page. (Xuan Gong via zjshen)
+ YARN-3171. Sort by Application id, AppAttempt and ContainerID doesn't work
+ in ATS / RM web ui. (Naganarasimha G R via xgong)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85ec6eb3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java
index 384a976..5acabf5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java
@@ -44,7 +44,7 @@ public class WebPageUtils {
StringBuilder sb = new StringBuilder();
return sb
.append("[\n")
- .append("{'sType':'numeric', 'aTargets': [0]")
+ .append("{'sType':'string', 'aTargets': [0]")
.append(", 'mRender': parseHadoopID }")
.append("\n, {'sType':'numeric', 'aTargets': " +
(isFairSchedulerPage ? "[6, 7]": "[5, 6]"))
@@ -63,7 +63,7 @@ public class WebPageUtils {
private static String getAttemptsTableColumnDefs() {
StringBuilder sb = new StringBuilder();
- return sb.append("[\n").append("{'sType':'numeric', 'aTargets': [0]")
+ return sb.append("[\n").append("{'sType':'string', 'aTargets': [0]")
.append(", 'mRender': parseHadoopID }")
.append("\n, {'sType':'numeric', 'aTargets': [1]")
.append(", 'mRender': renderHadoopDate }]").toString();
@@ -79,7 +79,7 @@ public class WebPageUtils {
private static String getContainersTableColumnDefs() {
StringBuilder sb = new StringBuilder();
- return sb.append("[\n").append("{'sType':'numeric', 'aTargets': [0]")
+ return sb.append("[\n").append("{'sType':'string', 'aTargets': [0]")
.append(", 'mRender': parseHadoopID }]").toString();
}