You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2014/12/04 21:53:37 UTC
hadoop git commit: YARN-2301. Improved yarn container command.
Contributed by Naganarasimha G R
Repository: hadoop
Updated Branches:
refs/heads/trunk 565b0e60a -> 258623ff8
YARN-2301. Improved yarn container command. Contributed by Naganarasimha G R
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/258623ff
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/258623ff
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/258623ff
Branch: refs/heads/trunk
Commit: 258623ff8bb1a1057ae3501d4f20982d5a59ea34
Parents: 565b0e6
Author: Jian He <ji...@apache.org>
Authored: Thu Dec 4 12:51:15 2014 -0800
Committer: Jian He <ji...@apache.org>
Committed: Thu Dec 4 12:53:18 2014 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 2 +
.../hadoop/yarn/client/cli/ApplicationCLI.java | 8 +++-
.../hadoop/yarn/client/cli/TestYarnCLI.java | 41 +++++++++++++-------
.../yarn/server/resourcemanager/RMContext.java | 3 ++
.../server/resourcemanager/RMContextImpl.java | 11 ++++++
.../server/resourcemanager/ResourceManager.java | 2 +
.../rmcontainer/RMContainerImpl.java | 9 ++++-
.../resourcemanager/TestClientRMService.java | 1 +
.../rmcontainer/TestRMContainerImpl.java | 6 ++-
9 files changed, 63 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/258623ff/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 30b9260..f032b4f 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -121,6 +121,8 @@ Release 2.7.0 - UNRELEASED
YARN-1156. Enhance NodeManager AllocatedGB and AvailableGB metrics
for aggregation of decimal values. (Tsuyoshi OZAWA via junping_du)
+ YARN-2301. Improved yarn container command. (Naganarasimha G R via jianhe)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/258623ff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
index a847cd5..83d212d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
@@ -44,6 +44,7 @@ import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.hadoop.yarn.util.Times;
import com.google.common.annotations.VisibleForTesting;
@@ -536,8 +537,11 @@ public class ApplicationCLI extends YarnCLI {
writer.printf(CONTAINER_PATTERN, "Container-Id", "Start Time",
"Finish Time", "State", "Host", "LOG-URL");
for (ContainerReport containerReport : appsReport) {
- writer.printf(CONTAINER_PATTERN, containerReport.getContainerId(),
- containerReport.getCreationTime(), containerReport.getFinishTime(),
+ writer.printf(
+ CONTAINER_PATTERN,
+ containerReport.getContainerId(),
+ Times.format(containerReport.getCreationTime()),
+ Times.format(containerReport.getFinishTime()),
containerReport.getContainerState(), containerReport
.getAssignedNode(), containerReport.getLogUrl());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/258623ff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
index 9d9a86a..194d7d1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
@@ -32,19 +32,17 @@ import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.PrintStream;
import java.io.PrintWriter;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.EnumSet;
-import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
-import java.util.Map;
import java.util.Set;
import org.apache.commons.cli.Options;
import org.apache.commons.lang.time.DateFormatUtils;
-import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -65,9 +63,7 @@ import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
-import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
import org.apache.hadoop.yarn.util.Records;
-import org.jboss.netty.logging.CommonsLoggerFactory;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -257,25 +253,31 @@ public class TestYarnCLI {
applicationId, 1);
ContainerId containerId = ContainerId.newContainerId(attemptId, 1);
ContainerId containerId1 = ContainerId.newContainerId(attemptId, 2);
+ ContainerId containerId2 = ContainerId.newContainerId(attemptId, 3);
+ long time1=1234,time2=5678;
ContainerReport container = ContainerReport.newInstance(containerId, null,
- NodeId.newInstance("host", 1234), Priority.UNDEFINED, 1234, 5678,
+ NodeId.newInstance("host", 1234), Priority.UNDEFINED, time1, time2,
"diagnosticInfo", "logURL", 0, ContainerState.COMPLETE);
ContainerReport container1 = ContainerReport.newInstance(containerId1, null,
- NodeId.newInstance("host", 1234), Priority.UNDEFINED, 1234, 5678,
+ NodeId.newInstance("host", 1234), Priority.UNDEFINED, time1, time2,
"diagnosticInfo", "logURL", 0, ContainerState.COMPLETE);
+ ContainerReport container2 = ContainerReport.newInstance(containerId2, null,
+ NodeId.newInstance("host", 1234), Priority.UNDEFINED, time1,0,
+ "diagnosticInfo", "", 0, ContainerState.RUNNING);
List<ContainerReport> reports = new ArrayList<ContainerReport>();
reports.add(container);
reports.add(container1);
+ reports.add(container2);
+ DateFormat dateFormat=new SimpleDateFormat("EEE MMM dd HH:mm:ss Z yyyy");
when(client.getContainers(any(ApplicationAttemptId.class))).thenReturn(
reports);
int result = cli.run(new String[] { "container", "-list",
attemptId.toString() });
assertEquals(0, result);
verify(client).getContainers(attemptId);
- Log.info(sysOutStream.toString());
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintWriter pw = new PrintWriter(baos);
- pw.println("Total number of containers :2");
+ pw.println("Total number of containers :3");
pw.print(" Container-Id");
pw.print("\t Start Time");
pw.print("\t Finish Time");
@@ -283,19 +285,30 @@ public class TestYarnCLI {
pw.print("\t Host");
pw.println("\t LOG-URL");
pw.print(" container_1234_0005_01_000001");
- pw.print("\t 1234");
- pw.print("\t 5678");
+ pw.print("\t"+dateFormat.format(new Date(time1)));
+ pw.print("\t"+dateFormat.format(new Date(time2)));
pw.print("\t COMPLETE");
pw.print("\t host:1234");
pw.println("\t logURL");
pw.print(" container_1234_0005_01_000002");
- pw.print("\t 1234");
- pw.print("\t 5678");
+ pw.print("\t"+dateFormat.format(new Date(time1)));
+ pw.print("\t"+dateFormat.format(new Date(time2)));
pw.print("\t COMPLETE");
pw.print("\t host:1234");
pw.println("\t logURL");
+ pw.print(" container_1234_0005_01_000003");
+ pw.print("\t"+dateFormat.format(new Date(time1)));
+ pw.print("\t N/A");
+ pw.print("\t RUNNING");
+ pw.print("\t host:1234");
+ pw.println("\t ");
pw.close();
String appReportStr = baos.toString("UTF-8");
+ Log.info("ExpectedOutput");
+ Log.info("["+appReportStr+"]");
+ Log.info("OutputFrom command");
+ String actualOutput = sysOutStream.toString();
+ Log.info("["+actualOutput+"]");
Assert.assertEquals(appReportStr, sysOutStream.toString());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/258623ff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
index 56984e6..ecf6166 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager;
import java.nio.ByteBuffer;
import java.util.concurrent.ConcurrentMap;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.NodeId;
@@ -121,4 +122,6 @@ public interface RMContext {
ReservationSystem getReservationSystem();
boolean isSchedulerReadyForAllocatingContainers();
+
+ Configuration getYarnConfiguration();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/258623ff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
index 55d7667..ebf2fe4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
@@ -22,6 +22,7 @@ import java.nio.ByteBuffer;
import java.util.concurrent.ConcurrentMap;
import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.ha.HAServiceProtocol;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
@@ -65,6 +66,8 @@ public class RMContextImpl implements RMContext {
private RMActiveServiceContext activeServiceContext;
+ private Configuration yarnConfiguration;
+
/**
* Default constructor. To be used in conjunction with setter methods for
* individual fields.
@@ -399,4 +402,12 @@ public class RMContextImpl implements RMContext {
this.activeServiceContext = activeServiceContext;
}
+ @Override
+ public Configuration getYarnConfiguration() {
+ return this.yarnConfiguration;
+ }
+
+ public void setYarnConfiguration(Configuration yarnConfiguration) {
+ this.yarnConfiguration=yarnConfiguration;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/258623ff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index 9662c39..775d08a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -246,6 +246,8 @@ public class ResourceManager extends CompositeService implements Recoverable {
addService(adminService);
rmContext.setRMAdminService(adminService);
+ rmContext.setYarnConfiguration(conf);
+
createAndInitActiveServices();
webAppAddress = WebAppUtils.getWebAppBindURL(this.conf,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/258623ff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
index 479734a..e37d8fd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
@@ -285,8 +285,13 @@ public class RMContainerImpl implements RMContainer {
public String getLogURL() {
try {
readLock.lock();
- return WebAppUtils.getRunningLogURL("//" + container.getNodeHttpAddress(),
- ConverterUtils.toString(containerId), user);
+ StringBuilder logURL = new StringBuilder();
+ logURL.append(WebAppUtils.getHttpSchemePrefix(rmContext
+ .getYarnConfiguration()));
+ logURL.append(WebAppUtils.getRunningLogURL(
+ container.getNodeHttpAddress(), ConverterUtils.toString(containerId),
+ user));
+ return logURL.toString();
} finally {
readLock.unlock();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/258623ff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
index 15aca42..a344e9a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
@@ -443,6 +443,7 @@ public class TestClientRMService {
ConcurrentHashMap<ApplicationId, RMApp> apps = getRMApps(rmContext,
yarnScheduler);
when(rmContext.getRMApps()).thenReturn(apps);
+ when(rmContext.getYarnConfiguration()).thenReturn(new Configuration());
RMAppManager appManager = new RMAppManager(rmContext, yarnScheduler, null,
mock(ApplicationACLsManager.class), new Configuration());
when(rmContext.getDispatcher().getEventHandler()).thenReturn(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/258623ff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
index 76cdcae..21aba3b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
@@ -107,6 +107,7 @@ public class TestRMContainerImpl {
when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer);
when(rmContext.getRMApps()).thenReturn(rmApps);
when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher);
+ when(rmContext.getYarnConfiguration()).thenReturn(new YarnConfiguration());
RMContainer rmContainer = new RMContainerImpl(container, appAttemptId,
nodeId, "user", rmContext);
@@ -130,7 +131,7 @@ public class TestRMContainerImpl {
RMContainerEventType.LAUNCHED));
drainDispatcher.await();
assertEquals(RMContainerState.RUNNING, rmContainer.getState());
- assertEquals("//host:3465/node/containerlogs/container_1_0001_01_000001/user",
+ assertEquals("http://host:3465/node/containerlogs/container_1_0001_01_000001/user",
rmContainer.getLogURL());
// In RUNNING state. Verify RELEASED and associated actions.
@@ -196,6 +197,7 @@ public class TestRMContainerImpl {
when(rmContext.getContainerAllocationExpirer()).thenReturn(expirer);
when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer);
when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher);
+ when(rmContext.getYarnConfiguration()).thenReturn(new YarnConfiguration());
RMContainer rmContainer = new RMContainerImpl(container, appAttemptId,
nodeId, "user", rmContext);
@@ -220,7 +222,7 @@ public class TestRMContainerImpl {
RMContainerEventType.LAUNCHED));
drainDispatcher.await();
assertEquals(RMContainerState.RUNNING, rmContainer.getState());
- assertEquals("//host:3465/node/containerlogs/container_1_0001_01_000001/user",
+ assertEquals("http://host:3465/node/containerlogs/container_1_0001_01_000001/user",
rmContainer.getLogURL());
// In RUNNING state. Verify EXPIRE and associated actions.