You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by zj...@apache.org on 2014/01/17 04:12:48 UTC
svn commit: r1559001 - in
/hadoop/common/branches/YARN-321/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/
hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/
hadoop-yarn/h...
Author: zjshen
Date: Fri Jan 17 03:12:47 2014
New Revision: 1559001
URL: http://svn.apache.org/r1559001
Log:
YARN-1595. Made enabling history service configurable and fixed test failures on branch YARN-321. Contributed by Vinod Kumar Vavilapalli.
Modified:
hadoop/common/branches/YARN-321/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestMemoryApplicationHistoryStore.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/CHANGES.txt?rev=1559001&r1=1559000&r2=1559001&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/CHANGES.txt Fri Jan 17 03:12:47 2014
@@ -84,6 +84,9 @@ Branch YARN-321: Generic ApplicationHist
YARN-1597. Fixed Findbugs warnings on branch YARN-321. (Vinod Kumar Vavilapalli
via zjshen)
+ YARN-1595. Made enabling history service configurable and fixed test failures on
+ branch YARN-321. (Vinod Kumar Vavilapalli via zjshen)
+
Trunk - Unreleased
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1559001&r1=1559000&r2=1559001&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java Fri Jan 17 03:12:47 2014
@@ -263,11 +263,6 @@ public class YarnConfiguration extends C
RM_PREFIX + "nodemanagers.heartbeat-interval-ms";
public static final long DEFAULT_RM_NM_HEARTBEAT_INTERVAL_MS = 1000;
- /** The setting that controls whether RM writes history data. */
- public static final String RM_HISTORY_WRITER_ENABLED = RM_PREFIX
- + "history-writer.enabled";
- public static final boolean DEFAULT_RM_HISTORY_WRITER_ENABLED = false;
-
/** Number of worker threads that write the history data. */
public static final String RM_HISTORY_WRITER_MULTI_THREADED_DISPATCHER_POOL_SIZE =
RM_PREFIX + "history-writer.multi-threaded-dispatcher.pool-size";
@@ -950,6 +945,11 @@ public class YarnConfiguration extends C
public static final String AHS_PREFIX = YARN_PREFIX + "ahs.";
+ /** The setting that controls whether history-service is enabled or not.. */
+ public static final String YARN_HISTORY_SERVICE_ENABLED = AHS_PREFIX
+ + ".enabled";
+ public static final boolean DEFAULT_YARN_HISTORY_SERVICE_ENABLED = false;
+
/** URI for FileSystemApplicationHistoryStore */
public static final String FS_HISTORY_STORE_URI = AHS_PREFIX + "fs-history-store.uri";
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java?rev=1559001&r1=1559000&r2=1559001&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java Fri Jan 17 03:12:47 2014
@@ -87,6 +87,7 @@ public class YarnClientImpl extends Yarn
protected long submitPollIntervalMillis;
private long asyncApiPollIntervalMillis;
protected AHSClient historyClient;
+ private boolean historyServiceEnabled;
private static final String ROOT = "root";
@@ -107,8 +108,14 @@ public class YarnClientImpl extends Yarn
YarnConfiguration.YARN_CLIENT_APP_SUBMISSION_POLL_INTERVAL_MS,
YarnConfiguration.DEFAULT_YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_INTERVAL_MS);
}
- historyClient = AHSClientImpl.createAHSClient();
- historyClient.init(getConfig());
+
+ if (conf.getBoolean(YarnConfiguration.YARN_HISTORY_SERVICE_ENABLED,
+ YarnConfiguration.DEFAULT_YARN_HISTORY_SERVICE_ENABLED)) {
+ historyServiceEnabled = true;
+ historyClient = AHSClientImpl.createAHSClient();
+ historyClient.init(getConfig());
+ }
+
super.serviceInit(conf);
}
@@ -117,7 +124,9 @@ public class YarnClientImpl extends Yarn
try {
rmClient = ClientRMProxy.createRMProxy(getConfig(),
ApplicationClientProtocol.class);
- historyClient.start();
+ if (historyServiceEnabled) {
+ historyClient.start();
+ }
} catch (IOException e) {
throw new YarnRuntimeException(e);
}
@@ -129,7 +138,9 @@ public class YarnClientImpl extends Yarn
if (this.rmClient != null) {
RPC.stopProxy(this.rmClient);
}
- historyClient.stop();
+ if (historyServiceEnabled) {
+ historyClient.stop();
+ }
super.serviceStop();
}
@@ -225,11 +236,18 @@ public class YarnClientImpl extends Yarn
request.setApplicationId(appId);
response = rmClient.getApplicationReport(request);
} catch (YarnException e) {
+
+ if (!historyServiceEnabled) {
+ // Just throw it as usual if historyService is not enabled.
+ throw e;
+ }
+
+ // Even if history-service is enabled, treat all exceptions still the same
+ // except the following
if (!(e.getClass() == ApplicationNotFoundException.class)) {
throw e;
}
- }
- if (response == null || response.getApplicationReport() == null) {
+
return historyClient.getApplicationReport(appId);
}
return response.getApplicationReport();
@@ -397,25 +415,37 @@ public class YarnClientImpl extends Yarn
@Override
public ApplicationAttemptReport getApplicationAttemptReport(
ApplicationAttemptId appAttemptId) throws YarnException, IOException {
- return historyClient.getApplicationAttemptReport(appAttemptId);
+ if (historyServiceEnabled) {
+ return historyClient.getApplicationAttemptReport(appAttemptId);
+ }
+ throw new YarnException("History service is not enabled.");
}
@Override
public List<ApplicationAttemptReport> getApplicationAttempts(
ApplicationId appId) throws YarnException, IOException {
- return historyClient.getApplicationAttempts(appId);
+ if (historyServiceEnabled) {
+ return historyClient.getApplicationAttempts(appId);
+ }
+ throw new YarnException("History service is not enabled.");
}
@Override
public ContainerReport getContainerReport(ContainerId containerId)
throws YarnException, IOException {
- return historyClient.getContainerReport(containerId);
+ if (historyServiceEnabled) {
+ return historyClient.getContainerReport(containerId);
+ }
+ throw new YarnException("History service is not enabled.");
}
@Override
public List<ContainerReport> getContainers(
ApplicationAttemptId applicationAttemptId) throws YarnException,
IOException {
- return historyClient.getContainers(applicationAttemptId);
+ if (historyServiceEnabled) {
+ return historyClient.getContainers(applicationAttemptId);
+ }
+ throw new YarnException("History service is not enabled.");
}
}
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml?rev=1559001&r1=1559000&r2=1559001&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml Fri Jan 17 03:12:47 2014
@@ -565,9 +565,12 @@
</property>
<property>
- <description>Enable RM to write history data. If true, then
- yarn.resourcemanager.history-writer.class must be specified</description>
- <name>yarn.resourcemanager.history-writer.enabled</name>
+ <description>Indicate to ResourceManager as well as clients whether
+ history-service is enabled or not. If enabled, ResourceManager starts
+ recording historical data that ApplicationHistory service can consume.
+ Similarly, clients can redirect to the history service when applications
+ finish if this is enabled.</description>
+ <name>yarn.ahs.enabled</name>
<value>false</value>
</property>
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestMemoryApplicationHistoryStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestMemoryApplicationHistoryStore.java?rev=1559001&r1=1559000&r2=1559001&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestMemoryApplicationHistoryStore.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestMemoryApplicationHistoryStore.java Fri Jan 17 03:12:47 2014
@@ -201,7 +201,7 @@ public class TestMemoryApplicationHistor
writeContainerFinishData(containerId);
}
long usedMemoryAfter = (runtime.totalMemory() - runtime.freeMemory()) / mb;
- Assert.assertTrue((usedMemoryAfter - usedMemoryBefore) < 100);
+ Assert.assertTrue((usedMemoryAfter - usedMemoryBefore) < 200);
}
}
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java?rev=1559001&r1=1559000&r2=1559001&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/RMApplicationHistoryWriter.java Fri Jan 17 03:12:47 2014
@@ -68,6 +68,7 @@ public class RMApplicationHistoryWriter
private Dispatcher dispatcher;
private ApplicationHistoryWriter writer;
+ private boolean historyServiceEnabled;
public RMApplicationHistoryWriter() {
super(RMApplicationHistoryWriter.class.getName());
@@ -76,6 +77,11 @@ public class RMApplicationHistoryWriter
@Override
protected synchronized void serviceInit(
Configuration conf) throws Exception {
+
+ historyServiceEnabled = conf.getBoolean(
+ YarnConfiguration.YARN_HISTORY_SERVICE_ENABLED,
+ YarnConfiguration.DEFAULT_YARN_HISTORY_SERVICE_ENABLED);
+
writer = createApplicationHistoryStore(conf);
addIfService(writer);
@@ -96,12 +102,9 @@ public class RMApplicationHistoryWriter
protected ApplicationHistoryStore createApplicationHistoryStore(
Configuration conf) {
- boolean ahsEnabled = conf.getBoolean(
- YarnConfiguration.RM_HISTORY_WRITER_ENABLED,
- YarnConfiguration.DEFAULT_RM_HISTORY_WRITER_ENABLED);
// If the history writer is not enabled, a dummy store will be used to
// write nothing
- if (ahsEnabled) {
+ if (historyServiceEnabled) {
try {
Class<? extends ApplicationHistoryStore> storeClass =
conf.getClass(YarnConfiguration.RM_HISTORY_WRITER_CLASS,
@@ -225,42 +228,49 @@ public class RMApplicationHistoryWriter
@SuppressWarnings("unchecked")
public void applicationAttemptStarted(RMAppAttempt appAttempt) {
- dispatcher.getEventHandler().handle(
+ if (historyServiceEnabled) {
+ dispatcher.getEventHandler().handle(
new WritingApplicationAttemptStartEvent(appAttempt.getAppAttemptId(),
- ApplicationAttemptStartData.newInstance(
- appAttempt.getAppAttemptId(), appAttempt.getHost(),
- appAttempt.getRpcPort(), appAttempt.getMasterContainer()
- .getId())));
+ ApplicationAttemptStartData.newInstance(appAttempt.getAppAttemptId(),
+ appAttempt.getHost(), appAttempt.getRpcPort(), appAttempt
+ .getMasterContainer().getId())));
+ }
}
@SuppressWarnings("unchecked")
public void applicationAttemptFinished(RMAppAttempt appAttempt) {
- dispatcher.getEventHandler().handle(
+ if (historyServiceEnabled) {
+ dispatcher.getEventHandler().handle(
new WritingApplicationAttemptFinishEvent(appAttempt.getAppAttemptId(),
- ApplicationAttemptFinishData.newInstance(appAttempt
- .getAppAttemptId(), appAttempt.getDiagnostics().toString(),
- appAttempt.getTrackingUrl(), appAttempt
- .getFinalApplicationStatus(), appAttempt
- .createApplicationAttemptState())));
+ ApplicationAttemptFinishData.newInstance(
+ appAttempt.getAppAttemptId(), appAttempt.getDiagnostics()
+ .toString(), appAttempt.getTrackingUrl(), appAttempt
+ .getFinalApplicationStatus(), appAttempt
+ .createApplicationAttemptState())));
+ }
}
@SuppressWarnings("unchecked")
public void containerStarted(RMContainer container) {
- dispatcher.getEventHandler().handle(
+ if (historyServiceEnabled) {
+ dispatcher.getEventHandler().handle(
new WritingContainerStartEvent(container.getContainerId(),
- ContainerStartData.newInstance(container.getContainerId(),
- container.getAllocatedResource(), container.getAllocatedNode(),
- container.getAllocatedPriority(), container.getStartTime())));
+ ContainerStartData.newInstance(container.getContainerId(),
+ container.getAllocatedResource(), container.getAllocatedNode(),
+ container.getAllocatedPriority(), container.getStartTime())));
+ }
}
@SuppressWarnings("unchecked")
public void containerFinished(RMContainer container) {
- dispatcher.getEventHandler().handle(
+ if (historyServiceEnabled) {
+ dispatcher.getEventHandler().handle(
new WritingContainerFinishEvent(container.getContainerId(),
- ContainerFinishData.newInstance(container.getContainerId(),
- container.getFinishTime(), container.getDiagnosticsInfo(),
- container.getLogURL(), container.getContainerExitStatus(),
- container.getContainerState())));
+ ContainerFinishData.newInstance(container.getContainerId(),
+ container.getFinishTime(), container.getDiagnosticsInfo(),
+ container.getLogURL(), container.getContainerExitStatus(),
+ container.getContainerState())));
+ }
}
/**
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java?rev=1559001&r1=1559000&r2=1559001&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ahs/TestRMApplicationHistoryWriter.java Fri Jan 17 03:12:47 2014
@@ -77,6 +77,7 @@ public class TestRMApplicationHistoryWri
public void setup() {
store = new MemoryApplicationHistoryStore();
Configuration conf = new Configuration();
+ conf.setBoolean(YarnConfiguration.YARN_HISTORY_SERVICE_ENABLED, true);
writer = new RMApplicationHistoryWriter() {
@Override
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java?rev=1559001&r1=1559000&r2=1559001&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java Fri Jan 17 03:12:47 2014
@@ -105,7 +105,7 @@ public class TestRMContainerImpl {
drainDispatcher.await();
assertEquals(RMContainerState.RUNNING, rmContainer.getState());
assertEquals(
- "http://host:3465/node/containerlogs/container_1_0001_01_000001/user",
+ "http://host:3465/logs/host:3425/container_1_0001_01_000001/container_1_0001_01_000001/user",
rmContainer.getLogURL());
// In RUNNING state. Verify RELEASED and associated actions.
@@ -192,7 +192,7 @@ public class TestRMContainerImpl {
drainDispatcher.await();
assertEquals(RMContainerState.RUNNING, rmContainer.getState());
assertEquals(
- "http://host:3465/node/containerlogs/container_1_0001_01_000001/user",
+ "http://host:3465/logs/host:3425/container_1_0001_01_000001/container_1_0001_01_000001/user",
rmContainer.getLogURL());
// In RUNNING state. Verify EXPIRE and associated actions.