You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ro...@apache.org on 2019/01/04 08:59:36 UTC
[hadoop] branch trunk updated (f4906ac -> 8c6978c)
This is an automated email from the ASF dual-hosted git repository.
rohithsharmaks pushed a change to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git.
from f4906ac YARN-9038. [CSI] Add ability to publish/unpublish volumes on node managers. Contributed by Weiwei Yang.
new 573b158 YARN-8567. Fetching yarn logs fails for long running application if it is not present in timeline store. Contributed by Tarun Parimi.
new 8c6978c YARN-6149. Allow port range to be specified while starting NM Timeline collector manager. Contributed by Abhishek Modi.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../apache/hadoop/yarn/conf/YarnConfiguration.java | 4 ++
.../yarn/client/api/impl/YarnClientImpl.java | 2 +-
.../yarn/client/api/impl/TestYarnClient.java | 50 ++++++++++++++++++++++
.../collector/NodeTimelineCollectorManager.java | 15 ++++++-
.../collector/TestNMTimelineCollectorManager.java | 6 ++-
5 files changed, 72 insertions(+), 5 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 02/02: YARN-6149. Allow port range to be specified while
starting NM Timeline collector manager. Contributed by Abhishek Modi.
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rohithsharmaks pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 8c6978c3baef96a333ebd7e98e02098c99df7313
Author: Rohith Sharma K S <ro...@apache.org>
AuthorDate: Fri Jan 4 14:07:02 2019 +0530
YARN-6149. Allow port range to be specified while starting NM Timeline collector manager. Contributed by Abhishek Modi.
---
.../org/apache/hadoop/yarn/conf/YarnConfiguration.java | 4 ++++
.../collector/NodeTimelineCollectorManager.java | 15 +++++++++++++--
.../collector/TestNMTimelineCollectorManager.java | 6 ++++--
3 files changed, 21 insertions(+), 4 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 511edef..08f7f1a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -3869,6 +3869,10 @@ public class YarnConfiguration extends Configuration {
TIMELINE_SERVICE_COLLECTOR_PREFIX + "bind-host";
@Private
+ public static final String TIMELINE_SERVICE_COLLECTOR_BIND_PORT_RANGES =
+ TIMELINE_SERVICE_COLLECTOR_PREFIX + "bind-port-ranges";
+
+ @Private
public static final String TIMELINE_SERVICE_COLLECTOR_WEBAPP_ADDRESS =
TIMELINE_SERVICE_COLLECTOR_PREFIX + "webapp.address";
public static final String DEFAULT_TIMELINE_SERVICE_COLLECTOR_WEBAPP_ADDRESS =
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/NodeTimelineCollectorManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/NodeTimelineCollectorManager.java
index 696f4a3..0371d49 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/NodeTimelineCollectorManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/NodeTimelineCollectorManager.java
@@ -58,6 +58,7 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
/**
* Class on the NodeManager side that manages adding and removing collectors and
* their lifecycle. Also instantiates the per-node collector webapp.
@@ -280,14 +281,21 @@ public class NodeTimelineCollectorManager extends TimelineCollectorManager {
String bindAddress = null;
String host =
conf.getTrimmed(YarnConfiguration.TIMELINE_SERVICE_COLLECTOR_BIND_HOST);
+ Configuration.IntegerRanges portRanges = conf.getRange(
+ YarnConfiguration.TIMELINE_SERVICE_COLLECTOR_BIND_PORT_RANGES, "");
+ int startPort = 0;
+ if (portRanges != null && !portRanges.isEmpty()) {
+ startPort = portRanges.getRangeStart();
+ }
if (host == null || host.isEmpty()) {
// if collector bind-host is not set, fall back to
// timeline-service.bind-host to maintain compatibility
bindAddress =
conf.get(YarnConfiguration.DEFAULT_TIMELINE_SERVICE_BIND_HOST,
- YarnConfiguration.DEFAULT_TIMELINE_SERVICE_BIND_HOST) + ":0";
+ YarnConfiguration.DEFAULT_TIMELINE_SERVICE_BIND_HOST)
+ + ":" + startPort;
} else {
- bindAddress = host + ":0";
+ bindAddress = host + ":" + startPort;
}
try {
@@ -297,6 +305,9 @@ public class NodeTimelineCollectorManager extends TimelineCollectorManager {
.addEndpoint(URI.create(
(YarnConfiguration.useHttps(conf) ? "https://" : "http://") +
bindAddress));
+ if (portRanges != null && !portRanges.isEmpty()) {
+ builder.setPortRanges(portRanges);
+ }
if (YarnConfiguration.useHttps(conf)) {
builder = WebAppUtils.loadSslConfiguration(builder, conf);
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestNMTimelineCollectorManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestNMTimelineCollectorManager.java
index af9acce..0f2af21 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestNMTimelineCollectorManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestNMTimelineCollectorManager.java
@@ -60,6 +60,8 @@ public class TestNMTimelineCollectorManager {
Configuration conf = new YarnConfiguration();
conf.setClass(YarnConfiguration.TIMELINE_SERVICE_WRITER_CLASS,
FileSystemTimelineWriterImpl.class, TimelineWriter.class);
+ conf.set(YarnConfiguration.TIMELINE_SERVICE_COLLECTOR_BIND_PORT_RANGES,
+ "30000-30100");
collectorManager.init(conf);
collectorManager.start();
}
@@ -83,7 +85,8 @@ public class TestNMTimelineCollectorManager {
String[] parts = address.split(":");
assertEquals(2, parts.length);
assertNotNull(parts[0]);
- assertTrue(Integer.valueOf(parts[1]) > 0);
+ assertTrue(Integer.valueOf(parts[1]) >= 30000 &&
+ Integer.valueOf(parts[1]) <= 30100);
}
@Test(timeout=60000)
@@ -153,7 +156,6 @@ public class TestNMTimelineCollectorManager {
private NodeTimelineCollectorManager createCollectorManager() {
final NodeTimelineCollectorManager cm =
spy(new NodeTimelineCollectorManager());
- doReturn(new Configuration()).when(cm).getConfig();
CollectorNodemanagerProtocol nmCollectorService =
mock(CollectorNodemanagerProtocol.class);
GetTimelineCollectorContextResponse response =
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 01/02: YARN-8567. Fetching yarn logs fails for long
running application if it is not present in timeline store. Contributed by
Tarun Parimi.
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rohithsharmaks pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 573b1587918c4c0efdb7e9fff6f5be12bf31b619
Author: Rohith Sharma K S <ro...@apache.org>
AuthorDate: Fri Jan 4 14:01:24 2019 +0530
YARN-8567. Fetching yarn logs fails for long running application if it is not present in timeline store. Contributed by Tarun Parimi.
---
.../yarn/client/api/impl/YarnClientImpl.java | 2 +-
.../yarn/client/api/impl/TestYarnClient.java | 50 ++++++++++++++++++++++
2 files changed, 51 insertions(+), 1 deletion(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
index 28a7f85..86bda98 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
@@ -879,7 +879,7 @@ public class YarnClientImpl extends YarnClient {
try {
containersListFromAHS =
getContainerReportFromHistory(applicationAttemptId);
- } catch (IOException e) {
+ } catch (IOException | YarnException e) {
if (appNotFoundInRM) {
throw e;
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
index 17e43ca..7ad7e3a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
@@ -531,6 +531,44 @@ public class TestYarnClient extends ParameterizedSchedulerTestBase {
}
@Test(timeout = 10000)
+ public void testGetContainersOnAHSFail() throws YarnException, IOException {
+ Configuration conf = getConf();
+ conf.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED,
+ true);
+
+ final YarnClient client = new MockYarnClient() {
+ @Override
+ public List<ContainerReport> getContainers(
+ ApplicationAttemptId appAttemptId) throws YarnException,
+ IOException {
+ return getContainersOnAHSFail(appAttemptId);
+ }
+ };
+
+ client.init(conf);
+ client.start();
+
+ ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
+ ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(
+ applicationId, 1);
+ List<ContainerReport> reports = client.getContainers(appAttemptId);
+ Assert.assertNotNull(reports);
+ Assert.assertTrue(reports.size() == 2);
+ Assert.assertEquals(reports.get(0).getContainerId(),
+ (ContainerId.newContainerId(appAttemptId, 1)));
+ Assert.assertEquals(reports.get(1).getContainerId(),
+ (ContainerId.newContainerId(appAttemptId, 2)));
+
+ //Only 2 running containers from RM are present when AHS throws exception
+ Assert.assertEquals(ContainerState.RUNNING,
+ (reports.get(0).getContainerState()));
+ Assert.assertEquals(ContainerState.RUNNING,
+ (reports.get(1).getContainerState()));
+ client.stop();
+ }
+
+
+ @Test(timeout = 10000)
public void testGetContainerReport() throws YarnException, IOException {
Configuration conf = getConf();
conf.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED,
@@ -914,6 +952,18 @@ public class TestYarnClient extends ParameterizedSchedulerTestBase {
return super.getContainers(appAttemptId);
}
+ protected List<ContainerReport>
+ getContainersOnAHSFail(ApplicationAttemptId appAttemptId)
+ throws YarnException, IOException {
+ when(mockContainersResponse.getContainerList()).thenReturn(
+ getContainersReport(appAttemptId));
+ when(historyClient.getContainers(any(ApplicationAttemptId.class)))
+ .thenThrow(new ApplicationNotFoundException(
+ appAttemptId.getApplicationId() +
+ " does not exist in the timeline store"));
+ return super.getContainers(appAttemptId);
+ }
+
private List<ContainerReport> getContainersFromAHS(
ApplicationAttemptId appAttemptId) {
return containersFromAHS.get(appAttemptId);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org