You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sj...@apache.org on 2015/08/25 20:08:38 UTC
[09/50] [abbrv] hadoop git commit: YARN-3374. Collector's web server
should randomly bind an available port. Contributed by Zhijie Shen
YARN-3374. Collector's web server should randomly bind an available port. Contributed by Zhijie Shen
(cherry picked from commit 3aa898e734a1e4368ddf1d0bbd31f9b4de53ceba)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a9c81ac5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a9c81ac5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a9c81ac5
Branch: refs/heads/YARN-2928
Commit: a9c81ac5729e3e2cb39210d4679a8da9ca5c4e4e
Parents: e8b5ab6
Author: Junping Du <ju...@apache.org>
Authored: Thu Apr 2 11:59:59 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Tue Aug 25 10:38:44 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../hadoop/yarn/conf/YarnConfiguration.java | 1 +
.../collector/TimelineCollectorManager.java | 20 ++++++++++----------
.../collector/TestTimelineCollectorManager.java | 12 ++++++++++++
4 files changed, 26 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a9c81ac5/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index ed5dc88..76fa0a8 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -44,6 +44,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3040. Make putEntities operation be aware of the app's context. (Zhijie Shen
via junping_du)
+ YARN-3374. Collector's web server should randomly bind an available port. (
+ Zhijie Shen via junping_du)
+
IMPROVEMENTS
OPTIMIZATIONS
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a9c81ac5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 51f2b2d..1ba7f36 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1515,6 +1515,7 @@ public class YarnConfiguration extends Configuration {
/** The listening endpoint for the timeline service application.*/
public static final String TIMELINE_SERVICE_BIND_HOST =
TIMELINE_SERVICE_PREFIX + "bind-host";
+ public static final String DEFAULT_TIMELINE_SERVICE_BIND_HOST = "0.0.0.0";
/** The number of threads to handle client RPC API requests. */
public static final String TIMELINE_SERVICE_HANDLER_THREAD_COUNT =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a9c81ac5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
index 909027e..5f23c25 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
@@ -210,22 +210,17 @@ public class TimelineCollectorManager extends CompositeService {
*/
private void startWebApp() {
Configuration conf = getConfig();
- // use the same ports as the old ATS for now; we could create new properties
- // for the new timeline service if needed
- String bindAddress = WebAppUtils.getWebAppBindURL(conf,
- YarnConfiguration.TIMELINE_SERVICE_BIND_HOST,
- WebAppUtils.getAHSWebAppURLWithoutScheme(conf));
- this.timelineRestServerBindAddress = WebAppUtils.getResolvedAddress(
- NetUtils.createSocketAddr(bindAddress));
- LOG.info("Instantiating the per-node collector webapp at " +
- timelineRestServerBindAddress);
+ String bindAddress = conf.get(YarnConfiguration.TIMELINE_SERVICE_BIND_HOST,
+ YarnConfiguration.DEFAULT_TIMELINE_SERVICE_BIND_HOST) + ":0";
try {
Configuration confForInfoServer = new Configuration(conf);
confForInfoServer.setInt(HttpServer2.HTTP_MAX_THREADS, 10);
HttpServer2.Builder builder = new HttpServer2.Builder()
.setName("timeline")
.setConf(conf)
- .addEndpoint(URI.create("http://" + bindAddress));
+ .addEndpoint(URI.create(
+ (YarnConfiguration.useHttps(conf) ? "https://" : "http://") +
+ bindAddress));
timelineRestServer = builder.build();
// TODO: replace this by an authentication filter in future.
HashMap<String, String> options = new HashMap<>();
@@ -249,6 +244,11 @@ public class TimelineCollectorManager extends CompositeService {
LOG.error(msg, e);
throw new YarnRuntimeException(msg, e);
}
+ //TODO: We need to think of the case of multiple interfaces
+ this.timelineRestServerBindAddress = WebAppUtils.getResolvedAddress(
+ timelineRestServer.getConnectorAddress(0));
+ LOG.info("Instantiated the per-node collector webapp at " +
+ timelineRestServerBindAddress);
}
private void reportNewCollectorToNM(ApplicationId appId)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a9c81ac5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollectorManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollectorManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollectorManager.java
index 38227ca..36bda85 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollectorManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollectorManager.java
@@ -18,7 +18,9 @@
package org.apache.hadoop.yarn.server.timelineservice.collector;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.mockito.Mockito.any;
@@ -63,6 +65,16 @@ public class TestTimelineCollectorManager {
}
}
+ @Test
+ public void testStartWebApp() throws Exception {
+ assertNotNull(collectorManager.getRestServerBindAddress());
+ String address = collectorManager.getRestServerBindAddress();
+ String[] parts = address.split(":");
+ assertEquals(2, parts.length);
+ assertNotNull(parts[0]);
+ assertTrue(Integer.valueOf(parts[1]) > 0);
+ }
+
@Test(timeout=60000)
public void testMultithreadedAdd() throws Exception {
final int NUM_APPS = 5;