You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ju...@apache.org on 2015/04/02 20:43:09 UTC
hadoop git commit: YARN-3374. Collector's web server should randomly
bind an available port. Contributed by Zhijie Shen
Repository: hadoop
Updated Branches:
refs/heads/YARN-2928 471b1d936 -> 3aa898e73
YARN-3374. Collector's web server should randomly bind an available port. Contributed by Zhijie Shen
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3aa898e7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3aa898e7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3aa898e7
Branch: refs/heads/YARN-2928
Commit: 3aa898e734a1e4368ddf1d0bbd31f9b4de53ceba
Parents: 471b1d9
Author: Junping Du <ju...@apache.org>
Authored: Thu Apr 2 11:59:59 2015 -0700
Committer: Junping Du <ju...@apache.org>
Committed: Thu Apr 2 11:59:59 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../hadoop/yarn/conf/YarnConfiguration.java | 1 +
.../collector/TimelineCollectorManager.java | 20 ++++++++++----------
.../collector/TestTimelineCollectorManager.java | 12 ++++++++++++
4 files changed, 26 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3aa898e7/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 3c16f24..a3ded47 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -44,6 +44,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3040. Make putEntities operation be aware of the app's context. (Zhijie Shen
via junping_du)
+ YARN-3374. Collector's web server should randomly bind an available port. (
+ Zhijie Shen via junping_du)
+
IMPROVEMENTS
OPTIMIZATIONS
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3aa898e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 36dcd5d..e819d1d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1339,6 +1339,7 @@ public class YarnConfiguration extends Configuration {
/** The listening endpoint for the timeline service application.*/
public static final String TIMELINE_SERVICE_BIND_HOST =
TIMELINE_SERVICE_PREFIX + "bind-host";
+ public static final String DEFAULT_TIMELINE_SERVICE_BIND_HOST = "0.0.0.0";
/** The number of threads to handle client RPC API requests. */
public static final String TIMELINE_SERVICE_HANDLER_THREAD_COUNT =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3aa898e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
index 909027e..5f23c25 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
@@ -210,22 +210,17 @@ public class TimelineCollectorManager extends CompositeService {
*/
private void startWebApp() {
Configuration conf = getConfig();
- // use the same ports as the old ATS for now; we could create new properties
- // for the new timeline service if needed
- String bindAddress = WebAppUtils.getWebAppBindURL(conf,
- YarnConfiguration.TIMELINE_SERVICE_BIND_HOST,
- WebAppUtils.getAHSWebAppURLWithoutScheme(conf));
- this.timelineRestServerBindAddress = WebAppUtils.getResolvedAddress(
- NetUtils.createSocketAddr(bindAddress));
- LOG.info("Instantiating the per-node collector webapp at " +
- timelineRestServerBindAddress);
+ String bindAddress = conf.get(YarnConfiguration.TIMELINE_SERVICE_BIND_HOST,
+ YarnConfiguration.DEFAULT_TIMELINE_SERVICE_BIND_HOST) + ":0";
try {
Configuration confForInfoServer = new Configuration(conf);
confForInfoServer.setInt(HttpServer2.HTTP_MAX_THREADS, 10);
HttpServer2.Builder builder = new HttpServer2.Builder()
.setName("timeline")
.setConf(conf)
- .addEndpoint(URI.create("http://" + bindAddress));
+ .addEndpoint(URI.create(
+ (YarnConfiguration.useHttps(conf) ? "https://" : "http://") +
+ bindAddress));
timelineRestServer = builder.build();
// TODO: replace this by an authentication filter in future.
HashMap<String, String> options = new HashMap<>();
@@ -249,6 +244,11 @@ public class TimelineCollectorManager extends CompositeService {
LOG.error(msg, e);
throw new YarnRuntimeException(msg, e);
}
+ //TODO: We need to think of the case of multiple interfaces
+ this.timelineRestServerBindAddress = WebAppUtils.getResolvedAddress(
+ timelineRestServer.getConnectorAddress(0));
+ LOG.info("Instantiated the per-node collector webapp at " +
+ timelineRestServerBindAddress);
}
private void reportNewCollectorToNM(ApplicationId appId)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3aa898e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollectorManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollectorManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollectorManager.java
index 38227ca..36bda85 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollectorManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollectorManager.java
@@ -18,7 +18,9 @@
package org.apache.hadoop.yarn.server.timelineservice.collector;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.mockito.Mockito.any;
@@ -63,6 +65,16 @@ public class TestTimelineCollectorManager {
}
}
+ @Test
+ public void testStartWebApp() throws Exception {
+ assertNotNull(collectorManager.getRestServerBindAddress());
+ String address = collectorManager.getRestServerBindAddress();
+ String[] parts = address.split(":");
+ assertEquals(2, parts.length);
+ assertNotNull(parts[0]);
+ assertTrue(Integer.valueOf(parts[1]) > 0);
+ }
+
@Test(timeout=60000)
public void testMultithreadedAdd() throws Exception {
final int NUM_APPS = 5;