You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ju...@apache.org on 2017/02/06 03:41:16 UTC
hadoop git commit: HADOOP-12097. Allow port range to be specified
while starting webapp. Contributed by Varun Saxena.
Repository: hadoop
Updated Branches:
refs/heads/trunk d401e63b6 -> cce35c381
HADOOP-12097. Allow port range to be specified while starting webapp. Contributed by Varun Saxena.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cce35c38
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cce35c38
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cce35c38
Branch: refs/heads/trunk
Commit: cce35c38159b23eb55204b3c9afcaa3215f4f4ef
Parents: d401e63
Author: Junping Du <ju...@apache.org>
Authored: Sun Feb 5 19:42:11 2017 -0800
Committer: Junping Du <ju...@apache.org>
Committed: Sun Feb 5 19:42:11 2017 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/conf/Configuration.java | 12 ++
.../org/apache/hadoop/http/HttpServer2.java | 119 ++++++++++++++++---
.../org/apache/hadoop/http/TestHttpServer.java | 38 ++++++
.../org/apache/hadoop/yarn/webapp/WebApps.java | 38 ++++--
.../apache/hadoop/yarn/webapp/TestWebApp.java | 46 +++++++
5 files changed, 227 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cce35c38/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
index c0e42e5..bade06e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
@@ -1887,6 +1887,18 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
return result.toString();
}
+ /**
+ * Get range start for the first integer range.
+ * @return range start.
+ */
+ public int getRangeStart() {
+ if (ranges == null || ranges.isEmpty()) {
+ return -1;
+ }
+ Range r = ranges.get(0);
+ return r.start;
+ }
+
@Override
public Iterator<Integer> iterator() {
return new RangeNumberIterator(ranges);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cce35c38/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
index 06f493b..25a4037 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
@@ -60,6 +60,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.ConfServlet;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configuration.IntegerRanges;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.jmx.JMXJsonServlet;
import org.apache.hadoop.log.LogLevel;
@@ -151,6 +152,7 @@ public final class HttpServer2 implements FilterContainer {
protected final WebAppContext webAppContext;
protected final boolean findPort;
+ protected final IntegerRanges portRanges;
private final Map<ServletContextHandler, Boolean> defaultContexts =
new HashMap<>();
protected final List<String> filterNames = new ArrayList<>();
@@ -189,6 +191,7 @@ public final class HttpServer2 implements FilterContainer {
private String keyPassword;
private boolean findPort;
+ private IntegerRanges portRanges = null;
private String hostName;
private boolean disallowFallbackToRandomSignerSecretProvider;
@@ -261,6 +264,11 @@ public final class HttpServer2 implements FilterContainer {
return this;
}
+ public Builder setPortRanges(IntegerRanges ranges) {
+ this.portRanges = ranges;
+ return this;
+ }
+
public Builder setConf(Configuration conf) {
this.conf = conf;
return this;
@@ -496,6 +504,7 @@ public final class HttpServer2 implements FilterContainer {
}
this.findPort = b.findPort;
+ this.portRanges = b.portRanges;
initializeWebServer(b.name, b.hostName, b.conf, b.pathSpecs);
}
@@ -1080,6 +1089,93 @@ public final class HttpServer2 implements FilterContainer {
}
/**
+ * Bind listener by closing and opening the listener.
+ * @param listener
+ * @throws Exception
+ */
+ private static void bindListener(ServerConnector listener) throws Exception {
+ // jetty has a bug where you can't reopen a listener that previously
+ // failed to open w/o issuing a close first, even if the port is changed
+ listener.close();
+ listener.open();
+ LOG.info("Jetty bound to port " + listener.getLocalPort());
+ }
+
+ /**
+ * Create bind exception by wrapping the bind exception thrown.
+ * @param listener
+ * @param ex
+ * @return
+ */
+ private static BindException constructBindException(ServerConnector listener,
+ BindException ex) {
+ BindException be = new BindException("Port in use: "
+ + listener.getHost() + ":" + listener.getPort());
+ if (ex != null) {
+ be.initCause(ex);
+ }
+ return be;
+ }
+
+ /**
+ * Bind using single configured port. If findPort is true, we will try to bind
+ * after incrementing port till a free port is found.
+ * @param listener jetty listener.
+ * @param port port which is set in the listener.
+ * @throws Exception
+ */
+ private void bindForSinglePort(ServerConnector listener, int port)
+ throws Exception {
+ while (true) {
+ try {
+ bindListener(listener);
+ break;
+ } catch (BindException ex) {
+ if (port == 0 || !findPort) {
+ throw constructBindException(listener, ex);
+ }
+ }
+ // try the next port number
+ listener.setPort(++port);
+ Thread.sleep(100);
+ }
+ }
+
+ /**
+ * Bind using port ranges. Keep on looking for a free port in the port range
+ * and throw a bind exception if no port in the configured range binds.
+ * @param listener jetty listener.
+ * @param startPort initial port which is set in the listener.
+ * @throws Exception
+ */
+ private void bindForPortRange(ServerConnector listener, int startPort)
+ throws Exception {
+ BindException bindException = null;
+ try {
+ bindListener(listener);
+ return;
+ } catch (BindException ex) {
+ // Ignore exception.
+ bindException = ex;
+ }
+ for(Integer port : portRanges) {
+ if (port == startPort) {
+ continue;
+ }
+ Thread.sleep(100);
+ listener.setPort(port);
+ try {
+ bindListener(listener);
+ return;
+ } catch (BindException ex) {
+ // Ignore exception. Move to next port.
+ bindException = ex;
+ }
+ }
+ throw constructBindException(listener, bindException);
+ }
+
+ /**
* Open the main listener for the server
* @throws Exception
*/
@@ -1091,25 +1187,10 @@ public final class HttpServer2 implements FilterContainer {
continue;
}
int port = listener.getPort();
- while (true) {
- // jetty has a bug where you can't reopen a listener that previously
- // failed to open w/o issuing a close first, even if the port is changed
- try {
- listener.close();
- listener.open();
- LOG.info("Jetty bound to port " + listener.getLocalPort());
- break;
- } catch (BindException ex) {
- if (port == 0 || !findPort) {
- BindException be = new BindException("Port in use: "
- + listener.getHost() + ":" + listener.getPort());
- be.initCause(ex);
- throw be;
- }
- }
- // try the next port number
- listener.setPort(++port);
- Thread.sleep(100);
+ if (portRanges != null && port != 0) {
+ bindForPortRange(listener, port);
+ } else {
+ bindForSinglePort(listener, port);
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cce35c38/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java
index 6b87cd8..baa6f91 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java
@@ -20,10 +20,12 @@ package org.apache.hadoop.http;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configuration.IntegerRanges;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.http.HttpServer2.QuotingInputFilter.RequestQuoter;
import org.apache.hadoop.http.resource.JerseyResource;
import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.net.ServerSocketUtil;
import org.apache.hadoop.security.Groups;
import org.apache.hadoop.security.ShellBasedUnixGroupsMapping;
import org.apache.hadoop.security.UserGroupInformation;
@@ -644,4 +646,40 @@ public class TestHttpServer extends HttpServerFunctionalTest {
assertNotNull(conn.getHeaderField("Date"));
assertEquals(conn.getHeaderField("Expires"), conn.getHeaderField("Date"));
}
+
+ private static void stopHttpServer(HttpServer2 server) throws Exception {
+ if (server != null) {
+ server.stop();
+ }
+ }
+
+ @Test
+ public void testPortRanges() throws Exception {
+ Configuration conf = new Configuration();
+ int port = ServerSocketUtil.waitForPort(49000, 60);
+ int endPort = 49500;
+ conf.set("abc", "49000-49500");
+ HttpServer2.Builder builder = new HttpServer2.Builder()
+ .setName("test").setConf(new Configuration()).setFindPort(false);
+ IntegerRanges ranges = conf.getRange("abc", "");
+ int startPort = 0;
+ if (ranges != null && !ranges.isEmpty()) {
+ startPort = ranges.getRangeStart();
+ builder.setPortRanges(ranges);
+ }
+ builder.addEndpoint(URI.create("http://localhost:" + startPort));
+ HttpServer2 myServer = builder.build();
+ HttpServer2 myServer2 = null;
+ try {
+ myServer.start();
+ assertEquals(port, myServer.getConnectorAddress(0).getPort());
+ myServer2 = builder.build();
+ myServer2.start();
+ assertTrue(myServer2.getConnectorAddress(0).getPort() > port &&
+ myServer2.getConnectorAddress(0).getPort() <= endPort);
+ } finally {
+ stopHttpServer(myServer);
+ stopHttpServer(myServer2);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cce35c38/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
index 7ce0dfa..9c5e8c3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
@@ -35,6 +35,7 @@ import javax.servlet.http.HttpServlet;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configuration.IntegerRanges;
import org.apache.hadoop.http.HttpConfig.Policy;
import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.security.UserGroupInformation;
@@ -92,6 +93,7 @@ public class WebApps {
boolean findPort = false;
Configuration conf;
Policy httpPolicy = null;
+ String portRangeConfigKey = null;
boolean devMode = false;
private String spnegoPrincipalKey;
private String spnegoKeytabKey;
@@ -157,6 +159,19 @@ public class WebApps {
return this;
}
+ /**
+ * Set port range config key and associated configuration object.
+ * @param config configuration.
+ * @param portRangeConfKey port range config key.
+ * @return builder object.
+ */
+ public Builder<T> withPortRange(Configuration config,
+ String portRangeConfKey) {
+ this.conf = config;
+ this.portRangeConfigKey = portRangeConfKey;
+ return this;
+ }
+
public Builder<T> withHttpSpnegoPrincipalKey(String spnegoPrincipalKey) {
this.spnegoPrincipalKey = spnegoPrincipalKey;
return this;
@@ -265,15 +280,24 @@ public class WebApps {
: WebAppUtils.HTTP_PREFIX;
}
HttpServer2.Builder builder = new HttpServer2.Builder()
- .setName(name)
- .addEndpoint(
- URI.create(httpScheme + bindAddress
- + ":" + port)).setConf(conf).setFindPort(findPort)
+ .setName(name).setConf(conf).setFindPort(findPort)
.setACL(new AccessControlList(conf.get(
- YarnConfiguration.YARN_ADMIN_ACL,
- YarnConfiguration.DEFAULT_YARN_ADMIN_ACL)))
+ YarnConfiguration.YARN_ADMIN_ACL,
+ YarnConfiguration.DEFAULT_YARN_ADMIN_ACL)))
.setPathSpec(pathList.toArray(new String[0]));
-
+ // Get port ranges from config.
+ IntegerRanges ranges = null;
+ if (portRangeConfigKey != null) {
+ ranges = conf.getRange(portRangeConfigKey, "");
+ }
+ int startPort = port;
+ if (ranges != null && !ranges.isEmpty()) {
+ // Set port ranges if its configured.
+ startPort = ranges.getRangeStart();
+ builder.setPortRanges(ranges);
+ }
+ builder.addEndpoint(URI.create(httpScheme + bindAddress +
+ ":" + startPort));
boolean hasSpnegoConf = spnegoPrincipalKey != null
&& conf.get(spnegoPrincipalKey) != null && spnegoKeytabKey != null
&& conf.get(spnegoKeytabKey) != null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cce35c38/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestWebApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestWebApp.java
index deef855..9454002 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestWebApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestWebApp.java
@@ -35,6 +35,8 @@ import java.net.HttpURLConnection;
import java.net.URL;
import org.apache.commons.lang.ArrayUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.net.ServerSocketUtil;
import org.apache.hadoop.yarn.MockApps;
import org.apache.hadoop.yarn.webapp.view.HtmlPage;
import org.apache.hadoop.yarn.webapp.view.JQueryUI;
@@ -307,6 +309,50 @@ public class TestWebApp {
}
}
+ private static void stopWebApp(WebApp app) {
+ if (app != null) {
+ app.stop();
+ }
+ }
+
+ @Test
+ public void testPortRanges() throws Exception {
+ WebApp app = WebApps.$for("test", this).start();
+ String baseUrl = baseUrl(app);
+ WebApp app1 = null;
+ WebApp app2 = null;
+ WebApp app3 = null;
+ WebApp app4 = null;
+ WebApp app5 = null;
+ try {
+ int port = ServerSocketUtil.waitForPort(48000, 60);
+ assertEquals("foo", getContent(baseUrl +"test/foo").trim());
+ app1 = WebApps.$for("test", this).at(port).start();
+ assertEquals(port, app1.getListenerAddress().getPort());
+ app2 = WebApps.$for("test", this).at("0.0.0.0",port, true).start();
+ assertTrue(app2.getListenerAddress().getPort() > port);
+ Configuration conf = new Configuration();
+ port = ServerSocketUtil.waitForPort(47000, 60);
+ app3 = WebApps.$for("test", this).at(port).withPortRange(conf, "abc").
+ start();
+ assertEquals(port, app3.getListenerAddress().getPort());
+ ServerSocketUtil.waitForPort(46000, 60);
+ conf.set("abc", "46000-46500");
+ app4 = WebApps.$for("test", this).at(port).withPortRange(conf, "abc").
+ start();
+ assertEquals(46000, app4.getListenerAddress().getPort());
+ app5 = WebApps.$for("test", this).withPortRange(conf, "abc").start();
+ assertTrue(app5.getListenerAddress().getPort() > 46000);
+ } finally {
+ stopWebApp(app);
+ stopWebApp(app1);
+ stopWebApp(app2);
+ stopWebApp(app3);
+ stopWebApp(app4);
+ stopWebApp(app5);
+ }
+ }
+
static String baseUrl(WebApp app) {
return "http://localhost:"+ app.port() +"/";
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org