You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flume.apache.org by jm...@apache.org on 2011/08/15 15:27:40 UTC
svn commit: r1157838 - in /incubator/flume/trunk:
flume-core/src/main/java/com/cloudera/flume/agent/
flume-core/src/main/java/com/cloudera/flume/master/
flume-core/src/main/java/com/cloudera/util/
flume-core/src/test/java/com/cloudera/flume/agent/ flum...
Author: jmhsieh
Date: Mon Aug 15 13:27:39 2011
New Revision: 1157838
URL: http://svn.apache.org/viewvc?rev=1157838&view=rev
Log:
FLUME-721: Webapps 'autofindport' feature does not work
This refactors the internal http server so that context are created by a callback object.
Added:
incubator/flume/trunk/flume-core/src/main/java/com/cloudera/util/HttpServerTestUtils.java
Removed:
incubator/flume/trunk/flume-core/src/main/java/com/cloudera/util/StatusHttpServer.java
incubator/flume/trunk/flume-core/src/test/java/com/cloudera/util/TestStatusHttpServer.java
Modified:
incubator/flume/trunk/flume-core/src/main/java/com/cloudera/flume/agent/FlumeNode.java
incubator/flume/trunk/flume-core/src/main/java/com/cloudera/flume/master/FlumeMaster.java
incubator/flume/trunk/flume-core/src/main/java/com/cloudera/util/InternalHttpServer.java
incubator/flume/trunk/flume-core/src/test/java/com/cloudera/flume/agent/TestNodeJersey.java
incubator/flume/trunk/flume-core/src/test/java/com/cloudera/flume/master/TestMasterJersey.java
incubator/flume/trunk/flume-core/src/test/java/com/cloudera/util/InternalHttpServerTest.java
incubator/flume/trunk/flume-node-web/src/test/java/com/cloudera/flume/agent/TestBootstrap.java
Modified: incubator/flume/trunk/flume-core/src/main/java/com/cloudera/flume/agent/FlumeNode.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-core/src/main/java/com/cloudera/flume/agent/FlumeNode.java?rev=1157838&r1=1157837&r2=1157838&view=diff
==============================================================================
--- incubator/flume/trunk/flume-core/src/main/java/com/cloudera/flume/agent/FlumeNode.java (original)
+++ incubator/flume/trunk/flume-core/src/main/java/com/cloudera/flume/agent/FlumeNode.java Mon Aug 15 13:27:39 2011
@@ -34,6 +34,7 @@ import org.apache.commons.cli.ParseExcep
import org.apache.commons.cli.PosixParser;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
+import org.mortbay.jetty.handler.ContextHandlerCollection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -61,6 +62,7 @@ import com.cloudera.flume.util.SystemInf
import com.cloudera.util.CheckJavaVersion;
import com.cloudera.util.FileUtil;
import com.cloudera.util.InternalHttpServer;
+import com.cloudera.util.InternalHttpServer.ContextCreator;
import com.cloudera.util.NetUtils;
import com.cloudera.util.Pair;
import com.cloudera.util.StatusHttpServer.StackServlet;
@@ -258,24 +260,20 @@ public class FlumeNode implements Report
ReportManager.get().add(this);
if (startHttp) {
- try {
- http = new InternalHttpServer();
-
- http.addHandler(InternalHttpServer.createLogAppContext());
-
- http.addHandler(InternalHttpServer.createServletContext(
- StackServlet.class, "/stacks", "/*", "stacks"));
-
- http.setBindAddress("0.0.0.0");
- http.setPort(conf.getNodeStatusPort());
- String webAppRoot = FlumeConfiguration.get().getNodeWebappRoot();
- http.setWebappDir(new File(webAppRoot));
- http.setScanForApps(true);
-
- http.start();
- } catch (Throwable t) {
- LOG.error("Unexpected exception/error thrown! " + t.getMessage(), t);
- }
+ int nodePort = conf.getNodeStatusPort();
+ String bindAddress = "0.0.0.0";
+ ContextCreator cc = new ContextCreator() {
+ @Override
+ public void addContexts(ContextHandlerCollection handlers) {
+ handlers.addHandler(InternalHttpServer.createLogAppContext());
+ handlers.addHandler(InternalHttpServer.createStackSevletContext());
+ String webAppRoot = FlumeConfiguration.get().getNodeWebappRoot();
+ InternalHttpServer.addHandlersFromPaths(handlers,
+ new File(webAppRoot));
+ }
+ };
+ http = InternalHttpServer.startFindPortHttpServer(cc, bindAddress,
+ nodePort);
}
if (reportPusher != null) {
@@ -371,9 +369,9 @@ public class FlumeNode implements Report
/**
* This function checks the agent logs dir to make sure that the process has
- * the ability to the directory if necessary, that the path if it does exist is
- * a directory, and that it can in fact create files inside of the directory.
- * If it fails any of these, it throws an exception.
+ * the ability to the directory if necessary, that the path if it does exist
+ * is a directory, and that it can in fact create files inside of the
+ * directory. If it fails any of these, it throws an exception.
*
* Finally, it checks to see if the path is in /tmp and warns the user that
* this may not be the best idea.
Modified: incubator/flume/trunk/flume-core/src/main/java/com/cloudera/flume/master/FlumeMaster.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-core/src/main/java/com/cloudera/flume/master/FlumeMaster.java?rev=1157838&r1=1157837&r2=1157838&view=diff
==============================================================================
--- incubator/flume/trunk/flume-core/src/main/java/com/cloudera/flume/master/FlumeMaster.java (original)
+++ incubator/flume/trunk/flume-core/src/main/java/com/cloudera/flume/master/FlumeMaster.java Mon Aug 15 13:27:39 2011
@@ -35,6 +35,7 @@ import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.cli.PosixParser;
import org.apache.thrift.transport.TTransportException;
+import org.mortbay.jetty.handler.ContextHandlerCollection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -53,6 +54,7 @@ import com.cloudera.flume.util.FlumeVMIn
import com.cloudera.flume.util.SystemInfo;
import com.cloudera.util.CheckJavaVersion;
import com.cloudera.util.InternalHttpServer;
+import com.cloudera.util.InternalHttpServer.ContextCreator;
import com.cloudera.util.NetUtils;
import com.cloudera.util.StatusHttpServer.StackServlet;
@@ -249,18 +251,19 @@ public class FlumeMaster implements Repo
ReportManager.get().add(sysInfo);
if (doHttp) {
- http = new InternalHttpServer();
-
- http.addHandler(InternalHttpServer.createLogAppContext());
- http.addHandler(InternalHttpServer.createServletContext(
- StackServlet.class, "/stacks", "/*", "stacks"));
-
- http.setBindAddress("0.0.0.0");
- http.setPort(cfg.getMasterHttpPort());
- String webAppRoot = FlumeConfiguration.get().getMasterWebappRoot();
- http.setWebappDir(new File(webAppRoot));
+ String bindAddress = "0.0.0.0";
+ int port = cfg.getMasterHttpPort();
+ final String webAppRoot = FlumeConfiguration.get().getMasterWebappRoot();
LOG.info("Webserver root directory: " + webAppRoot);
- http.start();
+ ContextCreator cc = new ContextCreator() {
+ @Override
+ public void addContexts(ContextHandlerCollection handlers) {
+ handlers.addHandler(InternalHttpServer.createLogAppContext());
+ handlers.addHandler(InternalHttpServer.createStackSevletContext());
+ InternalHttpServer.addHandlersFromPaths(handlers, new File(webAppRoot));
+ }
+ };
+ http = InternalHttpServer.startHttpServer(cc, bindAddress, port);
}
controlServer = new MasterClientServer(this, FlumeConfiguration.get());
Added: incubator/flume/trunk/flume-core/src/main/java/com/cloudera/util/HttpServerTestUtils.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-core/src/main/java/com/cloudera/util/HttpServerTestUtils.java?rev=1157838&view=auto
==============================================================================
--- incubator/flume/trunk/flume-core/src/main/java/com/cloudera/util/HttpServerTestUtils.java (added)
+++ incubator/flume/trunk/flume-core/src/main/java/com/cloudera/util/HttpServerTestUtils.java Mon Aug 15 13:27:39 2011
@@ -0,0 +1,82 @@
+/**
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.cloudera.util;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.net.URLConnection;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class HttpServerTestUtils {
+ public static final Logger LOG = LoggerFactory
+ .getLogger(HttpServerTestUtils.class);
+
+ /**
+ * Grab a url's contents. This assumes that grabbed pages are small
+ *
+ * @param urlString
+ * @return
+ * @throws IOException
+ */
+ public static String curl(String urlString) throws IOException {
+ URL url = new URL(urlString);
+ URLConnection urlConn = url.openConnection();
+ urlConn.setDoInput(true);
+ urlConn.setUseCaches(false);
+
+ int len = urlConn.getContentLength();
+ String type = urlConn.getContentType();
+ LOG.info("pulled " + urlString + " [type=" + type + " len=" + len + "]");
+ InputStreamReader isr = new InputStreamReader(urlConn.getInputStream());
+ BufferedReader br = new BufferedReader(isr);
+ StringBuilder sb = new StringBuilder();
+ String s;
+ while ((s = br.readLine()) != null) {
+ sb.append(s);
+ sb.append('\n');
+ }
+ return sb.toString();
+ }
+
+ /**
+ * Grab a url's http response code. It if fails, it will throw an exception.
+ *
+ * @param urlString
+ * @return
+ * @throws IOException
+ */
+ public static int curlResp(String urlString) throws IOException {
+ URL url = new URL(urlString);
+ HttpURLConnection urlConn = (HttpURLConnection) url.openConnection();
+ urlConn.setDoInput(true);
+ urlConn.setUseCaches(false);
+
+ int len = urlConn.getContentLength();
+ String type = urlConn.getContentType();
+ LOG.info("pulled " + urlString + " [type=" + type + " len=" + len + "]");
+ return urlConn.getResponseCode();
+ }
+
+}
Modified: incubator/flume/trunk/flume-core/src/main/java/com/cloudera/util/InternalHttpServer.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-core/src/main/java/com/cloudera/util/InternalHttpServer.java?rev=1157838&r1=1157837&r2=1157838&view=diff
==============================================================================
--- incubator/flume/trunk/flume-core/src/main/java/com/cloudera/util/InternalHttpServer.java (original)
+++ incubator/flume/trunk/flume-core/src/main/java/com/cloudera/util/InternalHttpServer.java Mon Aug 15 13:27:39 2011
@@ -1,7 +1,38 @@
+/**
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.cloudera.util;
import java.io.File;
-
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintWriter;
+import java.net.BindException;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.util.ReflectionUtils;
import org.mortbay.jetty.Connector;
import org.mortbay.jetty.Server;
import org.mortbay.jetty.handler.ContextHandlerCollection;
@@ -13,63 +44,14 @@ import org.mortbay.jetty.webapp.WebAppCo
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.cloudera.util.StatusHttpServer.StackServlet;
-import com.google.common.base.Preconditions;
-
/**
* <p>
- * An embedded Jetty HTTP server that support both normal and exploded war file
- * deployment. Those that wish to expose HTTP services should create an instance
- * of this class, configure the server via accessor methods, and then call
- * {@link #start()}.
- * </p>
- * <p>
- * Resources internally are allocated upon the first call to {@link #start()}.
- * This includes scanning of the configured webapp directory for applications if
- * {@link #getScanForApps()} is true (the default). Mostly this class is a thin
- * wrapper around Jetty's {@link Server} class and behaves as Jetty does. Both
- * traditional and exploded war formats are supported in the webapp directory.
- * In the case of exploded directories, the directory name is used as the
- * context. For war files, everything from the first instance of ".war" to the
- * end of the file name (inclusive) is stripped and the remainder is used for
- * the context name.
- * </p>
- * <p>
- * Name examples:
- * </p>
- * <table>
- * <tr>
- * <td>Name</td>
- * <td>Type</td>
- * <td>Context</td>
- * </tr>
- * <tr>
- * <td>app.war</td>
- * <td>file</td>
- * <td>app</td>
- * </tr>
- * <tr>
- * <td>app</td>
- * <td>dir</td>
- * <td>app</td>
- * </tr>
- * <tr>
- * <td>app.war</td>
- * <td>dir</td>
- * <td>app.war</td>
- * </tr>
- * <tr>
- * <td>app.war.war</td>
- * <td>file</td>
- * <td>app</td>
- * </tr>
- * <tr>
- * <td>
- * </table>
- * <p>
- * Example usage:
- * </p>
- *
+ * An embedded Jetty HTTP server. It defers addition of contexts/handlers to
+ * a callback so that the we can provide a method that increments ports until
+ * a valid port is found. This is mostly a thin wrapper around Jetty's
+ * {@link Server} class and behaves as Jetty does.
+ *
+ * Here is an example usage:
* <pre>
* InternalHttpServer server = new InternalHttpServer();
*
@@ -77,9 +59,20 @@ import com.google.common.base.Preconditi
* server.setWebappDir(new File(applicationHome, "webapps"));
* server.setPort(8080);
* server.setBindAddress("0.0.0.0");
- *
+ * server.setContextCreator(new ContextCreator() {
+ * @Override
+ * public void addContexts(ContextHandlerCollection handlers) {
+ * handlers.addHandler(InternalHttpServer.createLogAppContext());
+ * handlers.addHandler(InternalHttpServer.createStackSevletContext(
+ * StackServlet.class, "/stacks", "/*", "stacks"));
+ * String webAppRoot = FlumeConfiguration.get().getNodeWebappRoot();
+ * InternalHttpServer.addHandlersFromPaths(handlers,
+ * new File(webAppRoot));
+ * }
+ * });
+ *
* server.start();
- *
+ *
* // at some later time...
* server.stop();
* </pre>
@@ -90,16 +83,15 @@ public class InternalHttpServer {
.getLogger(InternalHttpServer.class);
private Server server;
- private File webappDir;
private int port;
+ private int boundPort = -1;
private String bindAddress;
- private boolean scanForApps;
private ContextHandlerCollection handlers;
+ private ContextCreator contextCreator = null;
public InternalHttpServer() {
port = 0;
bindAddress = "0.0.0.0";
- scanForApps = true;
handlers = new ContextHandlerCollection();
}
@@ -113,51 +105,14 @@ public class InternalHttpServer {
connector.setHost(bindAddress);
server.addConnector(connector);
- server.addHandler(handlers);
- }
- }
- protected void registerApplications() {
- logger.debug("Registering webapps in {}", webappDir);
-
- if (webappDir.isDirectory()) {
- for (File entry : webappDir.listFiles()) {
- tryRegisterApplication(server, entry);
+ if (contextCreator != null) {
+ contextCreator.addContexts(handlers);
}
- } else {
- tryRegisterApplication(server, webappDir);
+ server.setHandler(handlers);
}
}
- private boolean tryRegisterApplication(Server server, File path) {
- String name;
-
- logger.debug("checking {}", path);
-
- if (path.isFile()) {
- int idx = path.getName().indexOf(".war");
-
- if (idx > -1) {
- name = path.getName().substring(0, idx);
- } else {
- return false;
- }
- } else {
- name = path.getName();
- }
-
- logger.debug("creating context {} -> {}", name, path);
-
- // WebAppContext is for loading war files.
- WebAppContext handler = new WebAppContext(path.getPath(), "/" + name);
-
- handler.setParentLoaderPriority(true);
-
- handlers.addHandler(handler);
-
- return true;
- }
-
/**
* <p>
* Start a configured HTTP server. Users should have already injected all the
@@ -168,26 +123,22 @@ public class InternalHttpServer {
* The configured webappDir is not scanned for applications until start() is
* called.
* </p>
- *
+ *
+ * @throws BindException
* @throws InternalHttpServerException
*/
- public void start() {
- Preconditions.checkState(webappDir != null, "Webapp dir can not be null");
+ public void start() throws BindException {
initialize();
- if (scanForApps) {
- registerApplications();
- } else {
- logger.info("Not scanning for webapps");
- }
-
logger.info("Starting internal HTTP server");
try {
server.start();
-
- logger.info("Server started");
+ boundPort = server.getConnectors()[0].getLocalPort();
+ logger.info("Server started on port " + boundPort);
+ } catch (BindException be) {
+ throw be;
} catch (Exception e) {
logger.warn("Caught exception during HTTP server start.", e);
@@ -220,9 +171,8 @@ public class InternalHttpServer {
@Override
public String toString() {
- return "{ bindAddress:" + bindAddress + " webappDir:" + webappDir
- + " port:" + port + " scanForApps:" + scanForApps + " server:" + server
- + " }";
+ return "{ bindAddress:" + bindAddress + " port:" + port + " boundPort:"
+ + boundPort + " server:" + server + " }";
}
public Server getServer() {
@@ -233,18 +183,14 @@ public class InternalHttpServer {
this.server = server;
}
- public File getWebappDir() {
- return webappDir;
- }
-
- public void setWebappDir(File webappDir) {
- this.webappDir = webappDir;
- }
-
public int getPort() {
return port;
}
+ public int getBoundPort() {
+ return boundPort;
+ }
+
public void setPort(int port) {
this.port = port;
}
@@ -257,14 +203,6 @@ public class InternalHttpServer {
this.bindAddress = bindAddress;
}
- public boolean getScanForApps() {
- return scanForApps;
- }
-
- public void setScanForApps(boolean scanForApps) {
- this.scanForApps = scanForApps;
- }
-
public static class InternalHttpServerException extends RuntimeException {
private static final long serialVersionUID = -4936285404574873547L;
@@ -287,7 +225,19 @@ public class InternalHttpServer {
}
- public void addHandler(Context ctx) {
+ public void setHandlers(ContextHandlerCollection ctx) {
+ if (ctx == null) {
+ logger.warn("Attempting to add null webapp context");
+ return;
+ }
+ handlers = ctx;
+ }
+
+ public ContextHandlerCollection getHandlers() {
+ return handlers;
+ }
+
+ protected void addHandler(Context ctx) {
if (ctx == null) {
logger.warn("Attempting to add null webapp context");
return;
@@ -295,6 +245,117 @@ public class InternalHttpServer {
handlers.addHandler(ctx);
}
+ public void setContextCreator(ContextCreator cc) {
+ this.contextCreator = cc;
+ }
+
+ /**
+ * The jetty server cannot properly reload contexts if it attempts to bind to
+ * a port and fails. To support automatically going finding a new port, we
+ * thus need to parameterize the creation and addition of context. This class
+ * provides a call back that gets a instance of the server's
+ * ContextHandlerCollection, and gives clients the opportunity to populate it.
+ */
+ public abstract static class ContextCreator {
+ public abstract void addContexts(ContextHandlerCollection handlers);
+ }
+
+ public static WebAppContext createWarContext(File path) {
+ logger.debug("checking {}", path);
+
+ String name;
+ if (path.isFile()) {
+ // if not a war file reject
+ int idx = path.getName().indexOf(".war");
+ if (idx < 0) {
+ return null;
+ }
+
+ // drop the .war suffix
+ name = path.getName().substring(0, idx);
+ } else {
+ // is a dir
+ name = path.getName();
+ }
+
+ // WebAppContext is for loading war files.
+ logger.debug("creating context {} -> {}", name, path);
+ WebAppContext handler = new WebAppContext(path.getPath(), "/" + name);
+ handler.setParentLoaderPriority(true);
+ return handler;
+ }
+
+ /**
+ * This method adds support for both normal and exploded war file deployment.
+ * <p>
+ * This scannings the specified webapp directory for applications
+ * Both traditional and exploded war formats are supported in the webapp
+ * directory. In the case of exploded directories, the directory name is used
+ * as the context. For war files, everything from the first instance of ".war"
+ * to the end of the file name (inclusive) is stripped and the remainder is
+ * used for the context name.
+ * </p>
+ * <p>
+ * Name examples:
+ * </p>
+ * <table>
+ * <tr>
+ * <td>Name</td>
+ * <td>Type</td>
+ * <td>Context</td>
+ * </tr>
+ * <tr>
+ * <td>app.war</td>
+ * <td>file</td>
+ * <td>app</td>
+ * </tr>
+ * <tr>
+ * <td>app</td>
+ * <td>dir</td>
+ * <td>app</td>
+ * </tr>
+ * <tr>
+ * <td>app.war</td>
+ * <td>dir</td>
+ * <td>app.war</td>
+ * </tr>
+ * <tr>
+ * <td>app.war.war</td>
+ * <td>file</td>
+ * <td>app</td>
+ * </tr>
+ * <tr>
+ * <td>
+ * </table>
+ * <p>
+ * Example usage:
+ * </p>
+ */
+ public static void addHandlersFromPaths(ContextHandlerCollection handlers,
+ File webappDir) {
+ logger.debug("Registering webapps in {}", webappDir);
+
+ if (webappDir.isDirectory()) {
+ for (File entry : webappDir.listFiles()) {
+ Context ctx = createWarContext(entry);
+ if (ctx != null) {
+ handlers.addHandler(ctx);
+ }
+ }
+ } else {
+ Context ctx = createWarContext(webappDir);
+ if (ctx != null) {
+ handlers.addHandler(ctx);
+ }
+ }
+ }
+
+ /**
+ * This creates file listing servlet context that is used to point to the log
+ * directory of the daemon via the web interface.
+ *
+ * @return
+ */
public static Context createLogAppContext() {
Context ctx = new Context();
// logs applet
@@ -308,8 +369,39 @@ public class InternalHttpServer {
return ctx;
}
- public static Context createServletContext(Class<?> sltClz, String contextPath,
- String pathSpec, String name) {
+ /**
+ * A very simple servlet to serve up a text representation of the current
+ * stack traces. It both returns the stacks to the caller and logs them.
+ * Currently the stack traces are done sequentially rather than exactly the
+ * same data.
+ */
+ public static class StackServlet extends HttpServlet {
+ private static final Log LOG = LogFactory.getLog(StatusHttpServer.class
+ .getName());
+ private static final long serialVersionUID = -6284183679759467039L;
+
+ @Override
+ public void doGet(HttpServletRequest request, HttpServletResponse response)
+ throws ServletException, IOException {
+
+ OutputStream outStream = response.getOutputStream();
+ ReflectionUtils.printThreadInfo(new PrintWriter(outStream), "");
+ outStream.close();
+ ReflectionUtils.logThreadInfo(LOG, "jsp requested", 1);
+ }
+ }
+
+ /**
+ * This creates a stack dumping servlet that can be used to debug a running
+ * daemon via the web interface.
+ *
+ * @param sltClz
+ * @param contextPath
+ * @param pathSpec
+ * @param name
+ * @return
+ */
+ public static Context createStackSevletContext() {
Context ctx = new Context();
ServletHolder holder = new ServletHolder(StackServlet.class);
ctx.setContextPath("/stacks");
@@ -319,4 +411,50 @@ public class InternalHttpServer {
return ctx;
}
+ /**
+ * If successful returns the port the http server successfully bound to. If it
+ * failed, returns -1
+ */
+ public static InternalHttpServer startFindPortHttpServer(ContextCreator cc,
+ String bindAddress, int nodePort) {
+ do {
+ try {
+ return startHttpServer(cc, bindAddress, nodePort);
+ } catch (BindException be) {
+ logger.error("Unable to start webserver on " + bindAddress + ":"
+ + nodePort + ". Trying next port...");
+ nodePort++;
+ }
+ } while (true);
+ }
+
+ /**
+ * Single attempt to create an http server for the node.
+ *
+ * @param bindAddress
+ * @param nodePort
+ * @return instance of a started http server or null if failed.
+ * @throws BindException
+ */
+ public static InternalHttpServer startHttpServer(ContextCreator cc,
+ String bindAddress, int nodePort) throws BindException {
+ InternalHttpServer http = null;
+ try {
+ http = new InternalHttpServer();
+ http.setBindAddress(bindAddress);
+ http.setPort(nodePort);
+ http.setContextCreator(cc);
+ http.start();
+ return http;
+ } catch (BindException be) {
+ http.stop();
+ http = null;
+ throw be;
+ } catch (Throwable t) {
+ logger.error("Unexpected exception/error thrown! " + t.getMessage(), t);
+ // if any exception happens bail out and cleanup.
+ http.stop();
+ return null;
+ }
+ }
}
Modified: incubator/flume/trunk/flume-core/src/test/java/com/cloudera/flume/agent/TestNodeJersey.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-core/src/test/java/com/cloudera/flume/agent/TestNodeJersey.java?rev=1157838&r1=1157837&r2=1157838&view=diff
==============================================================================
--- incubator/flume/trunk/flume-core/src/test/java/com/cloudera/flume/agent/TestNodeJersey.java (original)
+++ incubator/flume/trunk/flume-core/src/test/java/com/cloudera/flume/agent/TestNodeJersey.java Mon Aug 15 13:27:39 2011
@@ -17,7 +17,7 @@
*/
package com.cloudera.flume.agent;
-import static com.cloudera.flume.master.TestMasterJersey.curl;
+import static com.cloudera.util.HttpServerTestUtils.curl;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
Modified: incubator/flume/trunk/flume-core/src/test/java/com/cloudera/flume/master/TestMasterJersey.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-core/src/test/java/com/cloudera/flume/master/TestMasterJersey.java?rev=1157838&r1=1157837&r2=1157838&view=diff
==============================================================================
--- incubator/flume/trunk/flume-core/src/test/java/com/cloudera/flume/master/TestMasterJersey.java (original)
+++ incubator/flume/trunk/flume-core/src/test/java/com/cloudera/flume/master/TestMasterJersey.java Mon Aug 15 13:27:39 2011
@@ -17,14 +17,11 @@
*/
package com.cloudera.flume.master;
+import static com.cloudera.util.HttpServerTestUtils.curl;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
-import java.io.BufferedReader;
import java.io.IOException;
-import java.io.InputStreamReader;
-import java.net.URL;
-import java.net.URLConnection;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
@@ -43,33 +40,6 @@ public class TestMasterJersey extends Se
public static final Logger LOG = LoggerFactory
.getLogger(TestMasterJersey.class);
- /**
- * Gra b a url's contents. Since most are json, this should be small.
- *
- * @param urlString
- * @return
- * @throws IOException
- */
- public static String curl(String urlString) throws IOException {
- URL url = new URL(urlString);
- URLConnection urlConn = url.openConnection();
- urlConn.setDoInput(true);
- urlConn.setUseCaches(false);
-
- int len = urlConn.getContentLength();
- String type = urlConn.getContentType();
- LOG.info("pulled " + urlString + "[ type=" + type + " len=" + len + "]");
- InputStreamReader isr = new InputStreamReader(urlConn.getInputStream());
- BufferedReader br = new BufferedReader(isr);
- StringBuilder sb = new StringBuilder();
- String s;
- while ((s = br.readLine()) != null) {
- sb.append(s);
- sb.append('\n');
- }
- return sb.toString();
- }
-
@Ignore
@Test
public void testMaster() throws IOException, InterruptedException,
Modified: incubator/flume/trunk/flume-core/src/test/java/com/cloudera/util/InternalHttpServerTest.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-core/src/test/java/com/cloudera/util/InternalHttpServerTest.java?rev=1157838&r1=1157837&r2=1157838&view=diff
==============================================================================
--- incubator/flume/trunk/flume-core/src/test/java/com/cloudera/util/InternalHttpServerTest.java (original)
+++ incubator/flume/trunk/flume-core/src/test/java/com/cloudera/util/InternalHttpServerTest.java Mon Aug 15 13:27:39 2011
@@ -1,13 +1,41 @@
+/**
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.cloudera.util;
+import static com.cloudera.util.HttpServerTestUtils.curlResp;
+import static org.junit.Assert.assertEquals;
+
import java.io.File;
+import java.io.IOException;
+import java.net.BindException;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import org.mortbay.jetty.handler.ContextHandlerCollection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.cloudera.flume.conf.FlumeConfiguration;
+import com.cloudera.util.InternalHttpServer.ContextCreator;
+
public class InternalHttpServerTest {
private static final Logger logger = LoggerFactory
@@ -21,25 +49,16 @@ public class InternalHttpServerTest {
}
@Test
- public void testStartInvalidState() {
+ public void testStart() throws BindException {
boolean success = false;
- try {
- httpServer.start();
- success = true;
- } catch (IllegalStateException e) {
- logger.info("Caught expected exception: {}", e.getMessage());
- }
-
- Assert.assertFalse(success);
- }
-
- @Test
- public void testStart() {
- boolean success = false;
-
- httpServer.setWebappDir(new File(getClass().getClassLoader()
- .getResource("test-webroot").getFile()));
+ httpServer.setContextCreator(new ContextCreator() {
+ @Override
+ public void addContexts(ContextHandlerCollection handlers) {
+ InternalHttpServer.addHandlersFromPaths(handlers, new File(getClass()
+ .getClassLoader().getResource("test-webroot").getFile()));
+ }
+ });
try {
httpServer.start();
@@ -68,4 +87,40 @@ public class InternalHttpServerTest {
Assert.assertTrue(success);
}
+ /**
+ * This tests to make sure that auto find port works. Two http servers are
+ * assigned to the same port -- the second one should detect the conflict and
+ * then pick the next port to bind and serve from. curl will throw exception
+ * on failure.
+ */
+ @Test
+ public void testAutoFindPort() throws IOException, Exception {
+ int port = FlumeConfiguration.get().getNodeStatusPort();
+ String bindAddress = "0.0.0.0";
+ InternalHttpServer http = InternalHttpServer.startHttpServer(null,
+ bindAddress, port);
+ http.start();
+
+ InternalHttpServer http2 = InternalHttpServer.startFindPortHttpServer(null,
+ bindAddress, port);
+ http2.start();
+
+ // grab something from each server
+ int port1 = http.getBoundPort();
+ int resp1 = curlResp("http://localhost:" + port1);
+ logger.info("http1 port:" + port1);
+
+ int port2 = http2.getBoundPort();
+ int resp2 = curlResp("http://localhost:" + port2);
+ logger.info("http2 port:" + port2);
+
+ // shutdown
+ http.stop();
+ http2.stop();
+
+ assertEquals(404, resp1);
+ assertEquals(404, resp2);
+ assertEquals(port, port1);
+ assertEquals(port + 1, port2);
+ }
}
Modified: incubator/flume/trunk/flume-node-web/src/test/java/com/cloudera/flume/agent/TestBootstrap.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-node-web/src/test/java/com/cloudera/flume/agent/TestBootstrap.java?rev=1157838&r1=1157837&r2=1157838&view=diff
==============================================================================
--- incubator/flume/trunk/flume-node-web/src/test/java/com/cloudera/flume/agent/TestBootstrap.java (original)
+++ incubator/flume/trunk/flume-node-web/src/test/java/com/cloudera/flume/agent/TestBootstrap.java Mon Aug 15 13:27:39 2011
@@ -1,14 +1,39 @@
+/**
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package com.cloudera.flume.agent;
+import static org.junit.Assert.assertEquals;
+
import java.io.File;
+import java.io.IOException;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import org.mortbay.jetty.handler.ContextHandlerCollection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.cloudera.util.HttpServerTestUtils;
import com.cloudera.util.InternalHttpServer;
+import com.cloudera.util.InternalHttpServer.ContextCreator;
public class TestBootstrap {
@@ -23,19 +48,25 @@ public class TestBootstrap {
}
@Test
- public void testBootstrap() throws InterruptedException {
+ public void testBootstrap() throws InterruptedException, IOException {
Assert.assertNotNull(httpServer);
logger.debug("httpServer:{}", httpServer);
httpServer.setPort(0);
- httpServer.setWebappDir(new File("src/main"));
+ httpServer.setContextCreator(new ContextCreator() {
+ @Override
+ public void addContexts(ContextHandlerCollection handlers) {
+ InternalHttpServer.addHandlersFromPaths(handlers, new File("src/main"));
+ }
+ });
httpServer.start();
-
- Thread.sleep(3000);
-
+ int port = httpServer.getBoundPort();
+ String url = "http://localhost:" + port;
+ logger.debug("Grabbing http response from " + url);
+ int resp = HttpServerTestUtils.curlResp(url);
httpServer.stop();
+ assertEquals(resp, 200); // expect ok response code.
}
-
}