You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flume.apache.org by jm...@apache.org on 2011/08/17 14:54:20 UTC
svn commit: r1158677 - in /incubator/flume/trunk:
flume-core/src/main/java/com/cloudera/flume/agent/
flume-core/src/main/java/com/cloudera/flume/master/
flume-core/src/main/java/com/cloudera/util/
flume-core/src/test/java/com/cloudera/flume/agent/ flum...
Author: jmhsieh
Date: Wed Aug 17 12:54:20 2011
New Revision: 1158677
URL: http://svn.apache.org/viewvc?rev=1158677&view=rev
Log:
Revert "FLUME-721: Webapps 'autofindport' feature does not work"
This patch breaks project complilation.
This reverts commit 6c523bd372d093d8f68397fc71d14bf0c80e1964.
Added:
incubator/flume/trunk/flume-core/src/main/java/com/cloudera/util/StatusHttpServer.java
incubator/flume/trunk/flume-core/src/test/java/com/cloudera/util/TestStatusHttpServer.java
Removed:
incubator/flume/trunk/flume-core/src/main/java/com/cloudera/util/HttpServerTestUtils.java
Modified:
incubator/flume/trunk/flume-core/src/main/java/com/cloudera/flume/agent/FlumeNode.java
incubator/flume/trunk/flume-core/src/main/java/com/cloudera/flume/master/FlumeMaster.java
incubator/flume/trunk/flume-core/src/main/java/com/cloudera/util/InternalHttpServer.java
incubator/flume/trunk/flume-core/src/test/java/com/cloudera/flume/agent/TestNodeJersey.java
incubator/flume/trunk/flume-core/src/test/java/com/cloudera/flume/master/TestMasterJersey.java
incubator/flume/trunk/flume-core/src/test/java/com/cloudera/util/InternalHttpServerTest.java
incubator/flume/trunk/flume-node-web/src/test/java/com/cloudera/flume/agent/TestBootstrap.java
Modified: incubator/flume/trunk/flume-core/src/main/java/com/cloudera/flume/agent/FlumeNode.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-core/src/main/java/com/cloudera/flume/agent/FlumeNode.java?rev=1158677&r1=1158676&r2=1158677&view=diff
==============================================================================
--- incubator/flume/trunk/flume-core/src/main/java/com/cloudera/flume/agent/FlumeNode.java (original)
+++ incubator/flume/trunk/flume-core/src/main/java/com/cloudera/flume/agent/FlumeNode.java Wed Aug 17 12:54:20 2011
@@ -34,7 +34,6 @@ import org.apache.commons.cli.ParseExcep
import org.apache.commons.cli.PosixParser;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
-import org.mortbay.jetty.handler.ContextHandlerCollection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -62,7 +61,6 @@ import com.cloudera.flume.util.SystemInf
import com.cloudera.util.CheckJavaVersion;
import com.cloudera.util.FileUtil;
import com.cloudera.util.InternalHttpServer;
-import com.cloudera.util.InternalHttpServer.ContextCreator;
import com.cloudera.util.NetUtils;
import com.cloudera.util.Pair;
import com.cloudera.util.StatusHttpServer.StackServlet;
@@ -260,20 +258,24 @@ public class FlumeNode implements Report
ReportManager.get().add(this);
if (startHttp) {
- int nodePort = conf.getNodeStatusPort();
- String bindAddress = "0.0.0.0";
- ContextCreator cc = new ContextCreator() {
- @Override
- public void addContexts(ContextHandlerCollection handlers) {
- handlers.addHandler(InternalHttpServer.createLogAppContext());
- handlers.addHandler(InternalHttpServer.createStackSevletContext());
- String webAppRoot = FlumeConfiguration.get().getNodeWebappRoot();
- InternalHttpServer.addHandlersFromPaths(handlers,
- new File(webAppRoot));
- }
- };
- http = InternalHttpServer.startFindPortHttpServer(cc, bindAddress,
- nodePort);
+ try {
+ http = new InternalHttpServer();
+
+ http.addHandler(InternalHttpServer.createLogAppContext());
+
+ http.addHandler(InternalHttpServer.createServletContext(
+ StackServlet.class, "/stacks", "/*", "stacks"));
+
+ http.setBindAddress("0.0.0.0");
+ http.setPort(conf.getNodeStatusPort());
+ String webAppRoot = FlumeConfiguration.get().getNodeWebappRoot();
+ http.setWebappDir(new File(webAppRoot));
+ http.setScanForApps(true);
+
+ http.start();
+ } catch (Throwable t) {
+ LOG.error("Unexpected exception/error thrown! " + t.getMessage(), t);
+ }
}
if (reportPusher != null) {
@@ -369,9 +371,9 @@ public class FlumeNode implements Report
/**
* This function checks the agent logs dir to make sure that the process has
- * the ability to the directory if necessary, that the path if it does exist
- * is a directory, and that it can in fact create files inside of the
- * directory. If it fails any of these, it throws an exception.
+ * the ability to the directory if necessary, that the path if it does exist is
+ * a directory, and that it can in fact create files inside of the directory.
+ * If it fails any of these, it throws an exception.
*
* Finally, it checks to see if the path is in /tmp and warns the user that
* this may not be the best idea.
Modified: incubator/flume/trunk/flume-core/src/main/java/com/cloudera/flume/master/FlumeMaster.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-core/src/main/java/com/cloudera/flume/master/FlumeMaster.java?rev=1158677&r1=1158676&r2=1158677&view=diff
==============================================================================
--- incubator/flume/trunk/flume-core/src/main/java/com/cloudera/flume/master/FlumeMaster.java (original)
+++ incubator/flume/trunk/flume-core/src/main/java/com/cloudera/flume/master/FlumeMaster.java Wed Aug 17 12:54:20 2011
@@ -35,7 +35,6 @@ import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.cli.PosixParser;
import org.apache.thrift.transport.TTransportException;
-import org.mortbay.jetty.handler.ContextHandlerCollection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -54,7 +53,6 @@ import com.cloudera.flume.util.FlumeVMIn
import com.cloudera.flume.util.SystemInfo;
import com.cloudera.util.CheckJavaVersion;
import com.cloudera.util.InternalHttpServer;
-import com.cloudera.util.InternalHttpServer.ContextCreator;
import com.cloudera.util.NetUtils;
import com.cloudera.util.StatusHttpServer.StackServlet;
@@ -251,19 +249,18 @@ public class FlumeMaster implements Repo
ReportManager.get().add(sysInfo);
if (doHttp) {
- String bindAddress = "0.0.0.0";
- int port = cfg.getMasterHttpPort();
- final String webAppRoot = FlumeConfiguration.get().getMasterWebappRoot();
+ http = new InternalHttpServer();
+
+ http.addHandler(InternalHttpServer.createLogAppContext());
+ http.addHandler(InternalHttpServer.createServletContext(
+ StackServlet.class, "/stacks", "/*", "stacks"));
+
+ http.setBindAddress("0.0.0.0");
+ http.setPort(cfg.getMasterHttpPort());
+ String webAppRoot = FlumeConfiguration.get().getMasterWebappRoot();
+ http.setWebappDir(new File(webAppRoot));
LOG.info("Webserver root directory: " + webAppRoot);
- ContextCreator cc = new ContextCreator() {
- @Override
- public void addContexts(ContextHandlerCollection handlers) {
- handlers.addHandler(InternalHttpServer.createLogAppContext());
- handlers.addHandler(InternalHttpServer.createStackSevletContext());
- InternalHttpServer.addHandlersFromPaths(handlers, new File(webAppRoot));
- }
- };
- http = InternalHttpServer.startHttpServer(cc, bindAddress, port);
+ http.start();
}
controlServer = new MasterClientServer(this, FlumeConfiguration.get());
Modified: incubator/flume/trunk/flume-core/src/main/java/com/cloudera/util/InternalHttpServer.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-core/src/main/java/com/cloudera/util/InternalHttpServer.java?rev=1158677&r1=1158676&r2=1158677&view=diff
==============================================================================
--- incubator/flume/trunk/flume-core/src/main/java/com/cloudera/util/InternalHttpServer.java (original)
+++ incubator/flume/trunk/flume-core/src/main/java/com/cloudera/util/InternalHttpServer.java Wed Aug 17 12:54:20 2011
@@ -1,38 +1,7 @@
-/**
- * Copyright 2011 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
package com.cloudera.util;
import java.io.File;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.io.PrintWriter;
-import java.net.BindException;
-
-import javax.servlet.ServletException;
-import javax.servlet.http.HttpServlet;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.util.ReflectionUtils;
+
import org.mortbay.jetty.Connector;
import org.mortbay.jetty.Server;
import org.mortbay.jetty.handler.ContextHandlerCollection;
@@ -44,14 +13,63 @@ import org.mortbay.jetty.webapp.WebAppCo
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.cloudera.util.StatusHttpServer.StackServlet;
+import com.google.common.base.Preconditions;
+
/**
* <p>
- * An embedded Jetty HTTP server. It defers addition of contexts/handlers to
- * a callback so that the we can provide a method that increments ports until
- * a valid port is found. This is mostly a thin wrapper around Jetty's
- * {@link Server} class and behaves as Jetty does.
- *
- * Here is an example usage:
+ * An embedded Jetty HTTP server that support both normal and exploded war file
+ * deployment. Those that wish to expose HTTP services should create an instance
+ * of this class, configure the server via accessor methods, and then call
+ * {@link #start()}.
+ * </p>
+ * <p>
+ * Resources internally are allocated upon the first call to {@link #start()}.
+ * This includes scanning of the configured webapp directory for applications if
+ * {@link #getScanForApps()} is true (the default). Mostly this class is a thin
+ * wrapper around Jetty's {@link Server} class and behaves as Jetty does. Both
+ * traditional and exploded war formats are supported in the webapp directory.
+ * In the case of exploded directories, the directory name is used as the
+ * context. For war files, everything from the first instance of ".war" to the
+ * end of the file name (inclusive) is stripped and the remainder is used for
+ * the context name.
+ * </p>
+ * <p>
+ * Name examples:
+ * </p>
+ * <table>
+ * <tr>
+ * <td>Name</td>
+ * <td>Type</td>
+ * <td>Context</td>
+ * </tr>
+ * <tr>
+ * <td>app.war</td>
+ * <td>file</td>
+ * <td>app</td>
+ * </tr>
+ * <tr>
+ * <td>app</td>
+ * <td>dir</td>
+ * <td>app</td>
+ * </tr>
+ * <tr>
+ * <td>app.war</td>
+ * <td>dir</td>
+ * <td>app.war</td>
+ * </tr>
+ * <tr>
+ * <td>app.war.war</td>
+ * <td>file</td>
+ * <td>app</td>
+ * </tr>
+ * <tr>
+ * <td>
+ * </table>
+ * <p>
+ * Example usage:
+ * </p>
+ *
* <pre>
* InternalHttpServer server = new InternalHttpServer();
*
@@ -59,20 +77,9 @@ import org.slf4j.LoggerFactory;
* server.setWebappDir(new File(applicationHome, "webapps"));
* server.setPort(8080);
* server.setBindAddress("0.0.0.0");
- * server.setContextCreator(new ContextCreator() {
- * @Override
- * public void addContexts(ContextHandlerCollection handlers) {
- * handlers.addHandler(InternalHttpServer.createLogAppContext());
- * handlers.addHandler(InternalHttpServer.createStackSevletContext(
- * StackServlet.class, "/stacks", "/*", "stacks"));
- * String webAppRoot = FlumeConfiguration.get().getNodeWebappRoot();
- * InternalHttpServer.addHandlersFromPaths(handlers,
- * new File(webAppRoot));
- * }
- * });
- *
+ *
* server.start();
- *
+ *
* // at some later time...
* server.stop();
* </pre>
@@ -83,15 +90,16 @@ public class InternalHttpServer {
.getLogger(InternalHttpServer.class);
private Server server;
+ private File webappDir;
private int port;
- private int boundPort = -1;
private String bindAddress;
+ private boolean scanForApps;
private ContextHandlerCollection handlers;
- private ContextCreator contextCreator = null;
public InternalHttpServer() {
port = 0;
bindAddress = "0.0.0.0";
+ scanForApps = true;
handlers = new ContextHandlerCollection();
}
@@ -105,14 +113,51 @@ public class InternalHttpServer {
connector.setHost(bindAddress);
server.addConnector(connector);
+ server.addHandler(handlers);
+ }
+ }
- if (contextCreator != null) {
- contextCreator.addContexts(handlers);
+ protected void registerApplications() {
+ logger.debug("Registering webapps in {}", webappDir);
+
+ if (webappDir.isDirectory()) {
+ for (File entry : webappDir.listFiles()) {
+ tryRegisterApplication(server, entry);
}
- server.setHandler(handlers);
+ } else {
+ tryRegisterApplication(server, webappDir);
}
}
+ private boolean tryRegisterApplication(Server server, File path) {
+ String name;
+
+ logger.debug("checking {}", path);
+
+ if (path.isFile()) {
+ int idx = path.getName().indexOf(".war");
+
+ if (idx > -1) {
+ name = path.getName().substring(0, idx);
+ } else {
+ return false;
+ }
+ } else {
+ name = path.getName();
+ }
+
+ logger.debug("creating context {} -> {}", name, path);
+
+ // WebAppContext is for loading war files.
+ WebAppContext handler = new WebAppContext(path.getPath(), "/" + name);
+
+ handler.setParentLoaderPriority(true);
+
+ handlers.addHandler(handler);
+
+ return true;
+ }
+
/**
* <p>
* Start a configured HTTP server. Users should have already injected all the
@@ -123,22 +168,26 @@ public class InternalHttpServer {
* The configured webappDir is not scanned for applications until start() is
* called.
* </p>
- *
- * @throws BindException
+ *
* @throws InternalHttpServerException
*/
- public void start() throws BindException {
+ public void start() {
+ Preconditions.checkState(webappDir != null, "Webapp dir can not be null");
initialize();
+ if (scanForApps) {
+ registerApplications();
+ } else {
+ logger.info("Not scanning for webapps");
+ }
+
logger.info("Starting internal HTTP server");
try {
server.start();
- boundPort = server.getConnectors()[0].getLocalPort();
- logger.info("Server started on port " + boundPort);
- } catch (BindException be) {
- throw be;
+
+ logger.info("Server started");
} catch (Exception e) {
logger.warn("Caught exception during HTTP server start.", e);
@@ -171,8 +220,9 @@ public class InternalHttpServer {
@Override
public String toString() {
- return "{ bindAddress:" + bindAddress + " port:" + port + " boundPort:"
- + boundPort + " server:" + server + " }";
+ return "{ bindAddress:" + bindAddress + " webappDir:" + webappDir
+ + " port:" + port + " scanForApps:" + scanForApps + " server:" + server
+ + " }";
}
public Server getServer() {
@@ -183,12 +233,16 @@ public class InternalHttpServer {
this.server = server;
}
- public int getPort() {
- return port;
+ public File getWebappDir() {
+ return webappDir;
}
- public int getBoundPort() {
- return boundPort;
+ public void setWebappDir(File webappDir) {
+ this.webappDir = webappDir;
+ }
+
+ public int getPort() {
+ return port;
}
public void setPort(int port) {
@@ -203,6 +257,14 @@ public class InternalHttpServer {
this.bindAddress = bindAddress;
}
+ public boolean getScanForApps() {
+ return scanForApps;
+ }
+
+ public void setScanForApps(boolean scanForApps) {
+ this.scanForApps = scanForApps;
+ }
+
public static class InternalHttpServerException extends RuntimeException {
private static final long serialVersionUID = -4936285404574873547L;
@@ -225,19 +287,7 @@ public class InternalHttpServer {
}
- public void setHandlers(ContextHandlerCollection ctx) {
- if (ctx == null) {
- logger.warn("Attempting to add null webapp context");
- return;
- }
- handlers = ctx;
- }
-
- public ContextHandlerCollection getHandlers() {
- return handlers;
- }
-
- protected void addHandler(Context ctx) {
+ public void addHandler(Context ctx) {
if (ctx == null) {
logger.warn("Attempting to add null webapp context");
return;
@@ -245,117 +295,6 @@ public class InternalHttpServer {
handlers.addHandler(ctx);
}
- public void setContextCreator(ContextCreator cc) {
- this.contextCreator = cc;
- }
-
- /**
- * The jetty server cannot properly reload contexts if it attempts to bind to
- * a port and fails. To support automatically going finding a new port, we
- * thus need to parameterize the creation and addition of context. This class
- * provides a call back that gets a instance of the server's
- * ContextHandlerCollection, and gives clients the opportunity to populate it.
- */
- public abstract static class ContextCreator {
- public abstract void addContexts(ContextHandlerCollection handlers);
- }
-
- public static WebAppContext createWarContext(File path) {
- logger.debug("checking {}", path);
-
- String name;
- if (path.isFile()) {
- // if not a war file reject
- int idx = path.getName().indexOf(".war");
- if (idx < 0) {
- return null;
- }
-
- // drop the .war suffix
- name = path.getName().substring(0, idx);
- } else {
- // is a dir
- name = path.getName();
- }
-
- // WebAppContext is for loading war files.
- logger.debug("creating context {} -> {}", name, path);
- WebAppContext handler = new WebAppContext(path.getPath(), "/" + name);
- handler.setParentLoaderPriority(true);
- return handler;
- }
-
- /**
- * This method adds support for both normal and exploded war file deployment.
- * <p>
- * This scannings the specified webapp directory for applications
- * Both traditional and exploded war formats are supported in the webapp
- * directory. In the case of exploded directories, the directory name is used
- * as the context. For war files, everything from the first instance of ".war"
- * to the end of the file name (inclusive) is stripped and the remainder is
- * used for the context name.
- * </p>
- * <p>
- * Name examples:
- * </p>
- * <table>
- * <tr>
- * <td>Name</td>
- * <td>Type</td>
- * <td>Context</td>
- * </tr>
- * <tr>
- * <td>app.war</td>
- * <td>file</td>
- * <td>app</td>
- * </tr>
- * <tr>
- * <td>app</td>
- * <td>dir</td>
- * <td>app</td>
- * </tr>
- * <tr>
- * <td>app.war</td>
- * <td>dir</td>
- * <td>app.war</td>
- * </tr>
- * <tr>
- * <td>app.war.war</td>
- * <td>file</td>
- * <td>app</td>
- * </tr>
- * <tr>
- * <td>
- * </table>
- * <p>
- * Example usage:
- * </p>
- */
- public static void addHandlersFromPaths(ContextHandlerCollection handlers,
- File webappDir) {
- logger.debug("Registering webapps in {}", webappDir);
-
- if (webappDir.isDirectory()) {
- for (File entry : webappDir.listFiles()) {
- Context ctx = createWarContext(entry);
- if (ctx != null) {
- handlers.addHandler(ctx);
- }
- }
- } else {
- Context ctx = createWarContext(webappDir);
- if (ctx != null) {
- handlers.addHandler(ctx);
- }
- }
- }
-
- /**
- * This creates file listing servlet context that is used to point to the log
- * directory of the daemon via the web interface.
- *
- * @return
- */
public static Context createLogAppContext() {
Context ctx = new Context();
// logs applet
@@ -369,39 +308,8 @@ public class InternalHttpServer {
return ctx;
}
- /**
- * A very simple servlet to serve up a text representation of the current
- * stack traces. It both returns the stacks to the caller and logs them.
- * Currently the stack traces are done sequentially rather than exactly the
- * same data.
- */
- public static class StackServlet extends HttpServlet {
- private static final Log LOG = LogFactory.getLog(StatusHttpServer.class
- .getName());
- private static final long serialVersionUID = -6284183679759467039L;
-
- @Override
- public void doGet(HttpServletRequest request, HttpServletResponse response)
- throws ServletException, IOException {
-
- OutputStream outStream = response.getOutputStream();
- ReflectionUtils.printThreadInfo(new PrintWriter(outStream), "");
- outStream.close();
- ReflectionUtils.logThreadInfo(LOG, "jsp requested", 1);
- }
- }
-
- /**
- * This creates a stack dumping servlet that can be used to debug a running
- * daemon via the web interface.
- *
- * @param sltClz
- * @param contextPath
- * @param pathSpec
- * @param name
- * @return
- */
- public static Context createStackSevletContext() {
+ public static Context createServletContext(Class<?> sltClz, String contextPath,
+ String pathSpec, String name) {
Context ctx = new Context();
ServletHolder holder = new ServletHolder(StackServlet.class);
ctx.setContextPath("/stacks");
@@ -411,50 +319,4 @@ public class InternalHttpServer {
return ctx;
}
- /**
- * If successful returns the port the http server successfully bound to. If it
- * failed, returns -1
- */
- public static InternalHttpServer startFindPortHttpServer(ContextCreator cc,
- String bindAddress, int nodePort) {
- do {
- try {
- return startHttpServer(cc, bindAddress, nodePort);
- } catch (BindException be) {
- logger.error("Unable to start webserver on " + bindAddress + ":"
- + nodePort + ". Trying next port...");
- nodePort++;
- }
- } while (true);
- }
-
- /**
- * Single attempt to create an http server for the node.
- *
- * @param bindAddress
- * @param nodePort
- * @return instance of a started http server or null if failed.
- * @throws BindException
- */
- public static InternalHttpServer startHttpServer(ContextCreator cc,
- String bindAddress, int nodePort) throws BindException {
- InternalHttpServer http = null;
- try {
- http = new InternalHttpServer();
- http.setBindAddress(bindAddress);
- http.setPort(nodePort);
- http.setContextCreator(cc);
- http.start();
- return http;
- } catch (BindException be) {
- http.stop();
- http = null;
- throw be;
- } catch (Throwable t) {
- logger.error("Unexpected exception/error thrown! " + t.getMessage(), t);
- // if any exception happens bail out and cleanup.
- http.stop();
- return null;
- }
- }
}
Added: incubator/flume/trunk/flume-core/src/main/java/com/cloudera/util/StatusHttpServer.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-core/src/main/java/com/cloudera/util/StatusHttpServer.java?rev=1158677&view=auto
==============================================================================
--- incubator/flume/trunk/flume-core/src/main/java/com/cloudera/util/StatusHttpServer.java (added)
+++ incubator/flume/trunk/flume-core/src/main/java/com/cloudera/util/StatusHttpServer.java Wed Aug 17 12:54:20 2011
@@ -0,0 +1,251 @@
+/**
+ * Licensed to Cloudera, Inc. under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Cloudera, Inc. licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.cloudera.util;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintWriter;
+import java.net.BindException;
+import java.net.InetSocketAddress;
+
+import javax.servlet.Servlet;
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.mortbay.jetty.Server;
+import org.mortbay.jetty.nio.SelectChannelConnector;
+import org.mortbay.jetty.security.SslSocketConnector;
+import org.mortbay.jetty.servlet.ServletHolder;
+import org.mortbay.jetty.webapp.WebAppContext;
+
+import com.google.common.base.Preconditions;
+
+// jon: This is a shamelessly hacked version of the Http status server from the jobtracker,
+// simplified for my needs. Originally from apache licensed hadoop 0.18.3, o.a.h.mapred.StatusHttpServer
+
+/**
+ * Create a Jetty embedded server to answer http requests. The primary goal is
+ * to serve up status information for the server. There are three contexts:
+ * "/logs/" -> points to the log directory "/static/" -> points to common static
+ * files (src/webapps/static) "/" -> the jsp server code from
+ * (src/webapps/<name>)
+ */
+public class StatusHttpServer {
+ private Server webServer;
+ private SelectChannelConnector channelConnector;
+ private SslSocketConnector sslConnector;
+ private boolean findPort;
+ private WebAppContext webAppContext;
+ private static final Log LOG = LogFactory.getLog(StatusHttpServer.class
+ .getName());
+
+ /**
+ * Create a status server on the given port. The jsp scripts are taken from
+ * src/webapps/<name>.
+ *
+ * @param name
+ * The name of the server
+ * @param port
+ * The port to use on the server
+ * @param findPort
+ * whether the server should start at the given port and increment by
+ * 1 until it finds a free port.
+ */
+ public StatusHttpServer(String name, String webAppsPath, String bindAddress,
+ int port, boolean findPort) throws IOException {
+ webServer = new org.mortbay.jetty.Server();
+ this.findPort = findPort;
+ channelConnector = new SelectChannelConnector();
+ channelConnector.setPort(port);
+ channelConnector.setHost(bindAddress);
+ webServer.addConnector(channelConnector);
+
+ String appDir = webAppsPath;
+ // set up the context for "/" jsp files
+ String webapp = new File(appDir, name).getAbsolutePath();
+ LOG.info("starting web app in directory: " + webapp);
+ webAppContext = new WebAppContext(webapp, "/");
+ webServer.setHandler(webAppContext);
+ addServlet("stacks", "/stacks", StackServlet.class);
+ }
+
+ /**
+ * Sets a value in the webapp context. These values are available to the jsp
+ * pages as "application.getAttribute(name)".
+ *
+ * @param name
+ * The name of the attribute
+ * @param value
+ * The value of the attribute
+ */
+ public void setAttribute(String name, Object value) {
+ webAppContext.setAttribute(name, value);
+ }
+
+ /**
+ * Add a servlet in the server.
+ *
+ * @param name
+ * The name of the servlet (can be passed as null)
+ * @param pathSpec
+ * The path spec for the servlet
+ * @param servletClass
+ * The servlet class
+ */
+ public <T extends HttpServlet> void addServlet(String name, String pathSpec,
+ Class<T> servletClass) {
+
+ WebAppContext context = webAppContext;
+ if (name == null) {
+ context.addServlet(pathSpec, servletClass.getName());
+ } else {
+ context.addServlet(servletClass, pathSpec);
+ }
+ }
+
+ public void addServlet(Servlet servlet, String pathSpec) {
+ webAppContext.addServlet(new ServletHolder(servlet), pathSpec);
+ }
+
+ /**
+ * Get the value in the webapp context.
+ *
+ * @param name
+ * The name of the attribute
+ * @return The value of the attribute
+ */
+ public Object getAttribute(String name) {
+ return webAppContext.getAttribute(name);
+ }
+
+ /**
+ * Get the port that the server is on
+ *
+ * @return the port
+ */
+ public int getPort() {
+ return channelConnector.getPort();
+ }
+
+ /**
+ * Configure an ssl listener on the server.
+ *
+ * @param addr
+ * address to listen on
+ * @param keystore
+ * location of the keystore
+ * @param storPass
+ * password for the keystore
+ * @param keyPass
+ * password for the key
+ */
+ public void addSslListener(InetSocketAddress addr, String keystore,
+ String storPass, String keyPass) throws IOException {
+ if (sslConnector != null || webServer.isStarted()) {
+ throw new IOException("Failed to add ssl listener");
+ }
+ sslConnector = new SslSocketConnector();
+ sslConnector.setHost(addr.getHostName());
+ sslConnector.setPort(addr.getPort());
+ sslConnector.setKeystore(keystore);
+ sslConnector.setPassword(storPass);
+ sslConnector.setKeyPassword(keyPass);
+ webServer.addConnector(sslConnector);
+ }
+
+ /**
+ * Start the server. Does not wait for the server to start.
+ */
+ public void start() throws IOException {
+ try {
+ while (true) {
+ try {
+ webServer.start();
+ break;
+ } catch (BindException ex) {
+ // if the multi exception contains ONLY a bind exception,
+ // then try the next port number.
+ if (!findPort) {
+ throw ex;
+ }
+ // pick another port
+ webServer.stop();
+ channelConnector.setPort(channelConnector.getPort() + 1);
+ }
+ }
+ } catch (Exception e) {
+ IOException ie = new IOException("Problem starting http server");
+ ie.initCause(e);
+ throw ie;
+ }
+ }
+
+ /**
+ * stop the server
+ */
+ public void stop() throws Exception {
+ webServer.stop();
+ }
+
+ /**
+ * A very simple servlet to serve up a text representation of the current
+ * stack traces. It both returns the stacks to the caller and logs them.
+ * Currently the stack traces are done sequentially rather than exactly the
+ * same data.
+ */
+ public static class StackServlet extends HttpServlet {
+
+ private static final long serialVersionUID = -6284183679759467039L;
+
+ @Override
+ public void doGet(HttpServletRequest request, HttpServletResponse response)
+ throws ServletException, IOException {
+
+ OutputStream outStream = response.getOutputStream();
+ ReflectionUtils.printThreadInfo(new PrintWriter(outStream), "");
+ outStream.close();
+ ReflectionUtils.logThreadInfo(LOG, "jsp requested", 1);
+ }
+ }
+
+ /**
+ * Test harness to get precompiled jsps working.
+ *
+ * @param argv
+ */
+ public static void main(String[] argv) {
+ Preconditions.checkArgument(argv.length == 3);
+ String name = argv[0];
+ String path = argv[1];
+ int port = Integer.parseInt(argv[2]);
+
+ try {
+ StatusHttpServer http = new StatusHttpServer(name, path, "0.0.0.0", port,
+ false);
+ http.start();
+ } catch (IOException ioe) {
+ ioe.printStackTrace();
+ }
+ }
+}
Modified: incubator/flume/trunk/flume-core/src/test/java/com/cloudera/flume/agent/TestNodeJersey.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-core/src/test/java/com/cloudera/flume/agent/TestNodeJersey.java?rev=1158677&r1=1158676&r2=1158677&view=diff
==============================================================================
--- incubator/flume/trunk/flume-core/src/test/java/com/cloudera/flume/agent/TestNodeJersey.java (original)
+++ incubator/flume/trunk/flume-core/src/test/java/com/cloudera/flume/agent/TestNodeJersey.java Wed Aug 17 12:54:20 2011
@@ -17,7 +17,7 @@
*/
package com.cloudera.flume.agent;
-import static com.cloudera.util.HttpServerTestUtils.curl;
+import static com.cloudera.flume.master.TestMasterJersey.curl;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
Modified: incubator/flume/trunk/flume-core/src/test/java/com/cloudera/flume/master/TestMasterJersey.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-core/src/test/java/com/cloudera/flume/master/TestMasterJersey.java?rev=1158677&r1=1158676&r2=1158677&view=diff
==============================================================================
--- incubator/flume/trunk/flume-core/src/test/java/com/cloudera/flume/master/TestMasterJersey.java (original)
+++ incubator/flume/trunk/flume-core/src/test/java/com/cloudera/flume/master/TestMasterJersey.java Wed Aug 17 12:54:20 2011
@@ -17,11 +17,14 @@
*/
package com.cloudera.flume.master;
-import static com.cloudera.util.HttpServerTestUtils.curl;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
+import java.io.BufferedReader;
import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.URL;
+import java.net.URLConnection;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
@@ -40,6 +43,33 @@ public class TestMasterJersey extends Se
public static final Logger LOG = LoggerFactory
.getLogger(TestMasterJersey.class);
+ /**
+ * Gra b a url's contents. Since most are json, this should be small.
+ *
+ * @param urlString
+ * @return
+ * @throws IOException
+ */
+ public static String curl(String urlString) throws IOException {
+ URL url = new URL(urlString);
+ URLConnection urlConn = url.openConnection();
+ urlConn.setDoInput(true);
+ urlConn.setUseCaches(false);
+
+ int len = urlConn.getContentLength();
+ String type = urlConn.getContentType();
+ LOG.info("pulled " + urlString + "[ type=" + type + " len=" + len + "]");
+ InputStreamReader isr = new InputStreamReader(urlConn.getInputStream());
+ BufferedReader br = new BufferedReader(isr);
+ StringBuilder sb = new StringBuilder();
+ String s;
+ while ((s = br.readLine()) != null) {
+ sb.append(s);
+ sb.append('\n');
+ }
+ return sb.toString();
+ }
+
@Ignore
@Test
public void testMaster() throws IOException, InterruptedException,
Modified: incubator/flume/trunk/flume-core/src/test/java/com/cloudera/util/InternalHttpServerTest.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-core/src/test/java/com/cloudera/util/InternalHttpServerTest.java?rev=1158677&r1=1158676&r2=1158677&view=diff
==============================================================================
--- incubator/flume/trunk/flume-core/src/test/java/com/cloudera/util/InternalHttpServerTest.java (original)
+++ incubator/flume/trunk/flume-core/src/test/java/com/cloudera/util/InternalHttpServerTest.java Wed Aug 17 12:54:20 2011
@@ -1,41 +1,13 @@
-/**
- * Copyright 2011 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
package com.cloudera.util;
-import static com.cloudera.util.HttpServerTestUtils.curlResp;
-import static org.junit.Assert.assertEquals;
-
import java.io.File;
-import java.io.IOException;
-import java.net.BindException;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-import org.mortbay.jetty.handler.ContextHandlerCollection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.cloudera.flume.conf.FlumeConfiguration;
-import com.cloudera.util.InternalHttpServer.ContextCreator;
-
public class InternalHttpServerTest {
private static final Logger logger = LoggerFactory
@@ -49,16 +21,25 @@ public class InternalHttpServerTest {
}
@Test
- public void testStart() throws BindException {
+ public void testStartInvalidState() {
boolean success = false;
- httpServer.setContextCreator(new ContextCreator() {
- @Override
- public void addContexts(ContextHandlerCollection handlers) {
- InternalHttpServer.addHandlersFromPaths(handlers, new File(getClass()
- .getClassLoader().getResource("test-webroot").getFile()));
- }
- });
+ try {
+ httpServer.start();
+ success = true;
+ } catch (IllegalStateException e) {
+ logger.info("Caught expected exception: {}", e.getMessage());
+ }
+
+ Assert.assertFalse(success);
+ }
+
+ @Test
+ public void testStart() {
+ boolean success = false;
+
+ httpServer.setWebappDir(new File(getClass().getClassLoader()
+ .getResource("test-webroot").getFile()));
try {
httpServer.start();
@@ -87,40 +68,4 @@ public class InternalHttpServerTest {
Assert.assertTrue(success);
}
- /**
- * This tests to make sure that auto find port works. Two http servers are
- * assigned to the same port -- the second one should detect the conflict and
- * then pick the next port to bind and serve from. curl will throw exception
- * on failure.
- */
- @Test
- public void testAutoFindPort() throws IOException, Exception {
- int port = FlumeConfiguration.get().getNodeStatusPort();
- String bindAddress = "0.0.0.0";
- InternalHttpServer http = InternalHttpServer.startHttpServer(null,
- bindAddress, port);
- http.start();
-
- InternalHttpServer http2 = InternalHttpServer.startFindPortHttpServer(null,
- bindAddress, port);
- http2.start();
-
- // grab something from each server
- int port1 = http.getBoundPort();
- int resp1 = curlResp("http://localhost:" + port1);
- logger.info("http1 port:" + port1);
-
- int port2 = http2.getBoundPort();
- int resp2 = curlResp("http://localhost:" + port2);
- logger.info("http2 port:" + port2);
-
- // shutdown
- http.stop();
- http2.stop();
-
- assertEquals(404, resp1);
- assertEquals(404, resp2);
- assertEquals(port, port1);
- assertEquals(port + 1, port2);
- }
}
Added: incubator/flume/trunk/flume-core/src/test/java/com/cloudera/util/TestStatusHttpServer.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-core/src/test/java/com/cloudera/util/TestStatusHttpServer.java?rev=1158677&view=auto
==============================================================================
--- incubator/flume/trunk/flume-core/src/test/java/com/cloudera/util/TestStatusHttpServer.java (added)
+++ incubator/flume/trunk/flume-core/src/test/java/com/cloudera/util/TestStatusHttpServer.java Wed Aug 17 12:54:20 2011
@@ -0,0 +1,90 @@
+/**
+ * Licensed to Cloudera, Inc. under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Cloudera, Inc. licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.cloudera.util;
+
+import static com.cloudera.flume.master.TestMasterJersey.curl;
+
+import java.io.IOException;
+
+import org.junit.Ignore;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.cloudera.flume.agent.FlumeNode;
+import com.cloudera.flume.conf.FlumeConfiguration;
+
+/**
+ * This tests the behavior of rapidly opening and closing the http server.
+ * Things are sane.
+ */
+public class TestStatusHttpServer {
+ public static final Logger LOG = LoggerFactory
+ .getLogger(TestStatusHttpServer.class);
+
+ @Test
+ public void testOpenClose() throws Exception {
+ // Set directory of webapps to build-specific dir
+ FlumeConfiguration.get().set(FlumeConfiguration.WEBAPPS_PATH,
+ "build/webapps");
+
+ FlumeConfiguration conf = FlumeConfiguration.get();
+ String webPath = FlumeNode.getWebPath(conf);
+ int port = FlumeConfiguration.get().getNodeStatusPort();
+ StatusHttpServer http = new StatusHttpServer("flumeagent", webPath,
+ "0.0.0.0", port, false);
+
+ for (int i = 0; i < 50; i++) {
+ http.start();
+ http.stop();
+ }
+ }
+
+ /**
+ * This tests to make sure that auto find port works. Two http servers are
+ * assigned to the same port -- the second one should detect the conflict and
+ * then pick the next port to bind and serve from. curl will throw exception
+ * on failure.
+ */
+ @Test
+ @Ignore
+ public void testAutoFindPort() throws IOException, Exception {
+ // Set directory of webapps to build-specific dir
+ FlumeConfiguration.get().set(FlumeConfiguration.WEBAPPS_PATH,
+ "build/webapps");
+
+ FlumeConfiguration conf = FlumeConfiguration.get();
+ String webPath = FlumeNode.getWebPath(conf);
+ int port = FlumeConfiguration.get().getNodeStatusPort();
+ StatusHttpServer http = new StatusHttpServer("flumeagent", webPath,
+ "0.0.0.0", port, true);
+ http.start();
+
+ StatusHttpServer http2 = new StatusHttpServer("flumeagent", webPath,
+ "0.0.0.0", port, true);
+ http2.start();
+
+ String s1 = curl("http://localhost:35862");
+ LOG.info("http1:" + s1);
+ String s2 = curl("http://localhost:35863");
+ LOG.info("http2:" + s2);
+ http.stop();
+ http2.stop();
+ }
+}
Modified: incubator/flume/trunk/flume-node-web/src/test/java/com/cloudera/flume/agent/TestBootstrap.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-node-web/src/test/java/com/cloudera/flume/agent/TestBootstrap.java?rev=1158677&r1=1158676&r2=1158677&view=diff
==============================================================================
--- incubator/flume/trunk/flume-node-web/src/test/java/com/cloudera/flume/agent/TestBootstrap.java (original)
+++ incubator/flume/trunk/flume-node-web/src/test/java/com/cloudera/flume/agent/TestBootstrap.java Wed Aug 17 12:54:20 2011
@@ -1,39 +1,14 @@
-/**
- * Copyright 2011 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
package com.cloudera.flume.agent;
-import static org.junit.Assert.assertEquals;
-
import java.io.File;
-import java.io.IOException;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-import org.mortbay.jetty.handler.ContextHandlerCollection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.cloudera.util.HttpServerTestUtils;
import com.cloudera.util.InternalHttpServer;
-import com.cloudera.util.InternalHttpServer.ContextCreator;
public class TestBootstrap {
@@ -48,25 +23,19 @@ public class TestBootstrap {
}
@Test
- public void testBootstrap() throws InterruptedException, IOException {
+ public void testBootstrap() throws InterruptedException {
Assert.assertNotNull(httpServer);
logger.debug("httpServer:{}", httpServer);
httpServer.setPort(0);
- httpServer.setContextCreator(new ContextCreator() {
- @Override
- public void addContexts(ContextHandlerCollection handlers) {
- InternalHttpServer.addHandlersFromPaths(handlers, new File("src/main"));
- }
- });
+ httpServer.setWebappDir(new File("src/main"));
httpServer.start();
- int port = httpServer.getBoundPort();
- String url = "http://localhost:" + port;
- logger.debug("Grabbing http response from " + url);
- int resp = HttpServerTestUtils.curlResp(url);
+
+ Thread.sleep(3000);
+
httpServer.stop();
- assertEquals(resp, 200); // expect ok response code.
}
+
}