You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by bu...@apache.org on 2017/11/02 00:16:12 UTC
[07/13] hbase git commit: Revert "HBASE-19053 Split out o.a.h.h.http
from hbase-server into a separate module"
http://git-wip-us.apache.org/repos/asf/hbase/blob/851f239f/hbase-server/src/main/java/org/apache/hadoop/hbase/http/log/LogLevel.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/http/log/LogLevel.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/http/log/LogLevel.java
new file mode 100644
index 0000000..e23eecd
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/http/log/LogLevel.java
@@ -0,0 +1,175 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.http.log;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.PrintWriter;
+import java.net.URL;
+import java.net.URLConnection;
+import java.util.regex.Pattern;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.commons.logging.impl.Jdk14Logger;
+import org.apache.commons.logging.impl.Log4JLogger;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.apache.yetus.audience.InterfaceStability;
+import org.apache.hadoop.hbase.http.HttpServer;
+import org.apache.hadoop.util.ServletUtil;
+
+/**
+ * Change log level in runtime.
+ */
+@InterfaceStability.Evolving
+public class LogLevel {
+ public static final String USAGES = "\nUsage: General options are:\n"
+ + "\t[-getlevel <host:httpPort> <name>]\n"
+ + "\t[-setlevel <host:httpPort> <name> <level>]\n";
+
+ /**
+ * A command line implementation
+ */
+ public static void main(String[] args) {
+ if (args.length == 3 && "-getlevel".equals(args[0])) {
+ process("http://" + args[1] + "/logLevel?log=" + args[2]);
+ return;
+ }
+ else if (args.length == 4 && "-setlevel".equals(args[0])) {
+ process("http://" + args[1] + "/logLevel?log=" + args[2]
+ + "&level=" + args[3]);
+ return;
+ }
+
+ System.err.println(USAGES);
+ System.exit(-1);
+ }
+
+ private static void process(String urlstring) {
+ try {
+ URL url = new URL(urlstring);
+ System.out.println("Connecting to " + url);
+ URLConnection connection = url.openConnection();
+ connection.connect();
+ try (InputStreamReader streamReader = new InputStreamReader(connection.getInputStream());
+ BufferedReader bufferedReader = new BufferedReader(streamReader)) {
+ for(String line; (line = bufferedReader.readLine()) != null; ) {
+ if (line.startsWith(MARKER)) {
+ System.out.println(TAG.matcher(line).replaceAll(""));
+ }
+ }
+ }
+ } catch (IOException ioe) {
+ System.err.println("" + ioe);
+ }
+ }
+
+ static final String MARKER = "<!-- OUTPUT -->";
+ static final Pattern TAG = Pattern.compile("<[^>]*>");
+
+ /**
+ * A servlet implementation
+ */
+ @InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
+ @InterfaceStability.Unstable
+ public static class Servlet extends HttpServlet {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public void doGet(HttpServletRequest request, HttpServletResponse response
+ ) throws ServletException, IOException {
+
+ // Do the authorization
+ if (!HttpServer.hasAdministratorAccess(getServletContext(), request,
+ response)) {
+ return;
+ }
+
+ PrintWriter out = ServletUtil.initHTML(response, "Log Level");
+ String logName = ServletUtil.getParameter(request, "log");
+ String level = ServletUtil.getParameter(request, "level");
+
+ if (logName != null) {
+ out.println("<br /><hr /><h3>Results</h3>");
+ out.println(MARKER
+ + "Submitted Log Name: <b>" + logName + "</b><br />");
+
+ Log log = LogFactory.getLog(logName);
+ out.println(MARKER
+ + "Log Class: <b>" + log.getClass().getName() +"</b><br />");
+ if (level != null) {
+ out.println(MARKER + "Submitted Level: <b>" + level + "</b><br />");
+ }
+
+ if (log instanceof Log4JLogger) {
+ process(((Log4JLogger)log).getLogger(), level, out);
+ }
+ else if (log instanceof Jdk14Logger) {
+ process(((Jdk14Logger)log).getLogger(), level, out);
+ }
+ else {
+ out.println("Sorry, " + log.getClass() + " not supported.<br />");
+ }
+ }
+
+ out.println(FORMS);
+ out.println(ServletUtil.HTML_TAIL);
+ }
+
+ static final String FORMS = "\n<br /><hr /><h3>Get / Set</h3>"
+ + "\n<form>Log: <input type='text' size='50' name='log' /> "
+ + "<input type='submit' value='Get Log Level' />"
+ + "</form>"
+ + "\n<form>Log: <input type='text' size='50' name='log' /> "
+ + "Level: <input type='text' name='level' /> "
+ + "<input type='submit' value='Set Log Level' />"
+ + "</form>";
+
+ private static void process(org.apache.log4j.Logger log, String level,
+ PrintWriter out) throws IOException {
+ if (level != null) {
+ if (!level.equals(org.apache.log4j.Level.toLevel(level).toString())) {
+ out.println(MARKER + "Bad level : <b>" + level + "</b><br />");
+ } else {
+ log.setLevel(org.apache.log4j.Level.toLevel(level));
+ out.println(MARKER + "Setting Level to " + level + " ...<br />");
+ }
+ }
+ out.println(MARKER
+ + "Effective level: <b>" + log.getEffectiveLevel() + "</b><br />");
+ }
+
+ private static void process(java.util.logging.Logger log, String level,
+ PrintWriter out) throws IOException {
+ if (level != null) {
+ log.setLevel(java.util.logging.Level.parse(level));
+ out.println(MARKER + "Setting Level to " + level + " ...<br />");
+ }
+
+ java.util.logging.Level lev;
+ for(; (lev = log.getLevel()) == null; log = log.getParent());
+ out.println(MARKER + "Effective level: <b>" + lev + "</b><br />");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/851f239f/hbase-server/src/main/java/org/apache/hadoop/hbase/http/package-info.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/http/package-info.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/http/package-info.java
new file mode 100644
index 0000000..f55e24b
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/http/package-info.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * <p>
+ * Copied from hadoop source code.<br>
+ * See https://issues.apache.org/jira/browse/HADOOP-10232 to know why.
+ * </p>
+ */
+@InterfaceStability.Unstable
+package org.apache.hadoop.hbase.http;
+
+import org.apache.yetus.audience.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hbase/blob/851f239f/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DumpRegionServerMetrics.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DumpRegionServerMetrics.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DumpRegionServerMetrics.java
deleted file mode 100644
index 2b07a64..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DumpRegionServerMetrics.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.regionserver;
-
-import org.apache.hadoop.hbase.util.JSONBean;
-
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.io.StringWriter;
-import java.lang.management.ManagementFactory;
-import javax.management.MBeanServer;
-import javax.management.MalformedObjectNameException;
-import javax.management.ObjectName;
-
-/**
- * Utility for doing JSON and MBeans.
- */
-public class DumpRegionServerMetrics {
- /**
- * Dump out a subset of regionserver mbeans only, not all of them, as json on System.out.
- */
- public static String dumpMetrics() throws MalformedObjectNameException, IOException {
- StringWriter sw = new StringWriter(1024 * 100); // Guess this size
- try (PrintWriter writer = new PrintWriter(sw)) {
- JSONBean dumper = new JSONBean();
- try (JSONBean.Writer jsonBeanWriter = dumper.open(writer)) {
- MBeanServer mbeanServer = ManagementFactory.getPlatformMBeanServer();
- jsonBeanWriter.write(mbeanServer,
- new ObjectName("java.lang:type=Memory"), null, false);
- jsonBeanWriter.write(mbeanServer,
- new ObjectName("Hadoop:service=HBase,name=RegionServer,sub=IPC"), null, false);
- jsonBeanWriter.write(mbeanServer,
- new ObjectName("Hadoop:service=HBase,name=RegionServer,sub=Replication"), null, false);
- jsonBeanWriter.write(mbeanServer,
- new ObjectName("Hadoop:service=HBase,name=RegionServer,sub=Server"), null, false);
- }
- }
- sw.close();
- return sw.toString();
- }
-
- public static void main(String[] args) throws IOException, MalformedObjectNameException {
- String str = dumpMetrics();
- System.out.println(str);
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/851f239f/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index f384c1f..ef3f10d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -146,6 +146,7 @@ import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.FSTableDescriptors;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.HasThread;
+import org.apache.hadoop.hbase.util.JSONBean;
import org.apache.hadoop.hbase.util.JvmPauseMonitor;
import org.apache.hadoop.hbase.util.NettyEventLoopGroupConfig;
import org.apache.hadoop.hbase.util.Pair;
@@ -2400,7 +2401,7 @@ public class HRegionServer extends HasThread implements
CoprocessorHost.getLoadedCoprocessors());
// Try and dump metrics if abort -- might give clue as to how fatal came about....
try {
- LOG.info("Dump of metrics as JSON on abort: " + DumpRegionServerMetrics.dumpMetrics());
+ LOG.info("Dump of metrics as JSON on abort: " + JSONBean.dumpRegionServerMetrics());
} catch (MalformedObjectNameException | IOException e) {
LOG.warn("Failed dumping metrics", e);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/851f239f/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HttpServerUtil.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HttpServerUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HttpServerUtil.java
new file mode 100644
index 0000000..59c6ad6
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/HttpServerUtil.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import org.eclipse.jetty.security.ConstraintSecurityHandler;
+import org.eclipse.jetty.util.security.Constraint;
+import org.eclipse.jetty.security.ConstraintMapping;
+import org.eclipse.jetty.servlet.ServletContextHandler;
+
+/**
+ * HttpServer utility.
+ */
+public class HttpServerUtil {
+ /**
+ * Add constraints to a Jetty Context to disallow undesirable Http methods.
+ * @param ctxHandler The context to modify
+ */
+ public static void constrainHttpMethods(ServletContextHandler ctxHandler) {
+ Constraint c = new Constraint();
+ c.setAuthenticate(true);
+
+ ConstraintMapping cmt = new ConstraintMapping();
+ cmt.setConstraint(c);
+ cmt.setMethod("TRACE");
+ cmt.setPathSpec("/*");
+
+ ConstraintMapping cmo = new ConstraintMapping();
+ cmo.setConstraint(c);
+ cmo.setMethod("OPTIONS");
+ cmo.setPathSpec("/*");
+
+ ConstraintSecurityHandler securityHandler = new ConstraintSecurityHandler();
+ securityHandler.setConstraintMappings(new ConstraintMapping[]{ cmt, cmo });
+
+ ctxHandler.setSecurityHandler(securityHandler);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/851f239f/hbase-server/src/main/java/org/apache/hadoop/hbase/util/JSONBean.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/JSONBean.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/JSONBean.java
new file mode 100644
index 0000000..f4a146e
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/JSONBean.java
@@ -0,0 +1,387 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.lang.management.ManagementFactory;
+import java.lang.reflect.Array;
+import java.util.Iterator;
+import java.util.Set;
+
+import javax.management.AttributeNotFoundException;
+import javax.management.InstanceNotFoundException;
+import javax.management.IntrospectionException;
+import javax.management.MBeanAttributeInfo;
+import javax.management.MBeanException;
+import javax.management.MBeanInfo;
+import javax.management.MBeanServer;
+import javax.management.MalformedObjectNameException;
+import javax.management.ObjectName;
+import javax.management.ReflectionException;
+import javax.management.RuntimeErrorException;
+import javax.management.RuntimeMBeanException;
+import javax.management.openmbean.CompositeData;
+import javax.management.openmbean.CompositeType;
+import javax.management.openmbean.TabularData;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonGenerationException;
+import com.fasterxml.jackson.core.JsonGenerator;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * Utility for doing JSON and MBeans.
+ */
+public class JSONBean {
+ private static final Log LOG = LogFactory.getLog(JSONBean.class);
+ private final JsonFactory jsonFactory;
+
+ public JSONBean() {
+ this.jsonFactory = new JsonFactory();
+ }
+
+ /**
+ * Use dumping out mbeans as JSON.
+ */
+ public interface Writer extends Closeable {
+ void write(final String key, final String value) throws JsonGenerationException, IOException;
+ int write(final MBeanServer mBeanServer, ObjectName qry, String attribute,
+ final boolean description) throws IOException;
+ void flush() throws IOException;
+ }
+
+ public Writer open(final PrintWriter writer) throws IOException {
+ final JsonGenerator jg = jsonFactory.createJsonGenerator(writer);
+ jg.disable(JsonGenerator.Feature.AUTO_CLOSE_TARGET);
+ jg.useDefaultPrettyPrinter();
+ jg.writeStartObject();
+ return new Writer() {
+ @Override
+ public void flush() throws IOException {
+ jg.flush();
+ }
+
+ @Override
+ public void close() throws IOException {
+ jg.close();
+ }
+
+ @Override
+ public void write(String key, String value) throws JsonGenerationException, IOException {
+ jg.writeStringField(key, value);
+ }
+
+ @Override
+ public int write(MBeanServer mBeanServer, ObjectName qry, String attribute,
+ boolean description)
+ throws IOException {
+ return JSONBean.write(jg, mBeanServer, qry, attribute, description);
+ }
+ };
+ }
+
+ /**
+ * @param mBeanServer
+ * @param qry
+ * @param attribute
+ * @param description
+ * @return Return non-zero if failed to find bean. 0
+ * @throws IOException
+ */
+ private static int write(final JsonGenerator jg,
+ final MBeanServer mBeanServer, ObjectName qry, String attribute,
+ final boolean description)
+ throws IOException {
+ LOG.trace("Listing beans for "+qry);
+ Set<ObjectName> names = null;
+ names = mBeanServer.queryNames(qry, null);
+ jg.writeArrayFieldStart("beans");
+ Iterator<ObjectName> it = names.iterator();
+ while (it.hasNext()) {
+ ObjectName oname = it.next();
+ MBeanInfo minfo;
+ String code = "";
+ String descriptionStr = null;
+ Object attributeinfo = null;
+ try {
+ minfo = mBeanServer.getMBeanInfo(oname);
+ code = minfo.getClassName();
+ if (description) descriptionStr = minfo.getDescription();
+ String prs = "";
+ try {
+ if ("org.apache.commons.modeler.BaseModelMBean".equals(code)) {
+ prs = "modelerType";
+ code = (String) mBeanServer.getAttribute(oname, prs);
+ }
+ if (attribute != null) {
+ prs = attribute;
+ attributeinfo = mBeanServer.getAttribute(oname, prs);
+ }
+ } catch (RuntimeMBeanException e) {
+ // UnsupportedOperationExceptions happen in the normal course of business,
+ // so no need to log them as errors all the time.
+ if (e.getCause() instanceof UnsupportedOperationException) {
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Getting attribute " + prs + " of " + oname + " threw " + e);
+ }
+ } else {
+ LOG.error("Getting attribute " + prs + " of " + oname + " threw an exception", e);
+ }
+ return 0;
+ } catch (AttributeNotFoundException e) {
+ // If the modelerType attribute was not found, the class name is used
+ // instead.
+ LOG.error("getting attribute " + prs + " of " + oname
+ + " threw an exception", e);
+ } catch (MBeanException e) {
+ // The code inside the attribute getter threw an exception so log it,
+ // and fall back on the class name
+ LOG.error("getting attribute " + prs + " of " + oname
+ + " threw an exception", e);
+ } catch (RuntimeException e) {
+ // For some reason even with an MBeanException available to them
+ // Runtime exceptionscan still find their way through, so treat them
+ // the same as MBeanException
+ LOG.error("getting attribute " + prs + " of " + oname
+ + " threw an exception", e);
+ } catch (ReflectionException e) {
+ // This happens when the code inside the JMX bean (setter?? from the
+ // java docs) threw an exception, so log it and fall back on the
+ // class name
+ LOG.error("getting attribute " + prs + " of " + oname
+ + " threw an exception", e);
+ }
+ } catch (InstanceNotFoundException e) {
+ //Ignored for some reason the bean was not found so don't output it
+ continue;
+ } catch (IntrospectionException e) {
+ // This is an internal error, something odd happened with reflection so
+ // log it and don't output the bean.
+ LOG.error("Problem while trying to process JMX query: " + qry
+ + " with MBean " + oname, e);
+ continue;
+ } catch (ReflectionException e) {
+ // This happens when the code inside the JMX bean threw an exception, so
+ // log it and don't output the bean.
+ LOG.error("Problem while trying to process JMX query: " + qry
+ + " with MBean " + oname, e);
+ continue;
+ }
+
+ jg.writeStartObject();
+ jg.writeStringField("name", oname.toString());
+ if (description && descriptionStr != null && descriptionStr.length() > 0) {
+ jg.writeStringField("description", descriptionStr);
+ }
+ jg.writeStringField("modelerType", code);
+ if (attribute != null && attributeinfo == null) {
+ jg.writeStringField("result", "ERROR");
+ jg.writeStringField("message", "No attribute with name " + attribute + " was found.");
+ jg.writeEndObject();
+ jg.writeEndArray();
+ jg.close();
+ return -1;
+ }
+
+ if (attribute != null) {
+ writeAttribute(jg, attribute, descriptionStr, attributeinfo);
+ } else {
+ MBeanAttributeInfo[] attrs = minfo.getAttributes();
+ for (int i = 0; i < attrs.length; i++) {
+ writeAttribute(jg, mBeanServer, oname, description, attrs[i]);
+ }
+ }
+ jg.writeEndObject();
+ }
+ jg.writeEndArray();
+ return 0;
+ }
+
+ private static void writeAttribute(final JsonGenerator jg,
+ final MBeanServer mBeanServer, ObjectName oname,
+ final boolean description, final MBeanAttributeInfo attr)
+ throws IOException {
+ if (!attr.isReadable()) {
+ return;
+ }
+ String attName = attr.getName();
+ if ("modelerType".equals(attName)) {
+ return;
+ }
+ if (attName.indexOf("=") >= 0 || attName.indexOf(":") >= 0 || attName.indexOf(" ") >= 0) {
+ return;
+ }
+ String descriptionStr = description? attr.getDescription(): null;
+ Object value = null;
+ try {
+ value = mBeanServer.getAttribute(oname, attName);
+ } catch (RuntimeMBeanException e) {
+ // UnsupportedOperationExceptions happen in the normal course of business,
+ // so no need to log them as errors all the time.
+ if (e.getCause() instanceof UnsupportedOperationException) {
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Getting attribute " + attName + " of " + oname + " threw " + e);
+ }
+ } else {
+ LOG.error("getting attribute "+attName+" of "+oname+" threw an exception", e);
+ }
+ return;
+ } catch (RuntimeErrorException e) {
+ // RuntimeErrorException happens when an unexpected failure occurs in getAttribute
+ // for example https://issues.apache.org/jira/browse/DAEMON-120
+ LOG.debug("getting attribute "+attName+" of "+oname+" threw an exception", e);
+ return;
+ } catch (AttributeNotFoundException e) {
+ //Ignored the attribute was not found, which should never happen because the bean
+ //just told us that it has this attribute, but if this happens just don't output
+ //the attribute.
+ return;
+ } catch (MBeanException e) {
+ //The code inside the attribute getter threw an exception so log it, and
+ // skip outputting the attribute
+ LOG.error("getting attribute "+attName+" of "+oname+" threw an exception", e);
+ return;
+ } catch (RuntimeException e) {
+ //For some reason even with an MBeanException available to them Runtime exceptions
+ //can still find their way through, so treat them the same as MBeanException
+ LOG.error("getting attribute "+attName+" of "+oname+" threw an exception", e);
+ return;
+ } catch (ReflectionException e) {
+ //This happens when the code inside the JMX bean (setter?? from the java docs)
+ //threw an exception, so log it and skip outputting the attribute
+ LOG.error("getting attribute "+attName+" of "+oname+" threw an exception", e);
+ return;
+ } catch (InstanceNotFoundException e) {
+ //Ignored the mbean itself was not found, which should never happen because we
+ //just accessed it (perhaps something unregistered in-between) but if this
+ //happens just don't output the attribute.
+ return;
+ }
+
+ writeAttribute(jg, attName, descriptionStr, value);
+ }
+
+ private static void writeAttribute(JsonGenerator jg, String attName, final String descriptionStr,
+ Object value)
+ throws IOException {
+ boolean description = false;
+ if (descriptionStr != null && descriptionStr.length() > 0 && !attName.equals(descriptionStr)) {
+ description = true;
+ jg.writeFieldName(attName);
+ jg.writeStartObject();
+ jg.writeFieldName("description");
+ jg.writeString(descriptionStr);
+ jg.writeFieldName("value");
+ writeObject(jg, description, value);
+ jg.writeEndObject();
+ } else {
+ jg.writeFieldName(attName);
+ writeObject(jg, description, value);
+ }
+ }
+
+ private static void writeObject(final JsonGenerator jg, final boolean description, Object value)
+ throws IOException {
+ if(value == null) {
+ jg.writeNull();
+ } else {
+ Class<?> c = value.getClass();
+ if (c.isArray()) {
+ jg.writeStartArray();
+ int len = Array.getLength(value);
+ for (int j = 0; j < len; j++) {
+ Object item = Array.get(value, j);
+ writeObject(jg, description, item);
+ }
+ jg.writeEndArray();
+ } else if(value instanceof Number) {
+ Number n = (Number)value;
+ jg.writeNumber(n.toString());
+ } else if(value instanceof Boolean) {
+ Boolean b = (Boolean)value;
+ jg.writeBoolean(b);
+ } else if(value instanceof CompositeData) {
+ CompositeData cds = (CompositeData)value;
+ CompositeType comp = cds.getCompositeType();
+ Set<String> keys = comp.keySet();
+ jg.writeStartObject();
+ for (String key: keys) {
+ writeAttribute(jg, key, null, cds.get(key));
+ }
+ jg.writeEndObject();
+ } else if(value instanceof TabularData) {
+ TabularData tds = (TabularData)value;
+ jg.writeStartArray();
+ for(Object entry : tds.values()) {
+ writeObject(jg, description, entry);
+ }
+ jg.writeEndArray();
+ } else {
+ jg.writeString(value.toString());
+ }
+ }
+ }
+
+ /**
+ * Dump out a subset of regionserver mbeans only, not all of them, as json on System.out.
+ * @throws MalformedObjectNameException
+ * @throws IOException
+ */
+ public static String dumpRegionServerMetrics() throws MalformedObjectNameException, IOException {
+ StringWriter sw = new StringWriter(1024 * 100); // Guess this size
+ try (PrintWriter writer = new PrintWriter(sw)) {
+ JSONBean dumper = new JSONBean();
+ try (JSONBean.Writer jsonBeanWriter = dumper.open(writer)) {
+ MBeanServer mbeanServer = ManagementFactory.getPlatformMBeanServer();
+ jsonBeanWriter.write(mbeanServer,
+ new ObjectName("java.lang:type=Memory"), null, false);
+ jsonBeanWriter.write(mbeanServer,
+ new ObjectName("Hadoop:service=HBase,name=RegionServer,sub=IPC"), null, false);
+ jsonBeanWriter.write(mbeanServer,
+ new ObjectName("Hadoop:service=HBase,name=RegionServer,sub=Replication"), null, false);
+ jsonBeanWriter.write(mbeanServer,
+ new ObjectName("Hadoop:service=HBase,name=RegionServer,sub=Server"), null, false);
+ }
+ }
+ sw.close();
+ return sw.toString();
+ }
+
+ /**
+ * Dump out all registered mbeans as json on System.out.
+ * @throws IOException
+ * @throws MalformedObjectNameException
+ */
+ public static void dumpAllBeans() throws IOException, MalformedObjectNameException {
+ try (PrintWriter writer = new PrintWriter(System.out)) {
+ JSONBean dumper = new JSONBean();
+ try (JSONBean.Writer jsonBeanWriter = dumper.open(writer)) {
+ MBeanServer mbeanServer = ManagementFactory.getPlatformMBeanServer();
+ jsonBeanWriter.write(mbeanServer, new ObjectName("*:*"), null, false);
+ }
+ }
+ }
+
+ public static void main(String[] args) throws IOException, MalformedObjectNameException {
+ String str = dumpRegionServerMetrics();
+ System.out.println(str);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/851f239f/hbase-server/src/main/java/org/apache/hadoop/hbase/util/JSONMetricUtil.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/JSONMetricUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/JSONMetricUtil.java
new file mode 100644
index 0000000..879f32e
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/JSONMetricUtil.java
@@ -0,0 +1,214 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * */
+package org.apache.hadoop.hbase.util;
+
+import java.beans.IntrospectionException;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.lang.management.GarbageCollectorMXBean;
+import java.lang.management.ManagementFactory;
+import java.lang.management.MemoryPoolMXBean;
+import java.lang.management.RuntimeMXBean;
+import java.util.Hashtable;
+import java.util.List;
+import java.util.Set;
+
+import javax.management.InstanceNotFoundException;
+import javax.management.MBeanAttributeInfo;
+import javax.management.MBeanInfo;
+import javax.management.MBeanServer;
+import javax.management.MalformedObjectNameException;
+import javax.management.ObjectName;
+import javax.management.ReflectionException;
+import javax.management.openmbean.CompositeData;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.codehaus.jettison.json.JSONException;
+
+public final class JSONMetricUtil {
+
+ private static final Log LOG = LogFactory.getLog(JSONMetricUtil.class);
+
+ private static MBeanServer mbServer = ManagementFactory.getPlatformMBeanServer();
+ //MBeans ObjectName domain names
+ public static final String JAVA_LANG_DOMAIN = "java.lang";
+ public static final String JAVA_NIO_DOMAIN = "java.nio";
+ public static final String SUN_MGMT_DOMAIN = "com.sun.management";
+ public static final String HADOOP_DOMAIN = "Hadoop";
+
+ //MBeans ObjectName properties key names
+ public static final String TYPE_KEY = "type";
+ public static final String NAME_KEY = "name";
+ public static final String SERVICE_KEY = "service";
+ public static final String SUBSYSTEM_KEY = "sub";
+
+/**
+ * Utility for getting metric values. Collection of static methods intended for
+ * easier access to metric values.
+ */
+ private JSONMetricUtil() {
+ // Not to be called
+ }
+
+ public static MBeanAttributeInfo[] getMBeanAttributeInfo(ObjectName bean)
+ throws IntrospectionException, InstanceNotFoundException, ReflectionException,
+ IntrospectionException, javax.management.IntrospectionException {
+ MBeanInfo mbinfo = mbServer.getMBeanInfo(bean);
+ return mbinfo.getAttributes();
+ }
+
+ public static Object getValueFromMBean(ObjectName bean, String attribute) {
+ Object value = null;
+ try {
+ value = mbServer.getAttribute(bean, attribute);
+ }
+ catch(Exception e) {
+ LOG.error("Unable to get value from MBean= "+ bean.toString() +
+ "for attribute=" + attribute + " " + e.getMessage());
+ }
+ return value;
+ }
+
+ /**
+ * Returns a subset of mbeans defined by qry.
+ * Modeled after {@link JSONBean#dumpRegionServerMetrics()}
+ * Example: String qry= "java.lang:type=Memory"
+ * @throws MalformedObjectNameException if json have bad format
+ * @throws IOException /
+ * @return String representation of json array.
+ */
+ public static String dumpBeanToString(String qry) throws MalformedObjectNameException,
+ IOException {
+ StringWriter sw = new StringWriter(1024 * 100); // Guess this size
+ try (PrintWriter writer = new PrintWriter(sw)) {
+ JSONBean dumper = new JSONBean();
+ try (JSONBean.Writer jsonBeanWriter = dumper.open(writer)) {
+ MBeanServer mbeanServer = ManagementFactory.getPlatformMBeanServer();
+ jsonBeanWriter.write(mbeanServer,
+ new ObjectName(qry), null, false);
+ }
+ }
+ sw.close();
+ return sw.toString();
+ }
+
+ public static JsonNode mappStringToJsonNode(String jsonString)
+ throws JsonProcessingException, IOException {
+ ObjectMapper mapper = new ObjectMapper();
+ JsonNode node = mapper.readTree(jsonString);
+ return node;
+ }
+
+
+ public static JsonNode searchJson(JsonNode tree, String searchKey)
+ throws JsonProcessingException, IOException {
+ if (tree == null) {
+ return null;
+ }
+ if(tree.has(searchKey)) {
+ return tree.get(searchKey);
+ }
+ if(tree.isContainerNode()) {
+ for(JsonNode branch: tree) {
+ JsonNode branchResult = searchJson(branch, searchKey);
+ if (branchResult != null && !branchResult.isMissingNode()) {
+ return branchResult;
+ }
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Method for building hashtable used for constructing ObjectName.
+ * Mapping is done with arrays indices
+ * @param keys Hashtable keys
+ * @param values Hashtable values
+ * @return Hashtable or null if arrays are empty * or have different number of elements
+ */
+ public static Hashtable<String, String> buldKeyValueTable(String[] keys, String[] values) {
+ if (keys.length != values.length) {
+ LOG.error("keys and values arrays must be same size");
+ return null;
+ }
+ if (keys.length == 0 || values.length == 0) {
+ LOG.error("keys and values arrays can not be empty;");
+ return null;
+ }
+ Hashtable<String, String> table = new Hashtable<String, String>();
+ for(int i = 0; i < keys.length; i++) {
+ table.put(keys[i], values[i]);
+ }
+ return table;
+ }
+
+ public static ObjectName buildObjectName(String pattern) throws MalformedObjectNameException {
+ return new ObjectName(pattern);
+ }
+
+ public static ObjectName buildObjectName(String domain, Hashtable<String, String> keyValueTable)
+ throws MalformedObjectNameException {
+ return new ObjectName(domain, keyValueTable);
+ }
+
+ public static Set<ObjectName> getRegistredMBeans(ObjectName name, MBeanServer mbs) {
+ return mbs.queryNames(name, null);
+ }
+
+ public static String getProcessPID() {
+ return ManagementFactory.getRuntimeMXBean().getName().split("@")[0];
+ }
+
+ public static String getCommmand() throws MalformedObjectNameException,
+ IOException, JSONException {
+ RuntimeMXBean runtimeBean = ManagementFactory.getRuntimeMXBean();
+ return runtimeBean.getSystemProperties().get("sun.java.command");
+ }
+
+ public static List<GarbageCollectorMXBean> getGcCollectorBeans() {
+ List<GarbageCollectorMXBean> gcBeans = ManagementFactory.getGarbageCollectorMXBeans();
+ return gcBeans;
+ }
+
+ public static long getLastGcDuration(ObjectName gcCollector) {
+ long lastGcDuration = 0;
+ Object lastGcInfo = getValueFromMBean(gcCollector, "LastGcInfo");
+ if (lastGcInfo != null && lastGcInfo instanceof CompositeData) {
+ CompositeData cds = (CompositeData)lastGcInfo;
+ lastGcDuration = (long) cds.get("duration");
+ }
+ return lastGcDuration;
+ }
+
+ public static List<MemoryPoolMXBean> getMemoryPools() {
+ List<MemoryPoolMXBean> mPools = ManagementFactory.getMemoryPoolMXBeans();
+ return mPools;
+ }
+
+ public static float calcPercentage(long a, long b) {
+ if (a == 0 || b == 0) {
+ return 0;
+ }
+ return ((float)a / (float)b) *100;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/851f239f/hbase-server/src/test/java/org/apache/hadoop/hbase/GenericTestUtils.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/GenericTestUtils.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/GenericTestUtils.java
index 08565e0..2014b5b 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/GenericTestUtils.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/GenericTestUtils.java
@@ -35,6 +35,7 @@ import java.util.regex.Pattern;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Time;
import org.apache.log4j.Layout;
import org.apache.log4j.Logger;
@@ -69,14 +70,14 @@ public abstract class GenericTestUtils {
public static int uniqueSequenceId() {
return sequence.incrementAndGet();
}
-
+
/**
* Assert that a given file exists.
*/
public static void assertExists(File f) {
Assert.assertTrue("File " + f + " should exist", f.exists());
}
-
+
/**
* List all of the files in 'dir' that match the regex 'pattern'.
* Then check that this list is identical to 'expectedMatches'.
@@ -84,7 +85,7 @@ public abstract class GenericTestUtils {
*/
public static void assertGlobEquals(File dir, String pattern,
String ... expectedMatches) throws IOException {
-
+
Set<String> found = Sets.newTreeSet();
for (File f : FileUtil.listFiles(dir)) {
if (f.getName().matches(pattern)) {
@@ -97,6 +98,13 @@ public abstract class GenericTestUtils {
Joiner.on(",").join(expectedSet),
Joiner.on(",").join(found));
}
+
+ public static void assertExceptionContains(String string, Throwable t) {
+ String msg = t.getMessage();
+ Assert.assertTrue(
+ "Expected to find '" + string + "' but got unexpected exception:"
+ + StringUtils.stringifyException(t), msg.contains(string));
+ }
public static void waitFor(Supplier<Boolean> check,
int checkEveryMillis, int waitForMillis)
@@ -108,26 +116,26 @@ public abstract class GenericTestUtils {
if (result) {
return;
}
-
+
Thread.sleep(checkEveryMillis);
} while (Time.now() - st < waitForMillis);
-
+
throw new TimeoutException("Timed out waiting for condition. " +
"Thread diagnostics:\n" +
TimedOutTestsListener.buildThreadDiagnosticString());
}
-
+
public static class LogCapturer {
private StringWriter sw = new StringWriter();
private WriterAppender appender;
private Logger logger;
-
+
public static LogCapturer captureLogs(Log l) {
Logger logger = ((Log4JLogger)l).getLogger();
LogCapturer c = new LogCapturer(logger);
return c;
}
-
+
private LogCapturer(Logger logger) {
this.logger = logger;
@@ -135,36 +143,36 @@ public abstract class GenericTestUtils {
WriterAppender wa = new WriterAppender(layout, sw);
logger.addAppender(wa);
}
-
+
public String getOutput() {
return sw.toString();
}
-
+
public void stopCapturing() {
logger.removeAppender(appender);
}
}
-
-
+
+
/**
* Mockito answer helper that triggers one latch as soon as the
* method is called, then waits on another before continuing.
*/
public static class DelayAnswer implements Answer<Object> {
private final Log LOG;
-
+
private final CountDownLatch fireLatch = new CountDownLatch(1);
private final CountDownLatch waitLatch = new CountDownLatch(1);
private final CountDownLatch resultLatch = new CountDownLatch(1);
-
+
private final AtomicInteger fireCounter = new AtomicInteger(0);
private final AtomicInteger resultCounter = new AtomicInteger(0);
-
+
// Result fields set after proceed() is called.
private volatile Throwable thrown;
private volatile Object returnValue;
-
+
public DelayAnswer(Log log) {
this.LOG = log;
}
@@ -175,7 +183,7 @@ public abstract class GenericTestUtils {
public void waitForCall() throws InterruptedException {
fireLatch.await();
}
-
+
/**
* Tell the method to proceed.
* This should only be called after waitForCall()
@@ -183,7 +191,7 @@ public abstract class GenericTestUtils {
public void proceed() {
waitLatch.countDown();
}
-
+
@Override
public Object answer(InvocationOnMock invocation) throws Throwable {
LOG.info("DelayAnswer firing fireLatch");
@@ -212,7 +220,7 @@ public abstract class GenericTestUtils {
resultLatch.countDown();
}
}
-
+
/**
* After calling proceed(), this will wait until the call has
* completed and a result has been returned to the caller.
@@ -220,7 +228,7 @@ public abstract class GenericTestUtils {
public void waitForResult() throws InterruptedException {
resultLatch.await();
}
-
+
/**
* After the call has gone through, return any exception that
* was thrown, or null if no exception was thrown.
@@ -228,7 +236,7 @@ public abstract class GenericTestUtils {
public Throwable getThrown() {
return thrown;
}
-
+
/**
* After the call has gone through, return the call's return value,
* or null in case it was void or an exception was thrown.
@@ -236,20 +244,20 @@ public abstract class GenericTestUtils {
public Object getReturnValue() {
return returnValue;
}
-
+
public int getFireCount() {
return fireCounter.get();
}
-
+
public int getResultCount() {
return resultCounter.get();
}
}
-
+
/**
* An Answer implementation that simply forwards all calls through
* to a delegate.
- *
+ *
* This is useful as the default Answer for a mock object, to create
* something like a spy on an RPC proxy. For example:
* <code>
@@ -260,14 +268,14 @@ public abstract class GenericTestUtils {
* ...
* </code>
*/
- public static class DelegateAnswer implements Answer<Object> {
+ public static class DelegateAnswer implements Answer<Object> {
private final Object delegate;
private final Log log;
-
+
public DelegateAnswer(Object delegate) {
this(null, delegate);
}
-
+
public DelegateAnswer(Log log, Object delegate) {
this.log = log;
this.delegate = delegate;
@@ -297,11 +305,11 @@ public abstract class GenericTestUtils {
public static class SleepAnswer implements Answer<Object> {
private final int maxSleepTime;
private static Random r = new Random();
-
+
public SleepAnswer(int maxSleepTime) {
this.maxSleepTime = maxSleepTime;
}
-
+
@Override
public Object answer(InvocationOnMock invocation) throws Throwable {
boolean interrupted = false;
@@ -325,11 +333,11 @@ public abstract class GenericTestUtils {
" but got:\n" + output,
Pattern.compile(pattern).matcher(output).find());
}
-
+
public static void assertValueNear(long expected, long actual, long allowedError) {
assertValueWithinRange(expected - allowedError, expected + allowedError, actual);
}
-
+
public static void assertValueWithinRange(long expectedMin, long expectedMax,
long actual) {
Assert.assertTrue("Expected " + actual + " to be in range (" + expectedMin + ","
@@ -344,7 +352,7 @@ public abstract class GenericTestUtils {
public static void assertNoThreadsMatching(String regex) {
Pattern pattern = Pattern.compile(regex);
ThreadMXBean threadBean = ManagementFactory.getThreadMXBean();
-
+
ThreadInfo[] infos = threadBean.getThreadInfo(threadBean.getAllThreadIds(), 20);
for (ThreadInfo info : infos) {
if (info == null) continue;
http://git-wip-us.apache.org/repos/asf/hbase/blob/851f239f/hbase-server/src/test/java/org/apache/hadoop/hbase/http/HttpServerFunctionalTest.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/http/HttpServerFunctionalTest.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/http/HttpServerFunctionalTest.java
new file mode 100644
index 0000000..7d610e4
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/http/HttpServerFunctionalTest.java
@@ -0,0 +1,272 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.http;
+
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.authorize.AccessControlList;
+import org.junit.Assert;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.http.HttpServer.Builder;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.ServerSocket;
+import java.net.URI;
+import java.net.URL;
+import java.net.MalformedURLException;
+
+/**
+ * This is a base class for functional tests of the {@link HttpServer}.
+ * The methods are static for other classes to import statically.
+ */
+public class HttpServerFunctionalTest extends Assert {
+ /** JVM property for the webapp test dir : {@value} */
+ public static final String TEST_BUILD_WEBAPPS = "test.build.webapps";
+ /** expected location of the test.build.webapps dir: {@value} */
+ private static final String BUILD_WEBAPPS_DIR = "src/main/resources/hbase-webapps";
+
+ /** name of the test webapp: {@value} */
+ private static final String TEST = "test";
+
+ /**
+ * Create but do not start the test webapp server. The test webapp dir is
+ * prepared/checked in advance.
+ *
+ * @return the server instance
+ *
+ * @throws IOException if a problem occurs
+ * @throws AssertionError if a condition was not met
+ */
+ public static HttpServer createTestServer() throws IOException {
+ prepareTestWebapp();
+ return createServer(TEST);
+ }
+
+ /**
+ * Create but do not start the test webapp server. The test webapp dir is
+ * prepared/checked in advance.
+ * @param conf the server configuration to use
+ * @return the server instance
+ *
+ * @throws IOException if a problem occurs
+ * @throws AssertionError if a condition was not met
+ */
+ public static HttpServer createTestServer(Configuration conf)
+ throws IOException {
+ prepareTestWebapp();
+ return createServer(TEST, conf);
+ }
+
+ public static HttpServer createTestServer(Configuration conf, AccessControlList adminsAcl)
+ throws IOException {
+ prepareTestWebapp();
+ return createServer(TEST, conf, adminsAcl);
+ }
+
+ /**
+ * Create but do not start the test webapp server. The test webapp dir is
+ * prepared/checked in advance.
+ * @param conf the server configuration to use
+ * @return the server instance
+ *
+ * @throws IOException if a problem occurs
+ * @throws AssertionError if a condition was not met
+ */
+ public static HttpServer createTestServer(Configuration conf,
+ String[] pathSpecs) throws IOException {
+ prepareTestWebapp();
+ return createServer(TEST, conf, pathSpecs);
+ }
+
+ public static HttpServer createTestServerWithSecurity(Configuration conf) throws IOException {
+ prepareTestWebapp();
+ return localServerBuilder(TEST).setFindPort(true).setConf(conf).setSecurityEnabled(true)
+ // InfoServer normally sets these for us
+ .setUsernameConfKey(HttpServer.HTTP_SPNEGO_AUTHENTICATION_PRINCIPAL_KEY)
+ .setKeytabConfKey(HttpServer.HTTP_SPNEGO_AUTHENTICATION_KEYTAB_KEY)
+ .build();
+ }
+
+ /**
+ * Prepare the test webapp by creating the directory from the test properties
+ * fail if the directory cannot be created.
+ * @throws AssertionError if a condition was not met
+ */
+ protected static void prepareTestWebapp() {
+ String webapps = System.getProperty(TEST_BUILD_WEBAPPS, BUILD_WEBAPPS_DIR);
+ File testWebappDir = new File(webapps +
+ File.separatorChar + TEST);
+ try {
+ if (!testWebappDir.exists()) {
+ fail("Test webapp dir " + testWebappDir.getCanonicalPath() + " missing");
+ }
+ }
+ catch (IOException e) {
+ }
+ }
+
+ /**
+ * Create an HttpServer instance on the given address for the given webapp
+ * @param host to bind
+ * @param port to bind
+ * @return the server
+ * @throws IOException if it could not be created
+ */
+ public static HttpServer createServer(String host, int port)
+ throws IOException {
+ prepareTestWebapp();
+ return new HttpServer.Builder().setName(TEST)
+ .addEndpoint(URI.create("http://" + host + ":" + port))
+ .setFindPort(true).build();
+ }
+
+ /**
+ * Create an HttpServer instance for the given webapp
+ * @param webapp the webapp to work with
+ * @return the server
+ * @throws IOException if it could not be created
+ */
+ public static HttpServer createServer(String webapp) throws IOException {
+ return localServerBuilder(webapp).setFindPort(true).build();
+ }
+ /**
+ * Create an HttpServer instance for the given webapp
+ * @param webapp the webapp to work with
+ * @param conf the configuration to use for the server
+ * @return the server
+ * @throws IOException if it could not be created
+ */
+ public static HttpServer createServer(String webapp, Configuration conf)
+ throws IOException {
+ return localServerBuilder(webapp).setFindPort(true).setConf(conf).build();
+ }
+
+ public static HttpServer createServer(String webapp, Configuration conf, AccessControlList adminsAcl)
+ throws IOException {
+ return localServerBuilder(webapp).setFindPort(true).setConf(conf).setACL(adminsAcl).build();
+ }
+
+ private static Builder localServerBuilder(String webapp) {
+ return new HttpServer.Builder().setName(webapp).addEndpoint(
+ URI.create("http://localhost:0"));
+ }
+
+ /**
+ * Create an HttpServer instance for the given webapp
+ * @param webapp the webapp to work with
+ * @param conf the configuration to use for the server
+ * @param pathSpecs the paths specifications the server will service
+ * @return the server
+ * @throws IOException if it could not be created
+ */
+ public static HttpServer createServer(String webapp, Configuration conf,
+ String[] pathSpecs) throws IOException {
+ return localServerBuilder(webapp).setFindPort(true).setConf(conf).setPathSpec(pathSpecs).build();
+ }
+
+ /**
+ * Create and start a server with the test webapp
+ *
+ * @return the newly started server
+ *
+ * @throws IOException on any failure
+ * @throws AssertionError if a condition was not met
+ */
+ public static HttpServer createAndStartTestServer() throws IOException {
+ HttpServer server = createTestServer();
+ server.start();
+ return server;
+ }
+
+ /**
+ * If the server is non null, stop it
+ * @param server to stop
+ * @throws Exception on any failure
+ */
+ public static void stop(HttpServer server) throws Exception {
+ if (server != null) {
+ server.stop();
+ }
+ }
+
+ /**
+ * Pass in a server, return a URL bound to localhost and its port
+ * @param server server
+ * @return a URL bonded to the base of the server
+ * @throws MalformedURLException if the URL cannot be created.
+ */
+ public static URL getServerURL(HttpServer server)
+ throws MalformedURLException {
+ assertNotNull("No server", server);
+ return new URL("http://"
+ + NetUtils.getHostPortString(server.getConnectorAddress(0)));
+ }
+
+ /**
+ * Read in the content from a URL
+ * @param url URL To read
+ * @return the text from the output
+ * @throws IOException if something went wrong
+ */
+ protected static String readOutput(URL url) throws IOException {
+ StringBuilder out = new StringBuilder();
+ InputStream in = url.openConnection().getInputStream();
+ byte[] buffer = new byte[64 * 1024];
+ int len = in.read(buffer);
+ while (len > 0) {
+ out.append(new String(buffer, 0, len));
+ len = in.read(buffer);
+ }
+ return out.toString();
+ }
+
+ /**
+ * Recursively deletes a {@link File}.
+ */
+ protected static void deleteRecursively(File d) {
+ if (d.isDirectory()) {
+ for (String name : d.list()) {
+ File child = new File(d, name);
+ if (child.isFile()) {
+ child.delete();
+ } else {
+ deleteRecursively(child);
+ }
+ }
+ }
+ d.delete();
+ }
+
+ /**
+ * Picks a free port on the host by binding a Socket to '0'.
+ */
+ protected static int getFreePort() throws IOException {
+ ServerSocket s = new ServerSocket(0);
+ try {
+ s.setReuseAddress(true);
+ int port = s.getLocalPort();
+ return port;
+ } finally {
+ if (null != s) {
+ s.close();
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/851f239f/hbase-server/src/test/java/org/apache/hadoop/hbase/http/TestGlobalFilter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/http/TestGlobalFilter.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/http/TestGlobalFilter.java
new file mode 100644
index 0000000..acfe929
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/http/TestGlobalFilter.java
@@ -0,0 +1,151 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.http;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.URL;
+import java.net.URLConnection;
+import java.util.Set;
+import java.util.TreeSet;
+
+import javax.servlet.Filter;
+import javax.servlet.FilterChain;
+import javax.servlet.FilterConfig;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import javax.servlet.http.HttpServletRequest;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.testclassification.MiscTests;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.net.NetUtils;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category({MiscTests.class, SmallTests.class})
+public class TestGlobalFilter extends HttpServerFunctionalTest {
+ private static final Log LOG = LogFactory.getLog(HttpServer.class);
+ static final Set<String> RECORDS = new TreeSet<>();
+
+ /** A very simple filter that records accessed uri's */
+ static public class RecordingFilter implements Filter {
+ private FilterConfig filterConfig = null;
+
+ @Override
+ public void init(FilterConfig filterConfig) {
+ this.filterConfig = filterConfig;
+ }
+
+ @Override
+ public void destroy() {
+ this.filterConfig = null;
+ }
+
+ @Override
+ public void doFilter(ServletRequest request, ServletResponse response,
+ FilterChain chain) throws IOException, ServletException {
+ if (filterConfig == null)
+ return;
+
+ String uri = ((HttpServletRequest)request).getRequestURI();
+ LOG.info("filtering " + uri);
+ RECORDS.add(uri);
+ chain.doFilter(request, response);
+ }
+
+ /** Configuration for RecordingFilter */
+ static public class Initializer extends FilterInitializer {
+ public Initializer() {}
+
+ @Override
+ public void initFilter(FilterContainer container, Configuration conf) {
+ container.addGlobalFilter("recording", RecordingFilter.class.getName(), null);
+ }
+ }
+ }
+
+
+ /** access a url, ignoring some IOException such as the page does not exist */
+ static void access(String urlstring) throws IOException {
+ LOG.warn("access " + urlstring);
+ URL url = new URL(urlstring);
+ URLConnection connection = url.openConnection();
+ connection.connect();
+
+ try {
+ BufferedReader in = new BufferedReader(new InputStreamReader(
+ connection.getInputStream()));
+ try {
+ for(; in.readLine() != null; );
+ } finally {
+ in.close();
+ }
+ } catch(IOException ioe) {
+ LOG.warn("urlstring=" + urlstring, ioe);
+ }
+ }
+
+ @Test
+ public void testServletFilter() throws Exception {
+ Configuration conf = new Configuration();
+
+ //start a http server with CountingFilter
+ conf.set(HttpServer.FILTER_INITIALIZERS_PROPERTY,
+ RecordingFilter.Initializer.class.getName());
+ HttpServer http = createTestServer(conf);
+ http.start();
+
+ final String fsckURL = "/fsck";
+ final String stacksURL = "/stacks";
+ final String ajspURL = "/a.jsp";
+ final String listPathsURL = "/listPaths";
+ final String dataURL = "/data";
+ final String streamFile = "/streamFile";
+ final String rootURL = "/";
+ final String allURL = "/*";
+ final String outURL = "/static/a.out";
+ final String logURL = "/logs/a.log";
+
+ final String[] urls = {fsckURL, stacksURL, ajspURL, listPathsURL,
+ dataURL, streamFile, rootURL, allURL, outURL, logURL};
+
+ //access the urls
+ final String prefix = "http://"
+ + NetUtils.getHostPortString(http.getConnectorAddress(0));
+ try {
+ for(int i = 0; i < urls.length; i++) {
+ access(prefix + urls[i]);
+ }
+ } finally {
+ http.stop();
+ }
+
+ LOG.info("RECORDS = " + RECORDS);
+
+ //verify records
+ for(int i = 0; i < urls.length; i++) {
+ assertTrue(RECORDS.remove(urls[i]));
+ }
+ assertTrue(RECORDS.isEmpty());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/851f239f/hbase-server/src/test/java/org/apache/hadoop/hbase/http/TestHtmlQuoting.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/http/TestHtmlQuoting.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/http/TestHtmlQuoting.java
new file mode 100644
index 0000000..82fbe04
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/http/TestHtmlQuoting.java
@@ -0,0 +1,94 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.http;
+
+import static org.junit.Assert.*;
+
+import javax.servlet.http.HttpServletRequest;
+
+import org.apache.hadoop.hbase.testclassification.MiscTests;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.mockito.Mockito;
+
+@Category({MiscTests.class, SmallTests.class})
+public class TestHtmlQuoting {
+
+ @Test public void testNeedsQuoting() throws Exception {
+ assertTrue(HtmlQuoting.needsQuoting("abcde>"));
+ assertTrue(HtmlQuoting.needsQuoting("<abcde"));
+ assertTrue(HtmlQuoting.needsQuoting("abc'de"));
+ assertTrue(HtmlQuoting.needsQuoting("abcde\""));
+ assertTrue(HtmlQuoting.needsQuoting("&"));
+ assertFalse(HtmlQuoting.needsQuoting(""));
+ assertFalse(HtmlQuoting.needsQuoting("ab\ncdef"));
+ assertFalse(HtmlQuoting.needsQuoting(null));
+ }
+
+ @Test public void testQuoting() throws Exception {
+ assertEquals("ab<cd", HtmlQuoting.quoteHtmlChars("ab<cd"));
+ assertEquals("ab>", HtmlQuoting.quoteHtmlChars("ab>"));
+ assertEquals("&&&", HtmlQuoting.quoteHtmlChars("&&&"));
+ assertEquals(" '\n", HtmlQuoting.quoteHtmlChars(" '\n"));
+ assertEquals(""", HtmlQuoting.quoteHtmlChars("\""));
+ assertEquals(null, HtmlQuoting.quoteHtmlChars(null));
+ }
+
+ private void runRoundTrip(String str) throws Exception {
+ assertEquals(str,
+ HtmlQuoting.unquoteHtmlChars(HtmlQuoting.quoteHtmlChars(str)));
+ }
+
+ @Test public void testRoundtrip() throws Exception {
+ runRoundTrip("");
+ runRoundTrip("<>&'\"");
+ runRoundTrip("ab>cd<ef&ghi'\"");
+ runRoundTrip("A string\n with no quotable chars in it!");
+ runRoundTrip(null);
+ StringBuilder buffer = new StringBuilder();
+ for(char ch=0; ch < 127; ++ch) {
+ buffer.append(ch);
+ }
+ runRoundTrip(buffer.toString());
+ }
+
+
+ @Test
+ public void testRequestQuoting() throws Exception {
+ HttpServletRequest mockReq = Mockito.mock(HttpServletRequest.class);
+ HttpServer.QuotingInputFilter.RequestQuoter quoter =
+ new HttpServer.QuotingInputFilter.RequestQuoter(mockReq);
+
+ Mockito.doReturn("a<b").when(mockReq).getParameter("x");
+ assertEquals("Test simple param quoting",
+ "a<b", quoter.getParameter("x"));
+
+ Mockito.doReturn(null).when(mockReq).getParameter("x");
+ assertEquals("Test that missing parameters dont cause NPE",
+ null, quoter.getParameter("x"));
+
+ Mockito.doReturn(new String[]{"a<b", "b"}).when(mockReq).getParameterValues("x");
+ assertArrayEquals("Test escaping of an array",
+ new String[]{"a<b", "b"}, quoter.getParameterValues("x"));
+
+ Mockito.doReturn(null).when(mockReq).getParameterValues("x");
+ assertArrayEquals("Test that missing parameters dont cause NPE for array",
+ null, quoter.getParameterValues("x"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/851f239f/hbase-server/src/test/java/org/apache/hadoop/hbase/http/TestHttpRequestLog.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/http/TestHttpRequestLog.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/http/TestHttpRequestLog.java
new file mode 100644
index 0000000..b8d21d1
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/http/TestHttpRequestLog.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.http;
+
+import org.apache.hadoop.hbase.testclassification.MiscTests;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.log4j.Logger;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import org.eclipse.jetty.server.RequestLog;
+import org.eclipse.jetty.server.NCSARequestLog;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+
+@Category({MiscTests.class, SmallTests.class})
+public class TestHttpRequestLog {
+
+ @Test
+ public void testAppenderUndefined() {
+ RequestLog requestLog = HttpRequestLog.getRequestLog("test");
+ assertNull("RequestLog should be null", requestLog);
+ }
+
+ @Test
+ public void testAppenderDefined() {
+ HttpRequestLogAppender requestLogAppender = new HttpRequestLogAppender();
+ requestLogAppender.setName("testrequestlog");
+ Logger.getLogger("http.requests.test").addAppender(requestLogAppender);
+ RequestLog requestLog = HttpRequestLog.getRequestLog("test");
+ Logger.getLogger("http.requests.test").removeAppender(requestLogAppender);
+ assertNotNull("RequestLog should not be null", requestLog);
+ assertEquals("Class mismatch", NCSARequestLog.class, requestLog.getClass());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/851f239f/hbase-server/src/test/java/org/apache/hadoop/hbase/http/TestHttpRequestLogAppender.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/http/TestHttpRequestLogAppender.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/http/TestHttpRequestLogAppender.java
new file mode 100644
index 0000000..a17b9e9
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/http/TestHttpRequestLogAppender.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.http;
+
+import org.apache.hadoop.hbase.testclassification.MiscTests;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import static org.junit.Assert.assertEquals;
+
+@Category({MiscTests.class, SmallTests.class})
+public class TestHttpRequestLogAppender {
+
+ @Test
+ public void testParameterPropagation() {
+
+ HttpRequestLogAppender requestLogAppender = new HttpRequestLogAppender();
+ requestLogAppender.setFilename("jetty-namenode-yyyy_mm_dd.log");
+ requestLogAppender.setRetainDays(17);
+ assertEquals("Filename mismatch", "jetty-namenode-yyyy_mm_dd.log",
+ requestLogAppender.getFilename());
+ assertEquals("Retain days mismatch", 17,
+ requestLogAppender.getRetainDays());
+ }
+}