You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cd...@apache.org on 2011/06/04 01:40:53 UTC
svn commit: r1131289 - in /hadoop/common/trunk: ./
src/java/org/apache/hadoop/http/ src/java/org/apache/hadoop/jmx/
src/test/core/org/apache/hadoop/jmx/
Author: cdouglas
Date: Fri Jun 3 23:40:53 2011
New Revision: 1131289
URL: http://svn.apache.org/viewvc?rev=1131289&view=rev
Log:
HADOOP-7144. Expose JMX metrics via JSON servlet.
Contributed by Robert Joseph Evans
Added:
hadoop/common/trunk/src/java/org/apache/hadoop/jmx/
hadoop/common/trunk/src/java/org/apache/hadoop/jmx/JMXJsonServlet.java
hadoop/common/trunk/src/java/org/apache/hadoop/jmx/package-info.java
hadoop/common/trunk/src/test/core/org/apache/hadoop/jmx/
hadoop/common/trunk/src/test/core/org/apache/hadoop/jmx/TestJMXJsonServlet.java
Modified:
hadoop/common/trunk/CHANGES.txt
hadoop/common/trunk/src/java/org/apache/hadoop/http/HttpServer.java
Modified: hadoop/common/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/CHANGES.txt?rev=1131289&r1=1131288&r2=1131289&view=diff
==============================================================================
--- hadoop/common/trunk/CHANGES.txt (original)
+++ hadoop/common/trunk/CHANGES.txt Fri Jun 3 23:40:53 2011
@@ -38,6 +38,9 @@ Trunk (unreleased changes)
HADOOP-6832. Add an authentication plugin using a configurable static user
for the web UI. (Owen O'Malley and Todd Lipcon via cdouglas)
+ HADOOP-7144. Expose JMX metrics via JSON servlet. (Robert Joseph Evans via
+ cdouglas)
+
IMPROVEMENTS
HADOOP-7042. Updates to test-patch.sh to include failed test names and
Modified: hadoop/common/trunk/src/java/org/apache/hadoop/http/HttpServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/http/HttpServer.java?rev=1131289&r1=1131288&r2=1131289&view=diff
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/http/HttpServer.java (original)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/http/HttpServer.java Fri Jun 3 23:40:53 2011
@@ -52,6 +52,7 @@ import org.apache.hadoop.http.AdminAutho
import org.apache.hadoop.http.FilterContainer;
import org.apache.hadoop.http.FilterInitializer;
import org.apache.hadoop.http.HtmlQuoting;
+import org.apache.hadoop.jmx.JMXJsonServlet;
import org.apache.hadoop.log.LogLevel;
import org.apache.hadoop.metrics.MetricsServlet;
import org.apache.hadoop.security.Krb5AndCertsSslSocketConnector;
@@ -287,6 +288,7 @@ public class HttpServer implements Filte
addServlet("stacks", "/stacks", StackServlet.class);
addServlet("logLevel", "/logLevel", LogLevel.Servlet.class);
addServlet("metrics", "/metrics", MetricsServlet.class);
+ addServlet("jmx", "/jmx", JMXJsonServlet.class);
addServlet("conf", "/conf", ConfServlet.class);
}
Added: hadoop/common/trunk/src/java/org/apache/hadoop/jmx/JMXJsonServlet.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/jmx/JMXJsonServlet.java?rev=1131289&view=auto
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/jmx/JMXJsonServlet.java (added)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/jmx/JMXJsonServlet.java Fri Jun 3 23:40:53 2011
@@ -0,0 +1,327 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.jmx;
+
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.lang.management.ManagementFactory;
+import java.lang.reflect.Array;
+import java.util.Iterator;
+import java.util.Set;
+
+import javax.management.AttributeNotFoundException;
+import javax.management.InstanceNotFoundException;
+import javax.management.IntrospectionException;
+import javax.management.MBeanAttributeInfo;
+import javax.management.MBeanException;
+import javax.management.MBeanInfo;
+import javax.management.MBeanServer;
+import javax.management.MalformedObjectNameException;
+import javax.management.ObjectName;
+import javax.management.ReflectionException;
+import javax.management.openmbean.CompositeData;
+import javax.management.openmbean.CompositeType;
+import javax.management.openmbean.TabularData;
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.http.HttpServer;
+import org.codehaus.jackson.JsonFactory;
+import org.codehaus.jackson.JsonGenerator;
+
+/*
+ * This servlet is based off of the JMXProxyServlet from Tomcat 7.0.14. It has
+ * been rewritten to be read only and to output in a JSON format so it is not
+ * really that close to the original.
+ */
+/**
+ * Provides Read only web access to JMX.
+ * <p>
+ * This servlet generally will be placed under the /jmx URL for each
+ * HttpServer. It provides read only
+ * access to JMX metrics. The optional <code>qry</code> parameter
+ * may be used to query only a subset of the JMX Beans. This query
+ * functionality is provided through the
+ * {@link MBeanServer#queryNames(ObjectName, javax.management.QueryExp)}
+ * method.
+ * <p>
+ * For example <code>http://.../jmx?qry=Hadoop:*</code> will return
+ * all hadoop metrics exposed through JMX.
+ * <p>
+ * If the <code>qry</code> parameter is not formatted correctly then a
+ * 400 BAD REQUEST http response code will be returned.
+ * <p>
+ * The return format is JSON and in the form
+ * <p>
+ * <code><pre>
+ * {
+ * "beans" : [
+ * {
+ * "name":"bean-name"
+ * ...
+ * }
+ * ]
+ * }
+ * </pre></code>
+ * <p>
+ * The servlet attempts to convert the the JMXBeans into JSON. Each
+ * bean's attributes will be converted to a JSON object member.
+ *
+ * If the attribute is a boolean, a number, a string, or an array
+ * it will be converted to the JSON equivalent.
+ *
+ * If the value is a {@link CompositeData} then it will be converted
+ * to a JSON object with the keys as the name of the JSON member and
+ * the value is converted following these same rules.
+ *
+ * If the value is a {@link TabularData} then it will be converted
+ * to an array of the {@link CompositeData} elements that it contains.
+ *
+ * All other objects will be converted to a string and output as such.
+ *
+ * The bean's name and modelerType will be returned for all beans.
+ */
+public class JMXJsonServlet extends HttpServlet {
+ private static final Log LOG = LogFactory.getLog(JMXJsonServlet.class);
+
+ private static final long serialVersionUID = 1L;
+
+ // ----------------------------------------------------- Instance Variables
+ /**
+ * MBean server.
+ */
+ protected transient MBeanServer mBeanServer = null;
+
+ // --------------------------------------------------------- Public Methods
+ /**
+ * Initialize this servlet.
+ */
+ @Override
+ public void init() throws ServletException {
+ // Retrieve the MBean server
+ mBeanServer = ManagementFactory.getPlatformMBeanServer();
+ }
+
+ /**
+ * Process a GET request for the specified resource.
+ *
+ * @param request
+ * The servlet request we are processing
+ * @param response
+ * The servlet response we are creating
+ */
+ @Override
+ public void doGet(HttpServletRequest request, HttpServletResponse response) {
+ try {
+ // Do the authorization
+ if (!HttpServer.hasAdministratorAccess(getServletContext(), request,
+ response)) {
+ return;
+ }
+
+ response.setContentType("application/json; charset=utf8");
+
+ PrintWriter writer = response.getWriter();
+
+ JsonFactory jsonFactory = new JsonFactory();
+ JsonGenerator jg = jsonFactory.createJsonGenerator(writer);
+ jg.useDefaultPrettyPrinter();
+ jg.writeStartObject();
+ if (mBeanServer == null) {
+ jg.writeStringField("result", "ERROR");
+ jg.writeStringField("message", "No MBeanServer could be found");
+ jg.close();
+ return;
+ }
+ String qry = request.getParameter("qry");
+ if (qry == null) {
+ qry = "*:*";
+ }
+ listBeans(jg, new ObjectName(qry));
+ jg.close();
+ } catch (IOException e) {
+ LOG.error("Caught an exception while processing JMX request", e);
+ response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
+ } catch (MalformedObjectNameException e) {
+ LOG.error("Caught an exception while processing JMX request", e);
+ response.setStatus(HttpServletResponse.SC_BAD_REQUEST);
+ }
+ }
+
+ // --------------------------------------------------------- Private Methods
+ private void listBeans(JsonGenerator jg, ObjectName qry) throws IOException {
+ LOG.debug("Listing beans for "+qry);
+ Set<ObjectName> names = null;
+ names = mBeanServer.queryNames(qry, null);
+
+ jg.writeArrayFieldStart("beans");
+ Iterator<ObjectName> it = names.iterator();
+ while (it.hasNext()) {
+ ObjectName oname = it.next();
+ MBeanInfo minfo;
+ String code;
+ try {
+ minfo = mBeanServer.getMBeanInfo(oname);
+ code = minfo.getClassName();
+ try {
+ if ("org.apache.commons.modeler.BaseModelMBean".equals(code)) {
+ code = (String) mBeanServer.getAttribute(oname, "modelerType");
+ }
+ } catch (AttributeNotFoundException e) {
+ //Ignored the modelerType attribute was not found, so use the class name instead.
+ } catch (MBeanException e) {
+ //The code inside the attribute getter threw an exception so log it, and
+ // fall back on the class name
+ LOG.error("getting attribute modelerType of "+oname+" threw an exception", e);
+ } catch (RuntimeException e) {
+ //For some reason even with an MBeanException available to them Runtime exceptions
+ //can still find their way through, so treat them the same as MBeanException
+ LOG.error("getting attribute modelerType of "+oname+" threw an exception", e);
+ } catch (ReflectionException e) {
+ //This happens when the code inside the JMX bean (setter?? from the java docs)
+ //threw an exception, so log it and fall back on the class name
+ LOG.error("getting attribute modelerType of "+oname+" threw an exception", e);
+ }
+ } catch (InstanceNotFoundException e) {
+ //Ignored for some reason the bean was not found so don't output it
+ continue;
+ } catch (IntrospectionException e) {
+ //This is an internal error, something odd happened with reflection so log it and
+ //don't output the bean.
+ LOG.error("Problem while trying to process JMX query: "+qry+" with MBean "+oname, e);
+ continue;
+ } catch (ReflectionException e) {
+ //This happens when the code inside the JMX bean threw an exception, so log it and
+ //don't output the bean.
+ LOG.error("Problem while trying to process JMX query: "+qry+" with MBean "+oname, e);
+ continue;
+ }
+
+ jg.writeStartObject();
+ jg.writeStringField("name", oname.toString());
+ // can't be null - I think
+
+ jg.writeStringField("modelerType", code);
+
+ MBeanAttributeInfo attrs[] = minfo.getAttributes();
+ for (int i = 0; i < attrs.length; i++) {
+ writeAttribute(jg, oname, attrs[i]);
+ }
+ // LOG.error("Caught Error writing value ",t);
+ // ExceptionUtils.handleThrowable(t);
+ //}
+ jg.writeEndObject();
+ }
+ jg.writeEndArray();
+ }
+
+ private void writeAttribute(JsonGenerator jg, ObjectName oname, MBeanAttributeInfo attr) throws IOException {
+ if (!attr.isReadable()) {
+ return;
+ }
+ String attName = attr.getName();
+ if ("modelerType".equals(attName)) {
+ return;
+ }
+ if (attName.indexOf("=") >= 0 || attName.indexOf(":") >= 0
+ || attName.indexOf(" ") >= 0) {
+ return;
+ }
+ Object value = null;
+ try {
+ value = mBeanServer.getAttribute(oname, attName);
+ } catch (AttributeNotFoundException e) {
+ //Ignored the attribute was not found, which should never happen because the bean
+ //just told us that it has this attribute, but if this happens just don't output
+ //the attribute.
+ return;
+ } catch (MBeanException e) {
+ //The code inside the attribute getter threw an exception so log it, and
+ // skip outputting the attribute
+ LOG.error("getting attribute "+attName+" of "+oname+" threw an exception", e);
+ return;
+ } catch (RuntimeException e) {
+ //For some reason even with an MBeanException available to them Runtime exceptions
+ //can still find their way through, so treat them the same as MBeanException
+ LOG.error("getting attribute "+attName+" of "+oname+" threw an exception", e);
+ return;
+ } catch (ReflectionException e) {
+ //This happens when the code inside the JMX bean (setter?? from the java docs)
+ //threw an exception, so log it and skip outputting the attribute
+ LOG.error("getting attribute "+attName+" of "+oname+" threw an exception", e);
+ return;
+ } catch (InstanceNotFoundException e) {
+ //Ignored the mbean itself was not found, which should never happen because we
+ //just accessed it (perhaps something unregistered in-between) but if this
+ //happens just don't output the attribute.
+ return;
+ }
+
+ writeAttribute(jg, attName, value);
+ }
+
+ private void writeAttribute(JsonGenerator jg, String attName, Object value) throws IOException {
+ jg.writeFieldName(attName);
+ writeObject(jg, value);
+ }
+
+ private void writeObject(JsonGenerator jg, Object value) throws IOException {
+ if(value == null) {
+ jg.writeNull();
+ } else {
+ Class<?> c = value.getClass();
+ if (c.isArray()) {
+ jg.writeStartArray();
+ int len = Array.getLength(value);
+ for (int j = 0; j < len; j++) {
+ Object item = Array.get(value, j);
+ writeObject(jg, item);
+ }
+ jg.writeEndArray();
+ } else if(value instanceof Number) {
+ Number n = (Number)value;
+ jg.writeNumber(n.toString());
+ } else if(value instanceof Boolean) {
+ Boolean b = (Boolean)value;
+ jg.writeBoolean(b);
+ } else if(value instanceof CompositeData) {
+ CompositeData cds = (CompositeData)value;
+ CompositeType comp = cds.getCompositeType();
+ Set<String> keys = comp.keySet();
+ jg.writeStartObject();
+ for(String key: keys) {
+ writeAttribute(jg, key, cds.get(key));
+ }
+ jg.writeEndObject();
+ } else if(value instanceof TabularData) {
+ TabularData tds = (TabularData)value;
+ jg.writeStartArray();
+ for(Object entry : tds.values()) {
+ writeObject(jg, entry);
+ }
+ jg.writeEndArray();
+ } else {
+ jg.writeString(value.toString());
+ }
+ }
+ }
+}
Added: hadoop/common/trunk/src/java/org/apache/hadoop/jmx/package-info.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/java/org/apache/hadoop/jmx/package-info.java?rev=1131289&view=auto
==============================================================================
--- hadoop/common/trunk/src/java/org/apache/hadoop/jmx/package-info.java (added)
+++ hadoop/common/trunk/src/java/org/apache/hadoop/jmx/package-info.java Fri Jun 3 23:40:53 2011
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * This package provides access to JMX primarily through the
+ * {@link org.apache.hadoop.jmx.JMXJsonServlet} class.
+ */
+package org.apache.hadoop.jmx;
\ No newline at end of file
Added: hadoop/common/trunk/src/test/core/org/apache/hadoop/jmx/TestJMXJsonServlet.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/src/test/core/org/apache/hadoop/jmx/TestJMXJsonServlet.java?rev=1131289&view=auto
==============================================================================
--- hadoop/common/trunk/src/test/core/org/apache/hadoop/jmx/TestJMXJsonServlet.java (added)
+++ hadoop/common/trunk/src/test/core/org/apache/hadoop/jmx/TestJMXJsonServlet.java Fri Jun 3 23:40:53 2011
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.jmx;
+
+
+import java.net.URL;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.http.HttpServer;
+import org.apache.hadoop.http.HttpServerFunctionalTest;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestJMXJsonServlet extends HttpServerFunctionalTest {
+ private static final Log LOG = LogFactory.getLog(TestJMXJsonServlet.class);
+ private static HttpServer server;
+ private static URL baseUrl;
+
+ @BeforeClass public static void setup() throws Exception {
+ server = createTestServer();
+ server.start();
+ baseUrl = getServerURL(server);
+ }
+
+ @AfterClass public static void cleanup() throws Exception {
+ server.stop();
+ }
+
+ public static void assertReFind(String re, String value) {
+ Pattern p = Pattern.compile(re);
+ Matcher m = p.matcher(value);
+ assertTrue("'"+p+"' does not match "+value, m.find());
+ }
+
+ @Test public void testQury() throws Exception {
+ String result = readOutput(new URL(baseUrl, "/jmx?qry=java.lang:type=Runtime"));
+ LOG.info("/jmx?qry=java.lang:type=Runtime RESULT: "+result);
+ assertReFind("\"name\"\\s*:\\s*\"java.lang:type=Runtime\"", result);
+ assertReFind("\"modelerType\"", result);
+
+ result = readOutput(new URL(baseUrl, "/jmx?qry=java.lang:type=Memory"));
+ LOG.info("/jmx?qry=java.lang:type=Memory RESULT: "+result);
+ assertReFind("\"name\"\\s*:\\s*\"java.lang:type=Memory\"", result);
+ assertReFind("\"modelerType\"", result);
+
+ result = readOutput(new URL(baseUrl, "/jmx"));
+ LOG.info("/jmx RESULT: "+result);
+ assertReFind("\"name\"\\s*:\\s*\"java.lang:type=Memory\"", result);
+ }
+}