You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2014/06/02 18:19:28 UTC
git commit: HBASE-10289 Avoid random port usage by default JMX
Server. Create Custome JMX server (Qiang Tian)
Repository: hbase
Updated Branches:
refs/heads/master d56dfd2a8 -> 80557b872
HBASE-10289 Avoid random port usage by default JMX Server. Create Custome JMX server (Qiang Tian)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/80557b87
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/80557b87
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/80557b87
Branch: refs/heads/master
Commit: 80557b872fbf31fec94930fc4723d7a459fe52e7
Parents: d56dfd2
Author: Michael Stack <st...@duboce.net>
Authored: Mon Jun 2 09:19:16 2014 -0700
Committer: Michael Stack <st...@duboce.net>
Committed: Mon Jun 2 09:19:16 2014 -0700
----------------------------------------------------------------------
conf/hbase-env.sh | 4 +-
.../org/apache/hadoop/hbase/JMXListener.java | 195 +++++++++++++++++++
.../apache/hadoop/hbase/TestJMXListener.java | 95 +++++++++
src/main/docbkx/configuration.xml | 107 +++++++++-
4 files changed, 394 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/80557b87/conf/hbase-env.sh
----------------------------------------------------------------------
diff --git a/conf/hbase-env.sh b/conf/hbase-env.sh
index e6d1c9c..9059d12 100644
--- a/conf/hbase-env.sh
+++ b/conf/hbase-env.sh
@@ -76,7 +76,9 @@ export HBASE_OPTS="-XX:+UseConcMarkSweepGC"
# Uncomment and adjust to enable JMX exporting
# See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access.
# More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html
-#
+# NOTE: HBase provides an alternative JMX implementation to fix the random ports issue, please see JMX
+# section in HBase Reference Guide for instructions.
+
# export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false"
# export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10101"
# export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10102"
http://git-wip-us.apache.org/repos/asf/hbase/blob/80557b87/hbase-server/src/main/java/org/apache/hadoop/hbase/JMXListener.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/JMXListener.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/JMXListener.java
new file mode 100644
index 0000000..7a1ea11
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/JMXListener.java
@@ -0,0 +1,195 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.CoprocessorEnvironment;
+import org.apache.hadoop.hbase.coprocessor.*;
+
+import java.io.IOException;
+import java.lang.management.ManagementFactory;
+import java.rmi.registry.LocateRegistry;
+import java.rmi.server.RMIClientSocketFactory;
+import java.rmi.server.RMIServerSocketFactory;
+import java.util.HashMap;
+
+import javax.management.MBeanServer;
+import javax.management.remote.JMXConnectorServer;
+import javax.management.remote.JMXConnectorServerFactory;
+import javax.management.remote.JMXServiceURL;
+import javax.management.remote.rmi.RMIConnectorServer;
+import javax.rmi.ssl.SslRMIClientSocketFactory;
+import javax.rmi.ssl.SslRMIServerSocketFactory;
+
+/**
+ * Pluggable JMX Agent for HBase(to fix the 2 random TCP ports issue
+ * of the out-of-the-box JMX Agent):
+ * 1)connector port can share with the registry port if SSL is OFF
+ * 2)support password authentication
+ * 3)support subset of SSL (with default configuration)
+ */
+public class JMXListener implements Coprocessor {
+
+ public static final Log LOG = LogFactory.getLog(JMXListener.class);
+ public static final String RMI_REGISTRY_PORT_CONF_KEY = ".rmi.registry.port";
+ public static final String RMI_CONNECTOR_PORT_CONF_KEY = ".rmi.connector.port";
+ public static int defRMIRegistryPort = 10102;
+
+ /**
+ * workaround for HBASE-11146
+ * master and regionserver are in 1 JVM in standalone mode
+ * only 1 JMX instance is allowed, otherwise there is port conflict even if
+ * we only load regionserver coprocessor on master
+ */
+ private static JMXConnectorServer jmxCS = null;
+
+ public static JMXServiceURL buildJMXServiceURL(int rmiRegistryPort,
+ int rmiConnectorPort) throws IOException {
+ // Build jmxURL
+ StringBuilder url = new StringBuilder();
+ url.append("service:jmx:rmi://localhost:");
+ url.append(rmiConnectorPort);
+ url.append("/jndi/rmi://localhost:");
+ url.append(rmiRegistryPort);
+ url.append("/jmxrmi");
+
+ return new JMXServiceURL(url.toString());
+
+ }
+
+ public void startConnectorServer(int rmiRegistryPort, int rmiConnectorPort)
+ throws IOException {
+ boolean rmiSSL = false;
+ boolean authenticate = true;
+ String passwordFile = null;
+ String accessFile = null;
+
+ System.setProperty("java.rmi.server.randomIDs", "true");
+
+ String rmiSSLValue = System.getProperty("com.sun.management.jmxremote.ssl",
+ "false");
+ rmiSSL = Boolean.parseBoolean(rmiSSLValue);
+
+ String authenticateValue =
+ System.getProperty("com.sun.management.jmxremote.authenticate", "false");
+ authenticate = Boolean.parseBoolean(authenticateValue);
+
+ passwordFile = System.getProperty("com.sun.management.jmxremote.password.file");
+ accessFile = System.getProperty("com.sun.management.jmxremote.access.file");
+
+ LOG.info("rmiSSL:" + rmiSSLValue + ",authenticate:" + authenticateValue
+ + ",passwordFile:" + passwordFile + ",accessFile:" + accessFile);
+
+ // Environment map
+ HashMap<String, Object> jmxEnv = new HashMap<String, Object>();
+
+ RMIClientSocketFactory csf = null;
+ RMIServerSocketFactory ssf = null;
+
+ if (rmiSSL) {
+ if (rmiRegistryPort == rmiConnectorPort) {
+ throw new IOException("SSL is enabled. " +
+ "rmiConnectorPort cannot share with the rmiRegistryPort!");
+ }
+ csf = new SslRMIClientSocketFactory();
+ ssf = new SslRMIServerSocketFactory();
+ }
+
+ if (csf != null) {
+ jmxEnv.put(RMIConnectorServer.RMI_CLIENT_SOCKET_FACTORY_ATTRIBUTE, csf);
+ }
+ if (ssf != null) {
+ jmxEnv.put(RMIConnectorServer.RMI_SERVER_SOCKET_FACTORY_ATTRIBUTE, ssf);
+ }
+
+ // Configure authentication
+ if (authenticate) {
+ jmxEnv.put("jmx.remote.x.password.file", passwordFile);
+ jmxEnv.put("jmx.remote.x.access.file", accessFile);
+ }
+
+ // Create the RMI registry
+ LocateRegistry.createRegistry(rmiRegistryPort);
+ // Retrieve the PlatformMBeanServer.
+ MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+
+ // Build jmxURL
+ JMXServiceURL serviceUrl = buildJMXServiceURL(rmiRegistryPort, rmiConnectorPort);
+
+ try {
+ // Start the JMXListener with the connection string
+ jmxCS = JMXConnectorServerFactory.newJMXConnectorServer(serviceUrl, jmxEnv, mbs);
+ jmxCS.start();
+ LOG.info("ConnectorServer started!");
+ } catch (IOException e) {
+ LOG.error("fail to start connector server!", e);
+ }
+
+ }
+
+ public void stopConnectorServer() throws IOException {
+ synchronized(JMXListener.class) {
+ if (jmxCS != null) {
+ jmxCS.stop();
+ LOG.info("ConnectorServer stopped!");
+ jmxCS = null;
+ }
+ }
+ }
+
+
+ @Override
+ public void start(CoprocessorEnvironment env) throws IOException {
+ int rmiRegistryPort = -1;
+ int rmiConnectorPort = -1;
+ Configuration conf = env.getConfiguration();
+
+ if (env instanceof MasterCoprocessorEnvironment) {
+ LOG.error("JMXListener should not be loaded in Master Environment!");
+ } else if (env instanceof RegionServerCoprocessorEnvironment) {
+ // running on RegionServer --since 0.99 HMaster is also a HRegionServer
+ rmiRegistryPort =
+ conf.getInt("regionserver" + RMI_REGISTRY_PORT_CONF_KEY, defRMIRegistryPort);
+ rmiConnectorPort =
+ conf.getInt("regionserver" + RMI_CONNECTOR_PORT_CONF_KEY, rmiRegistryPort);
+ LOG.info("RegionServer rmiRegistryPort:" + rmiRegistryPort
+ + ",RegionServer rmiConnectorPort:" + rmiConnectorPort);
+
+ } else if (env instanceof RegionCoprocessorEnvironment) {
+ LOG.error("JMXListener should not be loaded in Region Environment!");
+ }
+
+ synchronized(JMXListener.class) {
+ if (jmxCS != null) {
+ LOG.info("JMXListener has been started at Registry port " + rmiRegistryPort);
+ }
+ else {
+ startConnectorServer(rmiRegistryPort, rmiConnectorPort);
+ }
+ }
+ }
+
+ @Override
+ public void stop(CoprocessorEnvironment env) throws IOException {
+ stopConnectorServer();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/80557b87/hbase-server/src/test/java/org/apache/hadoop/hbase/TestJMXListener.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestJMXListener.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestJMXListener.java
new file mode 100644
index 0000000..719e04f
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestJMXListener.java
@@ -0,0 +1,95 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import java.io.IOException;
+
+import javax.management.MBeanServerConnection;
+import javax.management.remote.JMXConnector;
+import javax.management.remote.JMXConnectorFactory;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.ExpectedException;
+
+
+
+@Category(MediumTests.class)
+public class TestJMXListener {
+ private static final Log LOG = LogFactory.getLog(TestJMXListener.class);
+ private static HBaseTestingUtility UTIL = new HBaseTestingUtility();
+ private static int connectorPort = 61120;
+
+ @BeforeClass
+ public static void setupBeforeClass() throws Exception {
+ Configuration conf = UTIL.getConfiguration();
+
+ conf.set(CoprocessorHost.REGIONSERVER_COPROCESSOR_CONF_KEY,
+ JMXListener.class.getName());
+ conf.setInt("regionserver.rmi.registry.port", connectorPort);
+
+ UTIL.startMiniCluster();
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ UTIL.shutdownMiniCluster();
+ }
+
+ @Test
+ public void testStart() throws Exception {
+ JMXConnector connector = JMXConnectorFactory.connect(
+ JMXListener.buildJMXServiceURL(connectorPort,connectorPort));
+
+ MBeanServerConnection mb = connector.getMBeanServerConnection();
+ String domain = mb.getDefaultDomain();
+ Assert.assertTrue("default domain is not correct",
+ !domain.isEmpty());
+ connector.close();
+
+ }
+
+ //shutdown hbase only. then try connect, IOException expected
+ @Rule
+ public ExpectedException expectedEx = ExpectedException.none();
+ @Test
+ public void testStop() throws Exception {
+ MiniHBaseCluster cluster = UTIL.getHBaseCluster();
+ LOG.info("shutdown hbase cluster...");
+ cluster.shutdown();
+ LOG.info("wait for the hbase cluster shutdown...");
+ cluster.waitUntilShutDown();
+
+ JMXConnector connector = JMXConnectorFactory.newJMXConnector(
+ JMXListener.buildJMXServiceURL(connectorPort,connectorPort), null);
+ expectedEx.expect(IOException.class);
+ connector.connect();
+
+ }
+
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/80557b87/src/main/docbkx/configuration.xml
----------------------------------------------------------------------
diff --git a/src/main/docbkx/configuration.xml b/src/main/docbkx/configuration.xml
index 1f4228a..2fbc3e8 100644
--- a/src/main/docbkx/configuration.xml
+++ b/src/main/docbkx/configuration.xml
@@ -1411,7 +1411,8 @@ index e70ebc6..96f8c27 100644
a late-version HDFS so you have the fixes he refers too and himself adds to HDFS that help
HBase MTTR (e.g. HDFS-3703, HDFS-3712, and HDFS-4791 -- hadoop 2 for sure has them and
late hadoop 1 has some). Set the following in the RegionServer. </para>
- <programlisting><![CDATA[<property>
+ <programlisting><![CDATA[
+<property>
<name>hbase.lease.recovery.dfs.timeout</name>
<value>23000</value>
<description>How much time we allow elapse between calls to recover lease.
@@ -1421,12 +1422,13 @@ index e70ebc6..96f8c27 100644
<name>dfs.client.socket-timeout</name>
<value>10000</value>
<description>Down the DFS timeout from 60 to 10 seconds.</description>
-</property>]]>
- </programlisting>
+</property>
+]]></programlisting>
<para>And on the namenode/datanode side, set the following to enable 'staleness' introduced
in HDFS-3703, HDFS-3912. </para>
- <programlisting><![CDATA[<property>
+ <programlisting><![CDATA[
+<property>
<name>dfs.client.socket-timeout</name>
<value>10000</value>
<description>Down the DFS timeout from 60 to 10 seconds.</description>
@@ -1460,12 +1462,105 @@ index e70ebc6..96f8c27 100644
<name>dfs.namenode.avoid.write.stale.datanode</name>
<value>true</value>
<description>Enable stale state in hdfs</description>
-</property>]]>
- </programlisting>
+</property>
+]]></programlisting>
</section>
+ <section
+ xml:id="JMX_config">
+ <title>JMX</title>
+ <para>JMX(Java Management Extensions) provides built-in instrumentation that enables you
+ to monitor and manage the Java VM. To enable monitoring and management from remote
+ systems, you need to set system property com.sun.management.jmxremote.port(the port
+ number through which you want to enable JMX RMI connections) when you start the Java VM.
+ See <link
+ xlink:href="http://docs.oracle.com/javase/6/docs/technotes/guides/management/agent.html">
+ official document</link> for more information. Historically, besides above port mentioned,
+ JMX opens 2 additional random TCP listening ports, which could lead to port conflict
+ problem.(See <link
+ xlink:href="https://issues.apache.org/jira/browse/HBASE-10289">HBASE-10289</link>
+ for details)
+ </para>
+ <para>As an alternative, You can use the coprocessor-based JMX implementation provided
+ by HBase. To enable it, add below property in <filename>hbase-site.xml</filename>:
+ <programlisting><![CDATA[
+<property>
+ <name>hbase.coprocessor.regionserver.classes</name>
+ <value>org.apache.hadoop.hbase.JMXListener</value>
+</property>
+]]></programlisting>
+ NOTE: DO NOT set com.sun.management.jmxremote.port for Java VM at the same time.
+ </para>
+ <para>Currently it supports Master and RegionServer Java VM. The reason why you only
+ configure coprocessor for 'regionserver' is that, starting from HBase 0.99,
+ a Master IS also a RegionServer. (See <link
+ xlink:href="https://issues.apache.org/jira/browse/HBASE-10569">HBASE-10569</link>
+ for more information.)
+ By default, the JMX listens on TCP port 10102, you can further configure the port
+ using below properties:
+
+ <programlisting><![CDATA[
+<property>
+ <name>regionserver.rmi.registry.port</name>
+ <value>61130</value>
+</property>
+<property>
+ <name>regionserver.rmi.connector.port</name>
+ <value>61140</value>
+</property>
+]]></programlisting>
+ The registry port can be shared with connector port in most cases, so you only
+ need to configure regionserver.rmi.registry.port. However if you want to use SSL
+ communication, the 2 ports must be configured to different values.
+ </para>
+
+ <para>By default the password authentication and SSL communication is disabled.
+ To enable password authentication, you need to update <filename>hbase-env.sh</filename>
+ like below:
+ <screen>
+export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.authenticate=true \
+ -Dcom.sun.management.jmxremote.password.file=your_password_file \
+ -Dcom.sun.management.jmxremote.access.file=your_access_file"
+
+export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS $HBASE_JMX_BASE "
+export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS $HBASE_JMX_BASE "
+ </screen>
+ See example password/access file under $JRE_HOME/lib/management.
+ </para>
+
+ <para>To enable SSL communication with password authentication, follow below steps:
+ <screen>
+#1. generate a key pair, stored in myKeyStore
+keytool -genkey -alias jconsole -keystore myKeyStore
+
+#2. export it to file jconsole.cert
+keytool -export -alias jconsole -keystore myKeyStore -file jconsole.cert
+
+#3. copy jconsole.cert to jconsole client machine, import it to jconsoleKeyStore
+keytool -import -alias jconsole -keystore jconsoleKeyStore -file jconsole.cert
+ </screen>
+ And then update <filename>hbase-env.sh</filename> like below:
+ <screen>
+export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=true \
+ -Djavax.net.ssl.keyStore=/home/tianq/myKeyStore \
+ -Djavax.net.ssl.keyStorePassword=your_password_in_step_#1 \
+ -Dcom.sun.management.jmxremote.authenticate=true \
+ -Dcom.sun.management.jmxremote.password.file=your_password file \
+ -Dcom.sun.management.jmxremote.access.file=your_access_file"
+
+export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS $HBASE_JMX_BASE "
+export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS $HBASE_JMX_BASE "
+ </screen>
+
+ Finally start jconsole on client using the key store:
+ <screen>
+jconsole -J-Djavax.net.ssl.trustStore=/home/tianq/jconsoleKeyStore
+ </screen>
+ </para>
</section>
+ </section>
+
</section>
<!-- important config -->