You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by re...@apache.org on 2019/05/17 02:50:17 UTC
[hbase] branch branch-2 updated: HBASE-22184 [security] Support
get|set LogLevel in HTTPS mode.
This is an automated email from the ASF dual-hosted git repository.
reidchan pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new a04cb3b HBASE-22184 [security] Support get|set LogLevel in HTTPS mode.
a04cb3b is described below
commit a04cb3b2a555af18cb3abfae11a27acec0b2fedd
Author: Wei-Chiu Chuang <we...@cloudera.com>
AuthorDate: Wed May 8 15:57:21 2019 -0700
HBASE-22184 [security] Support get|set LogLevel in HTTPS mode.
Signed-off-by: Reid Chan <re...@apache.org>
---
.../org/apache/hadoop/hbase/http/log/LogLevel.java | 66 ++++++++-
.../apache/hadoop/hbase/http/log/TestLogLevel.java | 163 ++++++++++++++++++---
2 files changed, 203 insertions(+), 26 deletions(-)
diff --git a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/log/LogLevel.java b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/log/LogLevel.java
index 7182a0b..003fa0d 100644
--- a/hbase-http/src/main/java/org/apache/hadoop/hbase/http/log/LogLevel.java
+++ b/hbase-http/src/main/java/org/apache/hadoop/hbase/http/log/LogLevel.java
@@ -27,6 +27,8 @@ import java.net.URLConnection;
import java.util.Objects;
import java.util.regex.Pattern;
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLSocketFactory;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
@@ -40,6 +42,7 @@ import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.hbase.http.HttpServer;
import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
import org.apache.hadoop.security.authentication.client.KerberosAuthenticator;
+import org.apache.hadoop.security.ssl.SSLFactory;
import org.apache.hadoop.util.ServletUtil;
import org.apache.hadoop.util.Tool;
import org.apache.log4j.LogManager;
@@ -59,10 +62,12 @@ import org.apache.hbase.thirdparty.com.google.common.base.Charsets;
@InterfaceAudience.Private
public final class LogLevel {
private static final String USAGES = "\nUsage: General options are:\n"
- + "\t[-getlevel <host:port> <classname>\n"
- + "\t[-setlevel <host:port> <classname> <level> ";
+ + "\t[-getlevel <host:port> <classname> [-protocol (http|https)]\n"
+ + "\t[-setlevel <host:port> <classname> <level> [-protocol (http|https)]";
public static final String PROTOCOL_HTTP = "http";
+ public static final String PROTOCOL_HTTPS = "https";
+
/**
* A command line implementation
*/
@@ -85,9 +90,15 @@ public final class LogLevel {
System.exit(-1);
}
+ public static boolean isValidProtocol(String protocol) {
+ return ((protocol.equals(PROTOCOL_HTTP) ||
+ protocol.equals(PROTOCOL_HTTPS)));
+ }
+
@VisibleForTesting
static class CLI extends Configured implements Tool {
private Operations operation = Operations.UNKNOWN;
+ private String protocol;
private String hostName;
private String className;
private String level;
@@ -141,6 +152,9 @@ public final class LogLevel {
case "-setlevel":
nextArgIndex = parseSetLevelArgs(args, nextArgIndex);
break;
+ case "-protocol":
+ nextArgIndex = parseProtocolArgs(args, nextArgIndex);
+ break;
default:
throw new HadoopIllegalArgumentException(
"Unexpected argument " + args[nextArgIndex]);
@@ -152,6 +166,11 @@ public final class LogLevel {
throw new HadoopIllegalArgumentException(
"Must specify either -getlevel or -setlevel");
}
+
+ // if protocol is unspecified, set it as http.
+ if (protocol == null) {
+ protocol = PROTOCOL_HTTP;
+ }
}
private int parseGetLevelArgs(String[] args, int index) throws
@@ -187,6 +206,27 @@ public final class LogLevel {
return index + 4;
}
+ private int parseProtocolArgs(String[] args, int index) throws
+ HadoopIllegalArgumentException {
+ // make sure only -protocol is specified
+ if (protocol != null) {
+ throw new HadoopIllegalArgumentException(
+ "Redundant -protocol command");
+ }
+ // check number of arguments is sufficient
+ if (index + 1 >= args.length) {
+ throw new HadoopIllegalArgumentException(
+ "-protocol needs one parameter");
+ }
+ // check protocol is valid
+ protocol = args[index + 1];
+ if (!isValidProtocol(protocol)) {
+ throw new HadoopIllegalArgumentException(
+ "Invalid protocol: " + protocol);
+ }
+ return index + 2;
+ }
+
/**
* Send HTTP request to get log level.
*
@@ -194,7 +234,7 @@ public final class LogLevel {
* @throws Exception if unable to connect
*/
private void doGetLevel() throws Exception {
- process(PROTOCOL_HTTP + "://" + hostName + "/logLevel?log=" + className);
+ process(protocol + "://" + hostName + "/logLevel?log=" + className);
}
/**
@@ -204,7 +244,7 @@ public final class LogLevel {
* @throws Exception if unable to connect
*/
private void doSetLevel() throws Exception {
- process(PROTOCOL_HTTP + "://" + hostName + "/logLevel?log=" + className
+ process(protocol + "://" + hostName + "/logLevel?log=" + className
+ "&level=" + level);
}
@@ -220,10 +260,22 @@ public final class LogLevel {
private URLConnection connect(URL url) throws Exception {
AuthenticatedURL.Token token = new AuthenticatedURL.Token();
AuthenticatedURL aUrl;
+ SSLFactory clientSslFactory;
URLConnection connection;
-
- aUrl = new AuthenticatedURL(new KerberosAuthenticator());
- connection = aUrl.openConnection(url, token);
+ // If https is chosen, configures SSL client.
+ if (PROTOCOL_HTTPS.equals(url.getProtocol())) {
+ clientSslFactory = new SSLFactory(SSLFactory.Mode.CLIENT, this.getConf());
+ clientSslFactory.init();
+ SSLSocketFactory sslSocketF = clientSslFactory.createSSLSocketFactory();
+
+ aUrl = new AuthenticatedURL(new KerberosAuthenticator(), clientSslFactory);
+ connection = aUrl.openConnection(url, token);
+ HttpsURLConnection httpsConn = (HttpsURLConnection) connection;
+ httpsConn.setSSLSocketFactory(sslSocketF);
+ } else {
+ aUrl = new AuthenticatedURL(new KerberosAuthenticator());
+ connection = aUrl.openConnection(url, token);
+ }
connection.connect();
return connection;
}
diff --git a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/log/TestLogLevel.java b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/log/TestLogLevel.java
index acd84db..1cfe23e 100644
--- a/hbase-http/src/test/java/org/apache/hadoop/hbase/http/log/TestLogLevel.java
+++ b/hbase-http/src/test/java/org/apache/hadoop/hbase/http/log/TestLogLevel.java
@@ -17,17 +17,19 @@
*/
package org.apache.hadoop.hbase.http.log;
-import static org.apache.hadoop.hbase.http.log.LogLevel.PROTOCOL_HTTP;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import java.io.File;
import java.net.BindException;
+import java.net.SocketException;
import java.net.URI;
import java.security.PrivilegedExceptionAction;
import java.util.Properties;
+import javax.net.ssl.SSLException;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.HadoopIllegalArgumentException;
@@ -37,14 +39,18 @@ import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseCommonTestingUtility;
+import org.apache.hadoop.hbase.http.HttpConfig;
import org.apache.hadoop.hbase.http.HttpServer;
import org.apache.hadoop.hbase.http.log.LogLevel.CLI;
+import org.apache.hadoop.hbase.http.ssl.KeyStoreTestUtil;
import org.apache.hadoop.hbase.testclassification.MiscTests;
import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.minikdc.MiniKdc;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AccessControlList;
+import org.apache.hadoop.security.ssl.SSLFactory;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.log4j.Level;
import org.apache.log4j.LogManager;
@@ -66,8 +72,11 @@ public class TestLogLevel {
HBaseClassTestRule.forClass(TestLogLevel.class);
private static File BASEDIR;
+ private static String keystoresDir;
+ private static String sslConfDir;
private static Configuration serverConf;
private static Configuration clientConf;
+ private static Configuration sslConf;
private static final String logName = TestLogLevel.class.getName();
private static final Logger log = LogManager.getLogger(logName);
private final static String PRINCIPAL = "loglevel.principal";
@@ -94,6 +103,8 @@ public class TestLogLevel {
serverConf = new Configuration();
clientConf = new Configuration();
+ setupSSL(BASEDIR);
+
kdc = setupMiniKdc();
// Create two principles: a client and a HTTP principal
kdc.createPrincipal(KEYTAB_FILE, clientPrincipal, HTTP_PRINCIPAL);
@@ -132,6 +143,35 @@ public class TestLogLevel {
return kdc;
}
+ static private void setupSSL(File base) throws Exception {
+ Configuration conf = new Configuration();
+ conf.set(DFSConfigKeys.DFS_HTTP_POLICY_KEY, HttpConfig.Policy.HTTPS_ONLY.name());
+ conf.set(DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY, "localhost:0");
+ conf.set(DFSConfigKeys.DFS_DATANODE_HTTPS_ADDRESS_KEY, "localhost:0");
+
+ keystoresDir = base.getAbsolutePath();
+ sslConfDir = KeyStoreTestUtil.getClasspathDir(TestLogLevel.class);
+ KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, conf, false);
+
+ sslConf = getSslConfig();
+ }
+
+ /**
+ * Get the SSL configuration.
+ * This method is copied from KeyStoreTestUtil#getSslConfig() in Hadoop.
+ * @return {@link Configuration} instance with ssl configs loaded.
+ */
+ private static Configuration getSslConfig(){
+ Configuration sslConf = new Configuration(false);
+ String sslServerConfFile = "ssl-server.xml";
+ String sslClientConfFile = "ssl-client.xml";
+ sslConf.addResource(sslServerConfFile);
+ sslConf.addResource(sslClientConfFile);
+ sslConf.set(SSLFactory.SSL_SERVER_CONF_KEY, sslServerConfFile);
+ sslConf.set(SSLFactory.SSL_CLIENT_CONF_KEY, sslClientConfFile);
+ return sslConf;
+ }
+
@AfterClass
public static void tearDown() {
if (kdc != null) {
@@ -199,15 +239,16 @@ public class TestLogLevel {
/**
* Creates and starts a Jetty server binding at an ephemeral port to run
* LogLevel servlet.
+ * @param protocol "http" or "https"
* @param isSpnego true if SPNEGO is enabled
* @return a created HttpServer object
* @throws Exception if unable to create or start a Jetty server
*/
- private HttpServer createServer(boolean isSpnego)
+ private HttpServer createServer(String protocol, boolean isSpnego)
throws Exception {
HttpServer.Builder builder = new HttpServer.Builder()
.setName("..")
- .addEndpoint(new URI(PROTOCOL_HTTP + "://localhost:0"))
+ .addEndpoint(new URI(protocol + "://localhost:0"))
.setFindPort(true)
.setConf(serverConf);
if (isSpnego) {
@@ -220,24 +261,46 @@ public class TestLogLevel {
.setACL(new AccessControlList("client"));
}
+ // if using HTTPS, configure keystore/truststore properties.
+ if (protocol.equals(LogLevel.PROTOCOL_HTTPS)) {
+ builder = builder.
+ keyPassword(sslConf.get("ssl.server.keystore.keypassword"))
+ .keyStore(sslConf.get("ssl.server.keystore.location"),
+ sslConf.get("ssl.server.keystore.password"),
+ sslConf.get("ssl.server.keystore.type", "jks"))
+ .trustStore(sslConf.get("ssl.server.truststore.location"),
+ sslConf.get("ssl.server.truststore.password"),
+ sslConf.get("ssl.server.truststore.type", "jks"));
+ }
+
HttpServer server = builder.build();
server.start();
return server;
}
- private void testDynamicLogLevel(final boolean isSpnego)
+ private void testDynamicLogLevel(final String bindProtocol, final String connectProtocol,
+ final boolean isSpnego)
throws Exception {
- testDynamicLogLevel(isSpnego, Level.DEBUG.toString());
+ testDynamicLogLevel(bindProtocol, connectProtocol, isSpnego, Level.DEBUG.toString());
}
/**
* Run both client and server using the given protocol.
*
+ * @param bindProtocol specify either http or https for server
+ * @param connectProtocol specify either http or https for client
* @param isSpnego true if SPNEGO is enabled
* @throws Exception if client can't accesss server.
*/
- private void testDynamicLogLevel(final boolean isSpnego, final String newLevel)
+ private void testDynamicLogLevel(final String bindProtocol, final String connectProtocol,
+ final boolean isSpnego, final String newLevel)
throws Exception {
+ if (!LogLevel.isValidProtocol(bindProtocol)) {
+ throw new Exception("Invalid server protocol " + bindProtocol);
+ }
+ if (!LogLevel.isValidProtocol(connectProtocol)) {
+ throw new Exception("Invalid client protocol " + connectProtocol);
+ }
Level oldLevel = log.getEffectiveLevel();
assertNotEquals("Get default Log Level which shouldn't be ERROR.",
Level.ERROR, oldLevel);
@@ -255,7 +318,7 @@ public class TestLogLevel {
UserGroupInformation.setConfiguration(serverConf);
}
- final HttpServer server = createServer(isSpnego);
+ final HttpServer server = createServer(bindProtocol, isSpnego);
// get server port
final String authority = NetUtils.getHostPortString(server.getConnectorAddress(0));
@@ -266,8 +329,8 @@ public class TestLogLevel {
try {
clientUGI.doAs((PrivilegedExceptionAction<Void>) () -> {
// client command line
- getLevel(authority);
- setLevel(authority, newLevel);
+ getLevel(connectProtocol, authority);
+ setLevel(connectProtocol, authority, newLevel);
return null;
});
} finally {
@@ -283,11 +346,12 @@ public class TestLogLevel {
* Run LogLevel command line to start a client to get log level of this test
* class.
*
+ * @param protocol specify either http or https
* @param authority daemon's web UI address
* @throws Exception if unable to connect
*/
- private void getLevel(String authority) throws Exception {
- String[] getLevelArgs = {"-getlevel", authority, logName};
+ private void getLevel(String protocol, String authority) throws Exception {
+ String[] getLevelArgs = {"-getlevel", authority, logName, "-protocol", protocol};
CLI cli = new CLI(clientConf);
cli.run(getLevelArgs);
}
@@ -296,12 +360,13 @@ public class TestLogLevel {
* Run LogLevel command line to start a client to set log level of this test
* class to debug.
*
+ * @param protocol specify either http or https
* @param authority daemon's web UI address
* @throws Exception if unable to run or log level does not change as expected
*/
- private void setLevel(String authority, String newLevel)
+ private void setLevel(String protocol, String authority, String newLevel)
throws Exception {
- String[] setLevelArgs = {"-setlevel", authority, logName, newLevel};
+ String[] setLevelArgs = {"-setlevel", authority, logName, newLevel, "-protocol", protocol};
CLI cli = new CLI(clientConf);
cli.run(setLevelArgs);
@@ -316,7 +381,7 @@ public class TestLogLevel {
*/
@Test(timeout=60000)
public void testInfoLogLevel() throws Exception {
- testDynamicLogLevel(true, "INFO");
+ testDynamicLogLevel(LogLevel.PROTOCOL_HTTP, LogLevel.PROTOCOL_HTTP, true, "INFO");
}
/**
@@ -326,26 +391,86 @@ public class TestLogLevel {
*/
@Test(timeout=60000)
public void testErrorLogLevel() throws Exception {
- testDynamicLogLevel(true, "ERROR");
+ testDynamicLogLevel(LogLevel.PROTOCOL_HTTP, LogLevel.PROTOCOL_HTTP, true, "ERROR");
}
/**
* Server runs HTTP, no SPNEGO.
*
- * @throws Exception if http client can't access http server.
+ * @throws Exception if http client can't access http server,
+ * or http client can access https server.
*/
@Test(timeout=60000)
public void testLogLevelByHttp() throws Exception {
- testDynamicLogLevel(false);
+ testDynamicLogLevel(LogLevel.PROTOCOL_HTTP, LogLevel.PROTOCOL_HTTP, false);
+ try {
+ testDynamicLogLevel(LogLevel.PROTOCOL_HTTP, LogLevel.PROTOCOL_HTTPS,
+ false);
+ fail("A HTTPS Client should not have succeeded in connecting to a " +
+ "HTTP server");
+ } catch (SSLException e) {
+ GenericTestUtils.assertExceptionContains("Unrecognized SSL message", e);
+ }
}
/**
* Server runs HTTP + SPNEGO.
*
- * @throws Exception if http client can't access http server.
+ * @throws Exception if http client can't access http server,
+ * or http client can access https server.
*/
@Test(timeout=60000)
public void testLogLevelByHttpWithSpnego() throws Exception {
- testDynamicLogLevel(true);
+ testDynamicLogLevel(LogLevel.PROTOCOL_HTTP, LogLevel.PROTOCOL_HTTP, true);
+ try {
+ testDynamicLogLevel(LogLevel.PROTOCOL_HTTP, LogLevel.PROTOCOL_HTTPS,
+ true);
+ fail("A HTTPS Client should not have succeeded in connecting to a " +
+ "HTTP server");
+ } catch (SSLException e) {
+ GenericTestUtils.assertExceptionContains("Unrecognized SSL message", e);
+ }
+ }
+
+ /**
+ * Server runs HTTPS, no SPNEGO.
+ *
+ * @throws Exception if https client can't access https server,
+ * or https client can access http server.
+ */
+ @Test(timeout=60000)
+ public void testLogLevelByHttps() throws Exception {
+ testDynamicLogLevel(LogLevel.PROTOCOL_HTTPS, LogLevel.PROTOCOL_HTTPS,
+ false);
+ try {
+ testDynamicLogLevel(LogLevel.PROTOCOL_HTTPS, LogLevel.PROTOCOL_HTTP,
+ false);
+ fail("A HTTP Client should not have succeeded in connecting to a " +
+ "HTTPS server");
+ } catch (SocketException e) {
+ GenericTestUtils.assertExceptionContains(
+ "Unexpected end of file from server", e);
+ }
+ }
+
+ /**
+ * Server runs HTTPS + SPNEGO.
+ *
+ * @throws Exception if https client can't access https server,
+ * or https client can access http server.
+ */
+ @Test(timeout=60000)
+ public void testLogLevelByHttpsWithSpnego() throws Exception {
+ testDynamicLogLevel(LogLevel.PROTOCOL_HTTPS, LogLevel.PROTOCOL_HTTPS,
+ true);
+ try {
+ testDynamicLogLevel(LogLevel.PROTOCOL_HTTPS, LogLevel.PROTOCOL_HTTP,
+ true);
+ fail("A HTTP Client should not have succeeded in connecting to a " +
+ "HTTPS server");
+ } catch (SocketException e) {
+ GenericTestUtils.assertExceptionContains(
+ "Unexpected end of file from server", e);
+ }
}
}