You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ae...@apache.org on 2016/06/01 19:55:09 UTC
[08/39] hadoop git commit: HADOOP-12847. hadoop daemonlog should
support https and SPNEGO for Kerberized cluster. (Wei-Chiu Chuang via Yongjun
Zhang)
HADOOP-12847. hadoop daemonlog should support https and SPNEGO for Kerberized cluster. (Wei-Chiu Chuang via Yongjun Zhang)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/34cc21f6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/34cc21f6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/34cc21f6
Branch: refs/heads/HDFS-1312
Commit: 34cc21f6d1a293d92613defba38e8ae810db4c71
Parents: 0a544f8
Author: Yongjun Zhang <yz...@cloudera.com>
Authored: Thu May 26 22:29:38 2016 -0700
Committer: Yongjun Zhang <yz...@cloudera.com>
Committed: Thu May 26 22:43:45 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/log/LogLevel.java | 285 +++++++++--
.../src/site/markdown/CommandsManual.md | 32 +-
.../org/apache/hadoop/log/TestLogLevel.java | 479 +++++++++++++++----
3 files changed, 674 insertions(+), 122 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34cc21f6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogLevel.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogLevel.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogLevel.java
index 8db0dce..4fa839f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogLevel.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogLevel.java
@@ -17,20 +17,38 @@
*/
package org.apache.hadoop.log;
-import java.io.*;
-import java.net.*;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.PrintWriter;
+import java.net.URL;
+import java.net.URLConnection;
import java.util.regex.Pattern;
-import javax.servlet.*;
-import javax.servlet.http.*;
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLSocketFactory;
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Charsets;
-import org.apache.commons.logging.*;
-import org.apache.commons.logging.impl.*;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.commons.logging.impl.Jdk14Logger;
+import org.apache.commons.logging.impl.Log4JLogger;
+import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.http.HttpServer2;
+import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
+import org.apache.hadoop.security.authentication.client.KerberosAuthenticator;
+import org.apache.hadoop.security.ssl.SSLFactory;
import org.apache.hadoop.util.ServletUtil;
+import org.apache.hadoop.util.Tool;
/**
* Change log level in runtime.
@@ -38,43 +56,252 @@ import org.apache.hadoop.util.ServletUtil;
@InterfaceStability.Evolving
public class LogLevel {
public static final String USAGES = "\nUsage: General options are:\n"
- + "\t[-getlevel <host:httpPort> <classname>]\n"
- + "\t[-setlevel <host:httpPort> <classname> <level>]\n";
+ + "\t[-getlevel <host:port> <classname> [-protocol (http|https)]\n"
+ + "\t[-setlevel <host:port> <classname> <level> "
+ + "[-protocol (http|https)]\n";
+ public static final String PROTOCOL_HTTP = "http";
+ public static final String PROTOCOL_HTTPS = "https";
/**
* A command line implementation
*/
- public static void main(String[] args) {
- if (args.length == 3 && "-getlevel".equals(args[0])) {
- process("http://" + args[1] + "/logLevel?log=" + args[2]);
- return;
- }
- else if (args.length == 4 && "-setlevel".equals(args[0])) {
- process("http://" + args[1] + "/logLevel?log=" + args[2]
- + "&level=" + args[3]);
- return;
- }
+ public static void main(String[] args) throws Exception {
+ CLI cli = new CLI(new Configuration());
+ System.exit(cli.run(args));
+ }
+
+ /**
+ * Valid command line options.
+ */
+ private enum Operations {
+ GETLEVEL,
+ SETLEVEL,
+ UNKNOWN
+ }
+ private static void printUsage() {
System.err.println(USAGES);
- System.exit(-1);
}
- private static void process(String urlstring) {
- try {
- URL url = new URL(urlstring);
- System.out.println("Connecting to " + url);
- URLConnection connection = url.openConnection();
+ public static boolean isValidProtocol(String protocol) {
+ return ((protocol.equals(PROTOCOL_HTTP) ||
+ protocol.equals(PROTOCOL_HTTPS)));
+ }
+
+ @VisibleForTesting
+ static class CLI extends Configured implements Tool {
+ private Operations operation = Operations.UNKNOWN;
+ private String protocol;
+ private String hostName;
+ private String className;
+ private String level;
+
+ CLI(Configuration conf) {
+ setConf(conf);
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+ try {
+ parseArguments(args);
+ sendLogLevelRequest();
+ } catch (HadoopIllegalArgumentException e) {
+ printUsage();
+ throw e;
+ }
+ return 0;
+ }
+
+ /**
+ * Send HTTP/HTTPS request to the daemon.
+ * @throws HadoopIllegalArgumentException if arguments are invalid.
+ * @throws Exception if unable to connect
+ */
+ private void sendLogLevelRequest()
+ throws HadoopIllegalArgumentException, Exception {
+ switch (operation) {
+ case GETLEVEL:
+ doGetLevel();
+ break;
+ case SETLEVEL:
+ doSetLevel();
+ break;
+ default:
+ throw new HadoopIllegalArgumentException(
+ "Expect either -getlevel or -setlevel");
+ }
+ }
+
+ public void parseArguments(String[] args) throws
+ HadoopIllegalArgumentException {
+ if (args.length == 0) {
+ throw new HadoopIllegalArgumentException("No arguments specified");
+ }
+ int nextArgIndex = 0;
+ while (nextArgIndex < args.length) {
+ if (args[nextArgIndex].equals("-getlevel")) {
+ nextArgIndex = parseGetLevelArgs(args, nextArgIndex);
+ } else if (args[nextArgIndex].equals("-setlevel")) {
+ nextArgIndex = parseSetLevelArgs(args, nextArgIndex);
+ } else if (args[nextArgIndex].equals("-protocol")) {
+ nextArgIndex = parseProtocolArgs(args, nextArgIndex);
+ } else {
+ throw new HadoopIllegalArgumentException(
+ "Unexpected argument " + args[nextArgIndex]);
+ }
+ }
+
+ // if operation is never specified in the arguments
+ if (operation == Operations.UNKNOWN) {
+ throw new HadoopIllegalArgumentException(
+ "Must specify either -getlevel or -setlevel");
+ }
+
+ // if protocol is unspecified, set it as http.
+ if (protocol == null) {
+ protocol = PROTOCOL_HTTP;
+ }
+ }
+
+ private int parseGetLevelArgs(String[] args, int index) throws
+ HadoopIllegalArgumentException {
+ // fail if multiple operations are specified in the arguments
+ if (operation != Operations.UNKNOWN) {
+ throw new HadoopIllegalArgumentException(
+ "Redundant -getlevel command");
+ }
+ // check number of arguments is sufficient
+ if (index+2 >= args.length) {
+ throw new HadoopIllegalArgumentException(
+ "-getlevel needs two parameters");
+ }
+ operation = Operations.GETLEVEL;
+ hostName = args[index+1];
+ className = args[index+2];
+ return index+3;
+ }
+
+ private int parseSetLevelArgs(String[] args, int index) throws
+ HadoopIllegalArgumentException {
+ // fail if multiple operations are specified in the arguments
+ if (operation != Operations.UNKNOWN) {
+ throw new HadoopIllegalArgumentException(
+ "Redundant -setlevel command");
+ }
+ // check number of arguments is sufficient
+ if (index+3 >= args.length) {
+ throw new HadoopIllegalArgumentException(
+ "-setlevel needs three parameters");
+ }
+ operation = Operations.SETLEVEL;
+ hostName = args[index+1];
+ className = args[index+2];
+ level = args[index+3];
+ return index+4;
+ }
+
+ private int parseProtocolArgs(String[] args, int index) throws
+ HadoopIllegalArgumentException {
+ // make sure only -protocol is specified
+ if (protocol != null) {
+ throw new HadoopIllegalArgumentException(
+ "Redundant -protocol command");
+ }
+ // check number of arguments is sufficient
+ if (index+1 >= args.length) {
+ throw new HadoopIllegalArgumentException(
+ "-protocol needs one parameter");
+ }
+ // check protocol is valid
+ protocol = args[index+1];
+ if (!isValidProtocol(protocol)) {
+ throw new HadoopIllegalArgumentException(
+ "Invalid protocol: " + protocol);
+ }
+ return index+2;
+ }
+
+ /**
+ * Send HTTP/HTTPS request to get log level.
+ *
+ * @throws HadoopIllegalArgumentException if arguments are invalid.
+ * @throws Exception if unable to connect
+ */
+ private void doGetLevel() throws Exception {
+ process(protocol + "://" + hostName + "/logLevel?log=" + className);
+ }
+
+ /**
+ * Send HTTP/HTTPS request to set log level.
+ *
+ * @throws HadoopIllegalArgumentException if arguments are invalid.
+ * @throws Exception if unable to connect
+ */
+ private void doSetLevel() throws Exception {
+ process(protocol + "://" + hostName + "/logLevel?log=" + className
+ + "&level=" + level);
+ }
+
+ /**
+ * Connect to the URL. Supports HTTP/HTTPS and supports SPNEGO
+ * authentication. It falls back to simple authentication if it fails to
+ * initiate SPNEGO.
+ *
+ * @param url the URL address of the daemon servlet
+ * @return a connected connection
+ * @throws Exception if it can not establish a connection.
+ */
+ private URLConnection connect(URL url) throws Exception {
+ AuthenticatedURL.Token token = new AuthenticatedURL.Token();
+ AuthenticatedURL aUrl;
+ SSLFactory clientSslFactory;
+ URLConnection connection;
+ // If https is chosen, configures SSL client.
+ if (PROTOCOL_HTTPS.equals(url.getProtocol())) {
+ clientSslFactory = new SSLFactory(
+ SSLFactory.Mode.CLIENT, this.getConf());
+ clientSslFactory.init();
+ SSLSocketFactory sslSocketF = clientSslFactory.createSSLSocketFactory();
+
+ aUrl = new AuthenticatedURL(
+ new KerberosAuthenticator(), clientSslFactory);
+ connection = aUrl.openConnection(url, token);
+ HttpsURLConnection httpsConn = (HttpsURLConnection) connection;
+ httpsConn.setSSLSocketFactory(sslSocketF);
+ } else {
+ aUrl = new AuthenticatedURL(new KerberosAuthenticator());
+ connection = aUrl.openConnection(url, token);
+ }
+
connection.connect();
+ return connection;
+ }
+
+ /**
+ * Configures the client to send HTTP/HTTPS request to the URL.
+ * Supports SPENGO for authentication.
+ * @param urlString URL and query string to the daemon's web UI
+ * @throws Exception if unable to connect
+ */
+ private void process(String urlString) throws Exception {
+ URL url = new URL(urlString);
+ System.out.println("Connecting to " + url);
- BufferedReader in = new BufferedReader(new InputStreamReader(
- connection.getInputStream(), Charsets.UTF_8));
- for(String line; (line = in.readLine()) != null; )
+ URLConnection connection = connect(url);
+
+ // read from the servlet
+ BufferedReader in = new BufferedReader(
+ new InputStreamReader(connection.getInputStream(), Charsets.UTF_8));
+ for (String line;;) {
+ line = in.readLine();
+ if (line == null) {
+ break;
+ }
if (line.startsWith(MARKER)) {
System.out.println(TAG.matcher(line).replaceAll(""));
}
+ }
in.close();
- } catch (IOException ioe) {
- System.err.println("" + ioe);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34cc21f6/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md b/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md
index 178a504..44ba5ea 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md
@@ -229,17 +229,37 @@ Commands useful for administrators of a hadoop cluster.
Usage:
- hadoop daemonlog -getlevel <host:httpport> <classname>
- hadoop daemonlog -setlevel <host:httpport> <classname> <level>
+ hadoop daemonlog -getlevel <host:port> <classname> [-protocol (http|https)]
+ hadoop daemonlog -setlevel <host:port> <classname> <level> [-protocol (http|https)]
| COMMAND\_OPTION | Description |
|:---- |:---- |
-| `-getlevel` *host:httpport* *classname* | Prints the log level of the log identified by a qualified *classname*, in the daemon running at *host:httpport*. This command internally connects to `http://<host:httpport>/logLevel?log=<classname>` |
-| `-setlevel` *host:httpport* *classname* *level* | Sets the log level of the log identified by a qualified *classname*, in the daemon running at *host:httpport*. This command internally connects to `http://<host:httpport>/logLevel?log=<classname>&level=<level>` |
+| `-getlevel` *host:port* *classname* [-protocol (http|https)] | Prints the log level of the log identified by a qualified *classname*, in the daemon running at *host:port*. The `-protocol` flag specifies the protocol for connection. |
+| `-setlevel` *host:port* *classname* *level* [-protocol (http|https)] | Sets the log level of the log identified by a qualified *classname*, in the daemon running at *host:port*. The `-protocol` flag specifies the protocol for connection. |
-Get/Set the log level for a Log identified by a qualified class name in the daemon.
+Get/Set the log level for a Log identified by a qualified class name in the daemon dynamically.
+By default, the command sends a HTTP request, but this can be overridden by using argument `-protocol https` to send a HTTPS request.
+
+Example:
+
+ $ bin/hadoop daemonlog -setlevel 127.0.0.1:9870 org.apache.hadoop.hdfs.server.namenode.NameNode DEBUG
+ $ bin/hadoop daemonlog -getlevel 127.0.0.1:9871 org.apache.hadoop.hdfs.server.namenode.NameNode DEBUG -protocol https
+
+Note that the setting is not permanent and will be reset when the daemon is restarted.
+This command works by sending a HTTP/HTTPS request to the daemon's internal Jetty servlet, so it supports the following daemons:
+
+* HDFS
+ * name node
+ * secondary name node
+ * data node
+ * journal node
+* YARN
+ * resource manager
+ * node manager
+ * Timeline server
+
+However, the command does not support KMS server, because its web interface is based on Tomcat, which does not support the servlet.
- Example: $ bin/hadoop daemonlog -setlevel 127.0.0.1:9870 org.apache.hadoop.hdfs.server.namenode.NameNode DEBUG
Files
-----
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34cc21f6/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/log/TestLogLevel.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/log/TestLogLevel.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/log/TestLogLevel.java
index e35323b..30bf726 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/log/TestLogLevel.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/log/TestLogLevel.java
@@ -17,99 +17,404 @@
*/
package org.apache.hadoop.log;
-import java.io.*;
-import java.net.*;
+import java.io.File;
+import java.net.SocketException;
+import java.net.URI;
+import java.util.concurrent.Callable;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.commons.logging.impl.Log4JLogger;
+import org.apache.hadoop.HadoopIllegalArgumentException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.http.HttpServer2;
+import org.apache.hadoop.log.LogLevel.CLI;
+import org.apache.hadoop.minikdc.KerberosSecurityTestcase;
import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authentication.KerberosTestUtils;
+import org.apache.hadoop.security.authentication.client.KerberosAuthenticator;
+import org.apache.hadoop.security.authorize.AccessControlList;
+import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
+import org.junit.Assert;
-import junit.framework.TestCase;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.LoggerFactory;
-import org.apache.commons.logging.*;
-import org.apache.commons.logging.impl.*;
-import org.apache.log4j.*;
-import org.junit.Assert;
+import javax.net.ssl.SSLException;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * Test LogLevel.
+ */
+public class TestLogLevel extends KerberosSecurityTestcase {
+ private static final File BASEDIR = GenericTestUtils.getRandomizedTestDir();
+ private static String keystoresDir;
+ private static String sslConfDir;
+ private static Configuration conf;
+ private static Configuration sslConf;
+ private final String logName = TestLogLevel.class.getName();
+ private String clientPrincipal;
+ private String serverPrincipal;
+ private final Log testlog = LogFactory.getLog(logName);
+ private final Logger log = ((Log4JLogger)testlog).getLogger();
+ private final static String PRINCIPAL = "loglevel.principal";
+ private final static String KEYTAB = "loglevel.keytab";
-public class TestLogLevel extends TestCase {
- static final PrintStream out = System.out;
-
- public void testDynamicLogLevel() throws Exception {
- String logName = TestLogLevel.class.getName();
- Log testlog = LogFactory.getLog(logName);
-
- //only test Log4JLogger
- if (testlog instanceof Log4JLogger) {
- Logger log = ((Log4JLogger)testlog).getLogger();
- log.debug("log.debug1");
- log.info("log.info1");
- log.error("log.error1");
- Assert.assertNotEquals("Get default Log Level which shouldn't be ERROR.",
- Level.ERROR, log.getEffectiveLevel());
-
- HttpServer2 server = new HttpServer2.Builder().setName("..")
- .addEndpoint(new URI("http://localhost:0")).setFindPort(true)
- .build();
-
- server.start();
- String authority = NetUtils.getHostPortString(server
- .getConnectorAddress(0));
-
- //servlet
- URL url = new URL("http://" + authority + "/logLevel?log=" + logName
- + "&level=" + Level.ERROR);
- out.println("*** Connecting to " + url);
- URLConnection connection = url.openConnection();
- connection.connect();
-
- BufferedReader in = new BufferedReader(new InputStreamReader(
- connection.getInputStream()));
- for(String line; (line = in.readLine()) != null; out.println(line));
- in.close();
-
- log.debug("log.debug2");
- log.info("log.info2");
- log.error("log.error2");
- assertEquals("Try setting log level: ERROR from servlet.", Level.ERROR,
- log.getEffectiveLevel());
-
- //command line
- String[] args = {"-setlevel", authority, logName, Level.DEBUG.toString()};
- LogLevel.main(args);
- log.debug("log.debug3");
- log.info("log.info3");
- log.error("log.error3");
- assertEquals("Try setting log level: DEBUG via command line", Level.DEBUG,
- log.getEffectiveLevel());
-
- // Test mixed upper case and lower case in level string.
- String[] args2 = {"-setlevel", authority, logName, "Info"};
- LogLevel.main(args2);
- log.debug("log.debug4");
- log.info("log.info4");
- log.error("log.error4");
- assertEquals("Try setting log level: Info via command line.", Level.INFO,
- log.getEffectiveLevel());
-
- // Test "Error" instead of "ERROR" should work for servlet
- URL newUrl = new URL("http://" + authority + "/logLevel?log=" + logName
- + "&level=" + "Error");
- out.println("*** Connecting to " + newUrl);
- connection = newUrl.openConnection();
- connection.connect();
-
- BufferedReader in2 = new BufferedReader(new InputStreamReader(
- connection.getInputStream()));
- for(String line; (line = in2.readLine()) != null; out.println(line));
- in2.close();
-
- log.debug("log.debug5");
- log.info("log.info5");
- log.error("log.error5");
- assertEquals("Try setting log level: Error via servlet.", Level.ERROR,
- log.getEffectiveLevel());
+ @BeforeClass
+ public static void setUp() throws Exception {
+ org.slf4j.Logger logger =
+ LoggerFactory.getLogger(KerberosAuthenticator.class);
+ GenericTestUtils.setLogLevel(logger, Level.DEBUG);
+ FileUtil.fullyDelete(BASEDIR);
+ if (!BASEDIR.mkdirs()) {
+ throw new Exception("unable to create the base directory for testing");
}
- else {
- out.println(testlog.getClass() + " not tested.");
+ conf = new Configuration();
+
+ setupSSL(BASEDIR);
+ }
+
+ static private void setupSSL(File base) throws Exception {
+ keystoresDir = base.getAbsolutePath();
+ sslConfDir = KeyStoreTestUtil.getClasspathDir(TestLogLevel.class);
+ KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, conf, false);
+
+ sslConf = KeyStoreTestUtil.getSslConfig();
+ }
+
+ @Before
+ public void setupKerberos() throws Exception {
+ File keytabFile = new File(KerberosTestUtils.getKeytabFile());
+ clientPrincipal = KerberosTestUtils.getClientPrincipal();
+ serverPrincipal = KerberosTestUtils.getServerPrincipal();
+ clientPrincipal = clientPrincipal.substring(0,
+ clientPrincipal.lastIndexOf("@"));
+ serverPrincipal = serverPrincipal.substring(0,
+ serverPrincipal.lastIndexOf("@"));
+ getKdc().createPrincipal(keytabFile, clientPrincipal, serverPrincipal);
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ KeyStoreTestUtil.cleanupSSLConfig(keystoresDir, sslConfDir);
+ FileUtil.fullyDelete(BASEDIR);
+ }
+
+ /**
+ * Test client command line options. Does not validate server behavior.
+ * @throws Exception
+ */
+ @Test(timeout=120000)
+ public void testCommandOptions() throws Exception {
+ final String className = this.getClass().getName();
+
+ assertFalse(validateCommand(new String[] {"-foo" }));
+ // fail due to insufficient number of arguments
+ assertFalse(validateCommand(new String[] {}));
+ assertFalse(validateCommand(new String[] {"-getlevel" }));
+ assertFalse(validateCommand(new String[] {"-setlevel" }));
+ assertFalse(validateCommand(new String[] {"-getlevel", "foo.bar:8080" }));
+
+ // valid command arguments
+ assertTrue(validateCommand(
+ new String[] {"-getlevel", "foo.bar:8080", className }));
+ assertTrue(validateCommand(
+ new String[] {"-setlevel", "foo.bar:8080", className, "DEBUG" }));
+ assertTrue(validateCommand(
+ new String[] {"-getlevel", "foo.bar:8080", className, "-protocol",
+ "http" }));
+ assertTrue(validateCommand(
+ new String[] {"-getlevel", "foo.bar:8080", className, "-protocol",
+ "https" }));
+ assertTrue(validateCommand(
+ new String[] {"-setlevel", "foo.bar:8080", className, "DEBUG",
+ "-protocol", "http" }));
+ assertTrue(validateCommand(
+ new String[] {"-setlevel", "foo.bar:8080", className, "DEBUG",
+ "-protocol", "https" }));
+
+ // fail due to the extra argument
+ assertFalse(validateCommand(
+ new String[] {"-getlevel", "foo.bar:8080", className, "-protocol",
+ "https", "blah" }));
+ assertFalse(validateCommand(
+ new String[] {"-setlevel", "foo.bar:8080", className, "DEBUG",
+ "-protocol", "https", "blah" }));
+ assertFalse(validateCommand(
+ new String[] {"-getlevel", "foo.bar:8080", className, "-protocol",
+ "https", "-protocol", "https" }));
+ assertFalse(validateCommand(
+ new String[] {"-getlevel", "foo.bar:8080", className,
+ "-setlevel", "foo.bar:8080", className }));
+ }
+
+ /**
+ * Check to see if a command can be accepted.
+ *
+ * @param args a String array of arguments
+ * @return true if the command can be accepted, false if not.
+ */
+ private boolean validateCommand(String[] args) throws Exception {
+ CLI cli = new CLI(sslConf);
+ try {
+ cli.parseArguments(args);
+ } catch (HadoopIllegalArgumentException e) {
+ return false;
+ } catch (Exception e) {
+ // this is used to verify the command arguments only.
+ // no HadoopIllegalArgumentException = the arguments are good.
+ return true;
+ }
+ return true;
+ }
+
+ /**
+ * Creates and starts a Jetty server binding at an ephemeral port to run
+ * LogLevel servlet.
+ * @param protocol "http" or "https"
+ * @param isSpnego true if SPNEGO is enabled
+ * @return a created HttpServer2 object
+ * @throws Exception if unable to create or start a Jetty server
+ */
+ private HttpServer2 createServer(String protocol, boolean isSpnego)
+ throws Exception {
+ HttpServer2.Builder builder = new HttpServer2.Builder()
+ .setName("..")
+ .addEndpoint(new URI(protocol + "://localhost:0"))
+ .setFindPort(true)
+ .setConf(conf);
+ if (isSpnego) {
+ // Set up server Kerberos credentials.
+ // Since the server may fall back to simple authentication,
+ // use ACL to make sure the connection is Kerberos/SPNEGO authenticated.
+ builder.setSecurityEnabled(true)
+ .setUsernameConfKey(PRINCIPAL)
+ .setKeytabConfKey(KEYTAB)
+ .setACL(new AccessControlList(clientPrincipal));
+ }
+
+ // if using HTTPS, configure keystore/truststore properties.
+ if (protocol.equals(LogLevel.PROTOCOL_HTTPS)) {
+ builder = builder.
+ keyPassword(sslConf.get("ssl.server.keystore.keypassword"))
+ .keyStore(sslConf.get("ssl.server.keystore.location"),
+ sslConf.get("ssl.server.keystore.password"),
+ sslConf.get("ssl.server.keystore.type", "jks"))
+ .trustStore(sslConf.get("ssl.server.truststore.location"),
+ sslConf.get("ssl.server.truststore.password"),
+ sslConf.get("ssl.server.truststore.type", "jks"));
+ }
+ HttpServer2 server = builder.build();
+ // Enable SPNEGO for LogLevel servlet
+ if (isSpnego) {
+ server.addInternalServlet("logLevel", "/logLevel", LogLevel.Servlet.class,
+ true);
+ }
+ server.start();
+ return server;
+ }
+
+ private void testDynamicLogLevel(final String bindProtocol,
+ final String connectProtocol, final boolean isSpnego)
+ throws Exception {
+ testDynamicLogLevel(bindProtocol, connectProtocol, isSpnego,
+ Level.DEBUG.toString());
+ }
+
+ /**
+ * Run both client and server using the given protocol.
+ *
+ * @param bindProtocol specify either http or https for server
+ * @param connectProtocol specify either http or https for client
+ * @param isSpnego true if SPNEGO is enabled
+ * @throws Exception
+ */
+ private void testDynamicLogLevel(final String bindProtocol,
+ final String connectProtocol, final boolean isSpnego,
+ final String newLevel) throws Exception {
+ if (!LogLevel.isValidProtocol(bindProtocol)) {
+ throw new Exception("Invalid server protocol " + bindProtocol);
+ }
+ if (!LogLevel.isValidProtocol(connectProtocol)) {
+ throw new Exception("Invalid client protocol " + connectProtocol);
+ }
+ Level oldLevel = log.getEffectiveLevel();
+ Assert.assertNotEquals("Get default Log Level which shouldn't be ERROR.",
+ Level.ERROR, oldLevel);
+
+ // configs needed for SPNEGO at server side
+ if (isSpnego) {
+ conf.set(PRINCIPAL, KerberosTestUtils.getServerPrincipal());
+ conf.set(KEYTAB, KerberosTestUtils.getKeytabFile());
+ conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
+ "kerberos");
+ conf.setBoolean(CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION,
+ true);
+ UserGroupInformation.setConfiguration(conf);
+ }
+
+ final HttpServer2 server = createServer(bindProtocol, isSpnego);
+ // get server port
+ final String authority = NetUtils.getHostPortString(server
+ .getConnectorAddress(0));
+
+ KerberosTestUtils.doAsClient(new Callable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ // client command line
+ getLevel(connectProtocol, authority);
+ setLevel(connectProtocol, authority, newLevel);
+ return null;
+ }
+ });
+ server.stop();
+ // restore log level
+ GenericTestUtils.setLogLevel(log, oldLevel);
+ }
+
+ /**
+ * Run LogLevel command line to start a client to get log level of this test
+ * class.
+ *
+ * @param protocol specify either http or https
+ * @param authority daemon's web UI address
+ * @throws Exception if unable to connect
+ */
+ private void getLevel(String protocol, String authority) throws Exception {
+ String[] getLevelArgs = {"-getlevel", authority, logName, "-protocol",
+ protocol};
+ CLI cli = new CLI(sslConf);
+ cli.run(getLevelArgs);
+ }
+
+ /**
+ * Run LogLevel command line to start a client to set log level of this test
+ * class to debug.
+ *
+ * @param protocol specify either http or https
+ * @param authority daemon's web UI address
+ * @throws Exception if unable to run or log level does not change as expected
+ */
+ private void setLevel(String protocol, String authority, String newLevel)
+ throws Exception {
+ String[] setLevelArgs = {"-setlevel", authority, logName,
+ newLevel, "-protocol", protocol};
+ CLI cli = new CLI(sslConf);
+ cli.run(setLevelArgs);
+
+ assertEquals("new level not equal to expected: ", newLevel.toUpperCase(),
+ log.getEffectiveLevel().toString());
+ }
+
+ /**
+ * Test setting log level to "Info".
+ *
+ * @throws Exception
+ */
+ @Test(timeout=60000)
+ public void testInfoLogLevel() throws Exception {
+ testDynamicLogLevel(LogLevel.PROTOCOL_HTTP, LogLevel.PROTOCOL_HTTP, false,
+ "Info");
+ }
+
+ /**
+ * Test setting log level to "Error".
+ *
+ * @throws Exception
+ */
+ @Test(timeout=60000)
+ public void testErrorLogLevel() throws Exception {
+ testDynamicLogLevel(LogLevel.PROTOCOL_HTTP, LogLevel.PROTOCOL_HTTP, false,
+ "Error");
+ }
+
+ /**
+ * Server runs HTTP, no SPNEGO.
+ *
+ * @throws Exception
+ */
+ @Test(timeout=60000)
+ public void testLogLevelByHttp() throws Exception {
+ testDynamicLogLevel(LogLevel.PROTOCOL_HTTP, LogLevel.PROTOCOL_HTTP, false);
+ try {
+ testDynamicLogLevel(LogLevel.PROTOCOL_HTTP, LogLevel.PROTOCOL_HTTPS,
+ false);
+ fail("A HTTPS Client should not have succeeded in connecting to a " +
+ "HTTP server");
+ } catch (SSLException e) {
+ GenericTestUtils.assertExceptionContains("Unrecognized SSL message", e);
+ }
+ }
+
+ /**
+ * Server runs HTTP + SPNEGO.
+ *
+ * @throws Exception
+ */
+ @Test(timeout=60000)
+ public void testLogLevelByHttpWithSpnego() throws Exception {
+ testDynamicLogLevel(LogLevel.PROTOCOL_HTTP, LogLevel.PROTOCOL_HTTP, true);
+ try {
+ testDynamicLogLevel(LogLevel.PROTOCOL_HTTP, LogLevel.PROTOCOL_HTTPS,
+ true);
+ fail("A HTTPS Client should not have succeeded in connecting to a " +
+ "HTTP server");
+ } catch (SSLException e) {
+ GenericTestUtils.assertExceptionContains("Unrecognized SSL message", e);
+ }
+ }
+
+ /**
+ * Server runs HTTPS, no SPNEGO.
+ *
+ * @throws Exception
+ */
+ @Test(timeout=60000)
+ public void testLogLevelByHttps() throws Exception {
+ testDynamicLogLevel(LogLevel.PROTOCOL_HTTPS, LogLevel.PROTOCOL_HTTPS,
+ false);
+ try {
+ testDynamicLogLevel(LogLevel.PROTOCOL_HTTPS, LogLevel.PROTOCOL_HTTP,
+ false);
+ fail("A HTTP Client should not have succeeded in connecting to a " +
+ "HTTPS server");
+ } catch (SocketException e) {
+ GenericTestUtils.assertExceptionContains(
+ "Unexpected end of file from server", e);
+ }
+ }
+
+ /**
+ * Server runs HTTPS + SPNEGO.
+ *
+ * @throws Exception
+ */
+ @Test(timeout=60000)
+ public void testLogLevelByHttpsWithSpnego() throws Exception {
+ testDynamicLogLevel(LogLevel.PROTOCOL_HTTPS, LogLevel.PROTOCOL_HTTPS,
+ true);
+ try {
+ testDynamicLogLevel(LogLevel.PROTOCOL_HTTPS, LogLevel.PROTOCOL_HTTP,
+ true);
+ fail("A HTTP Client should not have succeeded in connecting to a " +
+ "HTTPS server");
+ } catch (SocketException e) {
+ GenericTestUtils.assertExceptionContains(
+ "Unexpected end of file from server", e);
}
}
-}
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org