You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by tu...@apache.org on 2014/09/06 07:02:04 UTC
[1/2] git commit: HADOOP-11070. Create MiniKMS for testing. (tucu)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 c47d72d8d -> 8bf2a0de6
HADOOP-11070. Create MiniKMS for testing. (tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e98c2447
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e98c2447
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e98c2447
Branch: refs/heads/branch-2
Commit: e98c244730337477c0fe7c19c984ee4581ff567f
Parents: c47d72d
Author: Alejandro Abdelnur <tu...@apache.org>
Authored: Fri Sep 5 14:09:22 2014 -0700
Committer: Alejandro Abdelnur <tu...@apache.org>
Committed: Fri Sep 5 22:01:06 2014 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 2 +
hadoop-common-project/hadoop-kms/pom.xml | 4 +-
.../hadoop/crypto/key/kms/server/MiniKMS.java | 197 +++++++++++++++++++
.../hadoop/crypto/key/kms/server/TestKMS.java | 82 +-------
4 files changed, 211 insertions(+), 74 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e98c2447/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 724cfac..492d41a 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -168,6 +168,8 @@ Release 2.6.0 - UNRELEASED
HADOOP-11060. Create a CryptoCodec test that verifies interoperability
between the JCE and OpenSSL implementations. (hitliuyi via tucu)
+ HADOOP-11070. Create MiniKMS for testing. (tucu)
+
OPTIMIZATIONS
HADOOP-10838. Byte array native checksumming. (James Thomas via todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e98c2447/hadoop-common-project/hadoop-kms/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/pom.xml b/hadoop-common-project/hadoop-kms/pom.xml
index 527454b..481f80e 100644
--- a/hadoop-common-project/hadoop-kms/pom.xml
+++ b/hadoop-common-project/hadoop-kms/pom.xml
@@ -222,9 +222,9 @@
</goals>
<configuration>
<target>
- <mkdir dir="${project.build.directory}/test-classes/webapp"/>
+ <mkdir dir="${project.build.directory}/test-classes/kms-webapp"/>
- <copy todir="${project.build.directory}/test-classes/webapp">
+ <copy todir="${project.build.directory}/test-classes/kms-webapp">
<fileset dir="${basedir}/src/main/webapp"/>
</copy>
</target>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e98c2447/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
new file mode 100644
index 0000000..5a6d4c5
--- /dev/null
+++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/MiniKMS.java
@@ -0,0 +1,197 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.crypto.key.kms.server;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.mortbay.jetty.Connector;
+import org.mortbay.jetty.Server;
+import org.mortbay.jetty.security.SslSocketConnector;
+import org.mortbay.jetty.webapp.WebAppContext;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.Writer;
+import java.net.InetAddress;
+import java.net.MalformedURLException;
+import java.net.ServerSocket;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+
+public class MiniKMS {
+
+ private static Server createJettyServer(String keyStore, String password) {
+ try {
+ boolean ssl = keyStore != null;
+ InetAddress localhost = InetAddress.getByName("localhost");
+ String host = "localhost";
+ ServerSocket ss = new ServerSocket(0, 50, localhost);
+ int port = ss.getLocalPort();
+ ss.close();
+ Server server = new Server(0);
+ if (!ssl) {
+ server.getConnectors()[0].setHost(host);
+ server.getConnectors()[0].setPort(port);
+ } else {
+ SslSocketConnector c = new SslSocketConnector();
+ c.setHost(host);
+ c.setPort(port);
+ c.setNeedClientAuth(false);
+ c.setKeystore(keyStore);
+ c.setKeystoreType("jks");
+ c.setKeyPassword(password);
+ server.setConnectors(new Connector[]{c});
+ }
+ return server;
+ } catch (Exception ex) {
+ throw new RuntimeException("Could not start embedded servlet container, "
+ + ex.getMessage(), ex);
+ }
+ }
+
+ private static URL getJettyURL(Server server) {
+ boolean ssl = server.getConnectors()[0].getClass()
+ == SslSocketConnector.class;
+ try {
+ String scheme = (ssl) ? "https" : "http";
+ return new URL(scheme + "://" +
+ server.getConnectors()[0].getHost() + ":" +
+ server.getConnectors()[0].getPort());
+ } catch (MalformedURLException ex) {
+ throw new RuntimeException("It should never happen, " + ex.getMessage(),
+ ex);
+ }
+ }
+
+ public static class Builder {
+ private File kmsConfDir;
+ private String log4jConfFile;
+ private File keyStoreFile;
+ private String keyStorePassword;
+
+ public Builder() {
+ kmsConfDir = new File("target/test-classes").getAbsoluteFile();
+ log4jConfFile = "kms-log4j.properties";
+ }
+
+ public Builder setKmsConfDir(File confDir) {
+ Preconditions.checkNotNull(confDir, "KMS conf dir is NULL");
+ Preconditions.checkArgument(confDir.exists(),
+ "KMS conf dir does not exist");
+ kmsConfDir = confDir;
+ return this;
+ }
+
+ public Builder setLog4jConfFile(String log4jConfFile) {
+ Preconditions.checkNotNull(log4jConfFile, "log4jconf file is NULL");
+ this.log4jConfFile = log4jConfFile;
+ return this;
+ }
+
+ public Builder setSslConf(File keyStoreFile, String keyStorePassword) {
+ Preconditions.checkNotNull(keyStoreFile, "keystore file is NULL");
+ Preconditions.checkNotNull(keyStorePassword, "keystore password is NULL");
+ Preconditions.checkArgument(keyStoreFile.exists(),
+ "keystore file does not exist");
+ this.keyStoreFile = keyStoreFile;
+ this.keyStorePassword = keyStorePassword;
+ return this;
+ }
+
+ public MiniKMS build() {
+ Preconditions.checkArgument(kmsConfDir.exists(),
+ "KMS conf dir does not exist");
+ return new MiniKMS(kmsConfDir.getAbsolutePath(), log4jConfFile,
+ (keyStoreFile != null) ? keyStoreFile.getAbsolutePath() : null,
+ keyStorePassword);
+ }
+ }
+
+ private String kmsConfDir;
+ private String log4jConfFile;
+ private String keyStore;
+ private String keyStorePassword;
+ private Server jetty;
+ private URL kmsURL;
+
+ public MiniKMS(String kmsConfDir, String log4ConfFile, String keyStore,
+ String password) {
+ this.kmsConfDir = kmsConfDir;
+ this.log4jConfFile = log4ConfFile;
+ this.keyStore = keyStore;
+ this.keyStorePassword = password;
+ }
+
+ public void start() throws Exception {
+ System.setProperty(KMSConfiguration.KMS_CONFIG_DIR, kmsConfDir);
+ File aclsFile = new File(kmsConfDir, "kms-acls.xml");
+ if (!aclsFile.exists()) {
+ Configuration acls = new Configuration(false);
+ Writer writer = new FileWriter(aclsFile);
+ acls.writeXml(writer);
+ writer.close();
+ }
+ File coreFile = new File(kmsConfDir, "core-site.xml");
+ if (!coreFile.exists()) {
+ Configuration core = new Configuration();
+ Writer writer = new FileWriter(coreFile);
+ core.writeXml(writer);
+ writer.close();
+ }
+ File kmsFile = new File(kmsConfDir, "kms-site.xml");
+ if (!kmsFile.exists()) {
+ Configuration kms = new Configuration(false);
+ kms.set("hadoop.security.key.provider.path",
+ "jceks://file@" + kmsConfDir + "/kms.keystore");
+ kms.set("hadoop.kms.authentication.type", "simple");
+ Writer writer = new FileWriter(kmsFile);
+ kms.writeXml(writer);
+ writer.close();
+ }
+ System.setProperty("log4j.configuration", log4jConfFile);
+ jetty = createJettyServer(keyStore, keyStorePassword);
+ ClassLoader cl = Thread.currentThread().getContextClassLoader();
+ URL url = cl.getResource("kms-webapp");
+ if (url == null) {
+ throw new RuntimeException(
+ "Could not find kms-webapp/ dir in test classpath");
+ }
+ WebAppContext context = new WebAppContext(url.getPath(), "/kms");
+ jetty.addHandler(context);
+ jetty.start();
+ kmsURL = new URL(getJettyURL(jetty), "kms");
+ }
+
+ public URL getKMSUrl() {
+ return kmsURL;
+ }
+
+ public void stop() {
+ if (jetty != null && jetty.isRunning()) {
+ try {
+ jetty.stop();
+ jetty = null;
+ } catch (Exception ex) {
+ throw new RuntimeException("Could not stop MiniKMS embedded Jetty, " +
+ ex.getMessage(), ex);
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e98c2447/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
index 52f6354..f381fa0 100644
--- a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
+++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
@@ -36,10 +36,6 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
-import org.mortbay.jetty.Connector;
-import org.mortbay.jetty.Server;
-import org.mortbay.jetty.security.SslSocketConnector;
-import org.mortbay.jetty.webapp.WebAppContext;
import javax.security.auth.Subject;
import javax.security.auth.kerberos.KerberosPrincipal;
@@ -52,7 +48,6 @@ import java.io.IOException;
import java.io.Writer;
import java.net.InetAddress;
import java.net.InetSocketAddress;
-import java.net.MalformedURLException;
import java.net.ServerSocket;
import java.net.SocketTimeoutException;
import java.net.URI;
@@ -91,49 +86,6 @@ public class TestKMS {
return file;
}
- public static Server createJettyServer(String keyStore, String password) {
- try {
- boolean ssl = keyStore != null;
- InetAddress localhost = InetAddress.getByName("localhost");
- String host = "localhost";
- ServerSocket ss = new ServerSocket(0, 50, localhost);
- int port = ss.getLocalPort();
- ss.close();
- Server server = new Server(0);
- if (!ssl) {
- server.getConnectors()[0].setHost(host);
- server.getConnectors()[0].setPort(port);
- } else {
- SslSocketConnector c = new SslSocketConnector();
- c.setHost(host);
- c.setPort(port);
- c.setNeedClientAuth(false);
- c.setKeystore(keyStore);
- c.setKeystoreType("jks");
- c.setKeyPassword(password);
- server.setConnectors(new Connector[]{c});
- }
- return server;
- } catch (Exception ex) {
- throw new RuntimeException("Could not start embedded servlet container, "
- + ex.getMessage(), ex);
- }
- }
-
- public static URL getJettyURL(Server server) {
- boolean ssl = server.getConnectors()[0].getClass()
- == SslSocketConnector.class;
- try {
- String scheme = (ssl) ? "https" : "http";
- return new URL(scheme + "://" +
- server.getConnectors()[0].getHost() + ":" +
- server.getConnectors()[0].getPort());
- } catch (MalformedURLException ex) {
- throw new RuntimeException("It should never happen, " + ex.getMessage(),
- ex);
- }
- }
-
public static abstract class KMSCallable implements Callable<Void> {
private URL kmsUrl;
@@ -144,33 +96,19 @@ public class TestKMS {
protected void runServer(String keystore, String password, File confDir,
KMSCallable callable) throws Exception {
- System.setProperty(KMSConfiguration.KMS_CONFIG_DIR,
- confDir.getAbsolutePath());
- System.setProperty("log4j.configuration", "log4j.properties");
- Server jetty = createJettyServer(keystore, password);
+ MiniKMS.Builder miniKMSBuilder = new MiniKMS.Builder().setKmsConfDir(confDir)
+ .setLog4jConfFile("log4j.properties");
+ if (keystore != null) {
+ miniKMSBuilder.setSslConf(new File(keystore), password);
+ }
+ MiniKMS miniKMS = miniKMSBuilder.build();
+ miniKMS.start();
try {
- ClassLoader cl = Thread.currentThread().getContextClassLoader();
- URL url = cl.getResource("webapp");
- if (url == null) {
- throw new RuntimeException(
- "Could not find webapp/ dir in test classpath");
- }
- WebAppContext context = new WebAppContext(url.getPath(), "/kms");
- jetty.addHandler(context);
- jetty.start();
- url = new URL(getJettyURL(jetty), "kms");
- System.out.println("Test KMS running at: " + url);
- callable.kmsUrl = url;
+ System.out.println("Test KMS running at: " + miniKMS.getKMSUrl());
+ callable.kmsUrl = miniKMS.getKMSUrl();
callable.call();
} finally {
- if (jetty != null && jetty.isRunning()) {
- try {
- jetty.stop();
- } catch (Exception ex) {
- throw new RuntimeException("Could not stop embedded Jetty, " +
- ex.getMessage(), ex);
- }
- }
+ miniKMS.stop();
}
}
[2/2] git commit: HADOOP-11069. KMSClientProvider should use
getAuthenticationMethod() to determine if in proxyuser mode or not. (tucu)
Posted by tu...@apache.org.
HADOOP-11069. KMSClientProvider should use getAuthenticationMethod() to determine if in proxyuser mode or not. (tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8bf2a0de
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8bf2a0de
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8bf2a0de
Branch: refs/heads/branch-2
Commit: 8bf2a0de69547ac50b6e8c36ff7f13b028525641
Parents: e98c244
Author: Alejandro Abdelnur <tu...@apache.org>
Authored: Fri Sep 5 10:04:07 2014 -0700
Committer: Alejandro Abdelnur <tu...@apache.org>
Committed: Fri Sep 5 22:01:13 2014 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../org/apache/hadoop/crypto/key/kms/KMSClientProvider.java | 6 +++---
.../java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java | 6 +++---
3 files changed, 9 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bf2a0de/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 492d41a..c799e20 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -424,6 +424,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11067. warning message 'ssl.client.truststore.location has not
been set' gets printed for hftp command. (Xiaoyu Yao via Arpit Agarwal)
+ HADOOP-11069. KMSClientProvider should use getAuthenticationMethod() to
+ determine if in proxyuser mode or not. (tucu)
+
Release 2.5.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bf2a0de/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
index d459ba8..14593ed 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
@@ -385,9 +385,9 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
// if current UGI is different from UGI at constructor time, behave as
// proxyuser
UserGroupInformation currentUgi = UserGroupInformation.getCurrentUser();
- final String doAsUser =
- (loginUgi.getShortUserName().equals(currentUgi.getShortUserName()))
- ? null : currentUgi.getShortUserName();
+ final String doAsUser = (currentUgi.getAuthenticationMethod() ==
+ UserGroupInformation.AuthenticationMethod.PROXY)
+ ? currentUgi.getShortUserName() : null;
// creating the HTTP connection using the current UGI at constructor time
conn = loginUgi.doAs(new PrivilegedExceptionAction<HttpURLConnection>() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8bf2a0de/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
index f381fa0..b921c84 100644
--- a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
+++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
@@ -1157,7 +1157,7 @@ public class TestKMS {
final URI uri = createKMSUri(getKMSUrl());
// proxyuser client using kerberos credentials
- UserGroupInformation clientUgi = UserGroupInformation.
+ final UserGroupInformation clientUgi = UserGroupInformation.
loginUserFromKeytabAndReturnUGI("client", keytab.getAbsolutePath());
clientUgi.doAs(new PrivilegedExceptionAction<Void>() {
@Override
@@ -1167,7 +1167,7 @@ public class TestKMS {
// authorized proxyuser
UserGroupInformation fooUgi =
- UserGroupInformation.createRemoteUser("foo");
+ UserGroupInformation.createProxyUser("foo", clientUgi);
fooUgi.doAs(new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
@@ -1179,7 +1179,7 @@ public class TestKMS {
// unauthorized proxyuser
UserGroupInformation foo1Ugi =
- UserGroupInformation.createRemoteUser("foo1");
+ UserGroupInformation.createProxyUser("foo1", clientUgi);
foo1Ugi.doAs(new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {