You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by tu...@apache.org on 2014/08/21 20:58:39 UTC
svn commit: r1619513 - in
/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common: ./
src/main/java/org/apache/hadoop/crypto/key/
src/test/java/org/apache/hadoop/crypto/key/ src/test/resources/
Author: tucu
Date: Thu Aug 21 18:58:39 2014
New Revision: 1619513
URL: http://svn.apache.org/r1619513
Log:
HADOOP-10428. JavaKeyStoreProvider should accept keystore password via configuration falling back to ENV VAR. (tucu)
Conflicts:
hadoop-common-project/hadoop-common/CHANGES.txt
Added:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/resources/javakeystoreprovider.password
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/pom.xml
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java
Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1619513&r1=1619512&r2=1619513&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt Thu Aug 21 18:58:39 2014
@@ -102,6 +102,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-10427. KeyProvider implementations should be thread safe. (tucu)
+ HADOOP-10428. JavaKeyStoreProvider should accept keystore password via
+ configuration falling back to ENV VAR. (tucu)
+
OPTIMIZATIONS
HADOOP-10838. Byte array native checksumming. (James Thomas via todd)
Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/pom.xml?rev=1619513&r1=1619512&r2=1619513&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/pom.xml (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/pom.xml Thu Aug 21 18:58:39 2014
@@ -489,6 +489,7 @@
<exclude>src/test/resources/test.har/_index</exclude>
<exclude>src/test/resources/test.har/_masterindex</exclude>
<exclude>src/test/resources/test.har/part-0</exclude>
+ <exclude>src/test/resources/javakeystoreprovider.password</exclude>
</excludes>
</configuration>
</plugin>
Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java?rev=1619513&r1=1619512&r2=1619513&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java Thu Aug 21 18:58:39 2014
@@ -18,6 +18,7 @@
package org.apache.hadoop.crypto.key;
+import org.apache.commons.io.IOUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -27,10 +28,12 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import javax.crypto.spec.SecretKeySpec;
import java.io.IOException;
+import java.io.InputStream;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.Serializable;
import java.net.URI;
+import java.net.URL;
import java.security.Key;
import java.security.KeyStore;
import java.security.KeyStoreException;
@@ -52,10 +55,21 @@ import java.util.concurrent.locks.Reentr
* any Hadoop FileSystem using the following name mangling:
* jks://hdfs@nn1.example.com/my/keys.jks -> hdfs://nn1.example.com/my/keys.jks
* jks://file/home/owen/keys.jks -> file:///home/owen/keys.jks
- *
- * The password for the keystore is taken from the HADOOP_KEYSTORE_PASSWORD
- * environment variable with a default of 'none'.
- *
+ * <p/>
+ * If the <code>HADOOP_KEYSTORE_PASSWORD</code> environment variable is set,
+ * its value is used as the password for the keystore.
+ * <p/>
+ * If the <code>HADOOP_KEYSTORE_PASSWORD</code> environment variable is not set,
+ * the password for the keystore is read from file specified in the
+ * {@link #KEYSTORE_PASSWORD_FILE_KEY} configuration property. The password file
+ * is looked up in Hadoop's configuration directory via the classpath.
+ * <p/>
+ * <b>NOTE:</b> Make sure the password in the password file does not have an
+ * ENTER at the end, else it won't be valid for the Java KeyStore.
+ * <p/>
+ * If the environment variable, nor the property are not set, the password used
+ * is 'none'.
+ * <p/>
* It is expected for encrypted InputFormats and OutputFormats to copy the keys
* from the original provider into the job's Credentials object, which is
* accessed via the UserProvider. Therefore, this provider won't be used by
@@ -65,16 +79,20 @@ import java.util.concurrent.locks.Reentr
public class JavaKeyStoreProvider extends KeyProvider {
private static final String KEY_METADATA = "KeyMetadata";
public static final String SCHEME_NAME = "jceks";
- public static final String KEYSTORE_PASSWORD_NAME =
+
+ public static final String KEYSTORE_PASSWORD_FILE_KEY =
+ "hadoop.security.keystore.java-keystore-provider.password-file";
+
+ public static final String KEYSTORE_PASSWORD_ENV_VAR =
"HADOOP_KEYSTORE_PASSWORD";
- public static final String KEYSTORE_PASSWORD_DEFAULT = "none";
+ public static final char[] KEYSTORE_PASSWORD_DEFAULT = "none".toCharArray();
private final URI uri;
private final Path path;
private final FileSystem fs;
private final FsPermission permissions;
private final KeyStore keyStore;
- private final char[] password;
+ private char[] password;
private boolean changed = false;
private Lock readLock;
private Lock writeLock;
@@ -85,12 +103,29 @@ public class JavaKeyStoreProvider extend
this.uri = uri;
path = unnestUri(uri);
fs = path.getFileSystem(conf);
- // Get the password from the user's environment
- String pw = System.getenv(KEYSTORE_PASSWORD_NAME);
- if (pw == null) {
- pw = KEYSTORE_PASSWORD_DEFAULT;
+ // Get the password file from the conf, if not present from the user's
+ // environment var
+ if (System.getenv().containsKey(KEYSTORE_PASSWORD_ENV_VAR)) {
+ password = System.getenv(KEYSTORE_PASSWORD_ENV_VAR).toCharArray();
+ }
+ if (password == null) {
+ String pwFile = conf.get(KEYSTORE_PASSWORD_FILE_KEY);
+ if (pwFile != null) {
+ ClassLoader cl = Thread.currentThread().getContextClassLoader();
+ URL pwdFile = cl.getResource(pwFile);
+ if (pwdFile != null) {
+ InputStream is = pwdFile.openStream();
+ try {
+ password = IOUtils.toCharArray(is);
+ } finally {
+ is.close();
+ }
+ }
+ }
+ }
+ if (password == null) {
+ password = KEYSTORE_PASSWORD_DEFAULT;
}
- password = pw.toCharArray();
try {
keyStore = KeyStore.getInstance(SCHEME_NAME);
if (fs.exists(path)) {
Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java?rev=1619513&r1=1619512&r2=1619513&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java Thu Aug 21 18:58:39 2014
@@ -30,6 +30,7 @@ import org.apache.hadoop.fs.permission.F
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
+import org.junit.Assert;
import org.junit.Test;
import static org.junit.Assert.assertArrayEquals;
@@ -237,4 +238,48 @@ public class TestKeyProviderFactory {
FileStatus s = fs.getFileStatus(path);
assertTrue("Permissions should have been retained from the preexisting keystore.", s.getPermission().toString().equals("rwxrwxrwx"));
}
+
+ @Test
+ public void testJksProviderPasswordViaConfig() throws Exception {
+ Configuration conf = new Configuration();
+ final String ourUrl =
+ JavaKeyStoreProvider.SCHEME_NAME + "://file" + tmpDir + "/test.jks";
+ File file = new File(tmpDir, "test.jks");
+ file.delete();
+ try {
+ conf.set(KeyProviderFactory.KEY_PROVIDER_PATH, ourUrl);
+ conf.set(JavaKeyStoreProvider.KEYSTORE_PASSWORD_FILE_KEY,
+ "javakeystoreprovider.password");
+ KeyProvider provider = KeyProviderFactory.getProviders(conf).get(0);
+ provider.createKey("key3", new byte[32], KeyProvider.options(conf));
+ provider.flush();
+ } catch (Exception ex) {
+ Assert.fail("could not create keystore with password file");
+ }
+ KeyProvider provider = KeyProviderFactory.getProviders(conf).get(0);
+ Assert.assertNotNull(provider.getCurrentKey("key3"));
+
+ try {
+ conf.set(JavaKeyStoreProvider.KEYSTORE_PASSWORD_FILE_KEY, "bar");
+ KeyProviderFactory.getProviders(conf).get(0);
+ Assert.fail("using non existing password file, it should fail");
+ } catch (IOException ex) {
+ //NOP
+ }
+ try {
+ conf.set(JavaKeyStoreProvider.KEYSTORE_PASSWORD_FILE_KEY, "core-site.xml");
+ KeyProviderFactory.getProviders(conf).get(0);
+ Assert.fail("using different password file, it should fail");
+ } catch (IOException ex) {
+ //NOP
+ }
+ try {
+ conf.unset(JavaKeyStoreProvider.KEYSTORE_PASSWORD_FILE_KEY);
+ KeyProviderFactory.getProviders(conf).get(0);
+ Assert.fail("No password file property, env not set, it should fail");
+ } catch (IOException ex) {
+ //NOP
+ }
+ }
+
}
Added: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/resources/javakeystoreprovider.password
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/resources/javakeystoreprovider.password?rev=1619513&view=auto
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/resources/javakeystoreprovider.password (added)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/resources/javakeystoreprovider.password Thu Aug 21 18:58:39 2014
@@ -0,0 +1 @@
+foo
\ No newline at end of file