You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by ga...@apache.org on 2015/12/02 04:34:46 UTC
incubator-ranger git commit: RANGER-749 : Ranger KMS to support
multiple KMS instances with keys across multiple clusters
Repository: incubator-ranger
Updated Branches:
refs/heads/ranger-0.5 049723c9e -> 5bebb549a
RANGER-749 : Ranger KMS to support multiple KMS instances with keys across multiple clusters
Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/5bebb549
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/5bebb549
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/5bebb549
Branch: refs/heads/ranger-0.5
Commit: 5bebb549a823e19b3b9db87969e59f780593d283
Parents: 049723c
Author: Gautam Borad <ga...@apache.org>
Authored: Wed Nov 25 13:48:14 2015 +0530
Committer: Gautam Borad <ga...@apache.org>
Committed: Wed Dec 2 09:04:35 2015 +0530
----------------------------------------------------------------------
kms/scripts/exportKeysToJCEKS.sh | 19 +++
kms/scripts/importJCEKSKeys.sh | 2 +-
.../hadoop/crypto/key/Ranger2JKSUtil.java | 134 +++++++++++++++++++
.../hadoop/crypto/key/RangerKeyStore.java | 40 +++++-
src/main/assembly/kms.xml | 1 +
5 files changed, 188 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/5bebb549/kms/scripts/exportKeysToJCEKS.sh
----------------------------------------------------------------------
diff --git a/kms/scripts/exportKeysToJCEKS.sh b/kms/scripts/exportKeysToJCEKS.sh
new file mode 100644
index 0000000..03dbe48
--- /dev/null
+++ b/kms/scripts/exportKeysToJCEKS.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# -------------------------------------------------------------------------------------
+RANGER_KMS_HOME=`dirname $0`
+cp="${RANGER_KMS_HOME}/cred/lib/*:${RANGER_KMS_HOME}/./ews/webapp/WEB-INF/classes/conf/:${RANGER_KMS_HOME}/ews/webapp/config:${RANGER_KMS_HOME}/ews/lib/*:${RANGER_KMS_HOME}/ews/webapp/lib/*:${RANGER_KMS_HOME}/ews/webapp/META-INF"
+java -cp "${cp}" org.apache.hadoop.crypto.key.Ranger2JKSUtil ${1} ${2}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/5bebb549/kms/scripts/importJCEKSKeys.sh
----------------------------------------------------------------------
diff --git a/kms/scripts/importJCEKSKeys.sh b/kms/scripts/importJCEKSKeys.sh
index 9c2f9fb..57a2053 100755
--- a/kms/scripts/importJCEKSKeys.sh
+++ b/kms/scripts/importJCEKSKeys.sh
@@ -15,5 +15,5 @@
# limitations under the License.
# -------------------------------------------------------------------------------------
RANGER_KMS_HOME=`dirname $0`
-cp="${RANGER_KMS_HOME}/ews/webapp/config:${RANGER_KMS_HOME}/ews/lib/*:${RANGER_KMS_HOME}/ews/webapp/lib/*:${RANGER_KMS_HOME}/ews/webapp/META-INF:"
+cp="${RANGER_KMS_HOME}/cred/lib/*:${RANGER_KMS_HOME}/./ews/webapp/WEB-INF/classes/conf/:${RANGER_KMS_HOME}/ews/webapp/config:${RANGER_KMS_HOME}/ews/lib/*:${RANGER_KMS_HOME}/ews/webapp/lib/*:${RANGER_KMS_HOME}/ews/webapp/META-INF"
java -cp "${cp}" org.apache.hadoop.crypto.key.JKS2RangerUtil ${1} ${2}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/5bebb549/kms/src/main/java/org/apache/hadoop/crypto/key/Ranger2JKSUtil.java
----------------------------------------------------------------------
diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/Ranger2JKSUtil.java b/kms/src/main/java/org/apache/hadoop/crypto/key/Ranger2JKSUtil.java
new file mode 100644
index 0000000..0def5e5
--- /dev/null
+++ b/kms/src/main/java/org/apache/hadoop/crypto/key/Ranger2JKSUtil.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.crypto.key;
+
+import java.io.Console;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.charset.Charset;
+import java.security.KeyStore;
+import java.security.KeyStoreException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.ranger.kms.dao.DaoManager;
+
+public class Ranger2JKSUtil {
+
+ private static final String DEFAULT_KEYSTORE_TYPE = "jceks" ;
+ private static final String ENCRYPTION_KEY = "ranger.db.encrypt.key.password" ;
+
+ public static void showUsage() {
+ System.err.println("USAGE: java " + Ranger2JKSUtil.class.getName() + " <KMS_FileName> [KeyStoreType]") ;
+ System.err.println(" If KeyStoreType is not provided, it will be considered as " + DEFAULT_KEYSTORE_TYPE) ;
+ System.err.println(" When execution of this utility, it will prompt for both keystore password and key password.") ;
+ }
+
+
+ public static void main(String[] args) throws IOException {
+ if (args.length == 0) {
+ System.err.println("Invalid number of parameters found.") ;
+ showUsage() ;
+ System.exit(1) ;
+ }
+ else {
+ String keyStoreFileName = args[0] ;
+ File f = new File(keyStoreFileName) ;
+ if (! f.exists()) {
+ f.createNewFile();
+ }
+ String keyStoreType = (args.length == 2 ? args[1] : DEFAULT_KEYSTORE_TYPE) ;
+ try {
+ KeyStore.getInstance(keyStoreType) ;
+ } catch (KeyStoreException e) {
+ System.err.println("ERROR: Unable to get valid keystore for the type [" + keyStoreType + "]") ;
+ showUsage() ;
+ System.exit(1) ;
+ }
+
+ new Ranger2JKSUtil().doExportKeysFromJKS(keyStoreFileName, keyStoreType);
+
+ System.out.println("Keys from Ranger KMS Database has been successfully exported into " + keyStoreFileName);
+
+ System.exit(0) ;
+
+ }
+ }
+
+ private void doExportKeysFromJKS(String keyStoreFileName, String keyStoreType) {
+ try {
+ char[] keyStorePassword = getPasswordFromConsole("Enter Password for the keystore FILE :") ;
+ char[] keyPassword = getPasswordFromConsole("Enter Password for the KEY(s) stored in the keystore:") ;
+ Configuration conf = RangerKeyStoreProvider.getDBKSConf();
+ RangerKMSDB rangerkmsDb = new RangerKMSDB(conf);
+ DaoManager daoManager = rangerkmsDb.getDaoManager();
+ RangerKeyStore dbStore = new RangerKeyStore(daoManager);
+ String password = conf.get(ENCRYPTION_KEY);
+ RangerMasterKey rangerMasterKey = new RangerMasterKey(daoManager);
+ char[] masterKey = rangerMasterKey.getMasterKey(password).toCharArray();
+ OutputStream out = null;
+ try {
+ out = new FileOutputStream(new File(keyStoreFileName));
+ dbStore.engineLoadToKeyStoreFile(out, keyStorePassword, keyPassword, masterKey, keyStoreType);
+ }
+ finally {
+ if (out != null) {
+ try {
+ out.close();
+ } catch (Exception e) {
+ throw new RuntimeException("ERROR: Unable to close file stream for [" + keyStoreFileName + "]", e) ;
+ }
+ }
+ }
+ }
+ catch(Throwable t) {
+ throw new RuntimeException("Unable to export keys to [" + keyStoreFileName + "] due to exception.", t) ;
+ }
+ }
+
+ private char[] getPasswordFromConsole(String prompt) throws IOException {
+ String ret = null ;
+ Console c=System.console();
+ if (c == null) {
+ System.out.print(prompt + " ");
+ InputStream in=System.in;
+ int max=50;
+ byte[] b=new byte[max];
+ int l= in.read(b);
+ l--; //last character is \n
+ if (l>0) {
+ byte[] e=new byte[l];
+ System.arraycopy(b,0, e, 0, l);
+ ret = new String(e, Charset.defaultCharset());
+ }
+ } else {
+ char[] pwd = c.readPassword(prompt + " ") ;
+ if (pwd == null) {
+ ret = null ;
+ }
+ else {
+ ret = new String(pwd);
+ }
+ }
+ if (ret == null) {
+ ret = "" ;
+ }
+ return ret.toCharArray() ;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/5bebb549/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStore.java
----------------------------------------------------------------------
diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStore.java b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStore.java
index ff82f53..850104f 100644
--- a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStore.java
+++ b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStore.java
@@ -42,6 +42,7 @@ import java.security.NoSuchAlgorithmException;
import java.security.UnrecoverableKeyException;
import java.security.cert.Certificate;
import java.security.cert.CertificateException;
+import java.util.ArrayList;
import java.util.Date;
import java.util.Enumeration;
import java.util.Hashtable;
@@ -223,10 +224,10 @@ public class RangerKeyStore extends KeyStoreSpi {
throw new IllegalArgumentException("Ranger Master Key can't be null");
}
- MessageDigest md = getKeyedMessageDigest(password);
+ MessageDigest md = getKeyedMessageDigest(password);
byte digest[] = md.digest();
- for (Enumeration<String> e = deltaEntries.keys(); e.hasMoreElements();) {
+ for (Enumeration<String> e = deltaEntries.keys(); e.hasMoreElements();) {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
DataOutputStream dos = new DataOutputStream(new DigestOutputStream(baos, md));
@@ -282,7 +283,6 @@ public class RangerKeyStore extends KeyStoreSpi {
xxRangerKeyStore = new XXRangerKeyStore();
keyStoreExists = false;
}
-
xxRangerKeyStore = mapToEntityBean(rangerKeyStore, xxRangerKeyStore, 0);
if (keyStoreExists) {
xxRangerKeyStore = rangerKMSDao.update(xxRangerKeyStore);
@@ -483,13 +483,13 @@ public class RangerKeyStore extends KeyStoreSpi {
public void engineLoadKeyStoreFile(InputStream stream, char[] storePass, char[] keyPass, char[] masterKey, String fileFormat)
throws IOException, NoSuchAlgorithmException, CertificateException
{
- synchronized(keyEntries) {
+ synchronized(deltaEntries) {
KeyStore ks;
try {
ks = KeyStore.getInstance(fileFormat);
ks.load(stream, storePass);
- keyEntries.clear();
+ deltaEntries.clear();
for (Enumeration<String> name = ks.aliases(); name.hasMoreElements();){
SecretKeyEntry entry = new SecretKeyEntry();
String alias = (String) name.nextElement();
@@ -532,8 +532,34 @@ public class RangerKeyStore extends KeyStoreSpi {
entry.date = ks.getCreationDate(alias);
entry.version = (alias.split("@").length == 2)?(Integer.parseInt(alias.split("@")[1])):0;
entry.description = k.getFormat()+" - "+ks.getType();
- keyEntries.put(alias, entry);
- }
+ deltaEntries.put(alias, entry);
+ }
+ } catch (Throwable t) {
+ logger.error("Unable to load keystore file ", t);
+ throw new IOException(t) ;
+ }
+ }
+ }
+
+ public void engineLoadToKeyStoreFile(OutputStream stream, char[] storePass, char[] keyPass, char[] masterKey, String fileFormat)
+ throws IOException, NoSuchAlgorithmException, CertificateException
+ {
+ synchronized(keyEntries) {
+ KeyStore ks;
+ try {
+ ks = KeyStore.getInstance(fileFormat);
+ ks.load(null, storePass);
+ String alias = null;
+ engineLoad(null, masterKey);
+ Enumeration<String> e = engineAliases();
+ Key key;
+ while (e.hasMoreElements()) {
+ alias = e.nextElement();
+ key = engineGetKey(alias, masterKey);
+ ks.setKeyEntry(alias, key, keyPass, null);
+ }
+
+ ks.store(stream, storePass);
} catch (Throwable t) {
logger.error("Unable to load keystore file ", t);
throw new IOException(t) ;
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/5bebb549/src/main/assembly/kms.xml
----------------------------------------------------------------------
diff --git a/src/main/assembly/kms.xml b/src/main/assembly/kms.xml
index 52ab5a0..5c9e11b 100755
--- a/src/main/assembly/kms.xml
+++ b/src/main/assembly/kms.xml
@@ -304,6 +304,7 @@
<include>setup.sh</include>
<include>install.properties</include>
<include>importJCEKSKeys.sh</include>
+ <include>exportKeysToJCEKS.sh</include>
</includes>
<fileMode>544</fileMode>
</fileSet>