You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by ga...@apache.org on 2016/04/14 06:58:15 UTC
incubator-ranger git commit: RANGER-868: Add Support for HSM to Ranger
Repository: incubator-ranger
Updated Branches:
refs/heads/master e85eb01d5 -> 73dc5031e
RANGER-868: Add Support for HSM to Ranger
Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/73dc5031
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/73dc5031
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/73dc5031
Branch: refs/heads/master
Commit: 73dc5031e3cfd8b9671b41e0f619b854e7aafd23
Parents: e85eb01
Author: Ankita Sinha <an...@freestoneinfotech.com>
Authored: Wed Apr 13 19:38:45 2016 +0530
Committer: Gautam Borad <ga...@apache.org>
Committed: Thu Apr 14 10:27:52 2016 +0530
----------------------------------------------------------------------
kms/config/kms-webapp/dbks-site.xml | 31 +++++
kms/scripts/DBMK2HSM.sh | 19 +++
kms/scripts/HSMMK2DB.sh | 19 +++
kms/scripts/install.properties | 6 +
kms/scripts/setup.sh | 64 ++++++++-
.../apache/hadoop/crypto/key/DB2HSMMKUtil.java | 129 ++++++++++++++++++
.../apache/hadoop/crypto/key/HSM2DBMKUtil.java | 127 ++++++++++++++++++
.../org/apache/hadoop/crypto/key/RangerHSM.java | 131 +++++++++++++++++++
.../apache/hadoop/crypto/key/RangerKMSMKI.java | 25 ++++
.../crypto/key/RangerKeyStoreProvider.java | 18 ++-
.../hadoop/crypto/key/RangerMasterKey.java | 40 +++++-
src/main/assembly/kms.xml | 2 +
12 files changed, 605 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/73dc5031/kms/config/kms-webapp/dbks-site.xml
----------------------------------------------------------------------
diff --git a/kms/config/kms-webapp/dbks-site.xml b/kms/config/kms-webapp/dbks-site.xml
index edaff93..f649264 100755
--- a/kms/config/kms-webapp/dbks-site.xml
+++ b/kms/config/kms-webapp/dbks-site.xml
@@ -113,4 +113,35 @@
</description>
</property>
+ <!-- HSM Config -->
+ <property>
+ <name>ranger.ks.hsm.type</name>
+ <value>LunaProvider</value>
+ <description></description>
+ </property>
+
+ <property>
+ <name>ranger.ks.hsm.enabled</name>
+ <value>false</value>
+ <description></description>
+ </property>
+
+ <property>
+ <name>ranger.ks.hsm.partition.name</name>
+ <value>par19</value>
+ <description></description>
+ </property>
+
+ <property>
+ <name>ranger.ks.hsm.partition.password</name>
+ <value>S@fenet123</value>
+ <description></description>
+ </property>
+
+ <property>
+ <name>ranger.ks.hsm.partition.password.alias</name>
+ <value>ranger.kms.hsm.partition.password</value>
+ <description></description>
+ </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/73dc5031/kms/scripts/DBMK2HSM.sh
----------------------------------------------------------------------
diff --git a/kms/scripts/DBMK2HSM.sh b/kms/scripts/DBMK2HSM.sh
new file mode 100644
index 0000000..e5beec3
--- /dev/null
+++ b/kms/scripts/DBMK2HSM.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# -------------------------------------------------------------------------------------
+RANGER_KMS_HOME=`dirname $0`
+cp="${RANGER_KMS_HOME}/cred/lib/*:${RANGER_KMS_HOME}/./ews/webapp/WEB-INF/classes/conf/:${RANGER_KMS_HOME}/ews/webapp/config:${RANGER_KMS_HOME}/ews/lib/*:${RANGER_KMS_HOME}/ews/webapp/lib/*:${RANGER_KMS_HOME}/ews/webapp/META-INF"
+java -cp "${cp}" org.apache.hadoop.crypto.key.DB2HSMMKUtil ${1} ${2}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/73dc5031/kms/scripts/HSMMK2DB.sh
----------------------------------------------------------------------
diff --git a/kms/scripts/HSMMK2DB.sh b/kms/scripts/HSMMK2DB.sh
new file mode 100644
index 0000000..03e9888
--- /dev/null
+++ b/kms/scripts/HSMMK2DB.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# -------------------------------------------------------------------------------------
+RANGER_KMS_HOME=`dirname $0`
+cp="${RANGER_KMS_HOME}/cred/lib/*:${RANGER_KMS_HOME}/./ews/webapp/WEB-INF/classes/conf/:${RANGER_KMS_HOME}/ews/webapp/config:${RANGER_KMS_HOME}/ews/lib/*:${RANGER_KMS_HOME}/ews/webapp/lib/*:${RANGER_KMS_HOME}/ews/webapp/META-INF"
+java -cp "${cp}" org.apache.hadoop.crypto.key.HSM2DBMKUtil ${1} ${2}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/73dc5031/kms/scripts/install.properties
----------------------------------------------------------------------
diff --git a/kms/scripts/install.properties b/kms/scripts/install.properties
index d30b28c..7762948 100755
--- a/kms/scripts/install.properties
+++ b/kms/scripts/install.properties
@@ -65,6 +65,12 @@ db_password=
#------------------------- RANGER KMS Master Key Crypt Key ------------------
KMS_MASTER_KEY_PASSWD=Str0ngPassw0rd
+#------------------------- Ranger KMS HSM CONFIG ------------------------------
+HSM_TYPE=LunaProvider
+HSM_ENABLED=false
+HSM_PARTITION_NAME=par19
+HSM_PARTITION_PASSWORD=S@fenet123
+
#
# ------- UNIX User CONFIG ----------------
#
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/73dc5031/kms/scripts/setup.sh
----------------------------------------------------------------------
diff --git a/kms/scripts/setup.sh b/kms/scripts/setup.sh
index 64abcc7..6019526 100755
--- a/kms/scripts/setup.sh
+++ b/kms/scripts/setup.sh
@@ -84,6 +84,10 @@ sqlanywhere_core_file=$(get_prop 'sqlanywhere_core_file' $PROPFILE)
cred_keystore_filename=$(eval echo "$(get_prop 'cred_keystore_filename' $PROPFILE)")
KMS_BLACKLIST_DECRYPT_EEK=$(get_prop 'KMS_BLACKLIST_DECRYPT_EEK' $PROPFILE)
RANGER_KMS_LOG_DIR=$(eval echo "$(get_prop 'RANGER_KMS_LOG_DIR' $PROPFILE)")
+HSM_TYPE=$(get_prop 'HSM_TYPE' $PROPFILE)
+HSM_ENABLED=$(get_prop 'HSM_ENABLED' $PROPFILE)
+HSM_PARTITION_NAME=$(get_prop 'HSM_PARTITION_NAME' $PROPFILE)
+HSM_PARTITION_PASSWORD=$(get_prop 'HSM_PARTITION_PASSWORD' $PROPFILE)
DB_HOST="${db_host}"
@@ -159,7 +163,7 @@ updatePropertyToFile(){
#$1 -> propertyName $2 -> newPropertyValue $3 -> fileName
updatePropertyToFilePy(){
python update_property.py $1 $2 $3
- check_ret_status $? "Update property failed for: " $1
+ check_ret_status $? "Update property failed for: {'$1'}"
}
@@ -207,7 +211,12 @@ init_variables(){
DB_FLAVOR="MYSQL"
fi
log "[I] DB_FLAVOR=${DB_FLAVOR}"
- password_validation "$KMS_MASTER_KEY_PASSWD" "KMS Master key"
+ ########## HSM Config ##########
+
+ propertyName=ranger.ks.hsm.enabled
+ HSM_ENABLED=`echo $HSM_ENABLED | tr '[:lower:]' '[:upper:]'`
+ password_validation "$KMS_MASTER_KEY_PASSWD" "KMS Master key"
+
#getPropertyFromFile 'db_root_user' $PROPFILE db_root_user
#getPropertyFromFile 'db_root_password' $PROPFILE db_user
#getPropertyFromFile 'db_user' $PROPFILE db_user
@@ -494,6 +503,11 @@ update_properties() {
MK_CREDENTIAL_ALIAS="ranger.ks.masterkey.password"
DB_CREDENTIAL_ALIAS="ranger.ks.jpa.jdbc.credential.alias"
+ HSM_PARTITION_PASSWD="ranger.ks.hsm.partition.password"
+ HSM_PARTITION_PASSWORD_ALIAS="ranger.kms.hsm.partition.password"
+
+ HSM_ENABLED=`echo $HSM_ENABLED | tr '[:lower:]' '[:upper:]'`
+
if [ "${keystore}" != "" ]
then
mkdir -p `dirname "${keystore}"`
@@ -503,6 +517,21 @@ update_properties() {
#$JAVA_HOME/bin/java -cp "cred/lib/*" org.apache.ranger.credentialapi.buildks create "${DB_CREDENTIAL_ALIAS}" -value "$db_password" -provider jceks://file$keystore
#$JAVA_HOME/bin/java -cp "cred/lib/*" org.apache.ranger.credentialapi.buildks create "${MK_CREDENTIAL_ALIAS}" -value "${KMS_MASTER_KEY_PASSWD}" -provider jceks://file$keystore
+ if [ "${HSM_ENABLED}" == "TRUE" ]
+ then
+ password_validation "$HSM_PARTITION_PASSWORD" "HSM Partition Password"
+
+ $PYTHON_COMMAND_INVOKER ranger_credential_helper.py -l "cred/lib/*" -f "$keystore" -k "${HSM_PARTITION_PASSWORD_ALIAS}" -v "${HSM_PARTITION_PASSWORD}" -c 1
+
+ propertyName=ranger.ks.hsm.partition.password.alias
+ newPropertyValue="${HSM_PARTITION_PASSWORD_ALIAS}"
+ updatePropertyToFilePy $propertyName $newPropertyValue $to_file
+
+ propertyName=ranger.ks.hsm.partition.password
+ newPropertyValue="_"
+ updatePropertyToFilePy $propertyName $newPropertyValue $to_file
+ fi
+
propertyName=ranger.ks.jpa.jdbc.credential.alias
newPropertyValue="${DB_CREDENTIAL_ALIAS}"
updatePropertyToFilePy $propertyName $newPropertyValue $to_file
@@ -530,6 +559,10 @@ update_properties() {
propertyName="${MK_CREDENTIAL_ATTR}"
newPropertyValue="${KMS_MASTER_KEY_PASSWD}"
updatePropertyToFilePy $propertyName $newPropertyValue $to_file
+
+ propertyName="${HSM_PARTITION_PASSWD}"
+ newPropertyValue="${HSM_PARTITION_PASSWORD}"
+ updatePropertyToFilePy $propertyName $newPropertyValue $to_file
fi
if test -f $keystore; then
@@ -546,13 +579,38 @@ update_properties() {
propertyName="${MK_CREDENTIAL_ATTR}"
newPropertyValue="${KMS_MASTER_KEY_PASSWD}"
updatePropertyToFilePy $propertyName $newPropertyValue $to_file
+
+ propertyName="${HSM_PARTITION_PASSWD}"
+ newPropertyValue="${HSM_PARTITION_PASSWORD}"
+ updatePropertyToFilePy $propertyName $newPropertyValue $to_file
fi
propertyName=hadoop.kms.blacklist.DECRYPT_EEK
newPropertyValue="${KMS_BLACKLIST_DECRYPT_EEK}"
updatePropertyToFilePy $propertyName $newPropertyValue $to_file
- ###########
+ ########### HSM CONFIG #################
+
+
+ if [ "${HSM_ENABLED}" != "TRUE" ]
+ then
+ propertyName=ranger.ks.hsm.enabled
+ newPropertyValue="false"
+ updatePropertyToFilePy $propertyName $newPropertyValue $to_file
+ else
+ propertyName=ranger.ks.hsm.enabled
+ newPropertyValue="true"
+ updatePropertyToFilePy $propertyName $newPropertyValue $to_file
+
+ propertyName=ranger.ks.hsm.type
+ newPropertyValue="${HSM_TYPE}"
+ updatePropertyToFilePy $propertyName $newPropertyValue $to_file
+
+ propertyName=ranger.ks.hsm.partition.name
+ newPropertyValue="${HSM_PARTITION_NAME}"
+ updatePropertyToFilePy $propertyName $newPropertyValue $to_file
+ fi
+
}
#=====================================================================
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/73dc5031/kms/src/main/java/org/apache/hadoop/crypto/key/DB2HSMMKUtil.java
----------------------------------------------------------------------
diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/DB2HSMMKUtil.java b/kms/src/main/java/org/apache/hadoop/crypto/key/DB2HSMMKUtil.java
new file mode 100644
index 0000000..ca69dc0
--- /dev/null
+++ b/kms/src/main/java/org/apache/hadoop/crypto/key/DB2HSMMKUtil.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.crypto.key;
+
+import java.io.Console;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.Charset;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.ranger.kms.dao.DaoManager;
+
+import com.sun.org.apache.xml.internal.security.utils.Base64;
+
+public class DB2HSMMKUtil {
+
+ private static final String ENCRYPTION_KEY = "ranger.db.encrypt.key.password" ;
+ private static final String PARTITION_PASSWORD = "ranger.ks.hsm.partition.password";
+ private static final String PARTITION_NAME = "ranger.ks.hsm.partition.name";
+ private static final String HSM_TYPE = "ranger.ks.hsm.type";
+
+ public static void showUsage() {
+ System.err.println("USAGE: java " + DB2HSMMKUtil.class.getName() + " <HSMType> <partitionName>") ;
+ }
+
+ public static void main(String[] args) {
+ if (args.length < 2) {
+ System.err.println("Invalid number of parameters found.") ;
+ showUsage() ;
+ System.exit(1) ;
+ }
+ else {
+ String hsmType = args[0];
+ if (hsmType == null || hsmType.trim().isEmpty()) {
+ System.err.println("HSM Type does not exists.") ;
+ showUsage() ;
+ System.exit(1);
+ }
+
+ String partitionName = args[1];
+ if (partitionName == null || partitionName.trim().isEmpty()) {
+ System.err.println("Partition name does not exists.") ;
+ showUsage() ;
+ System.exit(1) ;
+ }
+
+ boolean result = new DB2HSMMKUtil().doExportMKToHSM(hsmType, partitionName);
+ if(result){
+ System.out.println("Master Key from Ranger KMS DB has been successfully imported into HSM.") ;
+ }else{
+ System.out.println("Import of Master Key from DB has been unsuccessful.") ;
+ }
+ System.exit(0) ;
+
+ }
+ }
+
+ private boolean doExportMKToHSM(String hsmType, String partitionName) {
+ try {
+ String partitionPassword = getPasswordFromConsole("Enter Password for the Partition "+partitionName+" : ") ;
+ Configuration conf = RangerKeyStoreProvider.getDBKSConf();
+ conf.set(HSM_TYPE, hsmType);
+ conf.set(PARTITION_NAME, partitionName);
+ conf.set(PARTITION_PASSWORD, partitionPassword);
+
+ RangerKMSDB rangerkmsDb = new RangerKMSDB(conf);
+ DaoManager daoManager = rangerkmsDb.getDaoManager();
+ RangerKeyStore dbStore = new RangerKeyStore(daoManager);
+ String password = conf.get(ENCRYPTION_KEY);
+
+ // Get Master Key from Ranger DB
+ RangerMasterKey rangerMasterKey = new RangerMasterKey(daoManager);
+ String mkey = rangerMasterKey.getMasterKey(password);
+ byte[] key = Base64.decode(mkey);
+
+ // Put Master Key in HSM
+ RangerHSM rangerHSM = new RangerHSM(conf);
+
+ return rangerHSM.setMasterKey(password, key);
+ }
+ catch(Throwable t) {
+ throw new RuntimeException("Unable to import Master key from Ranger DB to HSM ", t) ;
+ }
+ }
+
+ private String getPasswordFromConsole(String prompt) throws IOException {
+ String ret = null ;
+ Console c=System.console();
+ if (c == null) {
+ System.out.print(prompt + " ");
+ InputStream in=System.in;
+ int max=50;
+ byte[] b=new byte[max];
+ int l= in.read(b);
+ l--; //last character is \n
+ if (l>0) {
+ byte[] e=new byte[l];
+ System.arraycopy(b,0, e, 0, l);
+ ret = new String(e, Charset.defaultCharset());
+ }
+ } else {
+ char[] pwd = c.readPassword(prompt + " ") ;
+ if (pwd == null) {
+ ret = null ;
+ }
+ else {
+ ret = new String(pwd);
+ }
+ }
+ if (ret == null) {
+ ret = "" ;
+ }
+ return ret;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/73dc5031/kms/src/main/java/org/apache/hadoop/crypto/key/HSM2DBMKUtil.java
----------------------------------------------------------------------
diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/HSM2DBMKUtil.java b/kms/src/main/java/org/apache/hadoop/crypto/key/HSM2DBMKUtil.java
new file mode 100644
index 0000000..73a5830
--- /dev/null
+++ b/kms/src/main/java/org/apache/hadoop/crypto/key/HSM2DBMKUtil.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.crypto.key;
+
+import java.io.Console;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.Charset;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.ranger.kms.dao.DaoManager;
+
+import com.sun.org.apache.xml.internal.security.utils.Base64;
+
+public class HSM2DBMKUtil {
+
+ private static final String ENCRYPTION_KEY = "ranger.db.encrypt.key.password" ;
+ private static final String PARTITION_PASSWORD = "ranger.ks.hsm.partition.password";
+ private static final String PARTITION_NAME = "ranger.ks.hsm.partition.name";
+ private static final String HSM_TYPE = "ranger.ks.hsm.type";
+
+ public static void showUsage() {
+ System.err.println("USAGE: java " + HSM2DBMKUtil.class.getName() + " <HSMType> <partitionName>") ;
+ }
+
+
+ public static void main(String[] args) {
+ if (args.length < 2) {
+ System.err.println("Invalid number of parameters found.") ;
+ showUsage() ;
+ System.exit(1) ;
+ }
+ else {
+ String hsmType = args[0];
+ if (hsmType == null || hsmType.trim().isEmpty()) {
+ System.err.println("HSM Type does not exists.") ;
+ showUsage() ;
+ System.exit(1);
+ }
+
+ String partitionName = args[1];
+ if (partitionName == null || partitionName.trim().isEmpty()) {
+ System.err.println("Partition name does not exists.") ;
+ showUsage() ;
+ System.exit(1) ;
+ }
+
+ new HSM2DBMKUtil().doImportMKFromHSM(hsmType, partitionName);
+
+ System.out.println("Master Key from HSM has been successfully imported into Ranger KMS DB.") ;
+
+ System.exit(0) ;
+
+ }
+ }
+
+ private void doImportMKFromHSM(String hsmType, String partitionName) {
+ try {
+ String partitionPassword = getPasswordFromConsole("Enter Password for the Partition "+partitionName+" : ") ;
+ Configuration conf = RangerKeyStoreProvider.getDBKSConf();
+ conf.set(HSM_TYPE, hsmType);
+ conf.set(PARTITION_NAME, partitionName);
+ conf.set(PARTITION_PASSWORD, partitionPassword);
+
+ RangerKMSDB rangerkmsDb = new RangerKMSDB(conf);
+ DaoManager daoManager = rangerkmsDb.getDaoManager();
+ RangerKeyStore dbStore = new RangerKeyStore(daoManager);
+ String password = conf.get(ENCRYPTION_KEY);
+
+ // Get Master Key from HSM
+ RangerHSM rangerHSM = new RangerHSM(conf);
+ String mKey = rangerHSM.getMasterKey(password);
+ byte[] key = Base64.decode(mKey);
+
+ // Put Master Key in Ranger DB
+ RangerMasterKey rangerMasterKey = new RangerMasterKey(daoManager);
+ rangerMasterKey.generateMKFromHSMMK(password, key);
+ }
+ catch(Throwable t) {
+ throw new RuntimeException("Unable to import Master key from HSM to Ranger DB", t) ;
+ }
+ }
+
+ private String getPasswordFromConsole(String prompt) throws IOException {
+ String ret = null ;
+ Console c=System.console();
+ if (c == null) {
+ System.out.print(prompt + " ");
+ InputStream in=System.in;
+ int max=50;
+ byte[] b=new byte[max];
+ int l= in.read(b);
+ l--; //last character is \n
+ if (l>0) {
+ byte[] e=new byte[l];
+ System.arraycopy(b,0, e, 0, l);
+ ret = new String(e, Charset.defaultCharset());
+ }
+ } else {
+ char[] pwd = c.readPassword(prompt + " ") ;
+ if (pwd == null) {
+ ret = null ;
+ }
+ else {
+ ret = new String(pwd);
+ }
+ }
+ if (ret == null) {
+ ret = "" ;
+ }
+ return ret;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/73dc5031/kms/src/main/java/org/apache/hadoop/crypto/key/RangerHSM.java
----------------------------------------------------------------------
diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerHSM.java b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerHSM.java
new file mode 100644
index 0000000..6ab91d9
--- /dev/null
+++ b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerHSM.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.crypto.key;
+
+import javax.crypto.KeyGenerator;
+import javax.crypto.SecretKey;
+import javax.crypto.spec.SecretKeySpec;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.log4j.Logger;
+
+import com.sun.org.apache.xml.internal.security.utils.Base64;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.security.Key;
+import java.security.KeyStore;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.security.cert.CertificateException;
+
+/**
+ * This Class is for HSM Keystore
+ */
+public class RangerHSM implements RangerKMSMKI {
+
+ static final Logger logger = Logger.getLogger(RangerHSM.class);
+
+ // Configure these as required.
+ private String passwd = null;
+ private String alias = "RangerKMSKey";
+ private String partitionName = null;
+ private KeyStore myStore = null;
+ private String hsm_keystore=null;
+ private static final String MK_CIPHER = "AES";
+ private static final int MK_KeySize = 128;
+ private static final String PARTITION_PASSWORD = "ranger.ks.hsm.partition.password";
+ private static final String PARTITION_NAME = "ranger.ks.hsm.partition.name";
+ private static final String HSM_TYPE = "ranger.ks.hsm.type";
+
+ public RangerHSM(){
+ }
+
+ public RangerHSM(Configuration conf) {
+ logger.info("RangerHSM provider");
+ /*
+ * We will log in to the HSM
+ */
+ passwd = conf.get(PARTITION_PASSWORD);
+ partitionName = conf.get(PARTITION_NAME);
+ hsm_keystore = conf.get(HSM_TYPE);
+ try {
+ ByteArrayInputStream is1 = new ByteArrayInputStream(("tokenlabel:" + partitionName).getBytes());
+ logger.debug("Loading HSM tokenlabel : "+partitionName);
+ myStore = KeyStore.getInstance("Luna");
+ myStore.load(is1, passwd.toCharArray());
+ } catch (KeyStoreException kse) {
+ logger.error("Unable to create keystore object : "+kse.getMessage());
+ } catch (NoSuchAlgorithmException nsae) {
+ logger.error("Unexpected NoSuchAlgorithmException while loading keystore : " + nsae.getMessage());
+ } catch (CertificateException e) {
+ logger.error("Unexpected CertificateException while loading keystore : "+e.getMessage());
+ } catch (IOException e) {
+ logger.error("Unexpected IOException while loading keystore : "+e.getMessage());
+ }
+ }
+
+ @Override
+ public boolean generateMasterKey(String password) throws Throwable {
+ if(myStore.size() < 1){
+ KeyGenerator keyGen = null;
+ SecretKey aesKey = null;
+ try {
+ logger.info("Generating AES Master Key for HSM Provider");
+ keyGen = KeyGenerator.getInstance(MK_CIPHER, hsm_keystore);
+ keyGen.init(MK_KeySize);
+ aesKey = keyGen.generateKey();
+ String masterKey = Base64.encode(aesKey.getEncoded()) ;
+ myStore.setKeyEntry(alias, aesKey, password.toCharArray(), (java.security.cert.Certificate[]) null);
+ return true;
+ } catch (Exception e) {
+ logger.error("generateMasterKey : Exception during Ranger Master Key Generation - " + e.getMessage());
+ return false;
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public String getMasterKey(String password) throws Throwable {
+ try {
+ logger.debug("Searching for Ranger Master Key in Luna Keystore");
+ boolean result = myStore.containsAlias(alias);
+ if (result == true) {
+ logger.debug("Ranger Master Key is present in Keystore");
+ SecretKey key = (SecretKey)myStore.getKey(alias, password.toCharArray());
+ String masterKey = Base64.encode(key.getEncoded()) ;
+ return masterKey;
+ }
+ } catch (Exception e) {
+ logger.error("getMasterKey : Exception searching for Ranger Master Key - " + e.getMessage());
+ }
+ return null;
+ }
+
+ public boolean setMasterKey(String password, byte[] key){
+ try {
+ Key aesKey = new SecretKeySpec(key, MK_CIPHER);
+ myStore.setKeyEntry(alias, aesKey, password.toCharArray(), (java.security.cert.Certificate[]) null);
+ return true;
+ } catch (KeyStoreException e) {
+ logger.error("setMasterKey : Exception while setting Master Key - " + e.getMessage());
+ }
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/73dc5031/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKMSMKI.java
----------------------------------------------------------------------
diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKMSMKI.java b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKMSMKI.java
new file mode 100644
index 0000000..75e70ff
--- /dev/null
+++ b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKMSMKI.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.crypto.key;
+
+public interface RangerKMSMKI {
+
+ boolean generateMasterKey(String password) throws Throwable;
+
+ String getMasterKey(String password) throws Throwable;
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/73dc5031/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStoreProvider.java
----------------------------------------------------------------------
diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStoreProvider.java b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStoreProvider.java
index a9e43fc..12c9ed4 100755
--- a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStoreProvider.java
+++ b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStoreProvider.java
@@ -64,6 +64,9 @@ public class RangerKeyStoreProvider extends KeyProvider{
private static final String MK_CREDENTIAL_ALIAS = "ranger.ks.masterkey.credential.alias";
private static final String DB_CREDENTIAL_ALIAS = "ranger.ks.jpa.jdbc.credential.alias";
private static final String DB_PASSWORD = "ranger.ks.jpa.jdbc.password";
+ private static final String HSM_ENABLED = "ranger.ks.hsm.enabled";
+ private static final String HSM_PARTITION_PASSWORD_ALIAS = "ranger.ks.hsm.partition.password.alias";
+ private static final String HSM_PARTITION_PASSWORD = "ranger.ks.hsm.partition.password";
private final RangerKeyStore dbStore;
private char[] masterKey;
@@ -78,14 +81,25 @@ public class RangerKeyStoreProvider extends KeyProvider{
conf = getDBKSConf();
getFromJceks(conf,CREDENTIAL_PATH, MK_CREDENTIAL_ALIAS, ENCRYPTION_KEY);
getFromJceks(conf,CREDENTIAL_PATH, DB_CREDENTIAL_ALIAS, DB_PASSWORD);
+ getFromJceks(conf,CREDENTIAL_PATH, HSM_PARTITION_PASSWORD_ALIAS, HSM_PARTITION_PASSWORD);
RangerKMSDB rangerKMSDB = new RangerKMSDB(conf);
daoManager = rangerKMSDB.getDaoManager();
- RangerMasterKey rangerMasterKey = new RangerMasterKey(daoManager);
- dbStore = new RangerKeyStore(daoManager);
+
+ RangerKMSMKI rangerMasterKey = null;
String password = conf.get(ENCRYPTION_KEY);
if(password == null || password.trim().equals("") || password.trim().equals("_") || password.trim().equals("crypted")){
throw new IOException("Master Key Jceks does not exists");
}
+ if(conf.get(HSM_ENABLED).equalsIgnoreCase("false")){
+ rangerMasterKey = new RangerMasterKey(daoManager);
+ }else{
+ rangerMasterKey = new RangerHSM(conf);
+ String partitionPasswd = conf.get(HSM_PARTITION_PASSWORD);
+ if(partitionPasswd == null || partitionPasswd.trim().equals("") || partitionPasswd.trim().equals("_") || partitionPasswd.trim().equals("crypted")){
+ throw new IOException("Partition Password doesn't exists");
+ }
+ }
+ dbStore = new RangerKeyStore(daoManager);
rangerMasterKey.generateMasterKey(password);
//code to retrieve rangerMasterKey password
masterKey = rangerMasterKey.getMasterKey(password).toCharArray();
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/73dc5031/kms/src/main/java/org/apache/hadoop/crypto/key/RangerMasterKey.java
----------------------------------------------------------------------
diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerMasterKey.java b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerMasterKey.java
index 75a34b2..d70ec4e 100755
--- a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerMasterKey.java
+++ b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerMasterKey.java
@@ -40,7 +40,7 @@ import org.apache.ranger.entity.XXRangerMasterKey;
import com.sun.org.apache.xml.internal.security.exceptions.Base64DecodingException;
import com.sun.org.apache.xml.internal.security.utils.Base64;
-public class RangerMasterKey {
+public class RangerMasterKey implements RangerKMSMKI{
static final Logger logger = Logger.getLogger(RangerMasterKey.class);
@@ -65,6 +65,7 @@ public class RangerMasterKey {
* @return Decrypted Master Key
* @throws Throwable
*/
+ @Override
public String getMasterKey(String password) throws Throwable{
logger.info("Getting Master Key");
byte masterKeyByte[] = getEncryptedMK();
@@ -76,6 +77,16 @@ public class RangerMasterKey {
}
}
+ public SecretKey getMasterSecretKey(String password) throws Throwable{
+ logger.info("Getting Master Key");
+ byte masterKeyByte[] = getEncryptedMK();
+ if(masterKeyByte != null && masterKeyByte.length > 0){
+ return decryptMasterKeySK(masterKeyByte, password);
+ }else{
+ throw new Exception("No Master Key Found");
+ }
+ }
+
/**
* Generate the master key encrypt's it and save it in database
* @param password password to be used for encryption
@@ -83,6 +94,7 @@ public class RangerMasterKey {
* false if master key generation was unsuccessful or already master key exists
* @throws Throwable
*/
+ @Override
public boolean generateMasterKey(String password) throws Throwable{
logger.info("Generating Master Key");
String encryptedMasterKey = encryptMasterKey(password);
@@ -93,6 +105,17 @@ public class RangerMasterKey {
}
return false;
}
+
+ public boolean generateMKFromHSMMK(String password, byte[] key) throws Throwable{
+ logger.info("Generating Master Key");
+ String encryptedMasterKey = encryptMasterKey(password, key);
+ String savedKey = saveEncryptedMK(encryptedMasterKey, daoManager);
+ if(savedKey != null && !savedKey.trim().equals("")){
+ logger.debug("Master Key Created with id = "+savedKey);
+ return true;
+ }
+ return false;
+ }
private String decryptMasterKey(byte masterKey[], String password) throws Throwable {
logger.debug("Decrypting Master Key");
@@ -101,6 +124,13 @@ public class RangerMasterKey {
SecretKey masterKeyFromDB = getMasterKeyFromBytes(masterKeyFromDBDecrypted) ;
return Base64.encode(masterKeyFromDB.getEncoded());
}
+
+ private SecretKey decryptMasterKeySK(byte masterKey[], String password) throws Throwable {
+ logger.debug("Decrypting Master Key");
+ PBEKeySpec pbeKeyspec = getPBEParameterSpec(password) ;
+ byte[] masterKeyFromDBDecrypted = decryptKey(masterKey, pbeKeyspec) ;
+ return getMasterKeyFromBytes(masterKeyFromDBDecrypted) ;
+ }
private byte[] getEncryptedMK() throws Base64DecodingException {
logger.debug("Retrieving Encrypted Master Key from database");
@@ -155,6 +185,14 @@ public class RangerMasterKey {
return masterKey;
}
+ private String encryptMasterKey(String password, byte[] secretKey) throws Throwable {
+ logger.debug("Encrypting Master Key");
+ PBEKeySpec pbeKeySpec = getPBEParameterSpec(password);
+ byte[] masterKeyToDB = encryptKey(secretKey, pbeKeySpec);
+ String masterKey = Base64.encode(masterKeyToDB) ;
+ return masterKey;
+ }
+
private Key generateMasterKey() throws NoSuchAlgorithmException{
KeyGenerator kg = KeyGenerator.getInstance(MK_CIPHER);
kg.init(MK_KeySize);
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/73dc5031/src/main/assembly/kms.xml
----------------------------------------------------------------------
diff --git a/src/main/assembly/kms.xml b/src/main/assembly/kms.xml
index e267687..01f14b9 100755
--- a/src/main/assembly/kms.xml
+++ b/src/main/assembly/kms.xml
@@ -304,6 +304,8 @@
<include>install.properties</include>
<include>importJCEKSKeys.sh</include>
<include>exportKeysToJCEKS.sh</include>
+ <include>HSMMK2DB.sh</include>
+ <include>DBMK2HSM.sh</include>
</includes>
<fileMode>544</fileMode>
</fileSet>