You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by dh...@apache.org on 2022/01/17 12:18:24 UTC
[ranger] branch ranger-2.3 updated: RANGER-3569 : Support Ranger KMS integration with Google cloud HSM
This is an automated email from the ASF dual-hosted git repository.
dhavalshah9131 pushed a commit to branch ranger-2.3
in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/ranger-2.3 by this push:
new 1803188 RANGER-3569 : Support Ranger KMS integration with Google cloud HSM
1803188 is described below
commit 1803188a72788870f5d8b3dbf2766a60e9d5dd5b
Author: mateenmansoori <ma...@gmail.com>
AuthorDate: Mon Jan 17 12:18:43 2022 +0530
RANGER-3569 : Support Ranger KMS integration with Google cloud HSM
---
distro/src/main/assembly/kms.xml | 39 +++-
kms/config/kms-webapp/dbks-site.xml | 35 +++-
kms/pom.xml | 14 ++
kms/scripts/MigrateMKeyStorageDbToGCP.sh | 46 +++++
kms/scripts/install.properties | 8 +-
kms/scripts/setup.sh | 56 ++++++
.../hadoop/crypto/key/MigrateDBMKeyToGCP.java | 118 +++++++++++
.../crypto/key/RangerGoogleCloudHSMProvider.java | 215 +++++++++++++++++++++
.../org/apache/hadoop/crypto/key/RangerKMSMKI.java | 8 +
.../apache/hadoop/crypto/key/RangerKeyStore.java | 43 +++--
.../hadoop/crypto/key/RangerKeyStoreProvider.java | 68 ++++---
.../crypto/key/RangerKeyVaultKeyGenerator.java | 4 +-
kms/src/main/resources/META-INF/context.xml | 20 ++
pom.xml | 4 +-
14 files changed, 629 insertions(+), 49 deletions(-)
diff --git a/distro/src/main/assembly/kms.xml b/distro/src/main/assembly/kms.xml
index 5f39f10..b119f50 100755
--- a/distro/src/main/assembly/kms.xml
+++ b/distro/src/main/assembly/kms.xml
@@ -100,7 +100,7 @@
<include>org.apache.httpcomponents:httpmime:jar:${httpcomponents.httpmime.version}</include>
<include>org.apache.httpcomponents:httpclient:jar:${kms.httpcomponents.httpclient.version}</include>
<include>org.noggit:noggit:jar:${noggit.version}</include>
- <include>com.google.protobuf:protobuf-java:jar:${protobuf-java.version}</include>
+ <include>com.google.protobuf:protobuf-java:jar:${gcp.protobuf-java.version}</include>
<include>org.apache.hadoop:hadoop-hdfs:jar:${hadoop.version}</include>
<include>org.apache.htrace:htrace-core4:jar:${htrace-core.version}</include>
<include>org.codehaus.woodstox:stax2-api</include>
@@ -159,6 +159,40 @@
<include>com.microsoft.azure:azure-mgmt-eventhub</include>
<include>com.microsoft.azure:azure-mgmt-eventhub</include>
<include>com.microsoft.azure:azure-keyvault-cryptography</include>
+ <!-- GCP -->
+ <include>com.google.cloud:google-cloud-kms</include>
+ <include>io.grpc:grpc-api</include>
+ <include>io.grpc:grpc-context</include>
+ <include>io.grpc:grpc-stub</include>
+ <include>io.grpc:grpc-protobuf</include>
+ <include>io.grpc:grpc-protobuf-lite</include>
+ <include>com.google.api:api-common</include>
+ <include>javax.annotation:javax.annotation-api</include>
+ <include>com.google.auto.value:auto-value-annotations</include>
+ <include>com.google.api.grpc:proto-google-common-protos</include>
+ <include>com.google.api.grpc:proto-google-cloud-kms-v1</include>
+ <include>com.google.api:gax</include>
+ <include>io.opencensus:opencensus-api</include>
+ <include>com.google.api:gax-grpc</include>
+ <include>com.google.auth:google-auth-library-credentials</include>
+ <include>io.grpc:grpc-netty-shaded</include>
+ <include>io.perfmark:perfmark-api</include>
+ <include>io.grpc:grpc-core</include>
+ <include>com.google.android:annotations</include>
+ <include>io.grpc:grpc-alts</include>
+ <include>io.grpc:grpc-grpclb</include>
+ <include>com.google.protobuf:protobuf-java-util</include>
+ <include>org.conscrypt:conscrypt-openjdk-uber</include>
+ <include>org.threeten:threetenbp</include>
+ <include>io.grpc:grpc-auth</include>
+ <include>com.google.api.grpc:proto-google-iam-v1</include>
+ <include>com.google.auth:google-auth-library-oauth2-http</include>
+ <include>com.google.http-client:google-http-client</include>
+ <include>io.opencensus:opencensus-contrib-http-util</include>
+ <include>com.google.http-client:google-http-client-gson</include>
+ <include>io.grpc:grpc-xds</include>
+ <include>io.grpc:grpc-services</include>
+ <include>io.opencensus:opencensus-proto</include>
</includes>
</binaries>
</moduleSet>
@@ -190,7 +224,7 @@
<include>org.apache.tomcat.embed:tomcat-embed*</include>
<include>org.apache.tomcat:tomcat-annotations-api*</include>
<include>org.eclipse.jdt.core.compiler:ecj:jar:P20140317-1600</include>
- <include>com.google.protobuf:protobuf-java:jar:${protobuf-java.version}</include>
+ <include>com.google.protobuf:protobuf-java:jar:${gcp.protobuf-java.version}</include>
<include>org.apache.hadoop:hadoop-hdfs:jar:${hadoop.version}</include>
<include>org.apache.htrace:htrace-core4:jar:${htrace-core.version}</include>
<include>org.apache.solr:solr-solrj:jar:${solr.version}</include>
@@ -391,6 +425,7 @@
<include>DBMKTOKEYSECURE.sh</include>
<include>DBMKTOAZUREKEYVAULT.sh</include>
<include>KEYSECUREMKTOKMSDB.sh</include>
+ <include>MigrateMKeyStorageDbToGCP.sh</include>
</includes>
<fileMode>544</fileMode>
</fileSet>
diff --git a/kms/config/kms-webapp/dbks-site.xml b/kms/config/kms-webapp/dbks-site.xml
index 75f21c8..07de4d4 100755
--- a/kms/config/kms-webapp/dbks-site.xml
+++ b/kms/config/kms-webapp/dbks-site.xml
@@ -293,7 +293,40 @@
<description>Azure Key Vault url</description>
</property>
<!--Azure Key Vault END-->
-
+
+ <!-- Google Cloud KMS start -->
+ <property>
+ <name>ranger.kms.gcp.enabled</name>
+ <value>false</value>
+ <description>Flag for Google Cloud HSM e.g - true or false</description>
+ </property>
+ <property>
+ <name>ranger.kms.gcp.keyring.id</name>
+ <value></value>
+ <description>Name of KeyRing.</description>
+ </property>
+ <property>
+ <name>ranger.kms.gcp.cred.file</name>
+ <value></value>
+ <description>Absolute path of downloaded json credential file, e.g - /path/to/credFile.json</description>
+ </property>
+ <property>
+ <name>ranger.kms.gcp.project.id</name>
+ <value></value>
+ <description>Name of project on Google Cloud HSM.</description>
+ </property>
+ <property>
+ <name>ranger.kms.gcp.location.id</name>
+ <value></value>
+ <description>GCP KeyRing location id, e.g - us-east1, global etc.</description>
+ </property>
+ <property>
+ <name>ranger.kms.gcp.masterkey.name</name>
+ <value></value>
+ <description>GCP Master Key Name.</description>
+ </property>
+ <!-- Google Cloud KMS end -->
+
<!-- HSM Config -->
<property>
<name>ranger.ks.hsm.type</name>
diff --git a/kms/pom.xml b/kms/pom.xml
index 04e79e1..8dccc9f 100644
--- a/kms/pom.xml
+++ b/kms/pom.xml
@@ -61,6 +61,10 @@
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -418,6 +422,16 @@
<artifactId>javax.persistence</artifactId>
<version>${javax.persistence.version}</version>
</dependency>
+ <dependency>
+ <groupId>com.google.cloud</groupId>
+ <artifactId>google-cloud-kms</artifactId>
+ <version>${google.cloud.kms}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ <version>${gcp.protobuf-java.version}</version>
+ </dependency>
</dependencies>
<build>
<pluginManagement>
diff --git a/kms/scripts/MigrateMKeyStorageDbToGCP.sh b/kms/scripts/MigrateMKeyStorageDbToGCP.sh
new file mode 100644
index 0000000..eb1e56c
--- /dev/null
+++ b/kms/scripts/MigrateMKeyStorageDbToGCP.sh
@@ -0,0 +1,46 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# -------------------------------------------------------------------------------------
+if [ -z "${JAVA_HOME}" ]; then
+ echo "PLEASE EXPORT VARAIBLE JAVA_HOME"
+exit;
+else
+ echo "JAVA_HOME : "$JAVA_HOME
+fi
+
+if [ -z "${RANGER_KMS_HOME}" ]; then
+ echo "PLEASE EXPORT VARAIBLE RANGER_KMS_HOME"
+exit;
+else
+ echo "RANGER_KMS_HOME : "$RANGER_KMS_HOME
+fi
+
+if [ -z "${RANGER_KMS_CONF}" ]; then
+ echo "PLEASE EXPORT VARAIBLE RANGER_KMS_CONF"
+exit;
+else
+ echo "RANGER_KMS_CONF : "$RANGER_KMS_CONF
+fi
+
+if [ -z "${SQL_CONNECTOR_JAR}" ]; then
+ echo "PLEASE EXPORT VARAIBLE SQL_CONNECTOR_JAR"
+exit;
+else
+ echo "SQL_CONNECTOR_JAR : "$SQL_CONNECTOR_JAR
+fi
+
+cp="${RANGER_KMS_HOME}/cred/lib/*:${RANGER_KMS_CONF}:${RANGER_KMS_HOME}/ews/webapp/WEB-INF/classes/lib/*:${SQL_CONNECTOR_JAR}:${RANGER_KMS_HOME}/ews/webapp/config:${RANGER_KMS_HOME}/ews/lib/*:${RANGER_KMS_HOME}/ews/webapp/lib/*:${RANGER_KMS_HOME}/ews/webapp/META-INF:${RANGER_KMS_CONF}/*"
+${JAVA_HOME}/bin/java -cp "${cp}" org.apache.hadoop.crypto.key.MigrateDBMKeyToGCP ${1} ${2} ${3} ${4} ${5}
diff --git a/kms/scripts/install.properties b/kms/scripts/install.properties
index 4935536..39f49de 100755
--- a/kms/scripts/install.properties
+++ b/kms/scripts/install.properties
@@ -124,8 +124,14 @@ AZURE_MASTER_KEY_TYPE=RSA
ZONE_KEY_ENCRYPTION_ALGO=RSA_OAEP
AZURE_KEYVAULT_URL=https://shahkeyvault.vault.azure.net/
+#------------------------- Ranger Google Cloud HSM ------------------------------
+IS_GCP_ENABLED=false
+GCP_KEYRING_ID=
+GCP_CRED_JSON_FILE=/full/path/to/credfile.json
+GCP_PROJECT_ID=
+GCP_LOCATION_ID=
+GCP_MASTER_KEY_NAME=MyMasterKeyNameChangeIt
-#
# ------- UNIX User CONFIG ----------------
#
unix_user=kms
diff --git a/kms/scripts/setup.sh b/kms/scripts/setup.sh
index 60c026b..2051df5 100755
--- a/kms/scripts/setup.sh
+++ b/kms/scripts/setup.sh
@@ -114,6 +114,13 @@ AZURE_MASTER_KEY_TYPE=$(get_prop 'AZURE_MASTER_KEY_TYPE' $PROPFILE)
ZONE_KEY_ENCRYPTION_ALGO=$(get_prop 'ZONE_KEY_ENCRYPTION_ALGO' $PROPFILE)
AZURE_KEYVAULT_URL=$(get_prop 'AZURE_KEYVAULT_URL' $PROPFILE)
+IS_GCP_ENABLED=$(get_prop 'IS_GCP_ENABLED' $PROPFILE)
+GCP_KEYRING_ID=$(get_prop 'GCP_KEYRING_ID' $PROPFILE)
+GCP_CRED_JSON_FILE=$(get_prop 'GCP_CRED_JSON_FILE' $PROPFILE)
+GCP_PROJECT_ID=$(get_prop 'GCP_PROJECT_ID' $PROPFILE)
+GCP_LOCATION_ID=$(get_prop 'GCP_LOCATION_ID' $PROPFILE)
+GCP_MASTER_KEY_NAME=$(get_prop 'GCP_MASTER_KEY_NAME' $PROPFILE)
+
kms_principal=$(get_prop 'kms_principal' $PROPFILE)
kms_keytab=$(get_prop 'kms_keytab' $PROPFILE)
hadoop_conf=$(get_prop 'hadoop_conf' $PROPFILE)
@@ -471,6 +478,14 @@ setup_kms(){
cd ${oldP}
}
+checkIfEmpty() {
+ if [ -z "$1" ]
+ then
+ echo "Error - Since GCP is enabled, Please provide valid value for '$2', Found : '$1'";
+ exit 1
+ fi
+}
+
update_properties() {
newPropertyValue=''
echo "export JAVA_HOME=${JAVA_HOME}" > ${WEBAPP_ROOT}/WEB-INF/classes/conf/java_home.sh
@@ -643,6 +658,7 @@ update_properties() {
HSM_ENABLED=`echo $HSM_ENABLED | tr '[:lower:]' '[:upper:]'`
KEYSECURE_ENABLED=`echo $KEYSECURE_ENABLED | tr '[:lower:]' '[:upper:]'`
AZURE_KEYVAULT_ENABLED=`echo $AZURE_KEYVAULT_ENABLED | tr '[:lower:]' '[:upper:]'`
+ IS_GCP_ENABLED=`echo $IS_GCP_ENABLED | tr '[:lower:]' '[:upper:]'`
if [ "${keystore}" != "" ]
then
@@ -888,6 +904,46 @@ update_properties() {
fi
+ ########### RANGER GCP #################
+ if [ "${IS_GCP_ENABLED}" != "TRUE" ]
+ then
+ propertyName=ranger.kms.gcp.enabled
+ newPropertyValue="false"
+ updatePropertyToFilePy $propertyName $newPropertyValue $to_file
+ else
+ propertyName=ranger.kms.gcp.enabled
+ newPropertyValue="true"
+ updatePropertyToFilePy $propertyName $newPropertyValue $to_file
+
+ propertyName=ranger.kms.gcp.keyring.id
+ newPropertyValue="${GCP_KEYRING_ID}"
+ checkIfEmpty "$newPropertyValue" "GCP_KEYRING_ID"
+ updatePropertyToFilePy $propertyName $newPropertyValue $to_file
+
+ propertyName=ranger.kms.gcp.cred.file
+ newPropertyValue="${GCP_CRED_JSON_FILE}"
+ if [ "${newPropertyValue: -5}" != ".json" ]
+ then
+ echo "Error - GCP Credential file must be in a json format, Provided file : ${newPropertyValue}";
+ exit 1
+ fi
+ updatePropertyToFilePy $propertyName $newPropertyValue $to_file
+
+ propertyName=ranger.kms.gcp.project.id
+ newPropertyValue="${GCP_PROJECT_ID}"
+ checkIfEmpty "$newPropertyValue" "GCP_PROJECT_ID"
+ updatePropertyToFilePy $propertyName $newPropertyValue $to_file
+
+ propertyName=ranger.kms.gcp.location.id
+ newPropertyValue="${GCP_LOCATION_ID}"
+ checkIfEmpty "$newPropertyValue" "GCP_LOCATION_ID"
+ updatePropertyToFilePy $propertyName $newPropertyValue $to_file
+
+ propertyName=ranger.kms.gcp.masterkey.name
+ newPropertyValue="${GCP_MASTER_KEY_NAME}"
+ checkIfEmpty "$newPropertyValue" "GCP_MASTER_KEY_NAME"
+ updatePropertyToFilePy $propertyName $newPropertyValue $to_file
+ fi
to_file_kms_site=$PWD/ews/webapp/WEB-INF/classes/conf/ranger-kms-site.xml
if test -f $to_file_kms_site; then
diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/MigrateDBMKeyToGCP.java b/kms/src/main/java/org/apache/hadoop/crypto/key/MigrateDBMKeyToGCP.java
new file mode 100644
index 0000000..d3b717a
--- /dev/null
+++ b/kms/src/main/java/org/apache/hadoop/crypto/key/MigrateDBMKeyToGCP.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.crypto.key;
+
+import java.io.IOException;
+import java.security.Key;
+import java.util.ArrayList;
+import java.util.Enumeration;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.ranger.entity.XXRangerKeyStore;
+import org.apache.ranger.kms.dao.DaoManager;
+
+public class MigrateDBMKeyToGCP {
+ private static final String ENCRYPTION_KEY = "ranger.db.encrypt.key.password";
+ private static RangerGoogleCloudHSMProvider rangerGcpProvider;
+ private RangerKeyStore dbStore;
+
+ public static void main(String[] args) throws Exception {
+ if (args == null || args.length != 5) {
+ System.err.println("Invalid number of parameters found.");
+ showUsage();
+ System.exit(1);
+ } else {
+ Configuration conf = RangerKeyStoreProvider.getDBKSConf();
+ final String gcpMasterKeyName = args[0];
+ final String gcpProjectId = args[1];
+ final String gcpKeyRingId = args[2];
+ final String gcpKeyRingLocationId = args[3];
+ final String pathOfJsonCredFile = args[4];
+
+ if (conf != null) {
+ conf.set(RangerGoogleCloudHSMProvider.GCP_MASTER_KEY_NAME, gcpMasterKeyName);
+ conf.set(RangerGoogleCloudHSMProvider.GCP_PROJECT_ID, gcpProjectId);
+ conf.set(RangerGoogleCloudHSMProvider.GCP_KEYRING_ID, gcpKeyRingId);
+ conf.set(RangerGoogleCloudHSMProvider.GCP_LOCATION_ID, gcpKeyRingLocationId);
+ conf.set(RangerGoogleCloudHSMProvider.GCP_CRED_JSON_FILE, pathOfJsonCredFile);
+
+ rangerGcpProvider = new RangerGoogleCloudHSMProvider(conf);
+ rangerGcpProvider.onInitialization();
+ boolean result = new MigrateDBMKeyToGCP().doExportMKToGcp(conf, gcpMasterKeyName);
+ if (result) {
+ System.out.println("Master Key from Ranger KMS DB has been successfully migrated to GCP.");
+ System.exit(0);
+ } else {
+ System.out.println("Migration of Master Key from Ranger KMS DB to GCP has been unsuccessful.");
+ System.exit(1);
+ }
+ } else {
+ System.out.println("Migration of Master Key from Ranger KMS DB to GCP failed, Error - Configuration is null.");
+ System.exit(1);
+ }
+ }
+ }
+
+ private boolean doExportMKToGcp(Configuration conf, final String masterKeyName) {
+ try {
+ String mKeyPass = conf.get(ENCRYPTION_KEY);
+ if (mKeyPass == null || mKeyPass.trim().equals("") || mKeyPass.trim().equals("_")
+ || mKeyPass.trim().equals("crypted")) {
+ throw new IOException("Master Key Jceks does not exists");
+ }
+
+ RangerKMSDB rangerkmsDb = new RangerKMSDB(conf);
+ DaoManager daoManager = rangerkmsDb.getDaoManager();
+
+ System.out.println("Creating masterkey with the name - " + masterKeyName);
+ boolean gcpMKSuccess = rangerGcpProvider.generateMasterKey(null);
+ if (gcpMKSuccess) {
+ System.out.println("Masterkey with the name '" + masterKeyName +"' created successfully on Google Cloud KMS.");
+ dbStore = new RangerKeyStore(daoManager, false, rangerGcpProvider);
+ // Get Master Key from Ranger DB
+ RangerMasterKey rangerMasterKey = new RangerMasterKey(daoManager);
+ char[] mkey = rangerMasterKey.getMasterKey(mKeyPass).toCharArray();
+ List<XXRangerKeyStore> rangerKeyStoreList = new ArrayList<XXRangerKeyStore>();
+ dbStore.engineLoad(null, mkey);
+ Enumeration<String> e = dbStore.engineAliases();
+ Key key;
+ String alias = null;
+ while (e.hasMoreElements()) {
+ alias = e.nextElement();
+ key = dbStore.engineGetKey(alias, mkey);
+ XXRangerKeyStore xxRangerKeyStore = dbStore.convertKeysBetweenRangerKMSAndGCP(alias, key, rangerGcpProvider);
+ rangerKeyStoreList.add(xxRangerKeyStore);
+ }
+ if (rangerKeyStoreList != null && !rangerKeyStoreList.isEmpty()) {
+ for (XXRangerKeyStore rangerKeyStore : rangerKeyStoreList) {
+ dbStore.dbOperationStore(rangerKeyStore);
+ }
+ }
+ return true;
+ }
+ return false;
+ } catch (Throwable t) {
+ throw new RuntimeException("Unable to migrate Master key from Ranger KMS DB to GCP ", t);
+ }
+ }
+
+ private static void showUsage() {
+ System.err.println("USAGE: java " + MigrateDBMKeyToGCP.class.getName() + " <gcpMasterKeyName> <gcpProjectName> <gcpKeyRingName> <gcpKeyRingLocationName> <pathOfJsonCredFile> ");
+ }
+}
diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerGoogleCloudHSMProvider.java b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerGoogleCloudHSMProvider.java
new file mode 100644
index 0000000..666a8c3
--- /dev/null
+++ b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerGoogleCloudHSMProvider.java
@@ -0,0 +1,215 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.crypto.key;
+
+import java.lang.reflect.Field;
+import java.security.Key;
+import java.util.Map;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.log4j.Logger;
+import org.bouncycastle.crypto.RuntimeCryptoException;
+
+import com.google.api.gax.rpc.AlreadyExistsException;
+import com.google.cloud.kms.v1.CryptoKey;
+import com.google.cloud.kms.v1.CryptoKey.CryptoKeyPurpose;
+import com.google.cloud.kms.v1.CryptoKeyName;
+import com.google.cloud.kms.v1.CryptoKeyVersion.CryptoKeyVersionAlgorithm;
+import com.google.cloud.kms.v1.CryptoKeyVersionTemplate;
+import com.google.cloud.kms.v1.DecryptResponse;
+import com.google.cloud.kms.v1.EncryptResponse;
+import com.google.cloud.kms.v1.KeyManagementServiceClient;
+import com.google.cloud.kms.v1.KeyRing;
+import com.google.cloud.kms.v1.KeyRingName;
+import com.google.cloud.kms.v1.ProtectionLevel;
+import com.google.protobuf.ByteString;
+
+public class RangerGoogleCloudHSMProvider implements RangerKMSMKI {
+
+ private static final Logger logger = Logger.getLogger(RangerGoogleCloudHSMProvider.class);
+ protected static final String GCP_KEYRING_ID = "ranger.kms.gcp.keyring.id";
+ protected static final String GCP_CRED_JSON_FILE = "ranger.kms.gcp.cred.file";
+ protected static final String GCP_PROJECT_ID = "ranger.kms.gcp.project.id";
+ protected static final String GCP_LOCATION_ID = "ranger.kms.gcp.location.id";
+ protected static final String GCP_MASTER_KEY_NAME = "ranger.kms.gcp.masterkey.name";
+ private static final String GCP_CRED_ENV_VARIABLE = "GOOGLE_APPLICATION_CREDENTIALS";
+
+ private String gcpKeyRingId;
+ private String gcpAppCredFile;
+ private String gcpProjectId;
+ private String gcpLocationId;
+ private String gcpMasterKeyName;
+ private KeyManagementServiceClient client = null;
+ private KeyRingName keyRingName = null;
+
+ public RangerGoogleCloudHSMProvider(Configuration conf) throws Exception {
+ this.gcpKeyRingId = conf.get(GCP_KEYRING_ID);
+ this.gcpAppCredFile = conf.get(GCP_CRED_JSON_FILE);
+ this.gcpLocationId = conf.get(GCP_LOCATION_ID);
+ this.gcpProjectId = conf.get(GCP_PROJECT_ID);
+ this.gcpMasterKeyName = conf.get(GCP_MASTER_KEY_NAME);
+ }
+
+ protected void validateGcpProps() {
+ if (StringUtils.isEmpty(this.gcpAppCredFile) || !this.gcpAppCredFile.endsWith(".json")) {
+ throw new RuntimeCryptoException("Error : Invalid GCP app Credential JSON file, Provided cred file : " + this.gcpAppCredFile);
+ } else if (StringUtils.isEmpty(this.gcpKeyRingId)) {
+ throw new RuntimeCryptoException("Error : Please provide GCP app KeyringId, Provided keyring ID : " + this.gcpKeyRingId);
+ } else if (StringUtils.isEmpty(this.gcpLocationId)) {
+ throw new RuntimeCryptoException("Error : Please provide the GCP app location Id, Provided location ID :" + this.gcpLocationId);
+ } else if (StringUtils.isEmpty(this.gcpProjectId)) {
+ throw new RuntimeCryptoException("Error : Please provide the GCP app project Id, Provided ID : " + this.gcpProjectId);
+ } else if (StringUtils.isEmpty(this.gcpMasterKeyName)) {
+ throw new RuntimeCryptoException("Error : Master key name must not be empty, Provided MasterKey Name : " + this.gcpMasterKeyName);
+ }
+ }
+
+ @Override
+ public void onInitialization() throws Exception {
+ this.validateGcpProps();
+ if (logger.isDebugEnabled()) {
+ logger.debug("==> onInitialization() : {gcpProjectId - " + this.gcpProjectId + ", gcpLocationId - "
+ + this.gcpLocationId + ", gcpKeyRingId - " + this.gcpKeyRingId + ", gcpAppCredFile Path - "
+ + this.gcpAppCredFile + "}");
+ }
+ String errorMessage = null;
+ client = getKeyClient(this.gcpAppCredFile);
+
+ KeyRing keyRingResponse = null;
+ if (client != null) {
+ this.keyRingName = KeyRingName.of(this.gcpProjectId, this.gcpLocationId, this.gcpKeyRingId);
+ if(this.keyRingName != null) {
+ keyRingResponse = this.client.getKeyRing(this.keyRingName.toString());
+ if (keyRingResponse == null) {
+ errorMessage = "Unable to get Key Ring response for Project : " + this.gcpProjectId + " and Location : " + this.gcpLocationId;
+ } else if (keyRingResponse != null && !keyRingResponse.getName().endsWith(this.gcpKeyRingId)) {
+ errorMessage = "Key Ring with name : " + this.gcpKeyRingId + " does not exist for Project : " + this.gcpProjectId + " and Location : " + this.gcpLocationId;
+ }
+ } else {
+ errorMessage = "Unable to get Key Ring response for Project : " + this.gcpProjectId + " and Location : " + this.gcpLocationId;
+ }
+ } else {
+ errorMessage = "Unable to create client object for Google Cloud HSM. Please check the Key HSM Log file OR Verify Google App Credential JSON file.";
+ }
+ if (logger.isDebugEnabled()) {
+ logger.debug("<== onInitialization() : {this.keyRingName - " + this.keyRingName + ", keyRingResponse - " + keyRingResponse + "}");
+ }
+ if (!StringUtils.isEmpty(errorMessage)) {
+ throw new RuntimeCryptoException(errorMessage);
+ }
+ }
+
+ private KeyManagementServiceClient getKeyClient(String credentialFileName) {
+ try {
+ if (StringUtils.isEmpty(System.getenv(GCP_CRED_ENV_VARIABLE))) {
+ updateEnv(GCP_CRED_ENV_VARIABLE, credentialFileName);
+ }
+ KeyManagementServiceClient client = KeyManagementServiceClient.create();
+ return client;
+ } catch (Exception ex) {
+ logger.error("Unable to create Google Cloud KMS Client, Error : ", ex);
+ }
+ return null;
+ }
+
+ @Override
+ public boolean generateMasterKey(String unused_password) throws Throwable {
+ //The ENCRYPT_DECRYPT key purpose enables symmetric encryption.
+ //All keys with key purpose ENCRYPT_DECRYPT use the GOOGLE_SYMMETRIC_ENCRYPTION algorithm.
+ //No parameters are used with this algorithm.
+ CryptoKey key = CryptoKey.newBuilder()
+ .setPurpose(CryptoKeyPurpose.ENCRYPT_DECRYPT)
+ .setVersionTemplate(CryptoKeyVersionTemplate.newBuilder()
+ .setProtectionLevel(ProtectionLevel.HSM)
+ .setAlgorithm(CryptoKeyVersionAlgorithm.GOOGLE_SYMMETRIC_ENCRYPTION))
+ .build();
+
+ // Create the key.
+ CryptoKey createdKey = null;
+ try {
+ createdKey = client.createCryptoKey(this.keyRingName, this.gcpMasterKeyName, key);
+ } catch (Exception e) {
+ if (e instanceof AlreadyExistsException) {
+ logger.info("MasterKey with the name '" + this.gcpMasterKeyName + "' already exist.");
+ return true;
+ } else {
+ throw new RuntimeCryptoException("Failed to create master key with name '" + this.gcpMasterKeyName + "', Error - " + e.getMessage());
+ }
+ }
+
+ if (createdKey == null) {
+ logger.info("Failed to create master key : " + this.gcpMasterKeyName);
+ return false;
+ }
+ logger.info("Master Key Created Successfully On Google Cloud HSM : " + this.gcpMasterKeyName);
+ return true;
+ }
+
+ @Override
+ public String getMasterKey(String password) throws Throwable {
+ // Not Allowed to get master key out side of the Google Cloud HSM i.e very similar to Azure Key Vault
+ return null;
+ }
+
+ @Override
+ public byte[] encryptZoneKey(Key zoneKey) throws Exception {
+ if(logger.isDebugEnabled()) {
+ logger.debug("==> GCP encryptZoneKey()");
+ }
+ byte[] primaryEncodedZoneKey = zoneKey.getEncoded(); // Data to encrypt i.e a zoneKey
+ CryptoKeyName keyName = CryptoKeyName.of(this.gcpProjectId, this.gcpLocationId, this.gcpKeyRingId, this.gcpMasterKeyName);
+
+ EncryptResponse encryptResponse = this.client.encrypt(keyName, ByteString.copyFrom(primaryEncodedZoneKey));
+ if (encryptResponse == null) {
+ throw new RuntimeCryptoException("Got null response for encrypt zone key operation, Please reverify/check configs!");
+ }
+ if(logger.isDebugEnabled()) {
+ logger.debug("<== GCP encryptZoneKey() : EncryptResponse - { " + encryptResponse + " }" );
+ }
+ return encryptResponse.getCiphertext().toByteArray();
+ }
+
+ @Override
+ public byte[] decryptZoneKey(byte[] encryptedByte) throws Exception {
+ CryptoKeyName keyName = CryptoKeyName.of(this.gcpProjectId, this.gcpLocationId, this.gcpKeyRingId, this.gcpMasterKeyName);
+ if(logger.isDebugEnabled()) {
+ logger.debug("==> GCP decryptZoneKey() : CryptoKeyName - { " + keyName + " }");
+ }
+
+ DecryptResponse response = client.decrypt(keyName, ByteString.copyFrom(encryptedByte));
+ if(response == null) {
+ throw new RuntimeCryptoException("Got null response for decrypt zone key operation!");
+ } else if(response.getPlaintext() == null || StringUtils.isEmpty(response.getPlaintext().toString())) {
+ throw new RuntimeCryptoException("Error - Received null or empty decrypted zone key : " + response.getPlaintext());
+ }
+ if(logger.isDebugEnabled()) {
+ logger.debug("<== GCP decryptZoneKey() : DecryptResponse - { " + response + " }" );
+ }
+ return response.getPlaintext().toByteArray();
+ }
+
+ @SuppressWarnings("unchecked")
+ private static void updateEnv(String name, String val) throws ReflectiveOperationException {
+ Map<String, String> env = System.getenv();
+ Field field = env.getClass().getDeclaredField("m");
+ field.setAccessible(true);
+
+ Map<String, String> writeAbleEnvMap = (Map<String, String>) field.get(env);
+ writeAbleEnvMap.put(name, val);
+ }
+}
diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKMSMKI.java b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKMSMKI.java
index 75e70ff..b09cd5b 100644
--- a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKMSMKI.java
+++ b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKMSMKI.java
@@ -17,9 +17,17 @@
package org.apache.hadoop.crypto.key;
+import java.security.Key;
+
public interface RangerKMSMKI {
boolean generateMasterKey(String password) throws Throwable;
String getMasterKey(String password) throws Throwable;
+
+ default byte[] decryptZoneKey(byte[] encryptedByte) throws Exception {return null;}
+
+ default byte[] encryptZoneKey(Key zoneKey) throws Exception {return null;}
+
+ default void onInitialization() throws Exception {}
}
diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStore.java b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStore.java
index b9e7cb2..12d485a 100644
--- a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStore.java
+++ b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStore.java
@@ -91,9 +91,10 @@ public class RangerKeyStore extends KeyStoreSpi {
private static final Pattern pattern = Pattern.compile(KEY_NAME_VALIDATION);
private static final String AZURE_KEYVAULT_ENABLED = "ranger.kms.azurekeyvault.enabled";
private boolean azureKeyVaultEnabled = false;
+ private boolean isGCPHSMEnabled = false;
private DaoManager daoManager;
- private RangerKeyVaultKeyGenerator kvKeyGen;
+ private RangerKMSMKI masterKeyProvider;
// keys
private static class KeyEntry {
@@ -130,10 +131,10 @@ public class RangerKeyStore extends KeyStoreSpi {
public RangerKeyStore(DaoManager daoManager) {
this.daoManager = daoManager;
}
-
+
public RangerKeyStore(DaoManager daoManager, Configuration conf, KeyVaultClient kvClient) {
this.daoManager = daoManager;
- this.kvKeyGen = new RangerKeyVaultKeyGenerator(conf, kvClient);
+ this.masterKeyProvider = new RangerKeyVaultKeyGenerator(conf, kvClient);
if(conf != null
&& StringUtils.isNotEmpty(conf
.get(AZURE_KEYVAULT_ENABLED))
@@ -143,6 +144,12 @@ public class RangerKeyStore extends KeyStoreSpi {
}
}
+ public RangerKeyStore(DaoManager daoManager, boolean isGcpEnabled, RangerKMSMKI rangerGCPProvider) {
+ this.daoManager = daoManager;
+ this.masterKeyProvider = rangerGCPProvider;
+ this.isGCPHSMEnabled = isGcpEnabled;
+ }
+
String convertAlias(String alias) {
return alias.toLowerCase();
}
@@ -176,7 +183,7 @@ public class RangerKeyStore extends KeyStoreSpi {
return null;
}
SecretKeyByteEntry key = (SecretKeyByteEntry) entry;
- byte[] decryptKeyByte = kvKeyGen.dencryptZoneKey(key.key);
+ byte[] decryptKeyByte = masterKeyProvider.decryptZoneKey(key.key);
return decryptKeyByte;
} catch (Exception ex) {
throw new Exception("Error while decrpting zone key. Name : "
@@ -224,7 +231,7 @@ public class RangerKeyStore extends KeyStoreSpi {
try {
entry.date = new Date();
// encrypt and store the key
- entry.key = kvKeyGen.encryptZoneKey(key);
+ entry.key = masterKeyProvider.encryptZoneKey(key);
entry.cipher_field = cipher;
entry.bit_length = bitLength;
entry.description = description;
@@ -396,7 +403,7 @@ public class RangerKeyStore extends KeyStoreSpi {
logger.debug("==> RangerKeyStore.engineStore()");
}
synchronized (deltaEntries) {
- if (azureKeyVaultEnabled) {
+ if (azureKeyVaultEnabled || isGCPHSMEnabled) {
for (Entry<String, Object> entry : deltaEntries.entrySet()) {
Long creationDate = ((SecretKeyByteEntry) entry.getValue()).date
.getTime();
@@ -528,7 +535,7 @@ public class RangerKeyStore extends KeyStoreSpi {
}
keyEntries.clear();
- if (azureKeyVaultEnabled) {
+ if (azureKeyVaultEnabled || isGCPHSMEnabled) {
for (XXRangerKeyStore rangerKey : rangerKeyDetails) {
String encodedStr = rangerKey.getEncoded();
byte[] encodedByte = DatatypeConverter
@@ -731,7 +738,7 @@ public class RangerKeyStore extends KeyStoreSpi {
}
synchronized (deltaEntries) {
KeyStore ks;
- if (azureKeyVaultEnabled) {
+ if (azureKeyVaultEnabled || isGCPHSMEnabled) {
try {
ks = KeyStore.getInstance(fileFormat);
ks.load(stream, storePass);
@@ -816,7 +823,7 @@ public class RangerKeyStore extends KeyStoreSpi {
validateKeyName(keyName);
entry.attributes = "{\"key.acl.name\":\"" + keyName
+ "\"}";
- entry.key = kvKeyGen.encryptZoneKey(secretKey);
+ entry.key = masterKeyProvider.encryptZoneKey(secretKey);
entry.date = ks.getCreationDate(alias);
entry.description = k.getFormat() + " - "
+ ks.getType();
@@ -921,7 +928,7 @@ public class RangerKeyStore extends KeyStoreSpi {
Key key;
while (e.hasMoreElements()) {
alias = e.nextElement();
- if(azureKeyVaultEnabled){
+ if(azureKeyVaultEnabled || isGCPHSMEnabled) {
key = engineGetDecryptedZoneKey(alias);
} else {
key = engineGetKey(alias, masterKey);
@@ -963,9 +970,7 @@ public class RangerKeyStore extends KeyStoreSpi {
return keyEntries.get(alias);
}
- public XXRangerKeyStore convertKeysBetweenRangerKMSAndAzureKeyVault(
- String alias, Key key,
- RangerKeyVaultKeyGenerator rangerKVKeyGenerator) {
+ private XXRangerKeyStore convertKeysBetweenRangerKMSAndHSM(String alias, Key key, RangerKMSMKI rangerMKeyProvider) {
try {
XXRangerKeyStore xxRangerKeyStore;
SecretKeyEntry secretKey = (SecretKeyEntry) getKeyEntry(alias);
@@ -977,7 +982,7 @@ public class RangerKeyStore extends KeyStoreSpi {
byte[] keyByte = keyGenerator.generateKey().getEncoded();
Key ezkey = new SecretKeySpec(keyByte,
getAlgorithm(meta.getCipher()));
- byte[] encryptedKey = rangerKVKeyGenerator
+ byte[] encryptedKey = rangerMKeyProvider
.encryptZoneKey(ezkey);
Long creationDate = new Date().getTime();
String attributes = secretKey.attributes;
@@ -986,7 +991,7 @@ public class RangerKeyStore extends KeyStoreSpi {
meta.getDescription(), meta.getVersions(),
attributes);
} else {
- byte[] encryptedKey = rangerKVKeyGenerator.encryptZoneKey(key);
+ byte[] encryptedKey = rangerMKeyProvider.encryptZoneKey(key);
Long creationDate = secretKey.date.getTime();
int version = secretKey.version;
if ((alias.split("@").length == 2)
@@ -1005,6 +1010,14 @@ public class RangerKeyStore extends KeyStoreSpi {
}
}
+ public XXRangerKeyStore convertKeysBetweenRangerKMSAndGCP(String alias, Key key, RangerKMSMKI rangerGCPProvider) {
+ return this.convertKeysBetweenRangerKMSAndHSM(alias, key, rangerGCPProvider);
+ }
+
+ public XXRangerKeyStore convertKeysBetweenRangerKMSAndAzureKeyVault(String alias, Key key, RangerKMSMKI rangerKVKeyGenerator) {
+ return this.convertKeysBetweenRangerKMSAndHSM(alias, key, rangerKVKeyGenerator);
+ }
+
public String getAlgorithm(String cipher) {
int slash = cipher.indexOf(47);
if (slash == -1) {
diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStoreProvider.java b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStoreProvider.java
index db8fa69..bd85c0d 100755
--- a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStoreProvider.java
+++ b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyStoreProvider.java
@@ -80,13 +80,17 @@ public class RangerKeyStoreProvider extends KeyProvider {
private static final String AZURE_CLIENT_SECRET = "ranger.kms.azure.client.secret";
private static final String AZURE_KEYVAULT_CERTIFICATE_PATH = "ranger.kms.azure.keyvault.certificate.path";
private static final String AZURE_KEYVAULT_CERTIFICATE_PASSWORD = "ranger.kms.azure.keyvault.certificate.password";
- private final RangerKeyStore dbStore;
+ private static final String IS_GCP_ENABLED = "ranger.kms.gcp.enabled";
+ private RangerKeyStore dbStore;
private char[] masterKey;
private boolean changed = false;
private final Map<String, Metadata> cache = new HashMap<String, Metadata>();
private DaoManager daoManager;
private Lock readLock;
+ private boolean isHSMEnabled = false;
private boolean azureKeyVaultEnabled = false;
+ private boolean isGCPEnabled = false;
+ private boolean isKeySecureEnabled = false;
public RangerKeyStoreProvider(Configuration conf) throws Throwable {
super(conf);
@@ -101,7 +105,7 @@ public class RangerKeyStoreProvider extends KeyProvider {
RangerKMSDB rangerKMSDB = new RangerKMSDB(conf);
daoManager = rangerKMSDB.getDaoManager();
- RangerKMSMKI rangerMasterKey = null;
+ RangerKMSMKI masterKeyProvider = null;
String password = conf.get(ENCRYPTION_KEY);
if (password == null || password.trim().equals("")
|| password.trim().equals("_")
@@ -109,36 +113,36 @@ public class RangerKeyStoreProvider extends KeyProvider {
throw new IOException(
"The Ranger MasterKey Password is empty or not a valid Password");
}
- if (StringUtils.isEmpty(conf.get(HSM_ENABLED))
- || conf.get(HSM_ENABLED).equalsIgnoreCase("false")) {
- logger.info("Ranger KMS Database is enabled for storing master key.");
- rangerMasterKey = new RangerMasterKey(daoManager);
- } else {
+
+ this.isHSMEnabled = conf.getBoolean(HSM_ENABLED, false);
+ this.azureKeyVaultEnabled = conf.getBoolean(AZURE_KEYVAULT_ENABLED, false);
+ this.isGCPEnabled = conf.getBoolean(IS_GCP_ENABLED, false);
+ this.isKeySecureEnabled = conf.getBoolean(KEYSECURE_ENABLED, false);
+
+ if(this.isHSMEnabled) {
logger.info("Ranger KMS HSM is enabled for storing master key.");
- rangerMasterKey = new RangerHSM(conf);
+ masterKeyProvider = new RangerHSM(conf);
String partitionPasswd = conf.get(HSM_PARTITION_PASSWORD);
if (partitionPasswd == null || partitionPasswd.trim().equals("")
|| partitionPasswd.trim().equals("_")
|| partitionPasswd.trim().equals("crypted")) {
throw new IOException("Partition Password doesn't exists");
}
- }
-
- if (conf != null && StringUtils.isNotEmpty(conf.get(KEYSECURE_ENABLED))
- && conf.get(KEYSECURE_ENABLED).equalsIgnoreCase("true")) {
+ } else if (this.isKeySecureEnabled) {
+ logger.info("KeySecure is enabled for storing the master key.");
getFromJceks(conf, CREDENTIAL_PATH, KEYSECURE_PASSWORD_ALIAS,
KEYSECURE_PASSWORD);
String keySecureLoginCred = conf.get(KEYSECURE_USERNAME).trim()
+ ":" + conf.get(KEYSECURE_PASSWORD);
conf.set(KEYSECURE_LOGIN, keySecureLoginCred);
- rangerMasterKey = new RangerSafenetKeySecure(conf);
+ masterKeyProvider = new RangerSafenetKeySecure(conf);
dbStore = new RangerKeyStore(daoManager);
// generate master key on key secure server
- rangerMasterKey.generateMasterKey(password);
+ masterKeyProvider.generateMasterKey(password);
try {
- masterKey = rangerMasterKey.getMasterKey(password)
+ masterKey = masterKeyProvider.getMasterKey(password)
.toCharArray();
} catch (Exception ex) {
throw new Exception(
@@ -146,9 +150,8 @@ public class RangerKeyStoreProvider extends KeyProvider {
+ ex);
}
- } else if (conf != null
- && StringUtils.isNotEmpty(conf.get(AZURE_KEYVAULT_ENABLED))
- && conf.get(AZURE_KEYVAULT_ENABLED).equalsIgnoreCase("true")) {
+ } else if (this.azureKeyVaultEnabled) {
+ logger.info("Azure Key Vault is enabled for storing the master key.");
azureKeyVaultEnabled = true;
getFromJceks(conf, CREDENTIAL_PATH, AZURE_CLIENT_SECRET_ALIAS,
AZURE_CLIENT_SECRET);
@@ -204,10 +207,10 @@ public class RangerKeyStoreProvider extends KeyProvider {
if (kvClient != null) {
try {
dbStore = new RangerKeyStore(daoManager, conf, kvClient);
- rangerMasterKey = new RangerKeyVaultKeyGenerator(conf,
+ masterKeyProvider = new RangerKeyVaultKeyGenerator(conf,
kvClient);
- if (rangerMasterKey != null) {
- success = rangerMasterKey.generateMasterKey(password);
+ if (masterKeyProvider != null) {
+ success = masterKeyProvider.generateMasterKey(password);
}
} catch (Exception ex) {
throw new Exception(
@@ -228,13 +231,22 @@ public class RangerKeyStoreProvider extends KeyProvider {
+ ex);
}
}
-
+ } else if(this.isGCPEnabled) {
+ logger.info("Google Cloud HSM is enabled for storing the master key.");
+ masterKeyProvider = new RangerGoogleCloudHSMProvider(conf);
+ masterKeyProvider.onInitialization();
+ if(masterKeyProvider != null) {
+ this.dbStore = new RangerKeyStore(daoManager, this.isGCPEnabled, masterKeyProvider);
+ masterKeyProvider.generateMasterKey(password);
+ }
} else {
+ logger.info("Ranger KMS Database is enabled for storing master key.");
+ masterKeyProvider = new RangerMasterKey(daoManager);
dbStore = new RangerKeyStore(daoManager);
- rangerMasterKey.generateMasterKey(password);
+ masterKeyProvider.generateMasterKey(password);
// code to retrieve rangerMasterKey password
try {
- masterKey = rangerMasterKey.getMasterKey(password)
+ masterKey = masterKeyProvider.getMasterKey(password)
.toCharArray();
} catch (Exception ex) {
throw new Exception("Error while getting Ranger Master key "
@@ -333,7 +345,7 @@ public class RangerKeyStoreProvider extends KeyProvider {
}
try {
String attribute = JsonUtilsV2.mapToJson(attributes);
- if (azureKeyVaultEnabled) {
+ if (azureKeyVaultEnabled || this.isGCPEnabled) {
dbStore.addSecureKeyByteEntry(versionName, new SecretKeySpec(
material, cipher), cipher, bitLength, description,
version, attribute);
@@ -396,7 +408,7 @@ public class RangerKeyStoreProvider extends KeyProvider {
Metadata metadata = entry.getValue();
String attributes = JsonUtilsV2.mapToJson(metadata
.getAttributes());
- if (azureKeyVaultEnabled) {
+ if (azureKeyVaultEnabled || this.isGCPEnabled) {
Key ezkey = new KeyMetadata(metadata);
if (ezkey.getEncoded().length == 0) {
KeyGenerator keyGenerator = KeyGenerator
@@ -444,7 +456,7 @@ public class RangerKeyStoreProvider extends KeyProvider {
readLock.lock();
try {
- if (azureKeyVaultEnabled) {
+ if (azureKeyVaultEnabled || this.isGCPEnabled) {
byte[] decryptKeyByte = null;
try {
if (!dbStore.engineContainsAlias(versionName)) {
@@ -557,7 +569,7 @@ public class RangerKeyStoreProvider extends KeyProvider {
return null;
}
}
- if (azureKeyVaultEnabled) {
+ if (azureKeyVaultEnabled || this.isGCPEnabled) {
Metadata meta = dbStore.engineGetKeyMetadata(name);
if (meta != null) {
cache.put(name, meta);
diff --git a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyVaultKeyGenerator.java b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyVaultKeyGenerator.java
index 854d7f0..d21d323 100644
--- a/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyVaultKeyGenerator.java
+++ b/kms/src/main/java/org/apache/hadoop/crypto/key/RangerKeyVaultKeyGenerator.java
@@ -120,6 +120,7 @@ public class RangerKeyVaultKeyGenerator implements RangerKMSMKI {
}
}
+ @Override
public byte[] encryptZoneKey(Key zoneKey) throws Exception {
JsonWebKeyEncryptionAlgorithm keyEncryptionAlgo = getZoneKeyEncryptionAlgo();
KeyOperationResult encryptResult = null;
@@ -139,7 +140,8 @@ public class RangerKeyVaultKeyGenerator implements RangerKMSMKI {
return encryptResult.result();
}
- public byte[] dencryptZoneKey(byte[] encryptedByte) throws Exception {
+ @Override
+ public byte[] decryptZoneKey(byte[] encryptedByte) throws Exception {
JsonWebKeyEncryptionAlgorithm keyEncryptionAlgo = getZoneKeyEncryptionAlgo();
if (masterKeyBundle == null) {
masterKeyBundle = keyVaultClient
diff --git a/kms/src/main/resources/META-INF/context.xml b/kms/src/main/resources/META-INF/context.xml
new file mode 100644
index 0000000..b13407a
--- /dev/null
+++ b/kms/src/main/resources/META-INF/context.xml
@@ -0,0 +1,20 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<Context useHttpOnly="true">
+ <JarScanner scanManifest="false" />
+</Context>
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
index 66a4139..d8c095f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -177,6 +177,7 @@
<poi.version>4.1.2</poi.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<protobuf-java.version>2.5.0</protobuf-java.version>
+ <gcp.protobuf-java.version>3.19.1</gcp.protobuf-java.version>
<scala.version>2.12.10</scala.version>
<scala.binary.version>2.12</scala.binary.version>
<scala.xml.version>1.0.4</scala.xml.version>
@@ -224,7 +225,8 @@
<lucene.version>8.4.0</lucene.version>
<hppc.version>0.8.0</hppc.version>
<joda.time.version>2.10.6</joda.time.version>
- <log4j.core.version>2.16.0</log4j.core.version>
+ <!-- GCP HSM -->
+ <google.cloud.kms>2.3.0</google.cloud.kms>
</properties>
<profiles>
<profile>