You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ng...@apache.org on 2023/09/15 14:38:26 UTC
[hive] branch master updated: HIVE-27675: Support keystore/truststore types for hive to zookeeper integration (Naveen Gangam) (#4691)
This is an automated email from the ASF dual-hosted git repository.
ngangam pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 59570d6202e HIVE-27675: Support keystore/truststore types for hive to zookeeper integration (Naveen Gangam) (#4691)
59570d6202e is described below
commit 59570d6202e6c29dad0824af80e241339cf89f83
Author: Naveen Gangam <ng...@cloudera.com>
AuthorDate: Fri Sep 15 10:38:20 2023 -0400
HIVE-27675: Support keystore/truststore types for hive to zookeeper integration (Naveen Gangam) (#4691)
---
.../java/org/apache/hadoop/hive/conf/HiveConf.java | 16 +++++++++++++--
.../hcatalog/templeton/tool/ZooKeeperStorage.java | 4 ++++
.../security/ZooKeeperTokenStoreTestBase.java | 5 +++++
.../org/apache/hive/jdbc/TestRestrictedList.java | 2 ++
.../InformationSchemaWithPrivilegeTestBase.java | 5 +++++
.../java/org/apache/hive/jdbc/HiveConnection.java | 4 +++-
jdbc/src/java/org/apache/hive/jdbc/Utils.java | 24 +++++++++++++++++++++-
.../hive/jdbc/ZooKeeperHiveClientHelper.java | 9 ++++++--
.../hadoop/hive/registry/impl/ZkRegistryBase.java | 2 ++
.../hadoop/hive/common/SSLZookeeperFactory.java | 14 ++++++++++---
.../hadoop/hive/common/ZooKeeperHiveHelper.java | 24 +++++++++++++++++++++-
.../hadoop/hive/metastore/conf/MetastoreConf.java | 14 ++++++++++++-
.../security/MetastoreDelegationTokenManager.java | 4 ++++
.../metastore/security/ZooKeeperTokenStore.java | 10 +++++++++
14 files changed, 126 insertions(+), 11 deletions(-)
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 10b8a34b76e..0d7a8d072e5 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -3012,6 +3012,10 @@ public class HiveConf extends Configuration {
"Keystore password when using a client-side certificate with TLS connectivity to ZooKeeper." +
"Overrides any explicit value set via the zookeeper.ssl.keyStore.password " +
"system property (note the camelCase)."),
+ HIVE_ZOOKEEPER_SSL_KEYSTORE_TYPE("hive.zookeeper.ssl.keystore.type", "",
+ "Keystore type when using a client-side certificate with TLS connectivity to ZooKeeper." +
+ "Overrides any explicit value set via the zookeeper.ssl.keyStore.type " +
+ "system property (note the camelCase)."),
HIVE_ZOOKEEPER_SSL_TRUSTSTORE_LOCATION("hive.zookeeper.ssl.truststore.location", "",
"Truststore location when using a client-side certificate with TLS connectivity to ZooKeeper. " +
"Overrides any explicit value set via the zookeeper.ssl.trustStore.location" +
@@ -3020,6 +3024,10 @@ public class HiveConf extends Configuration {
"Truststore password when using a client-side certificate with TLS connectivity to ZooKeeper." +
"Overrides any explicit value set via the zookeeper.ssl.trustStore.password " +
"system property (note the camelCase)."),
+ HIVE_ZOOKEEPER_SSL_TRUSTSTORE_TYPE("hive.zookeeper.ssl.truststore.type", "",
+ "Truststore type when using a client-side certificate with TLS connectivity to ZooKeeper." +
+ "Overrides any explicit value set via the zookeeper.ssl.trustStore.type " +
+ "system property (note the camelCase)."),
HIVE_ZOOKEEPER_KILLQUERY_ENABLE("hive.zookeeper.killquery.enable", true,
"Whether enabled kill query coordination with zookeeper, " +
"when hive.server2.support.dynamic.service.discovery is enabled."),
@@ -5550,8 +5558,10 @@ public class HiveConf extends Configuration {
"hive.driver.parallel.compilation.global.limit," +
"hive.zookeeper.ssl.keystore.location," +
"hive.zookeeper.ssl.keystore.password," +
+ "hive.zookeeper.ssl.keystore.type," +
"hive.zookeeper.ssl.truststore.location," +
- "hive.zookeeper.ssl.truststore.password",
+ "hive.zookeeper.ssl.truststore.password," +
+ "hive.zookeeper.ssl.truststore.type",
"Comma separated list of configuration options which are immutable at runtime"),
HIVE_CONF_HIDDEN_LIST("hive.conf.hidden.list",
METASTOREPWD.varname + "," + HIVE_SERVER2_SSL_KEYSTORE_PASSWORD.varname
@@ -6377,8 +6387,10 @@ public class HiveConf extends Configuration {
.sslEnabled(getBoolVar(ConfVars.HIVE_ZOOKEEPER_SSL_ENABLE))
.keyStoreLocation(getVar(ConfVars.HIVE_ZOOKEEPER_SSL_KEYSTORE_LOCATION))
.keyStorePassword(keyStorePassword)
+ .keyStoreType(getVar(ConfVars.HIVE_ZOOKEEPER_SSL_KEYSTORE_TYPE))
.trustStoreLocation(getVar(ConfVars.HIVE_ZOOKEEPER_SSL_TRUSTSTORE_LOCATION))
- .trustStorePassword(trustStorePassword).build();
+ .trustStorePassword(trustStorePassword)
+ .trustStoreType(getVar(ConfVars.HIVE_ZOOKEEPER_SSL_TRUSTSTORE_TYPE)).build();
}
public HiveConf() {
diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/ZooKeeperStorage.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/ZooKeeperStorage.java
index 2919038c78d..f54c866118d 100644
--- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/ZooKeeperStorage.java
+++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/ZooKeeperStorage.java
@@ -55,8 +55,10 @@ public class ZooKeeperStorage implements TempletonStorage {
public static final String ZK_SSL_ENABLE = "templeton.zookeeper.ssl.client.enable";
public static final String ZK_KEYSTORE_LOCATION = "templeton.zookeeper.keystore.location";
public static final String ZK_KEYSTORE_PASSWORD = "templeton.zookeeper.keystore.password";
+ public static final String ZK_KEYSTORE_TYPE = "templeton.zookeeper.keystore.type";
public static final String ZK_TRUSTSTORE_LOCATION = "templeton.zookeeper.truststore.location";
public static final String ZK_TRUSTSTORE_PASSWORD = "templeton.zookeeper.truststore.password";
+ public static final String ZK_TRUSTSTORE_TYPE = "templeton.zookeeper.truststore.type";
public static final String ENCODING = "UTF-8";
@@ -77,8 +79,10 @@ public class ZooKeeperStorage implements TempletonStorage {
.sslEnabled(conf.getBoolean(ZK_SSL_ENABLE, false))
.keyStoreLocation(conf.get(ZK_KEYSTORE_LOCATION, ""))
.keyStorePassword(conf.get(ZK_KEYSTORE_PASSWORD, ""))
+ .keyStoreType(conf.get(ZK_KEYSTORE_TYPE, ""))
.trustStoreLocation(conf.get(ZK_TRUSTSTORE_LOCATION, ""))
.trustStorePassword(conf.get(ZK_TRUSTSTORE_PASSWORD, ""))
+ .trustStoreType(conf.get(ZK_TRUSTSTORE_TYPE, ""))
.build();
CuratorFramework zk = xkHelper.getNewZookeeperClient();
zk.start();
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/security/ZooKeeperTokenStoreTestBase.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/security/ZooKeeperTokenStoreTestBase.java
index 35053e70b0a..9841a185dc0 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/security/ZooKeeperTokenStoreTestBase.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/security/ZooKeeperTokenStoreTestBase.java
@@ -54,6 +54,7 @@ public abstract class ZooKeeperTokenStoreTestBase {
private static final String LOCALHOST_KEY_STORE_NAME = "keystore.jks";
private static final String TRUST_STORE_NAME = "truststore.jks";
private static final String KEY_STORE_TRUST_STORE_PASSWORD = "HiveJdbc";
+ private static final String KEY_STORE_TRUST_STORE_TYPE = "JKS";
private static MiniZooKeeperCluster zkCluster = null;
private static int zkPort = -1;
@@ -96,10 +97,14 @@ public abstract class ZooKeeperTokenStoreTestBase {
dataFileDir + File.separator + LOCALHOST_KEY_STORE_NAME);
conf.set(MetastoreDelegationTokenManager.DELEGATION_TOKEN_STORE_ZK_KEYSTORE_PASSWORD,
KEY_STORE_TRUST_STORE_PASSWORD);
+ conf.set(MetastoreDelegationTokenManager.DELEGATION_TOKEN_STORE_ZK_KEYSTORE_TYPE,
+ KEY_STORE_TRUST_STORE_TYPE);
conf.set(MetastoreDelegationTokenManager.DELEGATION_TOKEN_STORE_ZK_TRUSTSTORE_LOCATION,
dataFileDir + File.separator + TRUST_STORE_NAME);
conf.set(MetastoreDelegationTokenManager.DELEGATION_TOKEN_STORE_ZK_TRUSTSTORE_PASSWORD,
KEY_STORE_TRUST_STORE_PASSWORD);
+ conf.set(MetastoreDelegationTokenManager.DELEGATION_TOKEN_STORE_ZK_TRUSTSTORE_TYPE,
+ KEY_STORE_TRUST_STORE_TYPE);
conf.set(MetastoreDelegationTokenManager.DELEGATION_TOKEN_STORE_ZK_SSL_ENABLE, "true");
}
diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestRestrictedList.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestRestrictedList.java
index 04780ad0546..aeec57757c2 100644
--- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestRestrictedList.java
+++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestRestrictedList.java
@@ -105,8 +105,10 @@ public class TestRestrictedList {
addToExpectedRestrictedMap("hive.driver.parallel.compilation.global.limit");
addToExpectedRestrictedMap("hive.zookeeper.ssl.keystore.location");
addToExpectedRestrictedMap("hive.zookeeper.ssl.keystore.password");
+ addToExpectedRestrictedMap("hive.zookeeper.ssl.keystore.type");
addToExpectedRestrictedMap("hive.zookeeper.ssl.truststore.location");
addToExpectedRestrictedMap("hive.zookeeper.ssl.truststore.password");
+ addToExpectedRestrictedMap("hive.zookeeper.ssl.truststore.type");
checkRestrictedListMatch();
}
diff --git a/itests/hive-unit/src/test/java/org/apache/hive/service/server/InformationSchemaWithPrivilegeTestBase.java b/itests/hive-unit/src/test/java/org/apache/hive/service/server/InformationSchemaWithPrivilegeTestBase.java
index 9573e5050c6..6f58d265e12 100644
--- a/itests/hive-unit/src/test/java/org/apache/hive/service/server/InformationSchemaWithPrivilegeTestBase.java
+++ b/itests/hive-unit/src/test/java/org/apache/hive/service/server/InformationSchemaWithPrivilegeTestBase.java
@@ -180,6 +180,7 @@ public abstract class InformationSchemaWithPrivilegeTestBase {
private static final String LOCALHOST_KEY_STORE_NAME = "keystore.jks";
private static final String TRUST_STORE_NAME = "truststore.jks";
private static final String KEY_STORE_TRUST_STORE_PASSWORD = "HiveJdbc";
+ private static final String KEY_STORE_TRUST_STORE_TYPE = "JKS";
private static MiniHS2 miniHS2 = null;
private static MiniZooKeeperCluster zkCluster = null;
@@ -223,10 +224,14 @@ public abstract class InformationSchemaWithPrivilegeTestBase {
dataFileDir + File.separator + LOCALHOST_KEY_STORE_NAME);
confOverlay.put(ConfVars.HIVE_ZOOKEEPER_SSL_KEYSTORE_PASSWORD.varname,
KEY_STORE_TRUST_STORE_PASSWORD);
+ confOverlay.put(ConfVars.HIVE_ZOOKEEPER_SSL_KEYSTORE_TYPE.varname,
+ KEY_STORE_TRUST_STORE_TYPE);
confOverlay.put(ConfVars.HIVE_ZOOKEEPER_SSL_TRUSTSTORE_LOCATION.varname,
dataFileDir + File.separator + TRUST_STORE_NAME);
confOverlay.put(ConfVars.HIVE_ZOOKEEPER_SSL_TRUSTSTORE_PASSWORD.varname,
KEY_STORE_TRUST_STORE_PASSWORD);
+ confOverlay.put(ConfVars.HIVE_ZOOKEEPER_SSL_TRUSTSTORE_TYPE.varname,
+ KEY_STORE_TRUST_STORE_TYPE);
confOverlay.put(ConfVars.HIVE_ZOOKEEPER_SSL_ENABLE.varname, "true");
}
miniHS2.start(confOverlay);
diff --git a/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java b/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java
index 5850105b5fa..197640cd388 100644
--- a/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java
+++ b/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java
@@ -1009,7 +1009,9 @@ public class HiveConnection implements java.sql.Connection {
JdbcConnectionParams.SUNJSSE_ALGORITHM_STRING);
String keyStorePath = sessConfMap.get(JdbcConnectionParams.SSL_KEY_STORE);
String keyStorePassword = Utils.getPassword(sessConfMap, JdbcConnectionParams.SSL_KEY_STORE_PASSWORD);
- KeyStore sslKeyStore = KeyStore.getInstance(JdbcConnectionParams.SSL_KEY_STORE_TYPE);
+ String keyStoreType = sessConfMap.get(JdbcConnectionParams.SSL_KEY_STORE_TYPE);
+ keyStoreType = (!StringUtils.isBlank(keyStoreType)) ? keyStoreType : KeyStore.getDefaultType();
+ KeyStore sslKeyStore = KeyStore.getInstance(keyStoreType);
if (keyStorePath == null || keyStorePath.isEmpty()) {
throw new IllegalArgumentException(JdbcConnectionParams.SSL_KEY_STORE
diff --git a/jdbc/src/java/org/apache/hive/jdbc/Utils.java b/jdbc/src/java/org/apache/hive/jdbc/Utils.java
index e6f07032e92..e3fb3edf810 100644
--- a/jdbc/src/java/org/apache/hive/jdbc/Utils.java
+++ b/jdbc/src/java/org/apache/hive/jdbc/Utils.java
@@ -146,8 +146,10 @@ public class Utils {
public static final String ZOOKEEPER_SSL_ENABLE = "zooKeeperSSLEnable";
public static final String ZOOKEEPER_KEYSTORE_LOCATION = "zooKeeperKeystoreLocation";
public static final String ZOOKEEPER_KEYSTORE_PASSWORD= "zooKeeperKeystorePassword";
+ public static final String ZOOKEEPER_KEYSTORE_TYPE= "zooKeeperKeystoreType";
public static final String ZOOKEEPER_TRUSTSTORE_LOCATION = "zooKeeperTruststoreLocation";
public static final String ZOOKEEPER_TRUSTSTORE_PASSWORD = "zooKeeperTruststorePassword";
+ public static final String ZOOKEEPER_TRUSTSTORE_TYPE = "zooKeeperTruststoreType";
// Default namespace value on ZooKeeper.
// This value is used if the param "zooKeeperNamespace" is not specified in the JDBC Uri.
static final String ZOOKEEPER_DEFAULT_NAMESPACE = "hiveserver2";
@@ -185,7 +187,7 @@ public class Utils {
static final String TRUE = "true";
static final String SSL_KEY_STORE = "sslKeyStore";
static final String SSL_KEY_STORE_PASSWORD = "keyStorePassword";
- static final String SSL_KEY_STORE_TYPE = "JKS";
+ static final String SSL_KEY_STORE_TYPE = "keyStoreType";
static final String SUNX509_ALGORITHM_STRING = "SunX509";
static final String SUNJSSE_ALGORITHM_STRING = "SunJSSE";
// --------------- End 2 way ssl options ----------------------------
@@ -207,8 +209,10 @@ public class Utils {
private boolean zooKeeperSslEnabled = false;
private String zookeeperKeyStoreLocation = "";
private String zookeeperKeyStorePassword = "";
+ private String zookeeperKeyStoreType;
private String zookeeperTrustStoreLocation = "";
private String zookeeperTrustStorePassword = "";
+ private String zookeeperTrustStoreType;
private String currentHostZnodePath;
private final List<String> rejectedHostZnodePaths = new ArrayList<String>();
@@ -233,8 +237,10 @@ public class Utils {
this.zooKeeperSslEnabled = params.zooKeeperSslEnabled;
this.zookeeperKeyStoreLocation = params.zookeeperKeyStoreLocation;
this.zookeeperKeyStorePassword = params.zookeeperKeyStorePassword;
+ this.zookeeperKeyStoreType = params.zookeeperKeyStoreType;
this.zookeeperTrustStoreLocation = params.zookeeperTrustStoreLocation;
this.zookeeperTrustStorePassword = params.zookeeperTrustStorePassword;
+ this.zookeeperTrustStoreType = params.zookeeperTrustStoreType;
this.currentHostZnodePath = params.currentHostZnodePath;
this.rejectedHostZnodePaths.addAll(rejectedHostZnodePaths);
@@ -291,6 +297,10 @@ public class Utils {
return zookeeperKeyStorePassword;
}
+ public String getZookeeperKeyStoreType() {
+ return zookeeperKeyStoreType;
+ }
+
public String getZookeeperTrustStoreLocation() {
return zookeeperTrustStoreLocation;
}
@@ -299,6 +309,10 @@ public class Utils {
return zookeeperTrustStorePassword;
}
+ public String getZookeeperTrustStoreType() {
+ return zookeeperTrustStoreType;
+ }
+
public List<String> getRejectedHostZnodePaths() {
return rejectedHostZnodePaths;
}
@@ -359,6 +373,10 @@ public class Utils {
this.zookeeperKeyStorePassword = zookeeperKeyStorePassword;
}
+ public void setZookeeperKeyStoreType(String zookeeperKeyStoreType) {
+ this.zookeeperKeyStoreType = zookeeperKeyStoreType;
+ }
+
public void setZookeeperTrustStoreLocation(String zookeeperTrustStoreLocation) {
this.zookeeperTrustStoreLocation = zookeeperTrustStoreLocation;
}
@@ -367,6 +385,10 @@ public class Utils {
this.zookeeperTrustStorePassword = zookeeperTrustStorePassword;
}
+ public void setZookeeperTrustStoreType(String zookeeperTrustStoreType) {
+ this.zookeeperTrustStoreType = zookeeperTrustStoreType;
+ }
+
public void setCurrentHostZnodePath(String currentHostZnodePath) {
this.currentHostZnodePath = currentHostZnodePath;
}
diff --git a/jdbc/src/java/org/apache/hive/jdbc/ZooKeeperHiveClientHelper.java b/jdbc/src/java/org/apache/hive/jdbc/ZooKeeperHiveClientHelper.java
index 70091343430..dfe21e71331 100644
--- a/jdbc/src/java/org/apache/hive/jdbc/ZooKeeperHiveClientHelper.java
+++ b/jdbc/src/java/org/apache/hive/jdbc/ZooKeeperHiveClientHelper.java
@@ -103,11 +103,15 @@ class ZooKeeperHiveClientHelper {
connParams.setZookeeperKeyStorePassword(
StringUtils.defaultString(Utils.getPassword(sessionConf, JdbcConnectionParams.ZOOKEEPER_KEYSTORE_PASSWORD),
""));
+ connParams.setZookeeperKeyStoreType(
+ StringUtils.defaultString(sessionConf.get(JdbcConnectionParams.ZOOKEEPER_KEYSTORE_TYPE),""));
connParams.setZookeeperTrustStoreLocation(
StringUtils.defaultString(sessionConf.get(JdbcConnectionParams.ZOOKEEPER_TRUSTSTORE_LOCATION), ""));
connParams.setZookeeperTrustStorePassword(
StringUtils.defaultString(Utils.getPassword(sessionConf, JdbcConnectionParams.ZOOKEEPER_TRUSTSTORE_PASSWORD),
""));
+ connParams.setZookeeperTrustStoreType(
+ StringUtils.defaultString(sessionConf.get(JdbcConnectionParams.ZOOKEEPER_TRUSTSTORE_TYPE),""));
}
}
@@ -119,8 +123,9 @@ class ZooKeeperHiveClientHelper {
.retryPolicy(new ExponentialBackoffRetry(1000, 3))
.zookeeperFactory(
new SSLZookeeperFactory(connParams.isZooKeeperSslEnabled(), connParams.getZookeeperKeyStoreLocation(),
- connParams.getZookeeperKeyStorePassword(), connParams.getZookeeperTrustStoreLocation(),
- connParams.getZookeeperTrustStorePassword()))
+ connParams.getZookeeperKeyStorePassword(), connParams.getZookeeperKeyStoreType(),
+ connParams.getZookeeperTrustStoreLocation(),
+ connParams.getZookeeperTrustStorePassword(), connParams.getZookeeperTrustStoreType()))
.build();
zooKeeperClient.start();
return zooKeeperClient;
diff --git a/llap-client/src/java/org/apache/hadoop/hive/registry/impl/ZkRegistryBase.java b/llap-client/src/java/org/apache/hadoop/hive/registry/impl/ZkRegistryBase.java
index 73290586b43..9da200f2e92 100644
--- a/llap-client/src/java/org/apache/hadoop/hive/registry/impl/ZkRegistryBase.java
+++ b/llap-client/src/java/org/apache/hadoop/hive/registry/impl/ZkRegistryBase.java
@@ -238,8 +238,10 @@ public abstract class ZkRegistryBase<InstanceType extends ServiceInstance> {
.sslEnabled(HiveConf.getBoolVar(conf, ConfVars.HIVE_ZOOKEEPER_SSL_ENABLE))
.keyStoreLocation(HiveConf.getVar(conf, ConfVars.HIVE_ZOOKEEPER_SSL_KEYSTORE_LOCATION))
.keyStorePassword(keyStorePassword)
+ .keyStoreType(HiveConf.getVar(conf, ConfVars.HIVE_ZOOKEEPER_SSL_KEYSTORE_TYPE))
.trustStoreLocation(HiveConf.getVar(conf, ConfVars.HIVE_ZOOKEEPER_SSL_TRUSTSTORE_LOCATION))
.trustStorePassword(trustStorePassword)
+ .trustStoreType(HiveConf.getVar(conf, ConfVars.HIVE_ZOOKEEPER_SSL_TRUSTSTORE_TYPE))
.build().getNewZookeeperClient(zooKeeperAclProvider, namespace);
}
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/SSLZookeeperFactory.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/SSLZookeeperFactory.java
index ee01731fa95..514199f2155 100644
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/SSLZookeeperFactory.java
+++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/SSLZookeeperFactory.java
@@ -27,6 +27,8 @@ import org.apache.zookeeper.common.ClientX509Util;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.security.KeyStore;
+
/**
* Factory to create Zookeeper clients with the zookeeper.client.secure enabled,
* allowing SSL communication with the Zookeeper server.
@@ -38,22 +40,26 @@ public class SSLZookeeperFactory implements ZookeeperFactory {
private boolean sslEnabled;
private String keyStoreLocation;
private String keyStorePassword;
+ private String keyStoreType;
private String trustStoreLocation;
private String trustStorePassword;
+ private String trustStoreType;
public SSLZookeeperFactory(boolean sslEnabled, String keyStoreLocation, String keyStorePassword,
- String trustStoreLocation, String trustStorePassword) {
+ String keyStoreType, String trustStoreLocation, String trustStorePassword, String trustStoreType) {
this.sslEnabled = sslEnabled;
this.keyStoreLocation = keyStoreLocation;
this.keyStorePassword = keyStorePassword;
+ this.keyStoreType = (!StringUtils.isBlank(keyStoreType)) ? keyStoreType : KeyStore.getDefaultType();
this.trustStoreLocation = trustStoreLocation;
this.trustStorePassword = trustStorePassword;
+ this.trustStoreType = (!StringUtils.isBlank(trustStoreType)) ? trustStoreType : KeyStore.getDefaultType();
if (sslEnabled) {
- if (StringUtils.isEmpty(keyStoreLocation)) {
+ if (StringUtils.isBlank(keyStoreLocation)) {
LOG.warn("Missing keystoreLocation parameter");
}
- if (StringUtils.isEmpty(trustStoreLocation)) {
+ if (StringUtils.isBlank(trustStoreLocation)) {
LOG.warn("Missing trustStoreLocation parameter");
}
}
@@ -71,8 +77,10 @@ public class SSLZookeeperFactory implements ZookeeperFactory {
ClientX509Util x509Util = new ClientX509Util();
clientConfig.setProperty(x509Util.getSslKeystoreLocationProperty(), this.keyStoreLocation);
clientConfig.setProperty(x509Util.getSslKeystorePasswdProperty(), this.keyStorePassword);
+ clientConfig.setProperty(x509Util.getSslKeystoreTypeProperty(), this.keyStoreType);
clientConfig.setProperty(x509Util.getSslTruststoreLocationProperty(), this.trustStoreLocation);
clientConfig.setProperty(x509Util.getSslTruststorePasswdProperty(), this.trustStorePassword);
+ clientConfig.setProperty(x509Util.getSslTruststoreTypeProperty(), this.trustStoreType);
return new ZooKeeper(connectString, sessionTimeout, watcher, canBeReadOnly, clientConfig);
}
}
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/ZooKeeperHiveHelper.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/ZooKeeperHiveHelper.java
index 1e35795d63c..c7da6259f53 100644
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/ZooKeeperHiveHelper.java
+++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/ZooKeeperHiveHelper.java
@@ -71,8 +71,10 @@ public class ZooKeeperHiveHelper {
private boolean sslEnabled = false;
private String keyStoreLocation = null;
private String keyStorePassword = null;
+ private String keyStoreType = null;
private String trustStoreLocation = null;
private String trustStorePassword = null;
+ private String trustStoreType = null;
public ZooKeeperHiveHelper build() {
return new ZooKeeperHiveHelper(this);
@@ -128,6 +130,11 @@ public class ZooKeeperHiveHelper {
return this;
}
+ public ZooKeeperHiveHelperBuilder keyStoreType(String keyStoreType) {
+ this.keyStoreType = keyStoreType;
+ return this;
+ }
+
public ZooKeeperHiveHelperBuilder trustStoreLocation(String trustStoreLocation) {
this.trustStoreLocation = trustStoreLocation;
return this;
@@ -138,6 +145,11 @@ public class ZooKeeperHiveHelper {
return this;
}
+ public ZooKeeperHiveHelperBuilder trustStoreType(String trustStoreType) {
+ this.trustStoreType = trustStoreType;
+ return this;
+ }
+
public String getQuorum() {
return quorum;
}
@@ -178,6 +190,10 @@ public class ZooKeeperHiveHelper {
return keyStorePassword;
}
+ public String getKeyStoreType() {
+ return keyStoreType;
+ }
+
public String getTrustStoreLocation() {
return trustStoreLocation;
}
@@ -185,6 +201,10 @@ public class ZooKeeperHiveHelper {
public String getTrustStorePassword() {
return trustStorePassword;
}
+
+ public String getTrustStoreType() {
+ return trustStoreType;
+ }
}
public static ZooKeeperHiveHelper.ZooKeeperHiveHelperBuilder builder() {
@@ -233,8 +253,10 @@ public class ZooKeeperHiveHelper {
new SSLZookeeperFactory(sslEnabled,
builder.getKeyStoreLocation(),
builder.getKeyStorePassword(),
+ builder.getKeyStoreType(),
builder.getTrustStoreLocation(),
- builder.getTrustStorePassword());
+ builder.getTrustStorePassword(),
+ builder.getTrustStoreType());
}
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
index 71ba8d520ff..27646119d6e 100644
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
+++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
@@ -1536,6 +1536,11 @@ public class MetastoreConf {
"Keystore password when using a client-side certificate with TLS connectivity to ZooKeeper." +
"Overrides any explicit value set via the zookeeper.ssl.keyStore.password" +
"system property (note the camelCase)."),
+ THRIFT_ZOOKEEPER_SSL_KEYSTORE_TYPE("metastore.zookeeper.ssl.keystore.type",
+ "hive.zookeeper.ssl.keystore.type", "",
+ "Keystore type when using a client-side certificate with TLS connectivity to ZooKeeper." +
+ "Overrides any explicit value set via the zookeeper.ssl.keyStore.type" +
+ "system property (note the camelCase)."),
THRIFT_ZOOKEEPER_SSL_TRUSTSTORE_LOCATION("metastore.zookeeper.ssl.truststore.location",
"hive.zookeeper.ssl.truststore.location", "",
"Truststore location when using a client-side certificate with TLS connectivity to ZooKeeper. " +
@@ -1546,6 +1551,11 @@ public class MetastoreConf {
"Truststore password when using a client-side certificate with TLS connectivity to ZooKeeper." +
"Overrides any explicit value set via the zookeeper.ssl.trustStore.password " +
"system property (note the camelCase)."),
+ THRIFT_ZOOKEEPER_SSL_TRUSTSTORE_TYPE("metastore.zookeeper.ssl.truststore.type",
+ "hive.zookeeper.ssl.truststore.type", "",
+ "Truststore type when using a client-side certificate with TLS connectivity to ZooKeeper." +
+ "Overrides any explicit value set via the zookeeper.ssl.trustStore.type" +
+ "system property (note the camelCase)."),
THRIFT_URI_SELECTION("metastore.thrift.uri.selection", "hive.metastore.uri.selection", "RANDOM",
new StringSetValidator("RANDOM", "SEQUENTIAL"),
"Determines the selection mechanism used by metastore client to connect to remote " +
@@ -2597,8 +2607,10 @@ public class MetastoreConf {
.sslEnabled(MetastoreConf.getBoolVar(conf, ConfVars.THRIFT_ZOOKEEPER_SSL_ENABLE))
.keyStoreLocation(MetastoreConf.getVar(conf, ConfVars.THRIFT_ZOOKEEPER_SSL_KEYSTORE_LOCATION))
.keyStorePassword(keyStorePassword)
+ .keyStoreType(MetastoreConf.getVar(conf, ConfVars.THRIFT_ZOOKEEPER_SSL_KEYSTORE_TYPE))
.trustStoreLocation(MetastoreConf.getVar(conf, ConfVars.THRIFT_ZOOKEEPER_SSL_TRUSTSTORE_LOCATION))
- .trustStorePassword(trustStorePassword).build();
+ .trustStorePassword(trustStorePassword)
+ .trustStoreType(MetastoreConf.getVar(conf, ConfVars.THRIFT_ZOOKEEPER_SSL_TRUSTSTORE_TYPE)).build();
}
/**
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/MetastoreDelegationTokenManager.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/MetastoreDelegationTokenManager.java
index 239bff6dc9d..e43d4b15a94 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/MetastoreDelegationTokenManager.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/MetastoreDelegationTokenManager.java
@@ -52,10 +52,14 @@ public class MetastoreDelegationTokenManager {
"hive.cluster.delegation.token.store.zookeeper.keystore.location";
public static final String DELEGATION_TOKEN_STORE_ZK_KEYSTORE_PASSWORD =
"hive.cluster.delegation.token.store.zookeeper.keystore.password";
+ public static final String DELEGATION_TOKEN_STORE_ZK_KEYSTORE_TYPE =
+ "hive.cluster.delegation.token.store.zookeeper.keystore.type";
public static final String DELEGATION_TOKEN_STORE_ZK_TRUSTSTORE_LOCATION =
"hive.cluster.delegation.token.store.zookeeper.truststore.location";
public static final String DELEGATION_TOKEN_STORE_ZK_TRUSTSTORE_PASSWORD =
"hive.cluster.delegation.token.store.zookeeper.truststore.password";
+ public static final String DELEGATION_TOKEN_STORE_ZK_TRUSTSTORE_TYPE =
+ "hive.cluster.delegation.token.store.zookeeper.truststore.type";
public MetastoreDelegationTokenManager() {
}
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/ZooKeeperTokenStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/ZooKeeperTokenStore.java
index 94066857398..aa2f0cccffa 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/ZooKeeperTokenStore.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/security/ZooKeeperTokenStore.java
@@ -69,8 +69,10 @@ public class ZooKeeperTokenStore implements DelegationTokenStore {
private boolean sslEnabled;
private String keyStoreLocation;
private String keyStorePassword;
+ private String keyStoreType;
private String trustStoreLocation;
private String trustStorePassword;
+ private String trustStoreType;
private List<ACL> newNodeAcl;
private Configuration conf;
@@ -144,8 +146,10 @@ public class ZooKeeperTokenStore implements DelegationTokenStore {
.sslEnabled(sslEnabled)
.keyStoreLocation(keyStoreLocation)
.keyStorePassword(keyStorePassword)
+ .keyStoreType(keyStoreType)
.trustStoreLocation(trustStoreLocation)
.trustStorePassword(trustStorePassword)
+ .trustStoreType(trustStoreType)
.build();
zkSession = zkHelper.getNewZookeeperClient(aclDefaultProvider);
zkSession.start();
@@ -499,10 +503,14 @@ public class ZooKeeperTokenStore implements DelegationTokenStore {
keyStoreLocation = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.THRIFT_ZOOKEEPER_SSL_KEYSTORE_LOCATION);
keyStorePassword =
MetastoreConf.getPassword(conf, MetastoreConf.ConfVars.THRIFT_ZOOKEEPER_SSL_KEYSTORE_PASSWORD);
+ keyStoreType =
+ MetastoreConf.getVar(conf, MetastoreConf.ConfVars.THRIFT_ZOOKEEPER_SSL_KEYSTORE_TYPE);
trustStoreLocation =
MetastoreConf.getVar(conf, MetastoreConf.ConfVars.THRIFT_ZOOKEEPER_SSL_TRUSTSTORE_LOCATION);
trustStorePassword =
MetastoreConf.getPassword(conf, MetastoreConf.ConfVars.THRIFT_ZOOKEEPER_SSL_TRUSTSTORE_PASSWORD);
+ trustStoreType =
+ MetastoreConf.getVar(conf, MetastoreConf.ConfVars.THRIFT_ZOOKEEPER_SSL_TRUSTSTORE_TYPE);
} catch (IOException ex) {
throw new RuntimeException("Failed to read zookeeper configuration passwords", ex);
}
@@ -517,10 +525,12 @@ public class ZooKeeperTokenStore implements DelegationTokenStore {
keyStoreLocation = conf.get(MetastoreDelegationTokenManager.DELEGATION_TOKEN_STORE_ZK_KEYSTORE_LOCATION, "");
char[] pwd = conf.getPassword(MetastoreDelegationTokenManager.DELEGATION_TOKEN_STORE_ZK_KEYSTORE_PASSWORD);
keyStorePassword = pwd == null ? null : new String(pwd);
+ keyStoreType = conf.get(MetastoreDelegationTokenManager.DELEGATION_TOKEN_STORE_ZK_KEYSTORE_TYPE, "");
trustStoreLocation =
conf.get(MetastoreDelegationTokenManager.DELEGATION_TOKEN_STORE_ZK_TRUSTSTORE_LOCATION, "");
pwd = conf.getPassword(MetastoreDelegationTokenManager.DELEGATION_TOKEN_STORE_ZK_TRUSTSTORE_PASSWORD);
trustStorePassword = pwd == null ? null : new String(pwd);
+ trustStoreType = conf.get(MetastoreDelegationTokenManager.DELEGATION_TOKEN_STORE_ZK_TRUSTSTORE_TYPE, "");
} catch (IOException ex) {
throw new RuntimeException("Failed to read zookeeper configuration passwords", ex);
}