You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by yc...@apache.org on 2020/10/15 21:00:16 UTC
[hive] branch master updated: HIVE-24253: HMS and HS2 needs to
support keystore/truststores types by config (#1580) (Yongzhi Chen,
reviewed by Kevin Risden and Naveen Gangam)
This is an automated email from the ASF dual-hosted git repository.
ychena pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 036ee74 HIVE-24253: HMS and HS2 needs to support keystore/truststores types by config (#1580) (Yongzhi Chen, reviewed by Kevin Risden and Naveen Gangam)
036ee74 is described below
commit 036ee740eb423714af99a06f32d76c9708f4201b
Author: Yongzhi Chen <yo...@hotmail.com>
AuthorDate: Thu Oct 15 17:00:00 2020 -0400
HIVE-24253: HMS and HS2 needs to support keystore/truststores types by config (#1580) (Yongzhi Chen, reviewed by Kevin Risden and Naveen Gangam)
HMS and HS2 needs to support keystore/truststores types besides JKS by config
Add following optional properties for SSL related configs:
HiveServer2:
Add new properties:
hive.server2.keystore.type
hive.server2.keymanagerfactory.algorithm
JDBC:
Remove hard-coded SSL_TRUST_STORE_TYPE(JKS) from jdbc connection for HS2
Add trustStoreType param for jdbc connection
Add trustManagerFactoryAlgorithm
Hive MetaStore:
New properties for service and client:
metastore.keystore.type
metastore.keymanagerfactory.algorithm
metastore.truststore.type
metastore.trustmanagerfactory.algorithm
Add bcfks into metastore.dbaccess.ssl.truststore.type stringvalidator
Tests:
Unit tests for HS2 and HMS
---
.../hadoop/hive/common/auth/HiveAuthUtils.java | 17 +++++--
.../java/org/apache/hadoop/hive/conf/HiveConf.java | 4 ++
.../java/org/hadoop/hive/jdbc/SSLTestUtils.java | 11 ++--
.../test/java/org/apache/hive/jdbc/TestSSL.java | 58 +++++++++++++++++++++-
.../java/org/apache/hive/jdbc/HiveConnection.java | 24 +++++++--
jdbc/src/java/org/apache/hive/jdbc/Utils.java | 6 +--
.../service/cli/thrift/ThriftBinaryCLIService.java | 4 +-
.../service/cli/thrift/ThriftHttpCLIService.java | 12 ++++-
.../hadoop/hive/metastore/HiveMetaStoreClient.java | 6 ++-
.../hadoop/hive/metastore/conf/MetastoreConf.java | 10 +++-
.../hadoop/hive/metastore/utils/SecurityUtils.java | 22 ++++++--
.../hadoop/hive/metastore/HiveMetaStore.java | 7 ++-
.../metastore/HiveMetaStoreClientPreCatalog.java | 7 ++-
.../hadoop/hive/metastore/tools/HMSClient.java | 6 ++-
14 files changed, 162 insertions(+), 32 deletions(-)
diff --git a/common/src/java/org/apache/hadoop/hive/common/auth/HiveAuthUtils.java b/common/src/java/org/apache/hadoop/hive/common/auth/HiveAuthUtils.java
index 44cde6f..938fd52 100644
--- a/common/src/java/org/apache/hadoop/hive/common/auth/HiveAuthUtils.java
+++ b/common/src/java/org/apache/hadoop/hive/common/auth/HiveAuthUtils.java
@@ -57,11 +57,14 @@ public class HiveAuthUtils {
}
public static TTransport getSSLSocket(String host, int port, int loginTimeout,
- String trustStorePath, String trustStorePassWord) throws TTransportException {
+ String trustStorePath, String trustStorePassWord, String trustStoreType,
+ String trustStoreAlgorithm) throws TTransportException {
TSSLTransportFactory.TSSLTransportParameters params =
new TSSLTransportFactory.TSSLTransportParameters();
- params.setTrustStore(trustStorePath, trustStorePassWord,
- TrustManagerFactory.getDefaultAlgorithm(), KeyStore.getDefaultType());
+ String tStoreType = trustStoreType.isEmpty()? KeyStore.getDefaultType() : trustStoreType;
+ String tStoreAlgorithm = trustStoreAlgorithm.isEmpty()?
+ TrustManagerFactory.getDefaultAlgorithm() : trustStoreAlgorithm;
+ params.setTrustStore(trustStorePath, trustStorePassWord, tStoreAlgorithm, tStoreType);
params.requireClientAuth(true);
// The underlying SSLSocket object is bound to host:port with the given SO_TIMEOUT and
// SSLContext created with the given params
@@ -92,12 +95,16 @@ public class HiveAuthUtils {
}
public static TServerSocket getServerSSLSocket(String hiveHost, int portNum, String keyStorePath,
- String keyStorePassWord, List<String> sslVersionBlacklist) throws TTransportException,
+ String keyStorePassWord, String keyStoreType, String keyStoreAlgorithm,
+ List<String> sslVersionBlacklist) throws TTransportException,
UnknownHostException {
TSSLTransportFactory.TSSLTransportParameters params =
new TSSLTransportFactory.TSSLTransportParameters();
+ String kStoreType = keyStoreType.isEmpty()? KeyStore.getDefaultType() : keyStoreType;
+ String kStoreAlgorithm = keyStoreAlgorithm.isEmpty()?
+ KeyManagerFactory.getDefaultAlgorithm() : keyStoreAlgorithm;
params.setKeyStore(keyStorePath, keyStorePassWord,
- KeyManagerFactory.getDefaultAlgorithm(), KeyStore.getDefaultType());
+ kStoreAlgorithm, kStoreType);
InetSocketAddress serverAddress;
if (hiveHost == null || hiveHost.isEmpty()) {
// Wildcard bind
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index d05336b..85fe3e4 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -3975,6 +3975,10 @@ public class HiveConf extends Configuration {
"SSL certificate keystore location."),
HIVE_SERVER2_SSL_KEYSTORE_PASSWORD("hive.server2.keystore.password", "",
"SSL certificate keystore password."),
+ HIVE_SERVER2_SSL_KEYSTORE_TYPE("hive.server2.keystore.type", "",
+ "SSL certificate keystore type."),
+ HIVE_SERVER2_SSL_KEYMANAGERFACTORY_ALGORITHM("hive.server2.keymanagerfactory.algorithm", "",
+ "SSL certificate keystore algorithm."),
HIVE_SERVER2_BUILTIN_UDF_WHITELIST("hive.server2.builtin.udf.whitelist", "",
"Comma separated list of builtin udf names allowed in queries.\n" +
"An empty whitelist allows all builtin udfs to be executed. " +
diff --git a/itests/hive-unit/src/main/java/org/hadoop/hive/jdbc/SSLTestUtils.java b/itests/hive-unit/src/main/java/org/hadoop/hive/jdbc/SSLTestUtils.java
index 78a23a2..b8e7e3d 100644
--- a/itests/hive-unit/src/main/java/org/hadoop/hive/jdbc/SSLTestUtils.java
+++ b/itests/hive-unit/src/main/java/org/hadoop/hive/jdbc/SSLTestUtils.java
@@ -26,6 +26,7 @@ import java.util.Map;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
public class SSLTestUtils {
@@ -55,14 +56,14 @@ public class SSLTestUtils {
}
public static void setMetastoreSslConf(HiveConf conf) {
- conf.setBoolVar(HiveConf.ConfVars.HIVE_METASTORE_USE_SSL, true);
- conf.setVar(HiveConf.ConfVars.HIVE_METASTORE_SSL_KEYSTORE_PATH,
+ MetastoreConf.setBoolVar(conf, MetastoreConf.ConfVars.USE_SSL, true);
+ MetastoreConf.setVar(conf, MetastoreConf.ConfVars.SSL_KEYSTORE_PATH,
dataFileDir + File.separator + LOCALHOST_KEY_STORE_NAME);
- conf.setVar(HiveConf.ConfVars.HIVE_METASTORE_SSL_KEYSTORE_PASSWORD,
+ MetastoreConf.setVar(conf, MetastoreConf.ConfVars.SSL_KEYSTORE_PASSWORD,
KEY_STORE_TRUST_STORE_PASSWORD);
- conf.setVar(HiveConf.ConfVars.HIVE_METASTORE_SSL_TRUSTSTORE_PATH,
+ MetastoreConf.setVar(conf, MetastoreConf.ConfVars.SSL_TRUSTSTORE_PATH,
dataFileDir + File.separator + TRUST_STORE_NAME);
- conf.setVar(HiveConf.ConfVars.HIVE_METASTORE_SSL_TRUSTSTORE_PASSWORD,
+ MetastoreConf.setVar(conf, MetastoreConf.ConfVars.SSL_TRUSTSTORE_PASSWORD,
KEY_STORE_TRUST_STORE_PASSWORD);
}
diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestSSL.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestSSL.java
index 3d63250..1d170ec 100644
--- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestSSL.java
+++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestSSL.java
@@ -32,6 +32,7 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
@@ -60,8 +61,10 @@ public class TestSSL {
private static final String EXAMPLEDOTCOM_KEY_STORE_NAME = "keystore_exampledotcom.jks";
private static final String TRUST_STORE_NAME = "truststore.jks";
private static final String KEY_STORE_TRUST_STORE_PASSWORD = "HiveJdbc";
+ private static final String KEY_STORE_TRUST_STORE_TYPE = "jks";
private static final String JAVA_TRUST_STORE_PROP = "javax.net.ssl.trustStore";
private static final String JAVA_TRUST_STORE_PASS_PROP = "javax.net.ssl.trustStorePassword";
+ private static final String JAVA_TRUST_STORE_TYPE_PROP = "javax.net.ssl.trustStoreType";
private MiniHS2 miniHS2 = null;
private static HiveConf conf = new HiveConf();
@@ -436,8 +439,28 @@ public class TestSSL {
*/
@Test
public void testMetastoreWithSSL() throws Exception {
+ testSSLHMS(true);
+ }
+
+ /**
+ * Test HMS server with SSL with input keystore type
+ * @throws Exception
+ */
+ @Test
+ public void testMetastoreWithSSLKeyStoreType() throws Exception {
+ testSSLHMS(false);
+ }
+
+ private void testSSLHMS(boolean useDefaultStoreType) throws Exception {
SSLTestUtils.setMetastoreSslConf(conf);
SSLTestUtils.setSslConfOverlay(confOverlay);
+ // Test in binary mode
+ SSLTestUtils.setBinaryConfOverlay(confOverlay);
+ if (!useDefaultStoreType) {
+ confOverlay.put(ConfVars.HIVE_SERVER2_SSL_KEYSTORE_TYPE.varname, KEY_STORE_TRUST_STORE_TYPE);
+ MetastoreConf.setVar(conf, MetastoreConf.ConfVars.SSL_KEYSTORE_TYPE, KEY_STORE_TRUST_STORE_TYPE);
+ }
+ //MetastoreConf.setBoolVar(conf, MetastoreConf.ConfVars.EVENT_DB_NOTIFICATION_API_AUTH, false);
// Test in http mode
SSLTestUtils.setHttpConfOverlay(confOverlay);
miniHS2 = new MiniHS2.Builder().withRemoteMetastore().withConf(conf).cleanupLocalDirOnStartup(false).build();
@@ -448,7 +471,7 @@ public class TestSSL {
// make SSL connection
hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL("default", SSLTestUtils.SSL_CONN_PARAMS),
- System.getProperty("user.name"), "bar");
+ System.getProperty("user.name"), "bar");
// Set up test data
SSLTestUtils.setupTestTableWithData(tableName, dataFilePath, hs2Conn);
@@ -464,7 +487,6 @@ public class TestSSL {
hs2Conn.close();
}
-
/**
* Verify the HS2 can't connect to HMS if the certificate doesn't match
* @throws Exception
@@ -484,4 +506,36 @@ public class TestSSL {
miniHS2.stop();
}
+
+ /***
+ * Test SSL client connection to SSL server
+ * @throws Exception
+ */
+ @Test
+ public void testSSLConnectionWithKeystoreType() throws Exception {
+ SSLTestUtils.setSslConfOverlay(confOverlay);
+ confOverlay.put(ConfVars.HIVE_SERVER2_SSL_KEYSTORE_TYPE.varname, KEY_STORE_TRUST_STORE_TYPE);
+ // Test in binary mode
+ SSLTestUtils.setBinaryConfOverlay(confOverlay);
+ // Start HS2 with SSL
+ miniHS2.start(confOverlay);
+
+ System.setProperty(JAVA_TRUST_STORE_PROP, dataFileDir + File.separator + TRUST_STORE_NAME );
+ System.setProperty(JAVA_TRUST_STORE_PASS_PROP, KEY_STORE_TRUST_STORE_PASSWORD);
+ System.setProperty(JAVA_TRUST_STORE_TYPE_PROP, KEY_STORE_TRUST_STORE_TYPE);
+ // make SSL connection
+ hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL() + ";ssl=true",
+ System.getProperty("user.name"), "bar");
+ hs2Conn.close();
+ miniHS2.stop();
+
+ // Test in http mode
+ SSLTestUtils.setHttpConfOverlay(confOverlay);
+ miniHS2.start(confOverlay);
+ // make SSL connection
+ hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL("default",
+ SSLTestUtils.SSL_CONN_PARAMS) + ";trustStoreType=" + KEY_STORE_TRUST_STORE_TYPE,
+ System.getProperty("user.name"), "bar");
+ hs2Conn.close();
+ }
}
diff --git a/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java b/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java
index e473521..ab7e2e0 100644
--- a/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java
+++ b/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java
@@ -618,7 +618,11 @@ public class HiveConnection implements java.sql.Connection {
socketFactory = SSLConnectionSocketFactory.getSocketFactory();
} else {
// Pick trust store config from the given path
- sslTrustStore = KeyStore.getInstance(JdbcConnectionParams.SSL_TRUST_STORE_TYPE);
+ String trustStoreType = sessConfMap.get(JdbcConnectionParams.SSL_TRUST_STORE_TYPE);
+ if (trustStoreType == null || trustStoreType.isEmpty()) {
+ trustStoreType = KeyStore.getDefaultType();
+ }
+ sslTrustStore = KeyStore.getInstance(trustStoreType);
try (FileInputStream fis = new FileInputStream(sslTrustStorePath)) {
sslTrustStore.load(fis, sslTrustStorePassword.toCharArray());
}
@@ -661,8 +665,18 @@ public class HiveConnection implements java.sql.Connection {
if (sslTrustStore == null || sslTrustStore.isEmpty()) {
transport = HiveAuthUtils.getSSLSocket(host, port, loginTimeout);
} else {
+ String trustStoreType =
+ sessConfMap.get(JdbcConnectionParams.SSL_TRUST_STORE_TYPE);
+ if (trustStoreType == null) {
+ trustStoreType = "";
+ }
+ String trustStoreAlgorithm =
+ sessConfMap.get(JdbcConnectionParams.SSL_TRUST_MANAGER_FACTORY_ALGORITHM);
+ if (trustStoreAlgorithm == null) {
+ trustStoreAlgorithm = "";
+ }
transport = HiveAuthUtils.getSSLSocket(host, port, loginTimeout,
- sslTrustStore, sslTrustStorePassword);
+ sslTrustStore, sslTrustStorePassword, trustStoreType, trustStoreAlgorithm);
}
} else {
// get non-SSL socket transport
@@ -760,7 +774,11 @@ public class HiveConnection implements java.sql.Connection {
String trustStorePath = sessConfMap.get(JdbcConnectionParams.SSL_TRUST_STORE);
String trustStorePassword = sessConfMap.get(
JdbcConnectionParams.SSL_TRUST_STORE_PASSWORD);
- KeyStore sslTrustStore = KeyStore.getInstance(JdbcConnectionParams.SSL_TRUST_STORE_TYPE);
+ String trustStoreType = sessConfMap.get(JdbcConnectionParams.SSL_TRUST_STORE_TYPE);
+ if (trustStoreType == null || trustStoreType.isEmpty()) {
+ trustStoreType = KeyStore.getDefaultType();
+ }
+ KeyStore sslTrustStore = KeyStore.getInstance(trustStoreType);
if (trustStorePath == null || trustStorePath.isEmpty()) {
throw new IllegalArgumentException(JdbcConnectionParams.SSL_TRUST_STORE
diff --git a/jdbc/src/java/org/apache/hive/jdbc/Utils.java b/jdbc/src/java/org/apache/hive/jdbc/Utils.java
index 737a5f7..4c8b06f 100644
--- a/jdbc/src/java/org/apache/hive/jdbc/Utils.java
+++ b/jdbc/src/java/org/apache/hive/jdbc/Utils.java
@@ -105,6 +105,8 @@ public class Utils {
public static final String USE_SSL = "ssl";
public static final String SSL_TRUST_STORE = "sslTrustStore";
public static final String SSL_TRUST_STORE_PASSWORD = "trustStorePassword";
+ public static final String SSL_TRUST_STORE_TYPE = "trustStoreType";
+ public static final String SSL_TRUST_MANAGER_FACTORY_ALGORITHM = "trustManagerFactoryAlgorithm";
// We're deprecating the name and placement of this in the parsed map (from hive conf vars to
// hive session vars).
static final String TRANSPORT_MODE_DEPRECATED = "hive.server2.transport.mode";
@@ -165,10 +167,6 @@ public class Utils {
static final String SUNJSSE_ALGORITHM_STRING = "SunJSSE";
// --------------- End 2 way ssl options ----------------------------
- // Non-configurable params:
- // Currently supports JKS keystore format
- static final String SSL_TRUST_STORE_TYPE = "JKS";
-
private static final String HIVE_VAR_PREFIX = "hivevar:";
public static final String HIVE_CONF_PREFIX = "hiveconf:";
private String host = null;
diff --git a/service/src/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java b/service/src/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java
index dfe99e6..95294ca 100644
--- a/service/src/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java
+++ b/service/src/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java
@@ -92,8 +92,10 @@ public class ThriftBinaryCLIService extends ThriftCLIService {
}
String keyStorePassword = ShimLoader.getHadoopShims().getPassword(hiveConf,
HiveConf.ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PASSWORD.varname);
+ String keyStoreType = hiveConf.getVar(ConfVars.HIVE_SERVER2_SSL_KEYSTORE_TYPE).trim();
+ String keyStoreAlgorithm = hiveConf.getVar(ConfVars.HIVE_SERVER2_SSL_KEYMANAGERFACTORY_ALGORITHM).trim();
serverSocket = HiveAuthUtils.getServerSSLSocket(hiveHost, portNum, keyStorePath, keyStorePassword,
- sslVersionBlacklist);
+ keyStoreType, keyStoreAlgorithm, sslVersionBlacklist);
}
// Server args
diff --git a/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java b/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
index 2757d60..03d3878 100644
--- a/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
+++ b/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java
@@ -25,6 +25,7 @@ import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
+import javax.net.ssl.KeyManagerFactory;
import javax.ws.rs.HttpMethod;
import org.apache.hadoop.hive.common.metrics.common.Metrics;
@@ -136,6 +137,14 @@ public class ThriftHttpCLIService extends ThriftCLIService {
ConfVars.HIVE_SERVER2_SSL_KEYSTORE_PATH.varname
+ " Not configured for SSL connection");
}
+ String keyStoreType = hiveConf.getVar(ConfVars.HIVE_SERVER2_SSL_KEYSTORE_TYPE).trim();
+ if (keyStoreType.isEmpty()) {
+ keyStoreType = KeyStore.getDefaultType();
+ }
+ String keyStoreAlgorithm = hiveConf.getVar(ConfVars.HIVE_SERVER2_SSL_KEYMANAGERFACTORY_ALGORITHM).trim();
+ if (keyStoreAlgorithm.isEmpty()) {
+ keyStoreAlgorithm = KeyManagerFactory.getDefaultAlgorithm();
+ }
SslContextFactory sslContextFactory = new SslContextFactory();
String[] excludedProtocols = hiveConf.getVar(ConfVars.HIVE_SSL_PROTOCOL_BLACKLIST).split(",");
LOG.info("HTTP Server SSL: adding excluded protocols: " + Arrays.toString(excludedProtocols));
@@ -144,7 +153,8 @@ public class ThriftHttpCLIService extends ThriftCLIService {
+ Arrays.toString(sslContextFactory.getExcludeProtocols()));
sslContextFactory.setKeyStorePath(keyStorePath);
sslContextFactory.setKeyStorePassword(keyStorePassword);
- sslContextFactory.setKeyStoreType(KeyStore.getDefaultType());
+ sslContextFactory.setKeyStoreType(keyStoreType);
+ sslContextFactory.setKeyManagerFactoryAlgorithm(keyStoreAlgorithm);
connector = new ServerConnector(server, sslContextFactory, http);
} else {
connector = new ServerConnector(server, http);
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
index dea55b3..f22a1a3 100644
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
+++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
@@ -600,10 +600,14 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
}
String trustStorePassword =
MetastoreConf.getPassword(conf, MetastoreConf.ConfVars.SSL_TRUSTSTORE_PASSWORD);
+ String trustStoreType =
+ MetastoreConf.getVar(conf, ConfVars.SSL_TRUSTSTORE_TYPE).trim();
+ String trustStoreAlgorithm =
+ MetastoreConf.getVar(conf, ConfVars.SSL_TRUSTMANAGERFACTORY_ALGORITHM).trim();
// Create an SSL socket and connect
transport = SecurityUtils.getSSLSocket(store.getHost(), store.getPort(), clientSocketTimeout,
- trustStorePath, trustStorePassword);
+ trustStorePath, trustStorePassword, trustStoreType, trustStoreAlgorithm);
final int newCount = connCount.incrementAndGet();
LOG.debug(
"Opened an SSL connection to metastore, current connections: {}",
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
index fe79659..c179ace 100644
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
+++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
@@ -518,7 +518,7 @@ public class MetastoreConf {
+ "metastore.dbaccess.ssl.use.SSL must be set to true for this property to take effect. \n"
+ "This directly maps to the javax.net.ssl.trustStore Java system property. Defaults to the default Java truststore file. \n"),
DBACCESS_SSL_TRUSTSTORE_TYPE("metastore.dbaccess.ssl.truststore.type", "hive.metastore.dbaccess.ssl.truststore.type", "jks",
- new StringSetValidator("jceks", "jks", "dks", "pkcs11", "pkcs12"),
+ new StringSetValidator("jceks", "jks", "dks", "pkcs11", "pkcs12", "bcfks"),
"File type for the Java truststore file that is used when encrypting the connection to the database store. \n"
+ "metastore.dbaccess.ssl.use.SSL must be set to true for this property to take effect. \n"
+ "This directly maps to the javax.net.ssl.trustStoreType Java system property. \n"
@@ -1049,12 +1049,20 @@ public class MetastoreConf {
"Metastore SSL certificate keystore password."),
SSL_KEYSTORE_PATH("metastore.keystore.path", "hive.metastore.keystore.path", "",
"Metastore SSL certificate keystore location."),
+ SSL_KEYSTORE_TYPE("metastore.keystore.type", "hive.metastore.keystore.type", "",
+ "Metastore SSL certificate keystore type."),
+ SSL_KEYMANAGERFACTORY_ALGORITHM("metastore.keymanagerfactory.algorithm", "hive.metastore.keymanagerfactory.algorithm", "",
+ "Metastore SSL certificate keystore algorithm."),
SSL_PROTOCOL_BLACKLIST("metastore.ssl.protocol.blacklist", "hive.ssl.protocol.blacklist",
"SSLv2,SSLv3", "SSL Versions to disable for all Hive Servers"),
SSL_TRUSTSTORE_PATH("metastore.truststore.path", "hive.metastore.truststore.path", "",
"Metastore SSL certificate truststore location."),
SSL_TRUSTSTORE_PASSWORD("metastore.truststore.password", "hive.metastore.truststore.password", "",
"Metastore SSL certificate truststore password."),
+ SSL_TRUSTSTORE_TYPE("metastore.truststore.type", "hive.metastore.truststore.type", "",
+ "Metastore SSL certificate truststore type."),
+ SSL_TRUSTMANAGERFACTORY_ALGORITHM("metastore.trustmanagerfactory.algorithm", "hive.metastore.trustmanagerfactory.algorithm", "",
+ "Metastore SSL certificate truststore algorithm."),
STATS_AUTO_GATHER("metastore.stats.autogather", "hive.stats.autogather", true,
"A flag to gather statistics (only basic) automatically during the INSERT OVERWRITE command."),
STATS_FETCH_BITVECTOR("metastore.stats.fetch.bitvector", "hive.stats.fetch.bitvector", false,
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/SecurityUtils.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/SecurityUtils.java
index 5ce340f..2b326b2 100644
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/SecurityUtils.java
+++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/SecurityUtils.java
@@ -36,15 +36,19 @@ import org.apache.zookeeper.client.ZooKeeperSaslClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import javax.net.ssl.KeyManagerFactory;
import javax.net.ssl.SSLParameters;
import javax.net.ssl.SSLServerSocket;
import javax.net.ssl.SSLSocket;
+import javax.net.ssl.TrustManagerFactory;
import javax.security.auth.login.AppConfigurationEntry;
import javax.security.auth.login.AppConfigurationEntry.LoginModuleControlFlag;
import javax.security.auth.login.LoginException;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.UnknownHostException;
+import java.security.KeyStore;
+
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
@@ -212,11 +216,14 @@ public class SecurityUtils {
}
public static TServerSocket getServerSSLSocket(String hiveHost, int portNum, String keyStorePath,
- String keyStorePassWord, List<String> sslVersionBlacklist) throws TTransportException,
- UnknownHostException {
+ String keyStorePassWord, String keyStoreType, String keyStoreAlgorithm, List<String> sslVersionBlacklist)
+ throws TTransportException, UnknownHostException {
TSSLTransportFactory.TSSLTransportParameters params =
new TSSLTransportFactory.TSSLTransportParameters();
- params.setKeyStore(keyStorePath, keyStorePassWord);
+ String kStoreType = keyStoreType.isEmpty()? KeyStore.getDefaultType() : keyStoreType;
+ String kStoreAlgorithm = keyStoreAlgorithm.isEmpty()?
+ KeyManagerFactory.getDefaultAlgorithm() : keyStoreAlgorithm;
+ params.setKeyStore(keyStorePath, keyStorePassWord, kStoreAlgorithm, kStoreType);
InetSocketAddress serverAddress;
if (hiveHost == null || hiveHost.isEmpty()) {
// Wildcard bind
@@ -248,10 +255,15 @@ public class SecurityUtils {
}
public static TTransport getSSLSocket(String host, int port, int loginTimeout,
- String trustStorePath, String trustStorePassWord) throws TTransportException {
+ String trustStorePath, String trustStorePassWord, String trustStoreType,
+ String trustStoreAlgorithm) throws TTransportException {
TSSLTransportFactory.TSSLTransportParameters params =
new TSSLTransportFactory.TSSLTransportParameters();
- params.setTrustStore(trustStorePath, trustStorePassWord);
+ String tStoreType = trustStoreType.isEmpty()? KeyStore.getDefaultType() : trustStoreType;
+ String tStoreAlgorithm = trustStoreAlgorithm.isEmpty()?
+ TrustManagerFactory.getDefaultAlgorithm() : trustStoreAlgorithm;
+ params.setTrustStore(trustStorePath, trustStorePassWord,
+ tStoreAlgorithm, tStoreType);
params.requireClientAuth(true);
// The underlying SSLSocket object is bound to host:port with the given SO_TIMEOUT and
// SSLContext created with the given params
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index 6a8178e..4a249fb 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -10589,7 +10589,10 @@ public class HiveMetaStore extends ThriftHiveMetastore {
}
String keyStorePassword =
MetastoreConf.getPassword(conf, MetastoreConf.ConfVars.SSL_KEYSTORE_PASSWORD);
-
+ String keyStoreType =
+ MetastoreConf.getVar(conf, ConfVars.SSL_KEYSTORE_TYPE).trim();
+ String keyStoreAlgorithm =
+ MetastoreConf.getVar(conf, ConfVars.SSL_KEYMANAGERFACTORY_ALGORITHM).trim();
// enable SSL support for HMS
List<String> sslVersionBlacklist = new ArrayList<>();
for (String sslVersion : MetastoreConf.getVar(conf, ConfVars.SSL_PROTOCOL_BLACKLIST).split(",")) {
@@ -10597,7 +10600,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
}
serverSocket = SecurityUtils.getServerSSLSocket(msHost, port, keyStorePath,
- keyStorePassword, sslVersionBlacklist);
+ keyStorePassword, keyStoreType, keyStoreAlgorithm, sslVersionBlacklist);
}
if (tcpKeepAlive) {
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java
index f65ea81..6ed4684 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java
@@ -455,10 +455,15 @@ public class HiveMetaStoreClientPreCatalog implements IMetaStoreClient, AutoClos
}
String trustStorePassword =
MetastoreConf.getPassword(conf, MetastoreConf.ConfVars.SSL_TRUSTSTORE_PASSWORD);
+ String trustStoreType =
+ MetastoreConf.getVar(conf, ConfVars.SSL_TRUSTSTORE_TYPE).trim();
+ String trustStoreAlgorithm =
+ MetastoreConf.getVar(conf, ConfVars.SSL_TRUSTMANAGERFACTORY_ALGORITHM).trim();
+
// Create an SSL socket and connect
transport = SecurityUtils.getSSLSocket(store.getHost(), store.getPort(), clientSocketTimeout,
- trustStorePath, trustStorePassword );
+ trustStorePath, trustStorePassword, trustStoreType, trustStoreAlgorithm );
LOG.info("Opened an SSL connection to metastore, current connections: " + connCount.incrementAndGet());
} catch(IOException e) {
throw new IllegalArgumentException(e);
diff --git a/standalone-metastore/metastore-tools/tools-common/src/main/java/org/apache/hadoop/hive/metastore/tools/HMSClient.java b/standalone-metastore/metastore-tools/tools-common/src/main/java/org/apache/hadoop/hive/metastore/tools/HMSClient.java
index d5fdfbb..dcb5cd4 100644
--- a/standalone-metastore/metastore-tools/tools-common/src/main/java/org/apache/hadoop/hive/metastore/tools/HMSClient.java
+++ b/standalone-metastore/metastore-tools/tools-common/src/main/java/org/apache/hadoop/hive/metastore/tools/HMSClient.java
@@ -414,10 +414,14 @@ final class HMSClient implements AutoCloseable {
}
String trustStorePassword =
MetastoreConf.getPassword(conf, MetastoreConf.ConfVars.SSL_TRUSTSTORE_PASSWORD);
+ String trustStoreType =
+ MetastoreConf.getVar(conf, MetastoreConf.ConfVars.SSL_TRUSTSTORE_TYPE).trim();
+ String trustStoreAlgorithm =
+ MetastoreConf.getVar(conf, MetastoreConf.ConfVars.SSL_TRUSTMANAGERFACTORY_ALGORITHM).trim();
// Create an SSL socket and connect
transport = SecurityUtils.getSSLSocket(host, port, clientSocketTimeout,
- trustStorePath, trustStorePassword);
+ trustStorePath, trustStorePassword, trustStoreType, trustStoreAlgorithm);
LOG.info("Opened an SSL connection to metastore, current connections");
}