You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by xi...@apache.org on 2024/02/29 00:24:35 UTC
(pinot) branch master updated: refactor TlsUtils class (#12515)
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 53b3bfa41c refactor TlsUtils class (#12515)
53b3bfa41c is described below
commit 53b3bfa41c017975554966c3f0335e274998b3d0
Author: Haitao Zhang <ha...@startree.ai>
AuthorDate: Wed Feb 28 16:24:30 2024 -0800
refactor TlsUtils class (#12515)
---
.../pinot/common/utils/grpc/GrpcQueryClient.java | 4 +-
.../common/utils/tls/JvmDefaultSslContext.java | 2 +-
.../pinot/common/utils/tls/RenewableTlsUtils.java | 268 +++++++++++++++++++++
.../apache/pinot/common/utils/tls/TlsUtils.java | 237 +-----------------
.../pinot/common/utils/tls/TlsUtilsTest.java | 26 +-
.../pinot/core/transport/grpc/GrpcQueryServer.java | 4 +-
.../apache/pinot/core/util/ListenerConfigUtil.java | 3 +-
7 files changed, 294 insertions(+), 250 deletions(-)
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/utils/grpc/GrpcQueryClient.java b/pinot-common/src/main/java/org/apache/pinot/common/utils/grpc/GrpcQueryClient.java
index af4ddf0181..3ba78d98d6 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/utils/grpc/GrpcQueryClient.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/utils/grpc/GrpcQueryClient.java
@@ -36,7 +36,7 @@ import org.apache.pinot.common.config.GrpcConfig;
import org.apache.pinot.common.config.TlsConfig;
import org.apache.pinot.common.proto.PinotQueryServerGrpc;
import org.apache.pinot.common.proto.Server;
-import org.apache.pinot.common.utils.tls.TlsUtils;
+import org.apache.pinot.common.utils.tls.RenewableTlsUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -73,7 +73,7 @@ public class GrpcQueryClient {
LOGGER.info("Building gRPC SSL context");
SslContext sslContext = CLIENT_SSL_CONTEXTS_CACHE.computeIfAbsent(tlsConfig.hashCode(), tlsConfigHashCode -> {
try {
- SSLFactory sslFactory = TlsUtils.createSSLFactoryAndEnableAutoRenewalWhenUsingFileStores(tlsConfig);
+ SSLFactory sslFactory = RenewableTlsUtils.createSSLFactoryAndEnableAutoRenewalWhenUsingFileStores(tlsConfig);
SslContextBuilder sslContextBuilder = SslContextBuilder.forClient();
sslFactory.getKeyManagerFactory().ifPresent(sslContextBuilder::keyManager);
sslFactory.getTrustManagerFactory().ifPresent(sslContextBuilder::trustManager);
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/utils/tls/JvmDefaultSslContext.java b/pinot-common/src/main/java/org/apache/pinot/common/utils/tls/JvmDefaultSslContext.java
index bc80b77084..ecd424ef45 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/utils/tls/JvmDefaultSslContext.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/utils/tls/JvmDefaultSslContext.java
@@ -99,7 +99,7 @@ public class JvmDefaultSslContext {
String jvmTrustStorePassword =
Optional.ofNullable(System.getProperty(JVM_TRUST_STORE_PASSWORD))
.map(String::trim).filter(StringUtils::isNotBlank).orElse(null);
- TlsUtils.enableAutoRenewalFromFileStoreForSSLFactory(jvmSslFactory, jvmKeystoreType, jvmKeyStorePath,
+ RenewableTlsUtils.enableAutoRenewalFromFileStoreForSSLFactory(jvmSslFactory, jvmKeystoreType, jvmKeyStorePath,
jvmKeystorePassword, jvmTrustStoreType, jvmTrustStorePath, jvmTrustStorePassword, null, null, false);
}
_initialized = true;
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/utils/tls/RenewableTlsUtils.java b/pinot-common/src/main/java/org/apache/pinot/common/utils/tls/RenewableTlsUtils.java
new file mode 100644
index 0000000000..e8cb6140c8
--- /dev/null
+++ b/pinot-common/src/main/java/org/apache/pinot/common/utils/tls/RenewableTlsUtils.java
@@ -0,0 +1,268 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.common.utils.tls;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.nio.file.FileSystems;
+import java.nio.file.Path;
+import java.nio.file.StandardWatchEventKinds;
+import java.nio.file.WatchEvent;
+import java.nio.file.WatchKey;
+import java.nio.file.WatchService;
+import java.security.SecureRandom;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Executors;
+import nl.altindag.ssl.SSLFactory;
+import nl.altindag.ssl.keymanager.HotSwappableX509ExtendedKeyManager;
+import nl.altindag.ssl.trustmanager.HotSwappableX509ExtendedTrustManager;
+import nl.altindag.ssl.util.SSLFactoryUtils;
+import org.apache.pinot.common.config.TlsConfig;
+import org.apache.pinot.spi.utils.retry.RetryPolicies;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Utility class for shared renewable TLS configuration logic
+ */
+public class RenewableTlsUtils {
+ private static final Logger LOGGER = LoggerFactory.getLogger(RenewableTlsUtils.class);
+ private static final String FILE_SCHEME = "file";
+
+ private RenewableTlsUtils() {
+ // left blank
+ }
+
+
+ /**
+ * Create a {@link SSLFactory} instance with identity material and trust material swappable for a given TlsConfig,
+ * and nables auto renewal of the {@link SSLFactory} instance when
+ * 1. the {@link SSLFactory} is created with a key manager and trust manager swappable
+ * 2. the key store is null or a local file
+ * 3. the trust store is null or a local file
+ * 4. the key store or trust store file changes.
+ * @param tlsConfig {@link TlsConfig}
+ * @return a {@link SSLFactory} instance with identity material and trust material swappable
+ */
+ public static SSLFactory createSSLFactoryAndEnableAutoRenewalWhenUsingFileStores(TlsConfig tlsConfig) {
+ SSLFactory sslFactory = createSSLFactory(tlsConfig);
+ if (TlsUtils.isKeyOrTrustStorePathNullOrHasFileScheme(tlsConfig.getKeyStorePath())
+ && TlsUtils.isKeyOrTrustStorePathNullOrHasFileScheme(tlsConfig.getTrustStorePath())) {
+ enableAutoRenewalFromFileStoreForSSLFactory(sslFactory, tlsConfig);
+ }
+ return sslFactory;
+ }
+
+ /**
+ * Create a {@link SSLFactory} instance with identity material and trust material swappable for a given TlsConfig
+ * @param tlsConfig {@link TlsConfig}
+ * @return a {@link SSLFactory} instance with identity material and trust material swappable
+ */
+ private static SSLFactory createSSLFactory(TlsConfig tlsConfig) {
+ return createSSLFactory(
+ tlsConfig.getKeyStoreType(), tlsConfig.getKeyStorePath(), tlsConfig.getKeyStorePassword(),
+ tlsConfig.getTrustStoreType(), tlsConfig.getTrustStorePath(), tlsConfig.getTrustStorePassword(),
+ null, null, true, tlsConfig.isInsecure());
+ }
+
+ static SSLFactory createSSLFactory(
+ String keyStoreType, String keyStorePath, String keyStorePassword,
+ String trustStoreType, String trustStorePath, String trustStorePassword,
+ String sslContextProtocol, SecureRandom secureRandom, boolean keyAndTrustMaterialSwappable, boolean isInsecure) {
+ try {
+ SSLFactory.Builder sslFactoryBuilder = SSLFactory.builder();
+ InputStream keyStoreStream = null;
+ InputStream trustStoreStream = null;
+ if (keyStorePath != null) {
+ Preconditions.checkNotNull(keyStorePassword, "key store password must not be null");
+ keyStoreStream = TlsUtils.makeKeyOrTrustStoreUrl(keyStorePath).openStream();
+ if (keyAndTrustMaterialSwappable) {
+ sslFactoryBuilder.withSwappableIdentityMaterial();
+ }
+ sslFactoryBuilder.withIdentityMaterial(keyStoreStream, keyStorePassword.toCharArray(), keyStoreType);
+ }
+ if (isInsecure) {
+ if (keyAndTrustMaterialSwappable) {
+ sslFactoryBuilder.withSwappableTrustMaterial();
+ }
+ sslFactoryBuilder.withUnsafeTrustMaterial();
+ } else if (trustStorePath != null) {
+ Preconditions.checkNotNull(trustStorePassword, "trust store password must not be null");
+ trustStoreStream = TlsUtils.makeKeyOrTrustStoreUrl(trustStorePath).openStream();
+ if (keyAndTrustMaterialSwappable) {
+ sslFactoryBuilder.withSwappableTrustMaterial();
+ }
+ sslFactoryBuilder.withTrustMaterial(trustStoreStream, trustStorePassword.toCharArray(), trustStoreType);
+ }
+ if (sslContextProtocol != null) {
+ sslFactoryBuilder.withSslContextAlgorithm(sslContextProtocol);
+ }
+ if (secureRandom != null) {
+ sslFactoryBuilder.withSecureRandom(secureRandom);
+ }
+ SSLFactory sslFactory = sslFactoryBuilder.build();
+ if (keyStoreStream != null) {
+ keyStoreStream.close();
+ }
+ if (trustStoreStream != null) {
+ trustStoreStream.close();
+ }
+ LOGGER.info("Successfully created SSLFactory {} with key store {} and trust store {}. "
+ + "Key and trust material swappable: {}",
+ sslFactory, keyStorePath, trustStorePath, keyAndTrustMaterialSwappable);
+ return sslFactory;
+ } catch (Exception e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ /**
+ * Enables auto renewal of SSLFactory when
+ * 1. the {@link SSLFactory} is created with a key manager and trust manager swappable
+ * 2. the key store is null or a local file
+ * 3. the trust store is null or a local file
+ * 4. the key store or trust store file changes.
+ * @param sslFactory the {@link SSLFactory} to enable key manager and trust manager auto renewal
+ * @param tlsConfig the {@link TlsConfig} to get the key store and trust store information
+ */
+ @VisibleForTesting
+ static void enableAutoRenewalFromFileStoreForSSLFactory(SSLFactory sslFactory, TlsConfig tlsConfig) {
+ enableAutoRenewalFromFileStoreForSSLFactory(sslFactory,
+ tlsConfig.getKeyStoreType(), tlsConfig.getKeyStorePath(), tlsConfig.getKeyStorePassword(),
+ tlsConfig.getTrustStoreType(), tlsConfig.getTrustStorePath(), tlsConfig.getTrustStorePassword(),
+ null, null, tlsConfig.isInsecure());
+ }
+
+ static void enableAutoRenewalFromFileStoreForSSLFactory(SSLFactory sslFactory, String keyStoreType,
+ String keyStorePath, String keyStorePassword, String trustStoreType, String trustStorePath,
+ String trustStorePassword, String sslContextProtocol, SecureRandom secureRandom, boolean isInsecure) {
+ try {
+ URL keyStoreURL = keyStorePath == null ? null : TlsUtils.makeKeyOrTrustStoreUrl(keyStorePath);
+ URL trustStoreURL = trustStorePath == null ? null : TlsUtils.makeKeyOrTrustStoreUrl(trustStorePath);
+ if (keyStoreURL != null) {
+ Preconditions.checkArgument(
+ keyStoreURL.toURI().getScheme().startsWith(FILE_SCHEME),
+ "key store path must be a local file path or null when SSL auto renew is enabled");
+ Preconditions.checkArgument(
+ sslFactory.getKeyManager().isPresent()
+ && sslFactory.getKeyManager().get() instanceof HotSwappableX509ExtendedKeyManager,
+ "key manager of the existing SSLFactory must be swappable"
+ );
+ }
+ if (trustStoreURL != null) {
+ Preconditions.checkArgument(
+ trustStoreURL.toURI().getScheme().startsWith(FILE_SCHEME),
+ "trust store path must be a local file path or null when SSL auto renew is enabled");
+ Preconditions.checkArgument(
+ sslFactory.getTrustManager().isPresent()
+ && sslFactory.getTrustManager().get() instanceof HotSwappableX509ExtendedTrustManager,
+ "trust manager of the existing SSLFactory must be swappable"
+ );
+ }
+ // The reloadSslFactoryWhenFileStoreChanges is a blocking call, so we need to create a new thread to run it.
+ // Creating a new thread to run the reloadSslFactoryWhenFileStoreChanges is costly; however, unless we
+ // invoke the createAutoRenewedSSLFactoryFromFileStore method crazily, this should not be a problem.
+ Executors.newSingleThreadExecutor().execute(() -> {
+ try {
+ reloadSslFactoryWhenFileStoreChanges(sslFactory,
+ keyStoreType, keyStorePath, keyStorePassword,
+ trustStoreType, trustStorePath, trustStorePassword,
+ sslContextProtocol, secureRandom, isInsecure);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ });
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @VisibleForTesting
+ static void reloadSslFactoryWhenFileStoreChanges(SSLFactory baseSslFactory,
+ String keyStoreType, String keyStorePath, String keyStorePassword,
+ String trustStoreType, String trustStorePath, String trustStorePassword,
+ String sslContextProtocol, SecureRandom secureRandom, boolean isInsecure)
+ throws IOException, URISyntaxException, InterruptedException {
+ LOGGER.info("Enable auto renewal of SSLFactory {} when key store {} or trust store {} changes",
+ baseSslFactory, keyStorePath, trustStorePath);
+ WatchService watchService = FileSystems.getDefault().newWatchService();
+ Map<WatchKey, Set<Path>> watchKeyPathMap = new HashMap<>();
+ registerFile(watchService, watchKeyPathMap, keyStorePath);
+ registerFile(watchService, watchKeyPathMap, trustStorePath);
+ int maxSslFactoryReloadingAttempts = 3;
+ int sslFactoryReloadingRetryDelayMs = 1000;
+ WatchKey key;
+ while ((key = watchService.take()) != null) {
+ for (WatchEvent<?> event : key.pollEvents()) {
+ Path changedFile = (Path) event.context();
+ if (watchKeyPathMap.get(key).contains(changedFile)) {
+ LOGGER.info("Detected change in file: {}, try to renew SSLFactory {} "
+ + "(built from key store {} and truststore {})",
+ changedFile, baseSslFactory, keyStorePath, trustStorePath);
+ try {
+ // Need to retry a few times because when one file (key store or trust store) is updated, the other file
+ // (trust store or key store) may not have been fully written yet, so we need to wait a bit and retry.
+ RetryPolicies.fixedDelayRetryPolicy(maxSslFactoryReloadingAttempts, sslFactoryReloadingRetryDelayMs)
+ .attempt(() -> {
+ try {
+ SSLFactory updatedSslFactory =
+ createSSLFactory(keyStoreType, keyStorePath, keyStorePassword, trustStoreType, trustStorePath,
+ trustStorePassword, sslContextProtocol, secureRandom, false, isInsecure);
+ SSLFactoryUtils.reload(baseSslFactory, updatedSslFactory);
+ LOGGER.info("Successfully renewed SSLFactory {} (built from key store {} and truststore {}) on file"
+ + " {} changes", baseSslFactory, keyStorePath, trustStorePath, changedFile);
+ return true;
+ } catch (Exception e) {
+ LOGGER.info(
+ "Encountered issues when renewing SSLFactory {} (built from key store {} and truststore {}) on "
+ + "file {} changes", baseSslFactory, keyStorePath, trustStorePath, changedFile, e);
+ return false;
+ }
+ });
+ } catch (Exception e) {
+ LOGGER.error(
+ "Failed to renew SSLFactory {} (built from key store {} and truststore {}) on file {} changes after {} "
+ + "retries", baseSslFactory, keyStorePath, trustStorePath, changedFile,
+ maxSslFactoryReloadingAttempts, e);
+ }
+ }
+ }
+ key.reset();
+ }
+ }
+
+ @VisibleForTesting
+ static void registerFile(WatchService watchService, Map<WatchKey, Set<Path>> keyPathMap, String filePath)
+ throws IOException, URISyntaxException {
+ if (filePath == null) {
+ return;
+ }
+ Path path = Path.of(TlsUtils.makeKeyOrTrustStoreUrl(filePath).getPath());
+ WatchKey key = path.getParent().register(watchService, StandardWatchEventKinds.ENTRY_MODIFY);
+ keyPathMap.computeIfAbsent(key, k -> new HashSet<>());
+ keyPathMap.get(key).add(path.getFileName());
+ }
+}
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/utils/tls/TlsUtils.java b/pinot-common/src/main/java/org/apache/pinot/common/utils/tls/TlsUtils.java
index 56a14a97d4..8ccf5c0e51 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/utils/tls/TlsUtils.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/utils/tls/TlsUtils.java
@@ -18,32 +18,19 @@
*/
package org.apache.pinot.common.utils.tls;
-import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import io.netty.handler.ssl.ClientAuth;
import io.netty.handler.ssl.SslContext;
import io.netty.handler.ssl.SslContextBuilder;
import io.netty.handler.ssl.SslProvider;
import io.netty.handler.ssl.util.InsecureTrustManagerFactory;
-import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
-import java.nio.file.FileSystems;
-import java.nio.file.Path;
-import java.nio.file.StandardWatchEventKinds;
-import java.nio.file.WatchEvent;
-import java.nio.file.WatchKey;
-import java.nio.file.WatchService;
import java.security.KeyStore;
import java.security.SecureRandom;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicReference;
import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.KeyManagerFactory;
@@ -51,14 +38,10 @@ import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManagerFactory;
import nl.altindag.ssl.SSLFactory;
import nl.altindag.ssl.exception.GenericSSLContextException;
-import nl.altindag.ssl.keymanager.HotSwappableX509ExtendedKeyManager;
-import nl.altindag.ssl.trustmanager.HotSwappableX509ExtendedTrustManager;
-import nl.altindag.ssl.util.SSLFactoryUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.http.ssl.SSLContexts;
import org.apache.pinot.common.config.TlsConfig;
import org.apache.pinot.spi.env.PinotConfiguration;
-import org.apache.pinot.spi.utils.retry.RetryPolicies;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -244,13 +227,13 @@ public final class TlsUtils {
String trustStoreType, String trustStorePath, String trustStorePassword) {
try {
SecureRandom secureRandom = new SecureRandom();
- SSLFactory sslFactory = createSSLFactory(keyStoreType, keyStorePath, keyStorePassword,
+ SSLFactory sslFactory = RenewableTlsUtils.createSSLFactory(keyStoreType, keyStorePath, keyStorePassword,
trustStoreType, trustStorePath, trustStorePassword,
"SSL", secureRandom, true, false);
if (isKeyOrTrustStorePathNullOrHasFileScheme(keyStorePath)
&& isKeyOrTrustStorePathNullOrHasFileScheme(trustStorePath)) {
- enableAutoRenewalFromFileStoreForSSLFactory(sslFactory, keyStoreType, keyStorePath, keyStorePassword,
- trustStoreType, trustStorePath, trustStorePassword, "SSL", secureRandom, false);
+ RenewableTlsUtils.enableAutoRenewalFromFileStoreForSSLFactory(sslFactory, keyStoreType, keyStorePath,
+ keyStorePassword, trustStoreType, trustStorePath, trustStorePassword, "SSL", secureRandom, false);
}
// HttpsURLConnection
HttpsURLConnection.setDefaultSSLSocketFactory(sslFactory.getSslSocketFactory());
@@ -317,7 +300,7 @@ public final class TlsUtils {
* @param tlsConfig TLS config
*/
public static SslContext buildClientContext(TlsConfig tlsConfig) {
- SSLFactory sslFactory = createSSLFactoryAndEnableAutoRenewalWhenUsingFileStores(tlsConfig);
+ SSLFactory sslFactory = RenewableTlsUtils.createSSLFactoryAndEnableAutoRenewalWhenUsingFileStores(tlsConfig);
SslContextBuilder sslContextBuilder =
SslContextBuilder.forClient().sslProvider(SslProvider.valueOf(tlsConfig.getSslProvider()));
sslFactory.getKeyManagerFactory().ifPresent(sslContextBuilder::keyManager);
@@ -338,7 +321,7 @@ public final class TlsUtils {
if (tlsConfig.getKeyStorePath() == null) {
throw new IllegalArgumentException("Must provide key store path for secured server");
}
- SSLFactory sslFactory = createSSLFactoryAndEnableAutoRenewalWhenUsingFileStores(tlsConfig);
+ SSLFactory sslFactory = RenewableTlsUtils.createSSLFactoryAndEnableAutoRenewalWhenUsingFileStores(tlsConfig);
SslContextBuilder sslContextBuilder = SslContextBuilder.forServer(sslFactory.getKeyManagerFactory().get())
.sslProvider(SslProvider.valueOf(tlsConfig.getSslProvider()));
sslFactory.getTrustManagerFactory().ifPresent(sslContextBuilder::trustManager);
@@ -357,7 +340,7 @@ public final class TlsUtils {
*
* @param keyOrTrustStorePath key store or trust store path in String format.
*/
- public static boolean isKeyOrTrustStorePathNullOrHasFileScheme(String keyOrTrustStorePath) {
+ static boolean isKeyOrTrustStorePathNullOrHasFileScheme(String keyOrTrustStorePath) {
try {
return keyOrTrustStorePath == null
|| makeKeyOrTrustStoreUrl(keyOrTrustStorePath).toURI().getScheme().startsWith(FILE_SCHEME);
@@ -365,212 +348,4 @@ public final class TlsUtils {
throw new RuntimeException(e);
}
}
-
- /**
- * Enables auto renewal of SSLFactory when
- * 1. the {@link SSLFactory} is created with a key manager and trust manager swappable
- * 2. the key store is null or a local file
- * 3. the trust store is null or a local file
- * 4. the key store or trust store file changes.
- * @param sslFactory the {@link SSLFactory} to enable key manager and trust manager auto renewal
- * @param tlsConfig the {@link TlsConfig} to get the key store and trust store information
- */
- public static void enableAutoRenewalFromFileStoreForSSLFactory(SSLFactory sslFactory, TlsConfig tlsConfig) {
- enableAutoRenewalFromFileStoreForSSLFactory(sslFactory,
- tlsConfig.getKeyStoreType(), tlsConfig.getKeyStorePath(), tlsConfig.getKeyStorePassword(),
- tlsConfig.getTrustStoreType(), tlsConfig.getTrustStorePath(), tlsConfig.getTrustStorePassword(),
- null, null, tlsConfig.isInsecure());
- }
-
- static void enableAutoRenewalFromFileStoreForSSLFactory(SSLFactory sslFactory, String keyStoreType,
- String keyStorePath, String keyStorePassword, String trustStoreType, String trustStorePath,
- String trustStorePassword, String sslContextProtocol, SecureRandom secureRandom, boolean isInsecure) {
- try {
- URL keyStoreURL = keyStorePath == null ? null : makeKeyOrTrustStoreUrl(keyStorePath);
- URL trustStoreURL = trustStorePath == null ? null : makeKeyOrTrustStoreUrl(trustStorePath);
- if (keyStoreURL != null) {
- Preconditions.checkArgument(
- keyStoreURL.toURI().getScheme().startsWith(FILE_SCHEME),
- "key store path must be a local file path or null when SSL auto renew is enabled");
- Preconditions.checkArgument(
- sslFactory.getKeyManager().isPresent()
- && sslFactory.getKeyManager().get() instanceof HotSwappableX509ExtendedKeyManager,
- "key manager of the existing SSLFactory must be swappable"
- );
- }
- if (trustStoreURL != null) {
- Preconditions.checkArgument(
- trustStoreURL.toURI().getScheme().startsWith(FILE_SCHEME),
- "trust store path must be a local file path or null when SSL auto renew is enabled");
- Preconditions.checkArgument(
- sslFactory.getTrustManager().isPresent()
- && sslFactory.getTrustManager().get() instanceof HotSwappableX509ExtendedTrustManager,
- "trust manager of the existing SSLFactory must be swappable"
- );
- }
- // The reloadSslFactoryWhenFileStoreChanges is a blocking call, so we need to create a new thread to run it.
- // Creating a new thread to run the reloadSslFactoryWhenFileStoreChanges is costly; however, unless we
- // invoke the createAutoRenewedSSLFactoryFromFileStore method crazily, this should not be a problem.
- Executors.newSingleThreadExecutor().execute(() -> {
- try {
- reloadSslFactoryWhenFileStoreChanges(sslFactory,
- keyStoreType, keyStorePath, keyStorePassword,
- trustStoreType, trustStorePath, trustStorePassword,
- sslContextProtocol, secureRandom, isInsecure);
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- });
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
-
- @VisibleForTesting
- static void reloadSslFactoryWhenFileStoreChanges(SSLFactory baseSslFactory,
- String keyStoreType, String keyStorePath, String keyStorePassword,
- String trustStoreType, String trustStorePath, String trustStorePassword,
- String sslContextProtocol, SecureRandom secureRandom, boolean isInsecure)
- throws IOException, URISyntaxException, InterruptedException {
- LOGGER.info("Enable auto renewal of SSLFactory {} when key store {} or trust store {} changes",
- baseSslFactory, keyStorePath, trustStorePath);
- WatchService watchService = FileSystems.getDefault().newWatchService();
- Map<WatchKey, Set<Path>> watchKeyPathMap = new HashMap<>();
- registerFile(watchService, watchKeyPathMap, keyStorePath);
- registerFile(watchService, watchKeyPathMap, trustStorePath);
- int maxSslFactoryReloadingAttempts = 3;
- int sslFactoryReloadingRetryDelayMs = 1000;
- WatchKey key;
- while ((key = watchService.take()) != null) {
- for (WatchEvent<?> event : key.pollEvents()) {
- Path changedFile = (Path) event.context();
- if (watchKeyPathMap.get(key).contains(changedFile)) {
- LOGGER.info("Detected change in file: {}, try to renew SSLFactory {} "
- + "(built from key store {} and truststore {})",
- changedFile, baseSslFactory, keyStorePath, trustStorePath);
- try {
- // Need to retry a few times because when one file (key store or trust store) is updated, the other file
- // (trust store or key store) may not have been fully written yet, so we need to wait a bit and retry.
- RetryPolicies.fixedDelayRetryPolicy(maxSslFactoryReloadingAttempts, sslFactoryReloadingRetryDelayMs)
- .attempt(() -> {
- try {
- SSLFactory updatedSslFactory =
- createSSLFactory(keyStoreType, keyStorePath, keyStorePassword, trustStoreType, trustStorePath,
- trustStorePassword, sslContextProtocol, secureRandom, false, isInsecure);
- SSLFactoryUtils.reload(baseSslFactory, updatedSslFactory);
- LOGGER.info("Successfully renewed SSLFactory {} (built from key store {} and truststore {}) on file"
- + " {} changes", baseSslFactory, keyStorePath, trustStorePath, changedFile);
- return true;
- } catch (Exception e) {
- LOGGER.info(
- "Encountered issues when renewing SSLFactory {} (built from key store {} and truststore {}) on "
- + "file {} changes", baseSslFactory, keyStorePath, trustStorePath, changedFile, e);
- return false;
- }
- });
- } catch (Exception e) {
- LOGGER.error(
- "Failed to renew SSLFactory {} (built from key store {} and truststore {}) on file {} changes after {} "
- + "retries", baseSslFactory, keyStorePath, trustStorePath, changedFile,
- maxSslFactoryReloadingAttempts, e);
- }
- }
- }
- key.reset();
- }
- }
-
- @VisibleForTesting
- static void registerFile(WatchService watchService, Map<WatchKey, Set<Path>> keyPathMap, String filePath)
- throws IOException, URISyntaxException {
- if (filePath == null) {
- return;
- }
- Path path = Path.of(makeKeyOrTrustStoreUrl(filePath).getPath());
- WatchKey key = path.getParent().register(watchService, StandardWatchEventKinds.ENTRY_MODIFY);
- keyPathMap.computeIfAbsent(key, k -> new HashSet<>());
- keyPathMap.get(key).add(path.getFileName());
- }
-
- /**
- * Create a {@link SSLFactory} instance with identity material and trust material swappable for a given TlsConfig,
- * and nables auto renewal of the {@link SSLFactory} instance when
- * 1. the {@link SSLFactory} is created with a key manager and trust manager swappable
- * 2. the key store is null or a local file
- * 3. the trust store is null or a local file
- * 4. the key store or trust store file changes.
- * @param tlsConfig {@link TlsConfig}
- * @return a {@link SSLFactory} instance with identity material and trust material swappable
- */
- public static SSLFactory createSSLFactoryAndEnableAutoRenewalWhenUsingFileStores(TlsConfig tlsConfig) {
- SSLFactory sslFactory = createSSLFactory(tlsConfig);
- if (isKeyOrTrustStorePathNullOrHasFileScheme(tlsConfig.getKeyStorePath())
- && isKeyOrTrustStorePathNullOrHasFileScheme(tlsConfig.getTrustStorePath())) {
- enableAutoRenewalFromFileStoreForSSLFactory(sslFactory, tlsConfig);
- }
- return sslFactory;
- }
-
- /**
- * Create a {@link SSLFactory} instance with identity material and trust material swappable for a given TlsConfig
- * @param tlsConfig {@link TlsConfig}
- * @return a {@link SSLFactory} instance with identity material and trust material swappable
- */
- public static SSLFactory createSSLFactory(TlsConfig tlsConfig) {
- return createSSLFactory(
- tlsConfig.getKeyStoreType(), tlsConfig.getKeyStorePath(), tlsConfig.getKeyStorePassword(),
- tlsConfig.getTrustStoreType(), tlsConfig.getTrustStorePath(), tlsConfig.getTrustStorePassword(),
- null, null, true, tlsConfig.isInsecure());
- }
-
- static SSLFactory createSSLFactory(
- String keyStoreType, String keyStorePath, String keyStorePassword,
- String trustStoreType, String trustStorePath, String trustStorePassword,
- String sslContextProtocol, SecureRandom secureRandom, boolean keyAndTrustMaterialSwappable, boolean isInsecure) {
- try {
- SSLFactory.Builder sslFactoryBuilder = SSLFactory.builder();
- InputStream keyStoreStream = null;
- InputStream trustStoreStream = null;
- if (keyStorePath != null) {
- Preconditions.checkNotNull(keyStorePassword, "key store password must not be null");
- keyStoreStream = makeKeyOrTrustStoreUrl(keyStorePath).openStream();
- if (keyAndTrustMaterialSwappable) {
- sslFactoryBuilder.withSwappableIdentityMaterial();
- }
- sslFactoryBuilder.withIdentityMaterial(keyStoreStream, keyStorePassword.toCharArray(), keyStoreType);
- }
- if (isInsecure) {
- if (keyAndTrustMaterialSwappable) {
- sslFactoryBuilder.withSwappableTrustMaterial();
- }
- sslFactoryBuilder.withUnsafeTrustMaterial();
- } else if (trustStorePath != null) {
- Preconditions.checkNotNull(trustStorePassword, "trust store password must not be null");
- trustStoreStream = makeKeyOrTrustStoreUrl(trustStorePath).openStream();
- if (keyAndTrustMaterialSwappable) {
- sslFactoryBuilder.withSwappableTrustMaterial();
- }
- sslFactoryBuilder.withTrustMaterial(trustStoreStream, trustStorePassword.toCharArray(), trustStoreType);
- }
- if (sslContextProtocol != null) {
- sslFactoryBuilder.withSslContextAlgorithm(sslContextProtocol);
- }
- if (secureRandom != null) {
- sslFactoryBuilder.withSecureRandom(secureRandom);
- }
- SSLFactory sslFactory = sslFactoryBuilder.build();
- if (keyStoreStream != null) {
- keyStoreStream.close();
- }
- if (trustStoreStream != null) {
- trustStoreStream.close();
- }
- LOGGER.info("Successfully created SSLFactory {} with key store {} and trust store {}. "
- + "Key and trust material swappable: {}",
- sslFactory, keyStorePath, trustStorePath, keyAndTrustMaterialSwappable);
- return sslFactory;
- } catch (Exception e) {
- throw new IllegalStateException(e);
- }
- }
}
diff --git a/pinot-common/src/test/java/org/apache/pinot/common/utils/tls/TlsUtilsTest.java b/pinot-common/src/test/java/org/apache/pinot/common/utils/tls/TlsUtilsTest.java
index 2f28bbedf6..2ae9a12839 100644
--- a/pinot-common/src/test/java/org/apache/pinot/common/utils/tls/TlsUtilsTest.java
+++ b/pinot-common/src/test/java/org/apache/pinot/common/utils/tls/TlsUtilsTest.java
@@ -130,7 +130,7 @@ public class TlsUtilsTest {
SSLContext sslContext = SSLContext.getInstance("TLS");
sslContext.init(keyManagerFactory.getKeyManagers(), trustManagerFactory.getTrustManagers(), secureRandom);
SSLFactory sslFactory =
- TlsUtils.createSSLFactory(KEYSTORE_TYPE, TLS_KEYSTORE_FILE_PATH, PASSWORD,
+ RenewableTlsUtils.createSSLFactory(KEYSTORE_TYPE, TLS_KEYSTORE_FILE_PATH, PASSWORD,
TRUSTSTORE_TYPE, TLS_TRUSTSTORE_FILE_PATH, PASSWORD, "TLS", secureRandom, true, false);
KeyManagerFactory swappableKeyManagerFactory = sslFactory.getKeyManagerFactory().get();
assertEquals(swappableKeyManagerFactory.getKeyManagers().length, keyManagerFactory.getKeyManagers().length);
@@ -172,8 +172,8 @@ public class TlsUtilsTest {
public void reloadSslFactoryWhenFileStoreChanges()
throws IOException, URISyntaxException, InterruptedException {
SecureRandom secureRandom = new SecureRandom();
- SSLFactory sslFactory = TlsUtils.createSSLFactory(KEYSTORE_TYPE, TLS_KEYSTORE_FILE_PATH, PASSWORD, TRUSTSTORE_TYPE,
- TLS_TRUSTSTORE_FILE_PATH, PASSWORD, "TLS", secureRandom, true, false);
+ SSLFactory sslFactory = RenewableTlsUtils.createSSLFactory(KEYSTORE_TYPE, TLS_KEYSTORE_FILE_PATH, PASSWORD,
+ TRUSTSTORE_TYPE, TLS_TRUSTSTORE_FILE_PATH, PASSWORD, "TLS", secureRandom, true, false);
X509ExtendedKeyManager x509ExtendedKeyManager = sslFactory.getKeyManager().get();
X509ExtendedTrustManager x509ExtendedTrustManager = sslFactory.getTrustManager().get();
SSLContext sslContext = sslFactory.getSslContext();
@@ -187,8 +187,8 @@ public class TlsUtilsTest {
executorService.execute(
() -> {
try {
- TlsUtils.reloadSslFactoryWhenFileStoreChanges(sslFactory, KEYSTORE_TYPE, TLS_KEYSTORE_FILE_PATH, PASSWORD,
- TRUSTSTORE_TYPE, TLS_TRUSTSTORE_FILE_PATH, PASSWORD, "TLS", secureRandom, false);
+ RenewableTlsUtils.reloadSslFactoryWhenFileStoreChanges(sslFactory, KEYSTORE_TYPE, TLS_KEYSTORE_FILE_PATH,
+ PASSWORD, TRUSTSTORE_TYPE, TLS_TRUSTSTORE_FILE_PATH, PASSWORD, "TLS", secureRandom, false);
} catch (Exception e) {
throw new RuntimeException(e);
}
@@ -196,8 +196,8 @@ public class TlsUtilsTest {
WatchService watchService = FileSystems.getDefault().newWatchService();
Map<WatchKey, Set<Path>> watchKeyPathMap = new HashMap<>();
- TlsUtils.registerFile(watchService, watchKeyPathMap, TLS_KEYSTORE_FILE_PATH);
- TlsUtils.registerFile(watchService, watchKeyPathMap, TLS_TRUSTSTORE_FILE_PATH);
+ RenewableTlsUtils.registerFile(watchService, watchKeyPathMap, TLS_KEYSTORE_FILE_PATH);
+ RenewableTlsUtils.registerFile(watchService, watchKeyPathMap, TLS_TRUSTSTORE_FILE_PATH);
// wait for the new thread to start
Thread.sleep(100);
@@ -230,7 +230,7 @@ public class TlsUtilsTest {
@Test
public void enableAutoRenewalFromFileStoreForSSLFactoryThrows() {
SSLFactory swappableSslFactory =
- TlsUtils.createSSLFactory(KEYSTORE_TYPE, TLS_KEYSTORE_FILE_PATH, PASSWORD, TRUSTSTORE_TYPE,
+ RenewableTlsUtils.createSSLFactory(KEYSTORE_TYPE, TLS_KEYSTORE_FILE_PATH, PASSWORD, TRUSTSTORE_TYPE,
TLS_TRUSTSTORE_FILE_PATH, PASSWORD, "TLS", null, true, false);
TlsConfig tlsConfig = new TlsConfig();
tlsConfig.setKeyStoreType(KEYSTORE_TYPE);
@@ -241,7 +241,7 @@ public class TlsUtilsTest {
tlsConfig.setTrustStorePassword(PASSWORD);
RuntimeException e = null;
try {
- TlsUtils.enableAutoRenewalFromFileStoreForSSLFactory(swappableSslFactory, tlsConfig);
+ RenewableTlsUtils.enableAutoRenewalFromFileStoreForSSLFactory(swappableSslFactory, tlsConfig);
} catch (RuntimeException ex) {
e = ex;
}
@@ -253,7 +253,7 @@ public class TlsUtilsTest {
tlsConfig.setTrustStorePath("ftp://" + TLS_TRUSTSTORE_FILE_PATH);
e = null;
try {
- TlsUtils.enableAutoRenewalFromFileStoreForSSLFactory(swappableSslFactory, tlsConfig);
+ RenewableTlsUtils.enableAutoRenewalFromFileStoreForSSLFactory(swappableSslFactory, tlsConfig);
} catch (RuntimeException ex) {
e = ex;
}
@@ -261,12 +261,12 @@ public class TlsUtilsTest {
+ "or null when SSL auto renew is enabled");
SSLFactory nonSwappableSslFactory =
- TlsUtils.createSSLFactory(KEYSTORE_TYPE, TLS_KEYSTORE_FILE_PATH, PASSWORD, TRUSTSTORE_TYPE,
+ RenewableTlsUtils.createSSLFactory(KEYSTORE_TYPE, TLS_KEYSTORE_FILE_PATH, PASSWORD, TRUSTSTORE_TYPE,
TLS_TRUSTSTORE_FILE_PATH, PASSWORD, "TLS", null, false, false);
e = null;
tlsConfig.setTrustStorePath(TLS_TRUSTSTORE_FILE_PATH);
try {
- TlsUtils.enableAutoRenewalFromFileStoreForSSLFactory(nonSwappableSslFactory, tlsConfig);
+ RenewableTlsUtils.enableAutoRenewalFromFileStoreForSSLFactory(nonSwappableSslFactory, tlsConfig);
} catch (RuntimeException ex) {
e = ex;
}
@@ -276,7 +276,7 @@ public class TlsUtilsTest {
tlsConfig.setKeyStorePath(null);
e = null;
try {
- TlsUtils.enableAutoRenewalFromFileStoreForSSLFactory(nonSwappableSslFactory, tlsConfig);
+ RenewableTlsUtils.enableAutoRenewalFromFileStoreForSSLFactory(nonSwappableSslFactory, tlsConfig);
} catch (RuntimeException ex) {
e = ex;
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/transport/grpc/GrpcQueryServer.java b/pinot-core/src/main/java/org/apache/pinot/core/transport/grpc/GrpcQueryServer.java
index bb16c0742d..24f952f8c9 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/transport/grpc/GrpcQueryServer.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/transport/grpc/GrpcQueryServer.java
@@ -42,7 +42,7 @@ import org.apache.pinot.common.metrics.ServerMetrics;
import org.apache.pinot.common.proto.PinotQueryServerGrpc;
import org.apache.pinot.common.proto.Server.ServerRequest;
import org.apache.pinot.common.proto.Server.ServerResponse;
-import org.apache.pinot.common.utils.tls.TlsUtils;
+import org.apache.pinot.common.utils.tls.RenewableTlsUtils;
import org.apache.pinot.core.operator.blocks.InstanceResponseBlock;
import org.apache.pinot.core.operator.streaming.StreamingResponseUtils;
import org.apache.pinot.core.query.executor.QueryExecutor;
@@ -98,7 +98,7 @@ public class GrpcQueryServer extends PinotQueryServerGrpc.PinotQueryServerImplBa
}
SslContext sslContext = SERVER_SSL_CONTEXTS_CACHE.computeIfAbsent(tlsConfig.hashCode(), tlsConfigHashCode -> {
try {
- SSLFactory sslFactory = TlsUtils.createSSLFactoryAndEnableAutoRenewalWhenUsingFileStores(tlsConfig);
+ SSLFactory sslFactory = RenewableTlsUtils.createSSLFactoryAndEnableAutoRenewalWhenUsingFileStores(tlsConfig);
SslContextBuilder sslContextBuilder = SslContextBuilder.forServer(sslFactory.getKeyManagerFactory().get())
.sslProvider(SslProvider.valueOf(tlsConfig.getSslProvider()));
sslFactory.getTrustManagerFactory().ifPresent(sslContextBuilder::trustManager);
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/util/ListenerConfigUtil.java b/pinot-core/src/main/java/org/apache/pinot/core/util/ListenerConfigUtil.java
index 41215a13ad..e617b0d8a6 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/util/ListenerConfigUtil.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/util/ListenerConfigUtil.java
@@ -38,6 +38,7 @@ import nl.altindag.ssl.SSLFactory;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.pinot.common.config.TlsConfig;
+import org.apache.pinot.common.utils.tls.RenewableTlsUtils;
import org.apache.pinot.common.utils.tls.TlsUtils;
import org.apache.pinot.core.transport.HttpServerThreadPoolConfig;
import org.apache.pinot.core.transport.ListenerConfig;
@@ -263,7 +264,7 @@ public final class ListenerConfigUtil {
}
private static SSLEngineConfigurator buildSSLEngineConfigurator(TlsConfig tlsConfig) {
- SSLFactory sslFactory = TlsUtils.createSSLFactoryAndEnableAutoRenewalWhenUsingFileStores(tlsConfig);
+ SSLFactory sslFactory = RenewableTlsUtils.createSSLFactoryAndEnableAutoRenewalWhenUsingFileStores(tlsConfig);
return new SSLEngineConfigurator(sslFactory.getSslContext()).setClientMode(false)
.setNeedClientAuth(tlsConfig.isClientAuthEnabled()).setEnabledProtocols(new String[]{"TLSv1.2"});
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org