You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by xi...@apache.org on 2024/02/08 20:29:55 UTC
(pinot) branch master updated: Fixing the multiple files concurrent write issue when reloading SSLFactory (#12384)
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 2b69d6afe2 Fixing the multiple files concurrent write issue when reloading SSLFactory (#12384)
2b69d6afe2 is described below
commit 2b69d6afe2e15ad5b7e683610d044a15f4650885
Author: Haitao Zhang <ha...@startree.ai>
AuthorDate: Thu Feb 8 12:29:49 2024 -0800
Fixing the multiple files concurrent write issue when reloading SSLFactory (#12384)
* add retry to SSLFactory reloading
* update comment
---
.../org/apache/pinot/common/utils/TlsUtils.java | 35 ++++++++++++++++++----
1 file changed, 29 insertions(+), 6 deletions(-)
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/utils/TlsUtils.java b/pinot-common/src/main/java/org/apache/pinot/common/utils/TlsUtils.java
index e905f7aafa..c75e171384 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/utils/TlsUtils.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/utils/TlsUtils.java
@@ -57,6 +57,7 @@ import org.apache.commons.lang.StringUtils;
import org.apache.http.ssl.SSLContexts;
import org.apache.pinot.common.config.TlsConfig;
import org.apache.pinot.spi.env.PinotConfiguration;
+import org.apache.pinot.spi.utils.retry.RetryPolicies;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -439,6 +440,8 @@ public final class TlsUtils {
Map<WatchKey, Set<Path>> watchKeyPathMap = new HashMap<>();
registerFile(watchService, watchKeyPathMap, keyStorePath);
registerFile(watchService, watchKeyPathMap, trustStorePath);
+ int maxSslFactoryReloadingAttempts = 3;
+ int sslFactoryReloadingRetryDelayMs = 1000;
WatchKey key;
while ((key = watchService.take()) != null) {
for (WatchEvent<?> event : key.pollEvents()) {
@@ -447,12 +450,32 @@ public final class TlsUtils {
LOGGER.info("Detected change in file: {}, try to renew SSLFactory {} "
+ "(built from key store {} and truststore {})",
changedFile, baseSslFactory, keyStorePath, trustStorePath);
- SSLFactory updatedSslFactory = createSSLFactory(
- keyStoreType, keyStorePath, keyStorePassword, trustStoreType, trustStorePath, trustStorePassword,
- sslContextProtocol, secureRandom, false);
- SSLFactoryUtils.reload(baseSslFactory, updatedSslFactory);
- LOGGER.info("Successfully renewed SSLFactory {} (built from key store {} and truststore {}) "
- + "on file {} changes", baseSslFactory, keyStorePath, trustStorePath, changedFile);
+ try {
+ // Need to retry a few times because when one file (key store or trust store) is updated, the other file
+ // (trust store or key store) may not have been fully written yet, so we need to wait a bit and retry.
+ RetryPolicies.fixedDelayRetryPolicy(maxSslFactoryReloadingAttempts, sslFactoryReloadingRetryDelayMs)
+ .attempt(() -> {
+ try {
+ SSLFactory updatedSslFactory =
+ createSSLFactory(keyStoreType, keyStorePath, keyStorePassword, trustStoreType, trustStorePath,
+ trustStorePassword, sslContextProtocol, secureRandom, false);
+ SSLFactoryUtils.reload(baseSslFactory, updatedSslFactory);
+ LOGGER.info("Successfully renewed SSLFactory {} (built from key store {} and truststore {}) on file"
+ + " {} changes", baseSslFactory, keyStorePath, trustStorePath, changedFile);
+ return true;
+ } catch (Exception e) {
+ LOGGER.info(
+ "Encountered issues when renewing SSLFactory {} (built from key store {} and truststore {}) on "
+ + "file {} changes", baseSslFactory, keyStorePath, trustStorePath, changedFile, e);
+ return false;
+ }
+ });
+ } catch (Exception e) {
+ LOGGER.error(
+ "Failed to renew SSLFactory {} (built from key store {} and truststore {}) on file {} changes after {} "
+ + "retries", baseSslFactory, keyStorePath, trustStorePath, changedFile,
+ maxSslFactoryReloadingAttempts, e);
+ }
}
}
key.reset();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org