You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by om...@apache.org on 2022/03/28 20:17:03 UTC
[hadoop] branch trunk updated: HDFS-16518: Add shutdownhook to invalidate the KeyProviders in the cache
This is an automated email from the ASF dual-hosted git repository.
omalley pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new a9b4396 HDFS-16518: Add shutdownhook to invalidate the KeyProviders in the cache
a9b4396 is described below
commit a9b43966c07fef4507015a7e63f2769f947a64ab
Author: Lei Yang <le...@linkedin.com>
AuthorDate: Wed Mar 23 12:47:32 2022 -0700
HDFS-16518: Add shutdownhook to invalidate the KeyProviders in the cache
Fixes #4100
Signed-off-by: Owen O'Malley <oo...@linkedin.com>
---
.../org/apache/hadoop/hdfs/KeyProviderCache.java | 25 ++++++++++++++++++++++
.../apache/hadoop/hdfs/TestKeyProviderCache.java | 9 ++++++++
2 files changed, 34 insertions(+)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/KeyProviderCache.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/KeyProviderCache.java
index c70081c..d8dd485 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/KeyProviderCache.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/KeyProviderCache.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.crypto.key.KeyProvider;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.util.KMSUtil;
import org.apache.hadoop.classification.VisibleForTesting;
@@ -34,6 +35,7 @@ import org.apache.hadoop.thirdparty.com.google.common.cache.CacheBuilder;
import org.apache.hadoop.thirdparty.com.google.common.cache.RemovalListener;
import org.apache.hadoop.thirdparty.com.google.common.cache.RemovalNotification;
+import org.apache.hadoop.util.ShutdownHookManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -65,6 +67,9 @@ public class KeyProviderCache {
}
})
.build();
+
+ ShutdownHookManager.get().addShutdownHook(new KeyProviderCacheFinalizer(),
+ SHUTDOWN_HOOK_PRIORITY);
}
public KeyProvider get(final Configuration conf,
@@ -85,6 +90,26 @@ public class KeyProviderCache {
}
}
+ public static final int SHUTDOWN_HOOK_PRIORITY = FileSystem.SHUTDOWN_HOOK_PRIORITY - 1;
+
+ private class KeyProviderCacheFinalizer implements Runnable {
+ @Override
+ public synchronized void run() {
+ invalidateCache();
+ }
+ }
+
+ /**
+ * Invalidate cache. KeyProviders in the cache will be closed by cache hook.
+ */
+ @VisibleForTesting
+ synchronized void invalidateCache() {
+ LOG.debug("Invalidating all cached KeyProviders.");
+ if (cache != null) {
+ cache.invalidateAll();
+ }
+ }
+
private URI createKeyProviderURI(Configuration conf) {
final String providerUriStr = conf.getTrimmed(
CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestKeyProviderCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestKeyProviderCache.java
index 9fc6b38..58011d7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestKeyProviderCache.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestKeyProviderCache.java
@@ -32,6 +32,8 @@ public class TestKeyProviderCache {
public static class DummyKeyProvider extends KeyProvider {
+ public static int CLOSE_CALL_COUNT = 0;
+
public DummyKeyProvider(Configuration conf) {
super(conf);
}
@@ -76,6 +78,10 @@ public class TestKeyProviderCache {
public void flush() throws IOException {
}
+ @Override
+ public void close() {
+ CLOSE_CALL_COUNT += 1;
+ }
}
public static class Factory extends KeyProviderFactory {
@@ -124,6 +130,9 @@ public class TestKeyProviderCache {
Assert.assertFalse("Same KeyProviders returned !!",
keyProvider1 == keyProvider4);
+ kpCache.invalidateCache();
+ Assert.assertEquals("Expected number of closing calls doesn't match",
+ 3, DummyKeyProvider.CLOSE_CALL_COUNT);
}
private URI getKeyProviderUriFromConf(Configuration conf) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org