You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by at...@apache.org on 2014/10/09 03:02:27 UTC
[3/3] git commit: HADOOP-11161. Expose close method in KeyProvider to
give clients of Provider implementations a hook to release resources.
Contribued by Arun Suresh. (cherry picked from commit
d9556e873ef4d3e68c4f0c991f856d1faa747f07)
HADOOP-11161. Expose close method in KeyProvider to give clients of Provider implementations a hook to release resources. Contribued by Arun Suresh.
(cherry picked from commit d9556e873ef4d3e68c4f0c991f856d1faa747f07)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/afaadd65
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/afaadd65
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/afaadd65
Branch: refs/heads/branch-2
Commit: afaadd65359ba54be38a118bfb5dcf4174416a27
Parents: f1feaae
Author: Aaron T. Myers <at...@apache.org>
Authored: Wed Oct 8 17:58:53 2014 -0700
Committer: Aaron T. Myers <at...@apache.org>
Committed: Wed Oct 8 18:02:00 2014 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../org/apache/hadoop/crypto/key/KeyProvider.java | 8 ++++++++
.../crypto/key/KeyProviderCryptoExtension.java | 7 +++++++
.../hadoop/crypto/key/kms/KMSClientProvider.java | 11 +++++++++++
.../apache/hadoop/crypto/key/kms/ValueQueue.java | 14 +++++++++++---
.../java/org/apache/hadoop/hdfs/DFSClient.java | 18 ++++++++++++------
6 files changed, 52 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/afaadd65/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index fe15cf5..819df99 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -470,6 +470,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-10404. Some accesses to DomainSocketWatcher#closed are not protected
by the lock (cmccabe)
+ HADOOP-11161. Expose close method in KeyProvider to give clients of
+ Provider implementations a hook to release resources. (Arun Suresh via atm)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HADOOP-10734. Implement high-performance secure random number sources.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/afaadd65/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
index a8b9414..9dd1d47 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
@@ -534,6 +534,14 @@ public abstract class KeyProvider {
) throws IOException;
/**
+ * Can be used by implementing classes to close any resources
+ * that require closing
+ */
+ public void close() throws IOException {
+ // NOP
+ }
+
+ /**
* Roll a new version of the given key generating the material for it.
* <p/>
* This implementation generates the key material and calls the
http://git-wip-us.apache.org/repos/asf/hadoop/blob/afaadd65/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
index f800689..73c9885 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
@@ -410,4 +410,11 @@ public class KeyProviderCryptoExtension extends
return new KeyProviderCryptoExtension(keyProvider, cryptoExtension);
}
+ @Override
+ public void close() throws IOException {
+ if (getKeyProvider() != null) {
+ getKeyProvider().close();
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/afaadd65/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
index 5b7f109..c4c7e0c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
@@ -791,4 +791,15 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
return tokens;
}
+ /**
+ * Shutdown valueQueue executor threads
+ */
+ @Override
+ public void close() throws IOException {
+ try {
+ encKeyVersionQueue.shutdown();
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/afaadd65/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java
index ee10483..aa0e624 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java
@@ -75,6 +75,8 @@ public class ValueQueue <E> {
private final int numValues;
private final float lowWatermark;
+ private volatile boolean executorThreadsStarted = false;
+
/**
* A <code>Runnable</code> which takes a string name.
*/
@@ -187,9 +189,6 @@ public class ValueQueue <E> {
TimeUnit.MILLISECONDS, queue, new ThreadFactoryBuilder()
.setDaemon(true)
.setNameFormat(REFILL_THREAD).build());
- // To ensure all requests are first queued, make coreThreads = maxThreads
- // and pre-start all the Core Threads.
- executor.prestartAllCoreThreads();
}
public ValueQueue(final int numValues, final float lowWaterMark, long expiry,
@@ -297,6 +296,15 @@ public class ValueQueue <E> {
private void submitRefillTask(final String keyName,
final Queue<E> keyQueue) throws InterruptedException {
+ if (!executorThreadsStarted) {
+ synchronized (this) {
+ // To ensure all requests are first queued, make coreThreads =
+ // maxThreads
+ // and pre-start all the Core Threads.
+ executor.prestartAllCoreThreads();
+ executorThreadsStarted = true;
+ }
+ }
// The submit/execute method of the ThreadPoolExecutor is bypassed and
// the Runnable is directly put in the backing BlockingQueue so that we
// can control exactly how the runnable is inserted into the queue.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/afaadd65/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 362c62d..94a68e2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -935,12 +935,18 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
*/
@Override
public synchronized void close() throws IOException {
- if(clientRunning) {
- closeAllFilesBeingWritten(false);
- clientRunning = false;
- getLeaseRenewer().closeClient(this);
- // close connections to the namenode
- closeConnectionToNamenode();
+ try {
+ if(clientRunning) {
+ closeAllFilesBeingWritten(false);
+ clientRunning = false;
+ getLeaseRenewer().closeClient(this);
+ // close connections to the namenode
+ closeConnectionToNamenode();
+ }
+ } finally {
+ if (provider != null) {
+ provider.close();
+ }
}
}