You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2014/10/10 21:01:13 UTC
[34/50] [abbrv] git commit: HADOOP-11161. Expose close method in
KeyProvider to give clients of Provider implementations a hook to release
resources. Contribued by Arun Suresh. (cherry picked from commit
d9556e873ef4d3e68c4f0c991f856d1faa747f07) (cherry
HADOOP-11161. Expose close method in KeyProvider to give clients of Provider implementations a hook to release resources. Contribued by Arun Suresh.
(cherry picked from commit d9556e873ef4d3e68c4f0c991f856d1faa747f07)
(cherry picked from commit 3a2565c7be80cf6e9cdfec0f5460ed8ed2252768)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5e5503b8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5e5503b8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5e5503b8
Branch: refs/heads/champlain
Commit: 5e5503b8d93b8413ba2268d7058575b1665a7db4
Parents: 3cbe727
Author: Aaron T. Myers <at...@apache.org>
Authored: Wed Oct 8 17:58:53 2014 -0700
Committer: Jitendra Pandey <Ji...@Jitendra-Pandeys-MacBook-Pro-4.local>
Committed: Thu Oct 9 00:30:12 2014 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../org/apache/hadoop/crypto/key/KeyProvider.java | 8 ++++++++
.../crypto/key/KeyProviderCryptoExtension.java | 7 +++++++
.../hadoop/crypto/key/kms/KMSClientProvider.java | 11 +++++++++++
.../apache/hadoop/crypto/key/kms/ValueQueue.java | 14 +++++++++++---
.../java/org/apache/hadoop/hdfs/DFSClient.java | 18 ++++++++++++------
6 files changed, 52 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e5503b8/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index db2795c..69b7622 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -469,6 +469,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-10404. Some accesses to DomainSocketWatcher#closed are not protected
by lock (cmccabe)
+ HADOOP-11161. Expose close method in KeyProvider to give clients of
+ Provider implementations a hook to release resources. (Arun Suresh via atm)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HADOOP-10734. Implement high-performance secure random number sources.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e5503b8/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
index a8b9414..9dd1d47 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
@@ -534,6 +534,14 @@ public abstract class KeyProvider {
) throws IOException;
/**
+ * Can be used by implementing classes to close any resources
+ * that require closing
+ */
+ public void close() throws IOException {
+ // NOP
+ }
+
+ /**
* Roll a new version of the given key generating the material for it.
* <p/>
* This implementation generates the key material and calls the
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e5503b8/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
index f800689..73c9885 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
@@ -410,4 +410,11 @@ public class KeyProviderCryptoExtension extends
return new KeyProviderCryptoExtension(keyProvider, cryptoExtension);
}
+ @Override
+ public void close() throws IOException {
+ if (getKeyProvider() != null) {
+ getKeyProvider().close();
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e5503b8/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
index 5b7f109..c4c7e0c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
@@ -791,4 +791,15 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
return tokens;
}
+ /**
+ * Shutdown valueQueue executor threads
+ */
+ @Override
+ public void close() throws IOException {
+ try {
+ encKeyVersionQueue.shutdown();
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e5503b8/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java
index ee10483..aa0e624 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java
@@ -75,6 +75,8 @@ public class ValueQueue <E> {
private final int numValues;
private final float lowWatermark;
+ private volatile boolean executorThreadsStarted = false;
+
/**
* A <code>Runnable</code> which takes a string name.
*/
@@ -187,9 +189,6 @@ public class ValueQueue <E> {
TimeUnit.MILLISECONDS, queue, new ThreadFactoryBuilder()
.setDaemon(true)
.setNameFormat(REFILL_THREAD).build());
- // To ensure all requests are first queued, make coreThreads = maxThreads
- // and pre-start all the Core Threads.
- executor.prestartAllCoreThreads();
}
public ValueQueue(final int numValues, final float lowWaterMark, long expiry,
@@ -297,6 +296,15 @@ public class ValueQueue <E> {
private void submitRefillTask(final String keyName,
final Queue<E> keyQueue) throws InterruptedException {
+ if (!executorThreadsStarted) {
+ synchronized (this) {
+ // To ensure all requests are first queued, make coreThreads =
+ // maxThreads
+ // and pre-start all the Core Threads.
+ executor.prestartAllCoreThreads();
+ executorThreadsStarted = true;
+ }
+ }
// The submit/execute method of the ThreadPoolExecutor is bypassed and
// the Runnable is directly put in the backing BlockingQueue so that we
// can control exactly how the runnable is inserted into the queue.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e5503b8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 66a4d0c..c26e3d1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -920,12 +920,18 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
*/
@Override
public synchronized void close() throws IOException {
- if(clientRunning) {
- closeAllFilesBeingWritten(false);
- clientRunning = false;
- getLeaseRenewer().closeClient(this);
- // close connections to the namenode
- closeConnectionToNamenode();
+ try {
+ if(clientRunning) {
+ closeAllFilesBeingWritten(false);
+ clientRunning = false;
+ getLeaseRenewer().closeClient(this);
+ // close connections to the namenode
+ closeConnectionToNamenode();
+ }
+ } finally {
+ if (provider != null) {
+ provider.close();
+ }
}
}