You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by at...@apache.org on 2014/10/09 03:02:26 UTC
[2/3] git commit: HADOOP-11161. Expose close method in KeyProvider to
give clients of Provider implementations a hook to release resources.
Contribued by Arun Suresh.
HADOOP-11161. Expose close method in KeyProvider to give clients of Provider implementations a hook to release resources. Contribued by Arun Suresh.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2a51494c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2a51494c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2a51494c
Branch: refs/heads/trunk
Commit: 2a51494ce1b05fc494fb3a818a7a3526f3f40070
Parents: d996235
Author: Aaron T. Myers <at...@apache.org>
Authored: Wed Oct 8 17:58:53 2014 -0700
Committer: Aaron T. Myers <at...@apache.org>
Committed: Wed Oct 8 18:01:51 2014 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../org/apache/hadoop/crypto/key/KeyProvider.java | 8 ++++++++
.../crypto/key/KeyProviderCryptoExtension.java | 9 ++++++++-
.../hadoop/crypto/key/kms/KMSClientProvider.java | 11 +++++++++++
.../apache/hadoop/crypto/key/kms/ValueQueue.java | 14 +++++++++++---
.../java/org/apache/hadoop/hdfs/DFSClient.java | 18 ++++++++++++------
6 files changed, 53 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a51494c/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 404d978..0f40caf 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -811,6 +811,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-10404. Some accesses to DomainSocketWatcher#closed are not protected
by the lock (cmccabe)
+ HADOOP-11161. Expose close method in KeyProvider to give clients of
+ Provider implementations a hook to release resources. (Arun Suresh via atm)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HADOOP-10734. Implement high-performance secure random number sources.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a51494c/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
index 36ccbad..dd2d5b9 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
@@ -534,6 +534,14 @@ public abstract class KeyProvider {
) throws IOException;
/**
+ * Can be used by implementing classes to close any resources
+ * that require closing
+ */
+ public void close() throws IOException {
+ // NOP
+ }
+
+ /**
* Roll a new version of the given key generating the material for it.
* <p/>
* This implementation generates the key material and calls the
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a51494c/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
index 968e341..7e95211 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
@@ -408,6 +408,13 @@ public class KeyProviderCryptoExtension extends
? (CryptoExtension) keyProvider
: new DefaultCryptoExtension(keyProvider);
return new KeyProviderCryptoExtension(keyProvider, cryptoExtension);
- }
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (getKeyProvider() != null) {
+ getKeyProvider().close();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a51494c/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
index d6abcbb..4f4e843 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
@@ -791,4 +791,15 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
return tokens;
}
+ /**
+ * Shutdown valueQueue executor threads
+ */
+ @Override
+ public void close() throws IOException {
+ try {
+ encKeyVersionQueue.shutdown();
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a51494c/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java
index ee10483..aa0e624 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java
@@ -75,6 +75,8 @@ public class ValueQueue <E> {
private final int numValues;
private final float lowWatermark;
+ private volatile boolean executorThreadsStarted = false;
+
/**
* A <code>Runnable</code> which takes a string name.
*/
@@ -187,9 +189,6 @@ public class ValueQueue <E> {
TimeUnit.MILLISECONDS, queue, new ThreadFactoryBuilder()
.setDaemon(true)
.setNameFormat(REFILL_THREAD).build());
- // To ensure all requests are first queued, make coreThreads = maxThreads
- // and pre-start all the Core Threads.
- executor.prestartAllCoreThreads();
}
public ValueQueue(final int numValues, final float lowWaterMark, long expiry,
@@ -297,6 +296,15 @@ public class ValueQueue <E> {
private void submitRefillTask(final String keyName,
final Queue<E> keyQueue) throws InterruptedException {
+ if (!executorThreadsStarted) {
+ synchronized (this) {
+ // To ensure all requests are first queued, make coreThreads =
+ // maxThreads
+ // and pre-start all the Core Threads.
+ executor.prestartAllCoreThreads();
+ executorThreadsStarted = true;
+ }
+ }
// The submit/execute method of the ThreadPoolExecutor is bypassed and
// the Runnable is directly put in the backing BlockingQueue so that we
// can control exactly how the runnable is inserted into the queue.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a51494c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 362c62d..94a68e2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -935,12 +935,18 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
*/
@Override
public synchronized void close() throws IOException {
- if(clientRunning) {
- closeAllFilesBeingWritten(false);
- clientRunning = false;
- getLeaseRenewer().closeClient(this);
- // close connections to the namenode
- closeConnectionToNamenode();
+ try {
+ if(clientRunning) {
+ closeAllFilesBeingWritten(false);
+ clientRunning = false;
+ getLeaseRenewer().closeClient(this);
+ // close connections to the namenode
+ closeConnectionToNamenode();
+ }
+ } finally {
+ if (provider != null) {
+ provider.close();
+ }
}
}