You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by le...@apache.org on 2017/09/21 23:12:16 UTC
hadoop git commit: HDFS-12523. Thread pools in ErasureCodingWorker do
not shutdown. (Huafeng Wang via Lei)
Repository: hadoop
Updated Branches:
refs/heads/branch-3.0 a294bfd80 -> 98612bb2d
HDFS-12523. Thread pools in ErasureCodingWorker do not shutdown. (Huafeng Wang via Lei)
(cherry picked from commit 8680131e4b38f0b876173cbee838fcdb91cb52b6)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/98612bb2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/98612bb2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/98612bb2
Branch: refs/heads/branch-3.0
Commit: 98612bb2d047cadce8b018c5f751a9291b0c2e43
Parents: a294bfd
Author: Lei Xu <le...@apache.org>
Authored: Thu Sep 21 16:10:32 2017 -0700
Committer: Lei Xu <le...@apache.org>
Committed: Thu Sep 21 16:12:10 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/server/datanode/DataNode.java | 6 +++++-
.../server/datanode/erasurecode/ErasureCodingWorker.java | 11 +++++++++--
.../datanode/erasurecode/StripedReconstructor.java | 8 +++-----
3 files changed, 17 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/98612bb2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index 042a627..6163d93 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -1115,7 +1115,7 @@ public class DataNode extends ReconfigurableBase
/**
* Shutdown disk balancer.
*/
- private void shutdownDiskBalancer() {
+ private void shutdownDiskBalancer() {
if (this.diskBalancer != null) {
this.diskBalancer.shutdown();
this.diskBalancer = null;
@@ -2077,6 +2077,10 @@ public class DataNode extends ReconfigurableBase
ipcServer.stop();
}
+ if (ecWorker != null) {
+ ecWorker.shutDown();
+ }
+
if(blockPoolManager != null) {
try {
this.blockPoolManager.shutDownAll(bposArray);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/98612bb2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java
index 0e00e11..70c5378 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/ErasureCodingWorker.java
@@ -27,6 +27,8 @@ import org.apache.hadoop.util.Daemon;
import org.slf4j.Logger;
import java.util.Collection;
+import java.util.concurrent.CompletionService;
+import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor;
@@ -149,7 +151,12 @@ public final class ErasureCodingWorker {
return conf;
}
- ThreadPoolExecutor getStripedReadPool() {
- return stripedReadPool;
+ CompletionService<Void> createReadService() {
+ return new ExecutorCompletionService<>(stripedReadPool);
+ }
+
+ public void shutDown() {
+ stripedReconstructionPool.shutdown();
+ stripedReadPool.shutdown();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/98612bb2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java
index 3202121..bbffcf5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/erasurecode/StripedReconstructor.java
@@ -39,8 +39,6 @@ import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
import java.util.BitSet;
import java.util.concurrent.CompletionService;
-import java.util.concurrent.ExecutorCompletionService;
-import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.atomic.AtomicLong;
/**
@@ -110,7 +108,7 @@ abstract class StripedReconstructor {
// position in striped internal block
private long positionInBlock;
private StripedReader stripedReader;
- private ThreadPoolExecutor stripedReadPool;
+ private ErasureCodingWorker erasureCodingWorker;
private final CachingStrategy cachingStrategy;
private long maxTargetLength = 0L;
private final BitSet liveBitSet;
@@ -122,7 +120,7 @@ abstract class StripedReconstructor {
StripedReconstructor(ErasureCodingWorker worker,
StripedReconstructionInfo stripedReconInfo) {
- this.stripedReadPool = worker.getStripedReadPool();
+ this.erasureCodingWorker = worker;
this.datanode = worker.getDatanode();
this.conf = worker.getConf();
this.ecPolicy = stripedReconInfo.getEcPolicy();
@@ -225,7 +223,7 @@ abstract class StripedReconstructor {
}
CompletionService<Void> createReadService() {
- return new ExecutorCompletionService<>(stripedReadPool);
+ return erasureCodingWorker.createReadService();
}
ExtendedBlock getBlockGroup() {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org