You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by we...@apache.org on 2019/08/19 16:02:25 UTC
[hadoop] branch branch-3.1 updated: HDFS-14476. lock too long when
fix inconsistent blocks between disk and in-memory. Contributed by Sean
Chow.
This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new e978c6c HDFS-14476. lock too long when fix inconsistent blocks between disk and in-memory. Contributed by Sean Chow.
e978c6c is described below
commit e978c6c9ed83301fcf8359c35e74ed68045c1a61
Author: Wei-Chiu Chuang <we...@apache.org>
AuthorDate: Mon Aug 19 08:38:06 2019 -0700
HDFS-14476. lock too long when fix inconsistent blocks between disk and in-memory. Contributed by Sean Chow.
(cherry picked from commit 8b802d6b444b95791ba156c7bb307dd584cc9dba)
---
.../hadoop/hdfs/server/datanode/DirectoryScanner.java | 14 ++++++++++++++
1 file changed, 14 insertions(+)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
index ab9743c..5a2f7c2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
@@ -67,6 +67,7 @@ public class DirectoryScanner implements Runnable {
+ " starting at %s with interval of %dms";
private static final String START_MESSAGE_WITH_THROTTLE = START_MESSAGE
+ " and throttle limit of %dms/s";
+ private static final int RECONCILE_BLOCKS_BATCH_SIZE = 1000;
private final FsDatasetSpi<?> dataset;
private final ExecutorService reportCompileThreadPool;
@@ -372,7 +373,11 @@ public class DirectoryScanner implements Runnable {
*/
@VisibleForTesting
public void reconcile() throws IOException {
+ LOG.debug("reconcile start DirectoryScanning");
scan();
+ // HDFS-14476: run checkAndUpadte with batch to avoid holding the lock too
+ // long
+ int loopCount = 0;
for (Entry<String, LinkedList<ScanInfo>> entry : diffs.entrySet()) {
String bpid = entry.getKey();
LinkedList<ScanInfo> diff = entry.getValue();
@@ -380,6 +385,15 @@ public class DirectoryScanner implements Runnable {
for (ScanInfo info : diff) {
dataset.checkAndUpdate(bpid, info);
}
+
+ if (loopCount % RECONCILE_BLOCKS_BATCH_SIZE == 0) {
+ try {
+ Thread.sleep(2000);
+ } catch (InterruptedException e) {
+ // do nothing
+ }
+ }
+ loopCount++;
}
if (!retainDiffs) clear();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org