[hadoop] branch branch-3.2 updated: HDFS-14476. lock too long when fix inconsistent blocks between disk and in-memory. Contributed by Sean Chow.

[we...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

weichiu pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.2 by this push:
     new 8b802d6  HDFS-14476. lock too long when fix inconsistent blocks between disk and in-memory. Contributed by Sean Chow.
8b802d6 is described below

commit 8b802d6b444b95791ba156c7bb307dd584cc9dba
Author: Wei-Chiu Chuang <we...@apache.org>
AuthorDate: Mon Aug 19 08:38:06 2019 -0700

    HDFS-14476. lock too long when fix inconsistent blocks between disk and in-memory. Contributed by Sean Chow.
---
 .../hadoop/hdfs/server/datanode/DirectoryScanner.java      | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
index 99584d9..2d9ccf9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
@@ -68,6 +68,7 @@ public class DirectoryScanner implements Runnable {
       + " starting at %s with interval of %dms";
   private static final String START_MESSAGE_WITH_THROTTLE = START_MESSAGE
       + " and throttle limit of %dms/s";
+  private static final int RECONCILE_BLOCKS_BATCH_SIZE = 1000;
 
   private final FsDatasetSpi<?> dataset;
   private final ExecutorService reportCompileThreadPool;
@@ -373,7 +374,11 @@ public class DirectoryScanner implements Runnable {
    */
   @VisibleForTesting
   public void reconcile() throws IOException {
+    LOG.debug("reconcile start DirectoryScanning");
     scan();
+    // HDFS-14476: run checkAndUpadte with batch to avoid holding the lock too
+    // long
+    int loopCount = 0;
     for (Entry<String, LinkedList<ScanInfo>> entry : diffs.entrySet()) {
       String bpid = entry.getKey();
       LinkedList<ScanInfo> diff = entry.getValue();
@@ -381,6 +386,15 @@ public class DirectoryScanner implements Runnable {
       for (ScanInfo info : diff) {
         dataset.checkAndUpdate(bpid, info);
       }
+
+      if (loopCount % RECONCILE_BLOCKS_BATCH_SIZE == 0) {
+        try {
+          Thread.sleep(2000);
+        } catch (InterruptedException e) {
+          // do nothing
+        }
+      }
+      loopCount++;
     }
     if (!retainDiffs) clear();
   }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org