You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by we...@apache.org on 2019/08/19 00:55:16 UTC
[hadoop] branch branch-2.9 updated: HDFS-14476. lock too long when
fix inconsistent blocks between disk and in-memory. Contributed by Sean
Chow.
This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch branch-2.9
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-2.9 by this push:
new 22321f1 HDFS-14476. lock too long when fix inconsistent blocks between disk and in-memory. Contributed by Sean Chow.
22321f1 is described below
commit 22321f1d3dfae5bf03b8724f5d74d292d551580f
Author: Wei-Chiu Chuang <we...@apache.org>
AuthorDate: Sun Aug 18 17:51:44 2019 -0700
HDFS-14476. lock too long when fix inconsistent blocks between disk and in-memory. Contributed by Sean Chow.
(cherry picked from commit 8755de108d0e711cbfa784a20bd89e28fe79266b)
---
.../hadoop/hdfs/server/datanode/DirectoryScanner.java | 16 +++++++++++++++-
1 file changed, 15 insertions(+), 1 deletion(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
index e8e8d0b..fc97d68 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
@@ -63,6 +63,7 @@ import org.apache.hadoop.util.Time;
public class DirectoryScanner implements Runnable {
private static final Log LOG = LogFactory.getLog(DirectoryScanner.class);
private static final int MILLIS_PER_SECOND = 1000;
+ private static final int RECONCILE_BLOCKS_BATCH_SIZE = 1000;
private static final String START_MESSAGE =
"Periodic Directory Tree Verification scan"
+ " starting at %s with interval of %dms";
@@ -564,14 +565,27 @@ public class DirectoryScanner implements Runnable {
*/
@VisibleForTesting
void reconcile() throws IOException {
+ LOG.debug("reconcile start DirectoryScanning");
scan();
for (Entry<String, LinkedList<ScanInfo>> entry : diffs.entrySet()) {
String bpid = entry.getKey();
LinkedList<ScanInfo> diff = entry.getValue();
-
+
+ // HDFS-14476: run checkAndUpadte with batch to avoid holding the lock too
+ // long
+ int loopCount = 0;
for (ScanInfo info : diff) {
dataset.checkAndUpdate(bpid, info.getBlockId(), info.getBlockFile(),
info.getMetaFile(), info.getVolume());
+
+ if (loopCount % RECONCILE_BLOCKS_BATCH_SIZE == 0) {
+ try {
+ Thread.sleep(2000);
+ } catch (InterruptedException e) {
+ // do nothing
+ }
+ }
+ loopCount++;
}
}
if (!retainDiffs) clear();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org