You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by iw...@apache.org on 2020/04/13 07:01:37 UTC
[hadoop] branch branch-2.10 updated (8fae3b1 -> 00b313d)
This is an automated email from the ASF dual-hosted git repository.
iwasakims pushed a change to branch branch-2.10
in repository https://gitbox.apache.org/repos/asf/hadoop.git.
from 8fae3b1 HADOOP-16869. Upgrade findbugs-maven-plugin to 3.0.5 to fix mvn findbugs:findbugs failure (#1855)
new 01ed683 HDFS-14476. lock too long when fix inconsistent blocks between disk and in-memory. Contributed by Sean Chow.
new 00b313d HDFS-15048. Fix findbug in DirectoryScanner. Contributed by Sean Chow.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../dev-support/findbugsExcludeFile.xml | 5 +++
.../hdfs/server/datanode/DirectoryScanner.java | 39 +++++++++++++++++-----
2 files changed, 35 insertions(+), 9 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 02/02: HDFS-15048. Fix findbug in DirectoryScanner.
Contributed by Sean Chow.
Posted by iw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
iwasakims pushed a commit to branch branch-2.10
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 00b313d6ffeadec14ba22484cbb880bdf8e07092
Author: Masatake Iwasaki <iw...@apache.org>
AuthorDate: Mon Apr 13 15:32:15 2020 +0900
HDFS-15048. Fix findbug in DirectoryScanner. Contributed by Sean Chow.
---
hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml b/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
index 53897ac..dad6451 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
@@ -246,4 +246,9 @@
<Class name="org.apache.hadoop.hdfs.server.datanode.checker.AbstractFuture" />
<Bug pattern="NS_DANGEROUS_NON_SHORT_CIRCUIT" />
</Match>
+ <Match>
+ <Class name="org.apache.hadoop.hdfs.server.datanode.DirectoryScanner" />
+ <Method name="reconcile" />
+ <Bug pattern="SWL_SLEEP_WITH_LOCK_HELD" />
+ </Match>
</FindBugsFilter>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 01/02: HDFS-14476. lock too long when fix inconsistent
blocks between disk and in-memory. Contributed by Sean Chow.
Posted by iw...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
iwasakims pushed a commit to branch branch-2.10
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 01ed6832f430a40ff209932eb605024326700cf6
Author: Masatake Iwasaki <iw...@apache.org>
AuthorDate: Mon Apr 13 15:11:37 2020 +0900
HDFS-14476. lock too long when fix inconsistent blocks between disk and in-memory. Contributed by Sean Chow.
---
.../hdfs/server/datanode/DirectoryScanner.java | 39 +++++++++++++++++-----
1 file changed, 30 insertions(+), 9 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
index e8e8d0b..d31e377 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
@@ -63,6 +63,7 @@ import org.apache.hadoop.util.Time;
public class DirectoryScanner implements Runnable {
private static final Log LOG = LogFactory.getLog(DirectoryScanner.class);
private static final int MILLIS_PER_SECOND = 1000;
+ private static final int RECONCILE_BLOCKS_BATCH_SIZE = 1000;
private static final String START_MESSAGE =
"Periodic Directory Tree Verification scan"
+ " starting at %s with interval of %dms";
@@ -491,7 +492,9 @@ public class DirectoryScanner implements Runnable {
* Clear the current cache of diffs and statistics.
*/
private void clear() {
- diffs.clear();
+ synchronized (diffs) {
+ diffs.clear();
+ }
stats.clear();
}
@@ -564,14 +567,30 @@ public class DirectoryScanner implements Runnable {
*/
@VisibleForTesting
void reconcile() throws IOException {
+ LOG.debug("reconcile start DirectoryScanning");
scan();
- for (Entry<String, LinkedList<ScanInfo>> entry : diffs.entrySet()) {
- String bpid = entry.getKey();
- LinkedList<ScanInfo> diff = entry.getValue();
-
- for (ScanInfo info : diff) {
- dataset.checkAndUpdate(bpid, info.getBlockId(), info.getBlockFile(),
- info.getMetaFile(), info.getVolume());
+
+ // HDFS-14476: run checkAndUpadte with batch to avoid holding the lock
+ // too long
+ int loopCount = 0;
+ synchronized (diffs) {
+ for (Entry<String, LinkedList<ScanInfo>> entry : diffs.entrySet()) {
+ String bpid = entry.getKey();
+ LinkedList<ScanInfo> diff = entry.getValue();
+
+ for (ScanInfo info : diff) {
+ dataset.checkAndUpdate(bpid, info.getBlockId(), info.getBlockFile(),
+ info.getMetaFile(), info.getVolume());
+
+ if (loopCount % RECONCILE_BLOCKS_BATCH_SIZE == 0) {
+ try {
+ Thread.sleep(2000);
+ } catch (InterruptedException e) {
+ // do nothing
+ }
+ }
+ loopCount++;
+ }
}
}
if (!retainDiffs) clear();
@@ -594,7 +613,9 @@ public class DirectoryScanner implements Runnable {
Stats statsRecord = new Stats(bpid);
stats.put(bpid, statsRecord);
LinkedList<ScanInfo> diffRecord = new LinkedList<ScanInfo>();
- diffs.put(bpid, diffRecord);
+ synchronized (diffs) {
+ diffs.put(bpid, diffRecord);
+ }
statsRecord.totalBlocks = blockpoolReport.length;
final List<FinalizedReplica> bl = dataset.getFinalizedBlocks(bpid);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org