You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by so...@apache.org on 2020/09/22 09:50:31 UTC
[hadoop] branch branch-3.2 updated: HDFS-15583. Backport
DirectoryScanner improvements HDFS-14476,
HDFS-14751 and HDFS-15048 to branch 3.2 and 3.1. Contributed by Stephen
O'Donnell
This is an automated email from the ASF dual-hosted git repository.
sodonnell pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 5f34e32 HDFS-15583. Backport DirectoryScanner improvements HDFS-14476, HDFS-14751 and HDFS-15048 to branch 3.2 and 3.1. Contributed by Stephen O'Donnell
5f34e32 is described below
commit 5f34e3214e42098c7515f0441697980303aab643
Author: S O'Donnell <so...@cloudera.com>
AuthorDate: Tue Sep 22 10:34:05 2020 +0100
HDFS-15583. Backport DirectoryScanner improvements HDFS-14476, HDFS-14751 and HDFS-15048 to branch 3.2 and 3.1. Contributed by Stephen O'Donnell
---
.../dev-support/findbugsExcludeFile.xml | 7 ++++-
.../hdfs/server/datanode/DirectoryScanner.java | 34 ++++++++++++++++------
2 files changed, 31 insertions(+), 10 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml b/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
index d408f2c..ae09b63 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
@@ -311,4 +311,9 @@
<Method name="setInteractiveFormat" />
<Bug pattern="ME_ENUM_FIELD_SETTER" />
</Match>
- </FindBugsFilter>
+ <Match>
+ <Class name="org.apache.hadoop.hdfs.server.datanode.DirectoryScanner" />
+ <Method name="reconcile" />
+ <Bug pattern="SWL_SLEEP_WITH_LOCK_HELD" />
+ </Match>
+</FindBugsFilter>
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
index aede03e..480a262 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
@@ -67,7 +67,7 @@ public class DirectoryScanner implements Runnable {
+ " starting at %s with interval of %dms";
private static final String START_MESSAGE_WITH_THROTTLE = START_MESSAGE
+ " and throttle limit of %dms/s";
-
+ private static final int RECONCILE_BLOCKS_BATCH_SIZE = 1000;
private final FsDatasetSpi<?> dataset;
private final ExecutorService reportCompileThreadPool;
private final ScheduledExecutorService masterThread;
@@ -299,7 +299,9 @@ public class DirectoryScanner implements Runnable {
* Clear the current cache of diffs and statistics.
*/
private void clear() {
- diffs.clear();
+ synchronized (diffs) {
+ diffs.clear();
+ }
stats.clear();
}
@@ -372,13 +374,25 @@ public class DirectoryScanner implements Runnable {
*/
@VisibleForTesting
public void reconcile() throws IOException {
+ LOG.debug("reconcile start DirectoryScanning");
scan();
- for (Entry<String, LinkedList<ScanInfo>> entry : diffs.entrySet()) {
- String bpid = entry.getKey();
- LinkedList<ScanInfo> diff = entry.getValue();
-
- for (ScanInfo info : diff) {
- dataset.checkAndUpdate(bpid, info);
+ int loopCount = 0;
+ synchronized (diffs) {
+ for (Entry<String, LinkedList<ScanInfo>> entry : diffs.entrySet()) {
+ String bpid = entry.getKey();
+ LinkedList<ScanInfo> diff = entry.getValue();
+
+ for (ScanInfo info : diff) {
+ dataset.checkAndUpdate(bpid, info);
+ if (loopCount % RECONCILE_BLOCKS_BATCH_SIZE == 0) {
+ try {
+ Thread.sleep(2000);
+ } catch (InterruptedException e) {
+ // do nothing
+ }
+ }
+ loopCount++;
+ }
}
}
if (!retainDiffs) clear();
@@ -401,7 +415,9 @@ public class DirectoryScanner implements Runnable {
Stats statsRecord = new Stats(bpid);
stats.put(bpid, statsRecord);
LinkedList<ScanInfo> diffRecord = new LinkedList<ScanInfo>();
- diffs.put(bpid, diffRecord);
+ synchronized(diffs) {
+ diffs.put(bpid, diffRecord);
+ }
statsRecord.totalBlocks = blockpoolReport.length;
final List<ReplicaInfo> bl = dataset.getSortedFinalizedBlocks(bpid);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org