You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by ki...@apache.org on 2013/10/23 23:31:23 UTC
svn commit: r1535190 - in
/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs: CHANGES.txt
src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
Author: kihwal
Date: Wed Oct 23 21:31:23 2013
New Revision: 1535190
URL: http://svn.apache.org/r1535190
Log:
svn merge -c 1535188 merging from trunk to branch-2 to fix HDFS-5341.
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1535190&r1=1535189&r2=1535190&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Wed Oct 23 21:31:23 2013
@@ -101,6 +101,9 @@ Release 2.3.0 - UNRELEASED
HDFS-5239. Allow FSNamesystem lock fairness to be configurable (daryn)
+ HDFS-5341. Reduce fsdataset lock duration during directory scanning.
+ (Qus-Jiawei via kihwal)
+
BUG FIXES
HDFS-5034. Remove debug prints from GetFileLinkInfo (Andrew Wang via Colin
Patrick McCabe)
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java?rev=1535190&r1=1535189&r2=1535190&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java Wed Oct 23 21:31:23 2013
@@ -191,6 +191,11 @@ public class DirectoryScanner implements
private final FsVolumeSpi volume;
+ /**
+ * Get the file's length in async block scan
+ */
+ private final long blockFileLength;
+
private final static Pattern CONDENSED_PATH_REGEX =
Pattern.compile("(?<!^)(\\\\|/){2,}");
@@ -235,6 +240,7 @@ public class DirectoryScanner implements
getCondensedPath(vol.getBasePath());
this.blockSuffix = blockFile == null ? null :
getSuffix(blockFile, condensedVolPath);
+ this.blockFileLength = (blockFile != null) ? blockFile.length() : 0;
if (metaFile == null) {
this.metaSuffix = null;
} else if (blockFile == null) {
@@ -251,6 +257,10 @@ public class DirectoryScanner implements
new File(volume.getBasePath(), blockSuffix);
}
+ long getBlockFileLength() {
+ return blockFileLength;
+ }
+
File getMetaFile() {
if (metaSuffix == null) {
return null;
@@ -458,7 +468,7 @@ public class DirectoryScanner implements
// Block metadata file exits and block file is missing
addDifference(diffRecord, statsRecord, info);
} else if (info.getGenStamp() != memBlock.getGenerationStamp()
- || info.getBlockFile().length() != memBlock.getNumBytes()) {
+ || info.getBlockFileLength() != memBlock.getNumBytes()) {
// Block metadata file is missing or has wrong generation stamp,
// or block file length is different than expected
statsRecord.mismatchBlocks++;