You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by ji...@apache.org on 2012/02/15 19:55:55 UTC
svn commit: r1244642 - in
/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs: ./
src/main/java/org/apache/hadoop/hdfs/server/datanode/
src/test/java/org/apache/hadoop/hdfs/
Author: jitendra
Date: Wed Feb 15 18:55:55 2012
New Revision: 1244642
URL: http://svn.apache.org/viewvc?rev=1244642&view=rev
Log:
Merged 1244640 from trunk for HDFS-2525.
Modified:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceScanner.java
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestAppendDifferentChecksum.java
Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1244642&r1=1244641&r2=1244642&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Wed Feb 15 18:55:55 2012
@@ -38,6 +38,9 @@ Release 0.23.2 - UNRELEASED
HDFS-2950. Secondary NN HTTPS address should be listed as a
NAMESERVICE_SPECIFIC_KEY. (todd)
+ HDFS-2525. Race between BlockPoolSliceScanner and append. (Brandon Li
+ via jitendra)
+
Release 0.23.1 - 2012-02-08
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceScanner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceScanner.java?rev=1244642&r1=1244641&r2=1244642&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceScanner.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceScanner.java Wed Feb 15 18:55:55 2012
@@ -51,11 +51,8 @@ import org.apache.hadoop.hdfs.util.DataT
import org.apache.hadoop.io.IOUtils;
/**
- * Performs two types of scanning:
- * <li> Gets block files from the data directories and reconciles the
- * difference between the blocks on the disk and in memory.</li>
- * <li> Scans the data directories for block files under a block pool
- * and verifies that the files are not corrupt</li>
+ * Scans the block files under a block pool and verifies that the
+ * files are not corrupt.
* This keeps track of blocks and their last verification times.
* Currently it does not modify the metadata for block.
*/
@@ -430,6 +427,19 @@ class BlockPoolSliceScanner {
return;
}
+ // If the block exists, the exception may due to a race with write:
+ // The BlockSender got an old block path in rbw. BlockReceiver removed
+ // the rbw block from rbw to finalized but BlockSender tried to open the
+ // file before BlockReceiver updated the VolumeMap. The state of the
+ // block can be changed again now, so ignore this error here. If there
+ // is a block really deleted by mistake, DirectoryScan should catch it.
+ if (e instanceof FileNotFoundException ) {
+ LOG.info("Verification failed for " + block +
+ ". It may be due to race with write.");
+ deleteBlock(block.getLocalBlock());
+ return;
+ }
+
LOG.warn((second ? "Second " : "First ") + "Verification failed for "
+ block, e);
Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestAppendDifferentChecksum.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestAppendDifferentChecksum.java?rev=1244642&r1=1244641&r2=1244642&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestAppendDifferentChecksum.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestAppendDifferentChecksum.java Wed Feb 15 18:55:55 2012
@@ -47,12 +47,6 @@ public class TestAppendDifferentChecksum
public static void setupCluster() throws IOException {
Configuration conf = new HdfsConfiguration();
conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 4096);
-
- // disable block scanner, since otherwise this test can trigger
- // HDFS-2525, which is a different bug than we're trying to unit test
- // here! When HDFS-2525 is fixed, this can be removed.
- conf.setInt(DFSConfigKeys.DFS_DATANODE_SCAN_PERIOD_HOURS_KEY, -1);
-
conf.set("fs.hdfs.impl.disable.cache", "true");
cluster = new MiniDFSCluster.Builder(conf)
.numDataNodes(1)