You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by ki...@apache.org on 2013/05/03 19:53:05 UTC
svn commit: r1478889 - in
/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs:
CHANGES.txt
src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
src/test/java/org/apache/hadoop/hdfs/TestFileAppendRestart.java
Author: kihwal
Date: Fri May 3 17:53:04 2013
New Revision: 1478889
URL: http://svn.apache.org/r1478889
Log:
svn merge -c 1341961 Merging from trunk to branch-0.23 to fix HDFS-3436.
Modified:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppendRestart.java
Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1478889&r1=1478888&r2=1478889&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Fri May 3 17:53:04 2013
@@ -17,6 +17,10 @@ Release 0.23.8 - UNRELEASED
HDFS-4699. TestPipelinesFailover#testPipelineRecoveryStress fails
sporadically (Chris Nauroth via kihwal)
+ HDFS-3436. In DataNode.transferReplicaForPipelineRecovery(..), it should
+ use the stored generation stamp to check if the block is valid. (Vinay
+ via szetszwo)
+
Release 0.23.7 - 2013-04-18
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java?rev=1478889&r1=1478888&r2=1478889&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java Fri May 3 17:53:04 2013
@@ -2123,6 +2123,18 @@ public class DataNode extends Configured
//get replica information
synchronized(data) {
+ Block storedBlock = data.getStoredBlock(b.getBlockPoolId(),
+ b.getBlockId());
+ if (null == storedBlock) {
+ throw new IOException(b + " not found in datanode.");
+ }
+ storedGS = storedBlock.getGenerationStamp();
+ if (storedGS < b.getGenerationStamp()) {
+ throw new IOException(storedGS
+ + " = storedGS < b.getGenerationStamp(), b=" + b);
+ }
+ // Update the genstamp with storedGS
+ b.setGenerationStamp(storedGS);
if (data.isValidRbw(b)) {
stage = BlockConstructionStage.TRANSFER_RBW;
} else if (data.isValidBlock(b)) {
@@ -2131,18 +2143,9 @@ public class DataNode extends Configured
final String r = data.getReplicaString(b.getBlockPoolId(), b.getBlockId());
throw new IOException(b + " is neither a RBW nor a Finalized, r=" + r);
}
-
- storedGS = data.getStoredBlock(b.getBlockPoolId(),
- b.getBlockId()).getGenerationStamp();
- if (storedGS < b.getGenerationStamp()) {
- throw new IOException(
- storedGS + " = storedGS < b.getGenerationStamp(), b=" + b);
- }
visible = data.getReplicaVisibleLength(b);
}
-
- //set storedGS and visible length
- b.setGenerationStamp(storedGS);
+ //set visible length
b.setNumBytes(visible);
if (targets.length > 0) {
Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppendRestart.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppendRestart.java?rev=1478889&r1=1478888&r2=1478889&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppendRestart.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppendRestart.java Fri May 3 17:53:04 2013
@@ -163,4 +163,32 @@ public class TestFileAppendRestart {
cluster.shutdown();
}
}
+
+ /**
+ * Test to append to the file, when one of datanode in the existing pipeline is down.
+ * @throws Exception
+ */
+ @Test
+ public void testAppendWithPipelineRecovery() throws Exception {
+ Configuration conf = new Configuration();
+ MiniDFSCluster cluster = null;
+ try {
+ cluster = new MiniDFSCluster.Builder(conf).manageDataDfsDirs(true)
+ .manageNameDfsDirs(true).numDataNodes(4)
+ .racks(new String[] { "/rack1", "/rack1", "/rack1", "/rack2" })
+ .build();
+ cluster.waitActive();
+
+ DistributedFileSystem fs = (DistributedFileSystem)cluster.getFileSystem();
+ Path path = new Path("/test1");
+ DFSTestUtil.createFile(fs, path, 1024, (short) 3, 1l);
+
+ cluster.stopDataNode(3);
+ DFSTestUtil.appendFile(fs, path, "hello");
+ } finally {
+ if (null != cluster) {
+ cluster.shutdown();
+ }
+ }
+ }
}