You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by el...@apache.org on 2010/11/20 04:54:29 UTC
svn commit: r1037110 - in /hadoop/hdfs/branches/branch-0.22: ./
src/c++/libhdfs/ src/contrib/hdfsproxy/ src/java/
src/java/org/apache/hadoop/hdfs/server/datanode/ src/test/hdfs/
src/test/hdfs/org/apache/hadoop/hdfs/ src/webapps/datanode/ src/webapps/hd...
Author: eli
Date: Sat Nov 20 03:54:29 2010
New Revision: 1037110
URL: http://svn.apache.org/viewvc?rev=1037110&view=rev
Log:
HDFS-1467. svn merge -c 1037109 from trunk
Modified:
hadoop/hdfs/branches/branch-0.22/ (props changed)
hadoop/hdfs/branches/branch-0.22/CHANGES.txt
hadoop/hdfs/branches/branch-0.22/build.xml (props changed)
hadoop/hdfs/branches/branch-0.22/src/c++/libhdfs/ (props changed)
hadoop/hdfs/branches/branch-0.22/src/contrib/hdfsproxy/ (props changed)
hadoop/hdfs/branches/branch-0.22/src/java/ (props changed)
hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java (props changed)
hadoop/hdfs/branches/branch-0.22/src/test/hdfs/ (props changed)
hadoop/hdfs/branches/branch-0.22/src/test/hdfs/org/apache/hadoop/hdfs/TestPipelines.java
hadoop/hdfs/branches/branch-0.22/src/webapps/datanode/ (props changed)
hadoop/hdfs/branches/branch-0.22/src/webapps/hdfs/ (props changed)
hadoop/hdfs/branches/branch-0.22/src/webapps/secondary/ (props changed)
Propchange: hadoop/hdfs/branches/branch-0.22/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Sat Nov 20 03:54:29 2010
@@ -1,4 +1,4 @@
/hadoop/core/branches/branch-0.19/hdfs:713112
/hadoop/hdfs/branches/HDFS-265:796829-820463
/hadoop/hdfs/branches/branch-0.21:820487
-/hadoop/hdfs/trunk:1036213,1036303,1036310,1036631,1036767,1037047
+/hadoop/hdfs/trunk:1036213,1036303,1036310,1036631,1036767,1037047,1037109
Modified: hadoop/hdfs/branches/branch-0.22/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/branch-0.22/CHANGES.txt?rev=1037110&r1=1037109&r2=1037110&view=diff
==============================================================================
--- hadoop/hdfs/branches/branch-0.22/CHANGES.txt (original)
+++ hadoop/hdfs/branches/branch-0.22/CHANGES.txt Sat Nov 20 03:54:29 2010
@@ -383,6 +383,9 @@ Release 0.22.0 - Unreleased
HDFS-1167. New property for local conf directory in system-test-hdfs.xml
file. (Vinay Thota via cos)
+ HDFS-1467. Append pipeline never succeeds with more than one replica.
+ (Todd Lipcon via eli)
+
Release 0.21.1 - Unreleased
IMPROVEMENTS
Propchange: hadoop/hdfs/branches/branch-0.22/build.xml
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Sat Nov 20 03:54:29 2010
@@ -2,4 +2,4 @@
/hadoop/core/trunk/build.xml:779102
/hadoop/hdfs/branches/HDFS-265/build.xml:796829-820463
/hadoop/hdfs/branches/branch-0.21/build.xml:820487
-/hadoop/hdfs/trunk/build.xml:1036213,1036303,1036310,1036631,1036767,1037047
+/hadoop/hdfs/trunk/build.xml:1036213,1036303,1036310,1036631,1036767,1037047,1037109
Propchange: hadoop/hdfs/branches/branch-0.22/src/c++/libhdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Sat Nov 20 03:54:29 2010
@@ -1,3 +1,3 @@
/hadoop/core/branches/branch-0.19/mapred/src/c++/libhdfs:713112
/hadoop/core/trunk/src/c++/libhdfs:776175-784663
-/hadoop/hdfs/trunk/src/c++/libhdfs:1036213,1036303,1036310,1036631,1036767,1037047
+/hadoop/hdfs/trunk/src/c++/libhdfs:1036213,1036303,1036310,1036631,1036767,1037047,1037109
Propchange: hadoop/hdfs/branches/branch-0.22/src/contrib/hdfsproxy/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Sat Nov 20 03:54:29 2010
@@ -2,4 +2,4 @@
/hadoop/core/trunk/src/contrib/hdfsproxy:776175-784663
/hadoop/hdfs/branches/HDFS-265/src/contrib/hdfsproxy:796829-820463
/hadoop/hdfs/branches/branch-0.21/src/contrib/hdfsproxy:820487
-/hadoop/hdfs/trunk/src/contrib/hdfsproxy:1036213,1036303,1036310,1036631,1036767,1037047
+/hadoop/hdfs/trunk/src/contrib/hdfsproxy:1036213,1036303,1036310,1036631,1036767,1037047,1037109
Propchange: hadoop/hdfs/branches/branch-0.22/src/java/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Sat Nov 20 03:54:29 2010
@@ -2,4 +2,4 @@
/hadoop/core/trunk/src/hdfs:776175-785643,785929-786278
/hadoop/hdfs/branches/HDFS-265/src/java:796829-820463
/hadoop/hdfs/branches/branch-0.21/src/java:820487
-/hadoop/hdfs/trunk/src/java:1036213,1036303,1036310,1036631,1036767,1037047
+/hadoop/hdfs/trunk/src/java:1036213,1036303,1036310,1036631,1036767,1037047,1037109
Modified: hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java?rev=1037110&r1=1037109&r2=1037110&view=diff
==============================================================================
--- hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java (original)
+++ hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java Sat Nov 20 03:54:29 2010
@@ -248,6 +248,11 @@ class DataXceiver extends DataTransferPr
" tcp no delay " + s.getTcpNoDelay());
}
+ // We later mutate block's generation stamp and length, but we need to
+ // forward the original version of the block to downstream mirrors, so
+ // make a copy here.
+ final Block originalBlock = new Block(block);
+
block.setNumBytes(dataXceiverServer.estimateBlockSize);
LOG.info("Receiving block " + block +
" src: " + remoteAddress +
@@ -322,7 +327,7 @@ class DataXceiver extends DataTransferPr
mirrorIn = new DataInputStream(NetUtils.getInputStream(mirrorSock));
// Write header: Copied from DFSClient.java!
- DataTransferProtocol.Sender.opWriteBlock(mirrorOut, block,
+ DataTransferProtocol.Sender.opWriteBlock(mirrorOut, originalBlock,
pipelineSize, stage, newGs, minBytesRcvd, maxBytesRcvd, client,
srcDataNode, targets, blockToken);
Propchange: hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Sat Nov 20 03:54:29 2010
@@ -4,4 +4,4 @@
/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:776175-785643,785929-786278
/hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:796829-820463
/hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:820487
-/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:1036213,1036303,1036310,1036631,1036767,1037047
+/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:1036213,1036303,1036310,1036631,1036767,1037047,1037109
Propchange: hadoop/hdfs/branches/branch-0.22/src/test/hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Sat Nov 20 03:54:29 2010
@@ -2,4 +2,4 @@
/hadoop/core/trunk/src/test/hdfs:776175-785643
/hadoop/hdfs/branches/HDFS-265/src/test/hdfs:796829-820463
/hadoop/hdfs/branches/branch-0.21/src/test/hdfs:820487
-/hadoop/hdfs/trunk/src/test/hdfs:1036213,1036303,1036310,1036631,1036767,1037047
+/hadoop/hdfs/trunk/src/test/hdfs:1036213,1036303,1036310,1036631,1036767,1037047,1037109
Modified: hadoop/hdfs/branches/branch-0.22/src/test/hdfs/org/apache/hadoop/hdfs/TestPipelines.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/branch-0.22/src/test/hdfs/org/apache/hadoop/hdfs/TestPipelines.java?rev=1037110&r1=1037109&r2=1037110&view=diff
==============================================================================
--- hadoop/hdfs/branches/branch-0.22/src/test/hdfs/org/apache/hadoop/hdfs/TestPipelines.java (original)
+++ hadoop/hdfs/branches/branch-0.22/src/test/hdfs/org/apache/hadoop/hdfs/TestPipelines.java Sat Nov 20 03:54:29 2010
@@ -103,12 +103,16 @@ public class TestPipelines {
List<LocatedBlock> lb = cluster.getNameNode().getBlockLocations(
filePath.toString(), FILE_SIZE - 1, FILE_SIZE).getLocatedBlocks();
- Replica r = DataNodeAdapter.fetchReplicaInfo(cluster.getDataNodes().get(0),
- lb.get(0).getBlock().getBlockId());
- assertTrue("Replica shouldn'e be null", r != null);
- assertEquals(
- "Should be RBW replica after sequence of calls append()/write()/hflush()",
- HdfsConstants.ReplicaState.RBW, r.getState());
+ for (DataNode dn : cluster.getDataNodes()) {
+ Replica r = DataNodeAdapter.fetchReplicaInfo(
+ dn, lb.get(0).getBlock().getBlockId());
+
+ assertTrue("Replica on DN " + dn + " shouldn't be null", r != null);
+ assertEquals(
+ "Should be RBW replica on " + dn + " after sequence of calls " +
+ "append()/write()/hflush()",
+ HdfsConstants.ReplicaState.RBW, r.getState());
+ }
ofs.close();
}
Propchange: hadoop/hdfs/branches/branch-0.22/src/webapps/datanode/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Sat Nov 20 03:54:29 2010
@@ -2,4 +2,4 @@
/hadoop/core/trunk/src/webapps/datanode:776175-784663
/hadoop/hdfs/branches/HDFS-265/src/webapps/datanode:796829-820463
/hadoop/hdfs/branches/branch-0.21/src/webapps/datanode:820487
-/hadoop/hdfs/trunk/src/webapps/datanode:1036213,1036303,1036310,1036631,1036767,1037047
+/hadoop/hdfs/trunk/src/webapps/datanode:1036213,1036303,1036310,1036631,1036767,1037047,1037109
Propchange: hadoop/hdfs/branches/branch-0.22/src/webapps/hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Sat Nov 20 03:54:29 2010
@@ -2,4 +2,4 @@
/hadoop/core/trunk/src/webapps/hdfs:776175-784663
/hadoop/hdfs/branches/HDFS-265/src/webapps/hdfs:796829-820463
/hadoop/hdfs/branches/branch-0.21/src/webapps/hdfs:820487
-/hadoop/hdfs/trunk/src/webapps/hdfs:1036213,1036303,1036310,1036631,1036767,1037047
+/hadoop/hdfs/trunk/src/webapps/hdfs:1036213,1036303,1036310,1036631,1036767,1037047,1037109
Propchange: hadoop/hdfs/branches/branch-0.22/src/webapps/secondary/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Sat Nov 20 03:54:29 2010
@@ -2,4 +2,4 @@
/hadoop/core/trunk/src/webapps/secondary:776175-784663
/hadoop/hdfs/branches/HDFS-265/src/webapps/secondary:796829-820463
/hadoop/hdfs/branches/branch-0.21/src/webapps/secondary:820487
-/hadoop/hdfs/trunk/src/webapps/secondary:1036213,1036303,1036310,1036631,1036767,1037047
+/hadoop/hdfs/trunk/src/webapps/secondary:1036213,1036303,1036310,1036631,1036767,1037047,1037109