You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by co...@apache.org on 2010/12/08 06:46:47 UTC
svn commit: r1043309 - in /hadoop/hdfs/branches/branch-0.22: ./
src/c++/libhdfs/ src/contrib/hdfsproxy/ src/java/
src/java/org/apache/hadoop/hdfs/server/datanode/ src/test/hdfs/
src/test/hdfs/org/apache/hadoop/hdfs/ src/webapps/datanode/ src/webapps/hd...
Author: cos
Date: Wed Dec 8 05:46:47 2010
New Revision: 1043309
URL: http://svn.apache.org/viewvc?rev=1043309&view=rev
Log:
HDFS-1527. svn merge -c 1043308 from trunk
Modified:
hadoop/hdfs/branches/branch-0.22/ (props changed)
hadoop/hdfs/branches/branch-0.22/CHANGES.txt
hadoop/hdfs/branches/branch-0.22/build.xml (props changed)
hadoop/hdfs/branches/branch-0.22/src/c++/libhdfs/ (props changed)
hadoop/hdfs/branches/branch-0.22/src/contrib/hdfsproxy/ (props changed)
hadoop/hdfs/branches/branch-0.22/src/java/ (props changed)
hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java (props changed)
hadoop/hdfs/branches/branch-0.22/src/test/hdfs/ (props changed)
hadoop/hdfs/branches/branch-0.22/src/test/hdfs/org/apache/hadoop/hdfs/TestLargeBlock.java
hadoop/hdfs/branches/branch-0.22/src/webapps/datanode/ (props changed)
hadoop/hdfs/branches/branch-0.22/src/webapps/hdfs/ (props changed)
hadoop/hdfs/branches/branch-0.22/src/webapps/secondary/ (props changed)
Propchange: hadoop/hdfs/branches/branch-0.22/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Dec 8 05:46:47 2010
@@ -1,4 +1,4 @@
/hadoop/core/branches/branch-0.19/hdfs:713112
/hadoop/hdfs/branches/HDFS-265:796829-820463
/hadoop/hdfs/branches/branch-0.21:820487
-/hadoop/hdfs/trunk:1036213,1036303,1036310,1036631,1036767,1037047,1037109,1037961,1039957,1040005,1040411,1041190,1041261,1042067,1042069,1042517,1043301
+/hadoop/hdfs/trunk:1036213,1036303,1036310,1036631,1036767,1037047,1037109,1037961,1039957,1040005,1040411,1041190,1041261,1042067,1042069,1042517,1043301,1043308
Modified: hadoop/hdfs/branches/branch-0.22/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/branch-0.22/CHANGES.txt?rev=1043309&r1=1043308&r2=1043309&view=diff
==============================================================================
--- hadoop/hdfs/branches/branch-0.22/CHANGES.txt (original)
+++ hadoop/hdfs/branches/branch-0.22/CHANGES.txt Wed Dec 8 05:46:47 2010
@@ -403,6 +403,9 @@ Release 0.22.0 - Unreleased
HDFS-1532. Exclude Findbugs warning in FSImageFormat$Saver. (Todd Lipcon
via cos)
+ HDFS-1527. SocketOutputStream.transferToFully fails for blocks >= 2GB on
+ 32 bit JVM. (Patrick Kling via cos)
+
Release 0.21.1 - Unreleased
IMPROVEMENTS
Propchange: hadoop/hdfs/branches/branch-0.22/build.xml
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Dec 8 05:46:47 2010
@@ -2,4 +2,4 @@
/hadoop/core/trunk/build.xml:779102
/hadoop/hdfs/branches/HDFS-265/build.xml:796829-820463
/hadoop/hdfs/branches/branch-0.21/build.xml:820487
-/hadoop/hdfs/trunk/build.xml:1036213,1036303,1036310,1036631,1036767,1037047,1037109,1037961,1039957,1040005,1040411,1041190,1041261,1042067,1042069,1042517,1043301
+/hadoop/hdfs/trunk/build.xml:1036213,1036303,1036310,1036631,1036767,1037047,1037109,1037961,1039957,1040005,1040411,1041190,1041261,1042067,1042069,1042517,1043301,1043308
Propchange: hadoop/hdfs/branches/branch-0.22/src/c++/libhdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Dec 8 05:46:47 2010
@@ -1,3 +1,3 @@
/hadoop/core/branches/branch-0.19/mapred/src/c++/libhdfs:713112
/hadoop/core/trunk/src/c++/libhdfs:776175-784663
-/hadoop/hdfs/trunk/src/c++/libhdfs:1036213,1036303,1036310,1036631,1036767,1037047,1037109,1037961,1039957,1040005,1040411,1041190,1041261,1042067,1042069,1042517,1043301
+/hadoop/hdfs/trunk/src/c++/libhdfs:1036213,1036303,1036310,1036631,1036767,1037047,1037109,1037961,1039957,1040005,1040411,1041190,1041261,1042067,1042069,1042517,1043301,1043308
Propchange: hadoop/hdfs/branches/branch-0.22/src/contrib/hdfsproxy/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Dec 8 05:46:47 2010
@@ -2,4 +2,4 @@
/hadoop/core/trunk/src/contrib/hdfsproxy:776175-784663
/hadoop/hdfs/branches/HDFS-265/src/contrib/hdfsproxy:796829-820463
/hadoop/hdfs/branches/branch-0.21/src/contrib/hdfsproxy:820487
-/hadoop/hdfs/trunk/src/contrib/hdfsproxy:1036213,1036303,1036310,1036631,1036767,1037047,1037109,1037961,1039957,1040005,1040411,1041190,1041261,1042067,1042069,1042517,1043301
+/hadoop/hdfs/trunk/src/contrib/hdfsproxy:1036213,1036303,1036310,1036631,1036767,1037047,1037109,1037961,1039957,1040005,1040411,1041190,1041261,1042067,1042069,1042517,1043301,1043308
Propchange: hadoop/hdfs/branches/branch-0.22/src/java/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Dec 8 05:46:47 2010
@@ -2,4 +2,4 @@
/hadoop/core/trunk/src/hdfs:776175-785643,785929-786278
/hadoop/hdfs/branches/HDFS-265/src/java:796829-820463
/hadoop/hdfs/branches/branch-0.21/src/java:820487
-/hadoop/hdfs/trunk/src/java:1036213,1036303,1036310,1036631,1036767,1037047,1037109,1037961,1039957,1040005,1040411,1041190,1041261,1042067,1042069,1042517,1043301
+/hadoop/hdfs/trunk/src/java:1036213,1036303,1036310,1036631,1036767,1037047,1037109,1037961,1039957,1040005,1040411,1041190,1041261,1042067,1042069,1042517,1043301,1043308
Modified: hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java?rev=1043309&r1=1043308&r2=1043309&view=diff
==============================================================================
--- hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java (original)
+++ hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java Wed Dec 8 05:46:47 2010
@@ -54,6 +54,8 @@ class BlockSender implements java.io.Clo
/** The visible length of a replica. */
private final long replicaVisibleLength;
+ private static final boolean is32Bit = System.getProperty("sun.arch.data.model").equals("32");
+
private InputStream blockIn; // data stream
private long blockInPosition = -1; // updated while using transferTo().
private DataInputStream checksumIn; // checksum datastream
@@ -143,7 +145,11 @@ class BlockSender implements java.io.Clo
this.chunkOffsetOK = chunkOffsetOK;
this.corruptChecksumOk = corruptChecksumOk;
this.verifyChecksum = verifyChecksum;
- this.transferToAllowed = datanode.transferToAllowed;
+
+ // transferToFully() fails on 32 bit platforms for block sizes >= 2GB,
+ // use normal transfer in those cases
+ this.transferToAllowed = datanode.transferToAllowed &&
+ (!is32Bit || length < (long) Integer.MAX_VALUE);
this.clientTraceFmt = clientTraceFmt;
if ( !corruptChecksumOk || datanode.data.metaFileExists(block) ) {
@@ -396,6 +402,7 @@ class BlockSender implements java.io.Clo
/* exception while writing to the client (well, with transferTo(),
* it could also be while reading from the local file).
*/
+ LOG.error("BlockSender.sendChunks() exception: " + StringUtils.stringifyException(e));
throw ioeToSocketException(e);
}
Propchange: hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Dec 8 05:46:47 2010
@@ -4,4 +4,4 @@
/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:776175-785643,785929-786278
/hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:796829-820463
/hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:820487
-/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:1036213,1036303,1036310,1036631,1036767,1037047,1037109,1037961,1039957,1040005,1040411,1041190,1041261,1042067,1042069,1042517,1043301
+/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:1036213,1036303,1036310,1036631,1036767,1037047,1037109,1037961,1039957,1040005,1040411,1041190,1041261,1042067,1042069,1042517,1043301,1043308
Propchange: hadoop/hdfs/branches/branch-0.22/src/test/hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Dec 8 05:46:47 2010
@@ -2,4 +2,4 @@
/hadoop/core/trunk/src/test/hdfs:776175-785643
/hadoop/hdfs/branches/HDFS-265/src/test/hdfs:796829-820463
/hadoop/hdfs/branches/branch-0.21/src/test/hdfs:820487
-/hadoop/hdfs/trunk/src/test/hdfs:1036213,1036303,1036310,1036631,1036767,1037047,1037109,1037961,1039957,1040005,1040411,1041190,1041261,1042067,1042069,1042517,1043301
+/hadoop/hdfs/trunk/src/test/hdfs:1036213,1036303,1036310,1036631,1036767,1037047,1037109,1037961,1039957,1040005,1040411,1041190,1041261,1042067,1042069,1042517,1043301,1043308
Modified: hadoop/hdfs/branches/branch-0.22/src/test/hdfs/org/apache/hadoop/hdfs/TestLargeBlock.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/branch-0.22/src/test/hdfs/org/apache/hadoop/hdfs/TestLargeBlock.java?rev=1043309&r1=1043308&r2=1043309&view=diff
==============================================================================
--- hadoop/hdfs/branches/branch-0.22/src/test/hdfs/org/apache/hadoop/hdfs/TestLargeBlock.java (original)
+++ hadoop/hdfs/branches/branch-0.22/src/test/hdfs/org/apache/hadoop/hdfs/TestLargeBlock.java Wed Dec 8 05:46:47 2010
@@ -37,7 +37,6 @@ import org.apache.hadoop.hdfs.server.nam
import org.junit.Test;
import static org.junit.Assert.assertTrue;
-import static org.junit.Assume.assumeTrue;
/**
* This class tests that blocks can be larger than 2GB
@@ -58,7 +57,6 @@ public class TestLargeBlock {
static final boolean verifyData = true;
static final byte[] pattern = { 'D', 'E', 'A', 'D', 'B', 'E', 'E', 'F'};
static final boolean simulatedStorage = false;
- private static final String ALLOWED_VM = "64";
// creates a file
static FSDataOutputStream createFile(FileSystem fileSys, Path name, int repl,
@@ -167,7 +165,6 @@ public class TestLargeBlock {
*/
@Test
public void testLargeBlockSize() throws IOException {
- assumeTrue(ALLOWED_VM.equals(System.getProperty("sun.arch.data.model")));
final long blockSize = 2L * 1024L * 1024L * 1024L + 512L; // 2GB + 512B
runTest(blockSize);
}
Propchange: hadoop/hdfs/branches/branch-0.22/src/webapps/datanode/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Dec 8 05:46:47 2010
@@ -2,4 +2,4 @@
/hadoop/core/trunk/src/webapps/datanode:776175-784663
/hadoop/hdfs/branches/HDFS-265/src/webapps/datanode:796829-820463
/hadoop/hdfs/branches/branch-0.21/src/webapps/datanode:820487
-/hadoop/hdfs/trunk/src/webapps/datanode:1036213,1036303,1036310,1036631,1036767,1037047,1037109,1037961,1039957,1040005,1040411,1041190,1041261,1042067,1042069,1042517,1043301
+/hadoop/hdfs/trunk/src/webapps/datanode:1036213,1036303,1036310,1036631,1036767,1037047,1037109,1037961,1039957,1040005,1040411,1041190,1041261,1042067,1042069,1042517,1043301,1043308
Propchange: hadoop/hdfs/branches/branch-0.22/src/webapps/hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Dec 8 05:46:47 2010
@@ -2,4 +2,4 @@
/hadoop/core/trunk/src/webapps/hdfs:776175-784663
/hadoop/hdfs/branches/HDFS-265/src/webapps/hdfs:796829-820463
/hadoop/hdfs/branches/branch-0.21/src/webapps/hdfs:820487
-/hadoop/hdfs/trunk/src/webapps/hdfs:1036213,1036303,1036310,1036631,1036767,1037047,1037109,1037961,1039957,1040005,1040411,1041190,1041261,1042067,1042069,1042517,1043301
+/hadoop/hdfs/trunk/src/webapps/hdfs:1036213,1036303,1036310,1036631,1036767,1037047,1037109,1037961,1039957,1040005,1040411,1041190,1041261,1042067,1042069,1042517,1043301,1043308
Propchange: hadoop/hdfs/branches/branch-0.22/src/webapps/secondary/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Dec 8 05:46:47 2010
@@ -2,4 +2,4 @@
/hadoop/core/trunk/src/webapps/secondary:776175-784663
/hadoop/hdfs/branches/HDFS-265/src/webapps/secondary:796829-820463
/hadoop/hdfs/branches/branch-0.21/src/webapps/secondary:820487
-/hadoop/hdfs/trunk/src/webapps/secondary:1036213,1036303,1036310,1036631,1036767,1037047,1037109,1037961,1039957,1040005,1040411,1041190,1041261,1042067,1042069,1042517,1043301
+/hadoop/hdfs/trunk/src/webapps/secondary:1036213,1036303,1036310,1036631,1036767,1037047,1037109,1037961,1039957,1040005,1040411,1041190,1041261,1042067,1042069,1042517,1043301,1043308