You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by da...@apache.org on 2012/07/16 23:13:51 UTC
svn commit: r1362259 - in
/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs:
CHANGES.txt src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java
Author: daryn
Date: Mon Jul 16 21:13:50 2012
New Revision: 1362259
URL: http://svn.apache.org/viewvc?rev=1362259&view=rev
Log:
svn merge -c 1301182 FIXES: HDFS-3067. NPE in DFSInputStream.readBuffer if read is repeated on corrupted block. Contributed by Henry Robinson.
Modified:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java
Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1362259&r1=1362258&r2=1362259&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Mon Jul 16 21:13:50 2012
@@ -66,6 +66,9 @@ Release 0.23.3 - UNRELEASED
HDFS-3541. Deadlock between recovery, xceiver and packet responder (Vinay via umamahesh)
+ HDFS-3067. NPE in DFSInputStream.readBuffer if read is repeated on
+ corrupted block. (Henry Robinson via atm)
+
Release 0.23.2 - UNRELEASED
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java?rev=1362259&r1=1362258&r2=1362259&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java Mon Jul 16 21:13:50 2012
@@ -538,7 +538,9 @@ public class DFSInputStream extends FSIn
int retries = 2;
while (retries > 0) {
try {
- if (pos > blockEnd) {
+ // currentNode can be left as null if previous read had a checksum
+ // error on the same block. See HDFS-3067
+ if (pos > blockEnd || currentNode == null) {
currentNode = blockSeekTo(pos);
}
int realLen = (int) Math.min((long) len, (blockEnd - pos + 1L));
Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java?rev=1362259&r1=1362258&r2=1362259&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java Mon Jul 16 21:13:50 2012
@@ -47,6 +47,7 @@ import org.apache.hadoop.fs.FSDataOutput
import org.apache.hadoop.fs.FileChecksum;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.UnresolvedLinkException;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.Block;
@@ -62,6 +63,7 @@ import org.apache.hadoop.ipc.RemoteExcep
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.test.GenericTestUtils;
import org.mockito.internal.stubbing.answers.ThrowsException;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
@@ -647,5 +649,52 @@ public class TestDFSClientRetries extend
server.stop();
}
}
+
+ /**
+ * Test that checksum failures are recovered from by the next read on the same
+ * DFSInputStream. Corruption information is not persisted from read call to
+ * read call, so the client should expect consecutive calls to behave the same
+ * way. See HDFS-3067.
+ */
+ public void testRetryOnChecksumFailure()
+ throws UnresolvedLinkException, IOException {
+ HdfsConfiguration conf = new HdfsConfiguration();
+ MiniDFSCluster cluster =
+ new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
+
+ try {
+ final short REPL_FACTOR = 1;
+ final long FILE_LENGTH = 512L;
+ cluster.waitActive();
+ FileSystem fs = cluster.getFileSystem();
+
+ Path path = new Path("/corrupted");
+
+ DFSTestUtil.createFile(fs, path, FILE_LENGTH, REPL_FACTOR, 12345L);
+ DFSTestUtil.waitReplication(fs, path, REPL_FACTOR);
+
+ ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, path);
+ int blockFilesCorrupted = cluster.corruptBlockOnDataNodes(block);
+ assertEquals("All replicas not corrupted", REPL_FACTOR,
+ blockFilesCorrupted);
+
+ InetSocketAddress nnAddr =
+ new InetSocketAddress("localhost", cluster.getNameNodePort());
+ DFSClient client = new DFSClient(nnAddr, conf);
+ DFSInputStream dis = client.open(path.toString());
+ byte[] arr = new byte[(int)FILE_LENGTH];
+ for (int i = 0; i < 2; ++i) {
+ try {
+ dis.read(arr, 0, (int)FILE_LENGTH);
+ fail("Expected ChecksumException not thrown");
+ } catch (Exception ex) {
+ GenericTestUtils.assertExceptionContains(
+ "Checksum error", ex);
+ }
+ }
+ } finally {
+ cluster.shutdown();
+ }
+ }
}