You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by bo...@apache.org on 2012/01/27 23:42:48 UTC
svn commit: r1236911 - in /hadoop/common/trunk/hadoop-common-project:
dev-support/ hadoop-common/ hadoop-common/src/main/java/org/apache/hadoop/fs/
hadoop-common/src/test/java/org/apache/hadoop/fs/
Author: bobby
Date: Fri Jan 27 22:42:47 2012
New Revision: 1236911
URL: http://svn.apache.org/viewvc?rev=1236911&view=rev
Log:
HADOOP-7998 CheckFileSystem does not correctly honor setVerifyChecksum (Daryn Sharp via bobby)
Modified:
hadoop/common/trunk/hadoop-common-project/dev-support/test-patch.properties
hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestChecksumFileSystem.java
Modified: hadoop/common/trunk/hadoop-common-project/dev-support/test-patch.properties
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/dev-support/test-patch.properties?rev=1236911&r1=1236910&r2=1236911&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/dev-support/test-patch.properties (original)
+++ hadoop/common/trunk/hadoop-common-project/dev-support/test-patch.properties Fri Jan 27 22:42:47 2012
@@ -18,4 +18,4 @@
OK_RELEASEAUDIT_WARNINGS=0
OK_FINDBUGS_WARNINGS=0
-OK_JAVADOC_WARNINGS=6
+OK_JAVADOC_WARNINGS=13
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1236911&r1=1236910&r2=1236911&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Fri Jan 27 22:42:47 2012
@@ -77,6 +77,8 @@ Trunk (unreleased changes)
HADOOP-7965. Support for protocol version and signature in PB. (jitendra)
BUGS
+ HADOOP-7998. CheckFileSystem does not correctly honor setVerifyChecksum
+ (Daryn Sharp via bobby)
HADOOP-7851. Configuration.getClasses() never returns the default value.
(Uma Maheswara Rao G via amarrk)
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java?rev=1236911&r1=1236910&r2=1236911&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ChecksumFileSystem.java Fri Jan 27 22:42:47 2012
@@ -304,8 +304,9 @@ public abstract class ChecksumFileSystem
*/
@Override
public FSDataInputStream open(Path f, int bufferSize) throws IOException {
- return new FSDataInputStream(
- new ChecksumFSInputChecker(this, f, bufferSize));
+ return verifyChecksum
+ ? new FSDataInputStream(new ChecksumFSInputChecker(this, f, bufferSize))
+ : getRawFileSystem().open(f, bufferSize);
}
/** {@inheritDoc} */
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestChecksumFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestChecksumFileSystem.java?rev=1236911&r1=1236910&r2=1236911&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestChecksumFileSystem.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestChecksumFileSystem.java Fri Jan 27 22:42:47 2012
@@ -22,12 +22,22 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FSDataOutputStream;
import static org.apache.hadoop.fs.FileSystemTestHelper.*;
import org.apache.hadoop.conf.Configuration;
-import junit.framework.TestCase;
+import org.junit.*;
+import static org.junit.Assert.*;
-public class TestChecksumFileSystem extends TestCase {
+public class TestChecksumFileSystem {
static final String TEST_ROOT_DIR
= System.getProperty("test.build.data","build/test/data/work-dir/localfs");
+ static LocalFileSystem localFs;
+
+ @Before
+ public void resetLocalFs() throws Exception {
+ localFs = FileSystem.getLocal(new Configuration());
+ localFs.setVerifyChecksum(true);
+ }
+
+ @Test
public void testgetChecksumLength() throws Exception {
assertEquals(8, ChecksumFileSystem.getChecksumLength(0L, 512));
assertEquals(12, ChecksumFileSystem.getChecksumLength(1L, 512));
@@ -40,9 +50,8 @@ public class TestChecksumFileSystem exte
ChecksumFileSystem.getChecksumLength(10000000000000L, 10));
}
+ @Test
public void testVerifyChecksum() throws Exception {
- Configuration conf = new Configuration();
- LocalFileSystem localFs = FileSystem.getLocal(conf);
Path testPath = new Path(TEST_ROOT_DIR, "testPath");
Path testPath11 = new Path(TEST_ROOT_DIR, "testPath11");
FSDataOutputStream fout = localFs.create(testPath);
@@ -68,7 +77,7 @@ public class TestChecksumFileSystem exte
//copying the wrong checksum file
FileUtil.copy(localFs, localFs.getChecksumFile(testPath11), localFs,
- localFs.getChecksumFile(testPath),false,true,conf);
+ localFs.getChecksumFile(testPath),false,true,localFs.getConf());
assertTrue("checksum exists", localFs.exists(localFs.getChecksumFile(testPath)));
boolean errorRead = false;
@@ -80,20 +89,13 @@ public class TestChecksumFileSystem exte
assertTrue("error reading", errorRead);
//now setting verify false, the read should succeed
- try {
- localFs.setVerifyChecksum(false);
- String str = readFile(localFs, testPath, 1024).toString();
- assertTrue("read", "testing".equals(str));
- } finally {
- // reset for other tests
- localFs.setVerifyChecksum(true);
- }
-
+ localFs.setVerifyChecksum(false);
+ String str = readFile(localFs, testPath, 1024).toString();
+ assertTrue("read", "testing".equals(str));
}
+ @Test
public void testMultiChunkFile() throws Exception {
- Configuration conf = new Configuration();
- LocalFileSystem localFs = FileSystem.getLocal(conf);
Path testPath = new Path(TEST_ROOT_DIR, "testMultiChunk");
FSDataOutputStream fout = localFs.create(testPath);
for (int i = 0; i < 1000; i++) {
@@ -116,9 +118,8 @@ public class TestChecksumFileSystem exte
* Test to ensure that if the checksum file is truncated, a
* ChecksumException is thrown
*/
+ @Test
public void testTruncatedChecksum() throws Exception {
- Configuration conf = new Configuration();
- LocalFileSystem localFs = FileSystem.getLocal(conf);
Path testPath = new Path(TEST_ROOT_DIR, "testtruncatedcrc");
FSDataOutputStream fout = localFs.create(testPath);
fout.write("testing truncation".getBytes());
@@ -146,14 +147,60 @@ public class TestChecksumFileSystem exte
}
// telling it not to verify checksums, should avoid issue.
+ localFs.setVerifyChecksum(false);
+ String str = readFile(localFs, testPath, 1024).toString();
+ assertTrue("read", "testing truncation".equals(str));
+ }
+
+ @Test
+ public void testStreamType() throws Exception {
+ Path testPath = new Path(TEST_ROOT_DIR, "testStreamType");
+ localFs.create(testPath).close();
+ FSDataInputStream in = null;
+
+ localFs.setVerifyChecksum(true);
+ in = localFs.open(testPath);
+ assertTrue("stream is input checker",
+ in.getWrappedStream() instanceof FSInputChecker);
+
+ localFs.setVerifyChecksum(false);
+ in = localFs.open(testPath);
+ assertFalse("stream is not input checker",
+ in.getWrappedStream() instanceof FSInputChecker);
+ }
+
+ @Test
+ public void testCorruptedChecksum() throws Exception {
+ Path testPath = new Path(TEST_ROOT_DIR, "testCorruptChecksum");
+ Path checksumPath = localFs.getChecksumFile(testPath);
+
+ // write a file to generate checksum
+ FSDataOutputStream out = localFs.create(testPath, true);
+ out.write("testing 1 2 3".getBytes());
+ out.close();
+ assertTrue(localFs.exists(checksumPath));
+ FileStatus stat = localFs.getFileStatus(checksumPath);
+
+ // alter file directly so checksum is invalid
+ out = localFs.getRawFileSystem().create(testPath, true);
+ out.write("testing stale checksum".getBytes());
+ out.close();
+ assertTrue(localFs.exists(checksumPath));
+ // checksum didn't change on disk
+ assertEquals(stat, localFs.getFileStatus(checksumPath));
+
+ Exception e = null;
try {
- localFs.setVerifyChecksum(false);
- String str = readFile(localFs, testPath, 1024).toString();
- assertTrue("read", "testing truncation".equals(str));
- } finally {
- // reset for other tests
localFs.setVerifyChecksum(true);
+ readFile(localFs, testPath, 1024);
+ } catch (ChecksumException ce) {
+ e = ce;
+ } finally {
+ assertNotNull("got checksum error", e);
}
+ localFs.setVerifyChecksum(false);
+ String str = readFile(localFs, testPath, 1024);
+ assertEquals("testing stale checksum", str);
}
}