You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by gi...@apache.org on 2018/10/17 14:53:25 UTC
[04/35] hbase-site git commit: Published site at
8cc56bd18c40ba9a7131336e97c74f8d97d8b2be.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/713132a3/testdevapidocs/src-html/org/apache/hadoop/hbase/io/hfile/TestChecksum.html
----------------------------------------------------------------------
diff --git a/testdevapidocs/src-html/org/apache/hadoop/hbase/io/hfile/TestChecksum.html b/testdevapidocs/src-html/org/apache/hadoop/hbase/io/hfile/TestChecksum.html
index 36bea79..2dd2eb6 100644
--- a/testdevapidocs/src-html/org/apache/hadoop/hbase/io/hfile/TestChecksum.html
+++ b/testdevapidocs/src-html/org/apache/hadoop/hbase/io/hfile/TestChecksum.html
@@ -307,105 +307,103 @@
<span class="sourceLineNo">299</span> long expectedChunks = ChecksumUtil.numChunks(<a name="line.299"></a>
<span class="sourceLineNo">300</span> dataSize + HConstants.HFILEBLOCK_HEADER_SIZE,<a name="line.300"></a>
<span class="sourceLineNo">301</span> bytesPerChecksum);<a name="line.301"></a>
-<span class="sourceLineNo">302</span> LOG.info("testChecksumChunks: pread=" + pread +<a name="line.302"></a>
-<span class="sourceLineNo">303</span> ", bytesPerChecksum=" + bytesPerChecksum +<a name="line.303"></a>
-<span class="sourceLineNo">304</span> ", fileSize=" + totalSize +<a name="line.304"></a>
-<span class="sourceLineNo">305</span> ", dataSize=" + dataSize +<a name="line.305"></a>
-<span class="sourceLineNo">306</span> ", expectedChunks=" + expectedChunks);<a name="line.306"></a>
-<span class="sourceLineNo">307</span><a name="line.307"></a>
-<span class="sourceLineNo">308</span> // Verify hbase checksums.<a name="line.308"></a>
-<span class="sourceLineNo">309</span> assertEquals(true, hfs.useHBaseChecksum());<a name="line.309"></a>
-<span class="sourceLineNo">310</span><a name="line.310"></a>
-<span class="sourceLineNo">311</span> // Read data back from file.<a name="line.311"></a>
-<span class="sourceLineNo">312</span> FSDataInputStream is = fs.open(path);<a name="line.312"></a>
-<span class="sourceLineNo">313</span> FSDataInputStream nochecksum = hfs.getNoChecksumFs().open(path);<a name="line.313"></a>
-<span class="sourceLineNo">314</span> meta = new HFileContextBuilder()<a name="line.314"></a>
-<span class="sourceLineNo">315</span> .withCompression(algo)<a name="line.315"></a>
-<span class="sourceLineNo">316</span> .withIncludesMvcc(true)<a name="line.316"></a>
-<span class="sourceLineNo">317</span> .withIncludesTags(useTags)<a name="line.317"></a>
-<span class="sourceLineNo">318</span> .withHBaseCheckSum(true)<a name="line.318"></a>
-<span class="sourceLineNo">319</span> .withBytesPerCheckSum(bytesPerChecksum)<a name="line.319"></a>
-<span class="sourceLineNo">320</span> .build();<a name="line.320"></a>
-<span class="sourceLineNo">321</span> HFileBlock.FSReader hbr = new HFileBlock.FSReaderImpl(new FSDataInputStreamWrapper(<a name="line.321"></a>
-<span class="sourceLineNo">322</span> is, nochecksum), totalSize, hfs, path, meta);<a name="line.322"></a>
-<span class="sourceLineNo">323</span> HFileBlock b = hbr.readBlockData(0, -1, pread, false);<a name="line.323"></a>
-<span class="sourceLineNo">324</span> is.close();<a name="line.324"></a>
-<span class="sourceLineNo">325</span> b.sanityCheck();<a name="line.325"></a>
-<span class="sourceLineNo">326</span> assertEquals(dataSize, b.getUncompressedSizeWithoutHeader());<a name="line.326"></a>
-<span class="sourceLineNo">327</span><a name="line.327"></a>
-<span class="sourceLineNo">328</span> // verify that we have the expected number of checksum chunks<a name="line.328"></a>
-<span class="sourceLineNo">329</span> assertEquals(totalSize, HConstants.HFILEBLOCK_HEADER_SIZE + dataSize +<a name="line.329"></a>
-<span class="sourceLineNo">330</span> expectedChunks * HFileBlock.CHECKSUM_SIZE);<a name="line.330"></a>
-<span class="sourceLineNo">331</span><a name="line.331"></a>
-<span class="sourceLineNo">332</span> // assert that we did not encounter hbase checksum verification failures<a name="line.332"></a>
-<span class="sourceLineNo">333</span> assertEquals(0, HFile.getAndResetChecksumFailuresCount());<a name="line.333"></a>
-<span class="sourceLineNo">334</span> }<a name="line.334"></a>
-<span class="sourceLineNo">335</span> }<a name="line.335"></a>
-<span class="sourceLineNo">336</span> }<a name="line.336"></a>
-<span class="sourceLineNo">337</span><a name="line.337"></a>
-<span class="sourceLineNo">338</span> private void validateData(DataInputStream in) throws IOException {<a name="line.338"></a>
-<span class="sourceLineNo">339</span> // validate data<a name="line.339"></a>
-<span class="sourceLineNo">340</span> for (int i = 0; i < 1234; i++) {<a name="line.340"></a>
-<span class="sourceLineNo">341</span> int val = in.readInt();<a name="line.341"></a>
-<span class="sourceLineNo">342</span> assertEquals("testChecksumCorruption: data mismatch at index " + i, i, val);<a name="line.342"></a>
-<span class="sourceLineNo">343</span> }<a name="line.343"></a>
-<span class="sourceLineNo">344</span> }<a name="line.344"></a>
-<span class="sourceLineNo">345</span><a name="line.345"></a>
-<span class="sourceLineNo">346</span> /**<a name="line.346"></a>
-<span class="sourceLineNo">347</span> * This class is to test checksum behavior when data is corrupted. It mimics the following<a name="line.347"></a>
-<span class="sourceLineNo">348</span> * behavior:<a name="line.348"></a>
-<span class="sourceLineNo">349</span> * - When fs checksum is disabled, hbase may get corrupted data from hdfs. If verifyChecksum<a name="line.349"></a>
-<span class="sourceLineNo">350</span> * is true, it means hbase checksum is on and fs checksum is off, so we corrupt the data.<a name="line.350"></a>
-<span class="sourceLineNo">351</span> * - When fs checksum is enabled, hdfs will get a different copy from another node, and will<a name="line.351"></a>
-<span class="sourceLineNo">352</span> * always return correct data. So we don't corrupt the data when verifyChecksum for hbase is<a name="line.352"></a>
-<span class="sourceLineNo">353</span> * off.<a name="line.353"></a>
-<span class="sourceLineNo">354</span> */<a name="line.354"></a>
-<span class="sourceLineNo">355</span> static private class CorruptedFSReaderImpl extends HFileBlock.FSReaderImpl {<a name="line.355"></a>
-<span class="sourceLineNo">356</span> /**<a name="line.356"></a>
-<span class="sourceLineNo">357</span> * If set to true, corrupt reads using readAtOffset(...).<a name="line.357"></a>
-<span class="sourceLineNo">358</span> */<a name="line.358"></a>
-<span class="sourceLineNo">359</span> boolean corruptDataStream = false;<a name="line.359"></a>
-<span class="sourceLineNo">360</span><a name="line.360"></a>
-<span class="sourceLineNo">361</span> public CorruptedFSReaderImpl(FSDataInputStreamWrapper istream, long fileSize, FileSystem fs,<a name="line.361"></a>
-<span class="sourceLineNo">362</span> Path path, HFileContext meta) throws IOException {<a name="line.362"></a>
-<span class="sourceLineNo">363</span> super(istream, fileSize, (HFileSystem) fs, path, meta);<a name="line.363"></a>
-<span class="sourceLineNo">364</span> }<a name="line.364"></a>
-<span class="sourceLineNo">365</span><a name="line.365"></a>
-<span class="sourceLineNo">366</span> @Override<a name="line.366"></a>
-<span class="sourceLineNo">367</span> protected HFileBlock readBlockDataInternal(FSDataInputStream is, long offset,<a name="line.367"></a>
-<span class="sourceLineNo">368</span> long onDiskSizeWithHeaderL, boolean pread, boolean verifyChecksum, boolean updateMetrics)<a name="line.368"></a>
-<span class="sourceLineNo">369</span> throws IOException {<a name="line.369"></a>
-<span class="sourceLineNo">370</span> if (verifyChecksum) {<a name="line.370"></a>
-<span class="sourceLineNo">371</span> corruptDataStream = true;<a name="line.371"></a>
-<span class="sourceLineNo">372</span> }<a name="line.372"></a>
-<span class="sourceLineNo">373</span> HFileBlock b = super.readBlockDataInternal(is, offset, onDiskSizeWithHeaderL, pread,<a name="line.373"></a>
-<span class="sourceLineNo">374</span> verifyChecksum, updateMetrics);<a name="line.374"></a>
-<span class="sourceLineNo">375</span> corruptDataStream = false;<a name="line.375"></a>
-<span class="sourceLineNo">376</span> return b;<a name="line.376"></a>
-<span class="sourceLineNo">377</span> }<a name="line.377"></a>
-<span class="sourceLineNo">378</span><a name="line.378"></a>
-<span class="sourceLineNo">379</span> @Override<a name="line.379"></a>
-<span class="sourceLineNo">380</span> protected int readAtOffset(FSDataInputStream istream, byte [] dest, int destOffset, int size,<a name="line.380"></a>
-<span class="sourceLineNo">381</span> boolean peekIntoNextBlock, long fileOffset, boolean pread) throws IOException {<a name="line.381"></a>
-<span class="sourceLineNo">382</span> int returnValue = super.readAtOffset(istream, dest, destOffset, size, peekIntoNextBlock,<a name="line.382"></a>
-<span class="sourceLineNo">383</span> fileOffset, pread);<a name="line.383"></a>
-<span class="sourceLineNo">384</span> if (!corruptDataStream) {<a name="line.384"></a>
-<span class="sourceLineNo">385</span> return returnValue;<a name="line.385"></a>
-<span class="sourceLineNo">386</span> }<a name="line.386"></a>
-<span class="sourceLineNo">387</span> // Corrupt 3rd character of block magic of next block's header.<a name="line.387"></a>
-<span class="sourceLineNo">388</span> if (peekIntoNextBlock) {<a name="line.388"></a>
-<span class="sourceLineNo">389</span> dest[destOffset + size + 3] = 0b00000000;<a name="line.389"></a>
-<span class="sourceLineNo">390</span> }<a name="line.390"></a>
-<span class="sourceLineNo">391</span> // We might be reading this block's header too, corrupt it.<a name="line.391"></a>
-<span class="sourceLineNo">392</span> dest[destOffset + 1] = 0b00000000;<a name="line.392"></a>
-<span class="sourceLineNo">393</span> // Corrupt non header data<a name="line.393"></a>
-<span class="sourceLineNo">394</span> if (size > hdrSize) {<a name="line.394"></a>
-<span class="sourceLineNo">395</span> dest[destOffset + hdrSize + 1] = 0b00000000;<a name="line.395"></a>
-<span class="sourceLineNo">396</span> }<a name="line.396"></a>
-<span class="sourceLineNo">397</span> return returnValue;<a name="line.397"></a>
-<span class="sourceLineNo">398</span> }<a name="line.398"></a>
-<span class="sourceLineNo">399</span> }<a name="line.399"></a>
-<span class="sourceLineNo">400</span>}<a name="line.400"></a>
+<span class="sourceLineNo">302</span> LOG.info("testChecksumChunks: pread={}, bytesPerChecksum={}, fileSize={}, "<a name="line.302"></a>
+<span class="sourceLineNo">303</span> + "dataSize={}, expectedChunks={}, compression={}", pread, bytesPerChecksum,<a name="line.303"></a>
+<span class="sourceLineNo">304</span> totalSize, dataSize, expectedChunks, algo.toString());<a name="line.304"></a>
+<span class="sourceLineNo">305</span><a name="line.305"></a>
+<span class="sourceLineNo">306</span> // Verify hbase checksums.<a name="line.306"></a>
+<span class="sourceLineNo">307</span> assertEquals(true, hfs.useHBaseChecksum());<a name="line.307"></a>
+<span class="sourceLineNo">308</span><a name="line.308"></a>
+<span class="sourceLineNo">309</span> // Read data back from file.<a name="line.309"></a>
+<span class="sourceLineNo">310</span> FSDataInputStream is = fs.open(path);<a name="line.310"></a>
+<span class="sourceLineNo">311</span> FSDataInputStream nochecksum = hfs.getNoChecksumFs().open(path);<a name="line.311"></a>
+<span class="sourceLineNo">312</span> meta = new HFileContextBuilder()<a name="line.312"></a>
+<span class="sourceLineNo">313</span> .withCompression(algo)<a name="line.313"></a>
+<span class="sourceLineNo">314</span> .withIncludesMvcc(true)<a name="line.314"></a>
+<span class="sourceLineNo">315</span> .withIncludesTags(useTags)<a name="line.315"></a>
+<span class="sourceLineNo">316</span> .withHBaseCheckSum(true)<a name="line.316"></a>
+<span class="sourceLineNo">317</span> .withBytesPerCheckSum(bytesPerChecksum)<a name="line.317"></a>
+<span class="sourceLineNo">318</span> .build();<a name="line.318"></a>
+<span class="sourceLineNo">319</span> HFileBlock.FSReader hbr = new HFileBlock.FSReaderImpl(new FSDataInputStreamWrapper(<a name="line.319"></a>
+<span class="sourceLineNo">320</span> is, nochecksum), totalSize, hfs, path, meta);<a name="line.320"></a>
+<span class="sourceLineNo">321</span> HFileBlock b = hbr.readBlockData(0, -1, pread, false);<a name="line.321"></a>
+<span class="sourceLineNo">322</span> is.close();<a name="line.322"></a>
+<span class="sourceLineNo">323</span> b.sanityCheck();<a name="line.323"></a>
+<span class="sourceLineNo">324</span> assertEquals(dataSize, b.getUncompressedSizeWithoutHeader());<a name="line.324"></a>
+<span class="sourceLineNo">325</span><a name="line.325"></a>
+<span class="sourceLineNo">326</span> // verify that we have the expected number of checksum chunks<a name="line.326"></a>
+<span class="sourceLineNo">327</span> assertEquals(totalSize, HConstants.HFILEBLOCK_HEADER_SIZE + dataSize +<a name="line.327"></a>
+<span class="sourceLineNo">328</span> expectedChunks * HFileBlock.CHECKSUM_SIZE);<a name="line.328"></a>
+<span class="sourceLineNo">329</span><a name="line.329"></a>
+<span class="sourceLineNo">330</span> // assert that we did not encounter hbase checksum verification failures<a name="line.330"></a>
+<span class="sourceLineNo">331</span> assertEquals(0, HFile.getAndResetChecksumFailuresCount());<a name="line.331"></a>
+<span class="sourceLineNo">332</span> }<a name="line.332"></a>
+<span class="sourceLineNo">333</span> }<a name="line.333"></a>
+<span class="sourceLineNo">334</span> }<a name="line.334"></a>
+<span class="sourceLineNo">335</span><a name="line.335"></a>
+<span class="sourceLineNo">336</span> private void validateData(DataInputStream in) throws IOException {<a name="line.336"></a>
+<span class="sourceLineNo">337</span> // validate data<a name="line.337"></a>
+<span class="sourceLineNo">338</span> for (int i = 0; i < 1234; i++) {<a name="line.338"></a>
+<span class="sourceLineNo">339</span> int val = in.readInt();<a name="line.339"></a>
+<span class="sourceLineNo">340</span> assertEquals("testChecksumCorruption: data mismatch at index " + i, i, val);<a name="line.340"></a>
+<span class="sourceLineNo">341</span> }<a name="line.341"></a>
+<span class="sourceLineNo">342</span> }<a name="line.342"></a>
+<span class="sourceLineNo">343</span><a name="line.343"></a>
+<span class="sourceLineNo">344</span> /**<a name="line.344"></a>
+<span class="sourceLineNo">345</span> * This class is to test checksum behavior when data is corrupted. It mimics the following<a name="line.345"></a>
+<span class="sourceLineNo">346</span> * behavior:<a name="line.346"></a>
+<span class="sourceLineNo">347</span> * - When fs checksum is disabled, hbase may get corrupted data from hdfs. If verifyChecksum<a name="line.347"></a>
+<span class="sourceLineNo">348</span> * is true, it means hbase checksum is on and fs checksum is off, so we corrupt the data.<a name="line.348"></a>
+<span class="sourceLineNo">349</span> * - When fs checksum is enabled, hdfs will get a different copy from another node, and will<a name="line.349"></a>
+<span class="sourceLineNo">350</span> * always return correct data. So we don't corrupt the data when verifyChecksum for hbase is<a name="line.350"></a>
+<span class="sourceLineNo">351</span> * off.<a name="line.351"></a>
+<span class="sourceLineNo">352</span> */<a name="line.352"></a>
+<span class="sourceLineNo">353</span> static private class CorruptedFSReaderImpl extends HFileBlock.FSReaderImpl {<a name="line.353"></a>
+<span class="sourceLineNo">354</span> /**<a name="line.354"></a>
+<span class="sourceLineNo">355</span> * If set to true, corrupt reads using readAtOffset(...).<a name="line.355"></a>
+<span class="sourceLineNo">356</span> */<a name="line.356"></a>
+<span class="sourceLineNo">357</span> boolean corruptDataStream = false;<a name="line.357"></a>
+<span class="sourceLineNo">358</span><a name="line.358"></a>
+<span class="sourceLineNo">359</span> public CorruptedFSReaderImpl(FSDataInputStreamWrapper istream, long fileSize, FileSystem fs,<a name="line.359"></a>
+<span class="sourceLineNo">360</span> Path path, HFileContext meta) throws IOException {<a name="line.360"></a>
+<span class="sourceLineNo">361</span> super(istream, fileSize, (HFileSystem) fs, path, meta);<a name="line.361"></a>
+<span class="sourceLineNo">362</span> }<a name="line.362"></a>
+<span class="sourceLineNo">363</span><a name="line.363"></a>
+<span class="sourceLineNo">364</span> @Override<a name="line.364"></a>
+<span class="sourceLineNo">365</span> protected HFileBlock readBlockDataInternal(FSDataInputStream is, long offset,<a name="line.365"></a>
+<span class="sourceLineNo">366</span> long onDiskSizeWithHeaderL, boolean pread, boolean verifyChecksum, boolean updateMetrics)<a name="line.366"></a>
+<span class="sourceLineNo">367</span> throws IOException {<a name="line.367"></a>
+<span class="sourceLineNo">368</span> if (verifyChecksum) {<a name="line.368"></a>
+<span class="sourceLineNo">369</span> corruptDataStream = true;<a name="line.369"></a>
+<span class="sourceLineNo">370</span> }<a name="line.370"></a>
+<span class="sourceLineNo">371</span> HFileBlock b = super.readBlockDataInternal(is, offset, onDiskSizeWithHeaderL, pread,<a name="line.371"></a>
+<span class="sourceLineNo">372</span> verifyChecksum, updateMetrics);<a name="line.372"></a>
+<span class="sourceLineNo">373</span> corruptDataStream = false;<a name="line.373"></a>
+<span class="sourceLineNo">374</span> return b;<a name="line.374"></a>
+<span class="sourceLineNo">375</span> }<a name="line.375"></a>
+<span class="sourceLineNo">376</span><a name="line.376"></a>
+<span class="sourceLineNo">377</span> @Override<a name="line.377"></a>
+<span class="sourceLineNo">378</span> protected int readAtOffset(FSDataInputStream istream, byte [] dest, int destOffset, int size,<a name="line.378"></a>
+<span class="sourceLineNo">379</span> boolean peekIntoNextBlock, long fileOffset, boolean pread) throws IOException {<a name="line.379"></a>
+<span class="sourceLineNo">380</span> int returnValue = super.readAtOffset(istream, dest, destOffset, size, peekIntoNextBlock,<a name="line.380"></a>
+<span class="sourceLineNo">381</span> fileOffset, pread);<a name="line.381"></a>
+<span class="sourceLineNo">382</span> if (!corruptDataStream) {<a name="line.382"></a>
+<span class="sourceLineNo">383</span> return returnValue;<a name="line.383"></a>
+<span class="sourceLineNo">384</span> }<a name="line.384"></a>
+<span class="sourceLineNo">385</span> // Corrupt 3rd character of block magic of next block's header.<a name="line.385"></a>
+<span class="sourceLineNo">386</span> if (peekIntoNextBlock) {<a name="line.386"></a>
+<span class="sourceLineNo">387</span> dest[destOffset + size + 3] = 0b00000000;<a name="line.387"></a>
+<span class="sourceLineNo">388</span> }<a name="line.388"></a>
+<span class="sourceLineNo">389</span> // We might be reading this block's header too, corrupt it.<a name="line.389"></a>
+<span class="sourceLineNo">390</span> dest[destOffset + 1] = 0b00000000;<a name="line.390"></a>
+<span class="sourceLineNo">391</span> // Corrupt non header data<a name="line.391"></a>
+<span class="sourceLineNo">392</span> if (size > hdrSize) {<a name="line.392"></a>
+<span class="sourceLineNo">393</span> dest[destOffset + hdrSize + 1] = 0b00000000;<a name="line.393"></a>
+<span class="sourceLineNo">394</span> }<a name="line.394"></a>
+<span class="sourceLineNo">395</span> return returnValue;<a name="line.395"></a>
+<span class="sourceLineNo">396</span> }<a name="line.396"></a>
+<span class="sourceLineNo">397</span> }<a name="line.397"></a>
+<span class="sourceLineNo">398</span>}<a name="line.398"></a>