You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by mb...@apache.org on 2012/03/08 23:55:50 UTC
svn commit: r1298641 [4/4] - in /hbase/trunk/src:
main/java/org/apache/hadoop/hbase/ main/java/org/apache/hadoop/hbase/fs/
main/java/org/apache/hadoop/hbase/io/hfile/
main/java/org/apache/hadoop/hbase/mapreduce/
main/java/org/apache/hadoop/hbase/region...
Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java?rev=1298641&r1=1298640&r2=1298641&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java Thu Mar 8 22:55:49 2012
@@ -33,6 +33,7 @@ import org.apache.hadoop.hbase.io.encodi
import org.apache.hadoop.hbase.io.encoding.RedundantKVGenerator;
import org.apache.hadoop.hbase.regionserver.metrics.SchemaConfigured;
import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
+import org.apache.hadoop.hbase.util.ChecksumType;
import org.apache.hadoop.hbase.util.Pair;
import org.junit.After;
import org.junit.Before;
@@ -123,12 +124,14 @@ public class TestHFileDataBlockEncoder {
HFileBlock block = getSampleHFileBlock();
Pair<ByteBuffer, BlockType> result =
blockEncoder.beforeWriteToDisk(block.getBufferWithoutHeader(),
- includesMemstoreTS);
+ includesMemstoreTS, HFileBlock.DUMMY_HEADER);
int size = result.getFirst().limit() - HFileBlock.HEADER_SIZE;
HFileBlock blockOnDisk = new HFileBlock(result.getSecond(),
size, size, -1, result.getFirst(), HFileBlock.FILL_HEADER, 0,
- includesMemstoreTS);
+ includesMemstoreTS, block.getMinorVersion(),
+ block.getBytesPerChecksum(), block.getChecksumType(),
+ block.getOnDiskDataSizeWithHeader());
if (blockEncoder.getEncodingOnDisk() !=
DataBlockEncoding.NONE) {
@@ -158,7 +161,8 @@ public class TestHFileDataBlockEncoder {
keyValues.rewind();
buf.put(keyValues);
HFileBlock b = new HFileBlock(BlockType.DATA, size, size, -1, buf,
- HFileBlock.FILL_HEADER, 0, includesMemstoreTS);
+ HFileBlock.FILL_HEADER, 0, includesMemstoreTS,
+ HFileReaderV2.MAX_MINOR_VERSION, 0, ChecksumType.NULL.getCode(), 0);
UNKNOWN_TABLE_AND_CF.passSchemaMetricsTo(b);
return b;
}
Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileReaderV1.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileReaderV1.java?rev=1298641&r1=1298640&r2=1298641&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileReaderV1.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileReaderV1.java Thu Mar 8 22:55:49 2012
@@ -76,7 +76,7 @@ public class TestHFileReaderV1 {
assertEquals(N, reader.getEntries());
assertEquals(N, trailer.getEntryCount());
- assertEquals(1, trailer.getVersion());
+ assertEquals(1, trailer.getMajorVersion());
assertEquals(Compression.Algorithm.GZ, trailer.getCompressionCodec());
for (boolean pread : new boolean[] { false, true }) {
Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java?rev=1298641&r1=1298640&r2=1298641&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java Thu Mar 8 22:55:49 2012
@@ -123,7 +123,7 @@ public class TestHFileWriterV2 {
FixedFileTrailer trailer =
FixedFileTrailer.readFromStream(fsdis, fileSize);
- assertEquals(2, trailer.getVersion());
+ assertEquals(2, trailer.getMajorVersion());
assertEquals(ENTRY_COUNT, trailer.getEntryCount());
HFileBlock.FSReader blockReader =
Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/CreateRandomStoreFile.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/CreateRandomStoreFile.java?rev=1298641&r1=1298640&r2=1298641&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/CreateRandomStoreFile.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/CreateRandomStoreFile.java Thu Mar 8 22:55:49 2012
@@ -189,6 +189,8 @@ public class CreateRandomStoreFile {
.withCompression(compr)
.withBloomType(bloomType)
.withMaxKeyCount(numKV)
+ .withChecksumType(HFile.DEFAULT_CHECKSUM_TYPE)
+ .withBytesPerChecksum(HFile.DEFAULT_BYTES_PER_CHECKSUM)
.build();
rand = new Random();
Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/HFileReadWriteTest.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/HFileReadWriteTest.java?rev=1298641&r1=1298640&r2=1298641&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/HFileReadWriteTest.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/HFileReadWriteTest.java Thu Mar 8 22:55:49 2012
@@ -356,6 +356,8 @@ public class HFileReadWriteTest {
.withDataBlockEncoder(dataBlockEncoder)
.withBloomType(bloomType)
.withMaxKeyCount(maxKeyCount)
+ .withChecksumType(HFile.DEFAULT_CHECKSUM_TYPE)
+ .withBytesPerChecksum(HFile.DEFAULT_BYTES_PER_CHECKSUM)
.build();
StatisticsPrinter statsPrinter = new StatisticsPrinter();
Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java?rev=1298641&r1=1298640&r2=1298641&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java Thu Mar 8 22:55:49 2012
@@ -298,6 +298,8 @@ public class TestCompoundBloomFilter {
BLOCK_SIZES[t])
.withOutputDir(TEST_UTIL.getDataTestDir())
.withBloomType(bt)
+ .withChecksumType(HFile.DEFAULT_CHECKSUM_TYPE)
+ .withBytesPerChecksum(HFile.DEFAULT_BYTES_PER_CHECKSUM)
.build();
assertTrue(w.hasGeneralBloom());
Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java?rev=1298641&r1=1298640&r2=1298641&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java Thu Mar 8 22:55:49 2012
@@ -41,6 +41,7 @@ import org.apache.hadoop.fs.PositionedRe
import org.apache.hadoop.hbase.*;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.fs.HFileSystem;
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hadoop.hbase.io.hfile.HFileScanner;
import org.apache.hadoop.hbase.io.hfile.NoOpDataBlockEncoder;
@@ -69,10 +70,12 @@ public class TestFSErrorsExposed {
Path hfilePath = new Path(new Path(
util.getDataTestDir("internalScannerExposesErrors"),
"regionname"), "familyname");
- FaultyFileSystem fs = new FaultyFileSystem(util.getTestFileSystem());
+ HFileSystem hfs = (HFileSystem)util.getTestFileSystem();
+ FaultyFileSystem faultyfs = new FaultyFileSystem(hfs.getBackingFs());
+ FileSystem fs = new HFileSystem(faultyfs);
CacheConfig cacheConf = new CacheConfig(util.getConfiguration());
StoreFile.Writer writer = new StoreFile.WriterBuilder(
- util.getConfiguration(), cacheConf, fs, 2*1024)
+ util.getConfiguration(), cacheConf, hfs, 2*1024)
.withOutputDir(hfilePath)
.build();
TestStoreFile.writeStoreFile(
@@ -85,14 +88,14 @@ public class TestFSErrorsExposed {
StoreFile.Reader reader = sf.createReader();
HFileScanner scanner = reader.getScanner(false, true);
- FaultyInputStream inStream = fs.inStreams.get(0).get();
+ FaultyInputStream inStream = faultyfs.inStreams.get(0).get();
assertNotNull(inStream);
scanner.seekTo();
// Do at least one successful read
assertTrue(scanner.next());
- inStream.startFaults();
+ faultyfs.startFaults();
try {
int scanned=0;
@@ -116,10 +119,12 @@ public class TestFSErrorsExposed {
Path hfilePath = new Path(new Path(
util.getDataTestDir("internalScannerExposesErrors"),
"regionname"), "familyname");
- FaultyFileSystem fs = new FaultyFileSystem(util.getTestFileSystem());
+ HFileSystem hfs = (HFileSystem)util.getTestFileSystem();
+ FaultyFileSystem faultyfs = new FaultyFileSystem(hfs.getBackingFs());
+ HFileSystem fs = new HFileSystem(faultyfs);
CacheConfig cacheConf = new CacheConfig(util.getConfiguration());
StoreFile.Writer writer = new StoreFile.WriterBuilder(
- util.getConfiguration(), cacheConf, fs, 2 * 1024)
+ util.getConfiguration(), cacheConf, hfs, 2 * 1024)
.withOutputDir(hfilePath)
.build();
TestStoreFile.writeStoreFile(
@@ -132,14 +137,13 @@ public class TestFSErrorsExposed {
Collections.singletonList(sf), false, true, false);
KeyValueScanner scanner = scanners.get(0);
- FaultyInputStream inStream = fs.inStreams.get(0).get();
+ FaultyInputStream inStream = faultyfs.inStreams.get(0).get();
assertNotNull(inStream);
scanner.seek(KeyValue.LOWESTKEY);
// Do at least one successful read
assertNotNull(scanner.next());
-
- inStream.startFaults();
+ faultyfs.startFaults();
try {
int scanned=0;
@@ -220,6 +224,15 @@ public class TestFSErrorsExposed {
inStreams.add(new SoftReference<FaultyInputStream>(faulty));
return faulty;
}
+
+ /**
+ * Starts to simulate faults on all streams opened so far
+ */
+ public void startFaults() {
+ for (SoftReference<FaultyInputStream> is: inStreams) {
+ is.get().startFaults();
+ }
+ }
}
static class FaultyInputStream extends FSDataInputStream {
Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java?rev=1298641&r1=1298640&r2=1298641&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java Thu Mar 8 22:55:49 2012
@@ -3173,7 +3173,7 @@ public class TestHRegion extends HBaseTe
// set up a cluster with 3 nodes
- MiniHBaseCluster cluster;
+ MiniHBaseCluster cluster = null;
String dataNodeHosts[] = new String[] { "host1", "host2", "host3" };
int regionServersCount = 3;
@@ -3221,7 +3221,9 @@ public class TestHRegion extends HBaseTe
ht.close();
} finally {
- htu.shutdownMiniCluster();
+ if (cluster != null) {
+ htu.shutdownMiniCluster();
+ }
}
}
Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java?rev=1298641&r1=1298640&r2=1298641&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java Thu Mar 8 22:55:49 2012
@@ -52,6 +52,7 @@ import org.apache.hadoop.hbase.regionser
import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
import org.apache.hadoop.hbase.util.BloomFilterFactory;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.ChecksumType;
import org.junit.experimental.categories.Category;
import org.mockito.Mockito;
@@ -69,6 +70,9 @@ public class TestStoreFile extends HBase
private String ROOT_DIR;
private Map<String, Long> startingMetrics;
+ private static final ChecksumType CKTYPE = ChecksumType.CRC32;
+ private static final int CKBYTES = 512;
+
@Override
public void setUp() throws Exception {
super.setUp();
@@ -401,6 +405,8 @@ public class TestStoreFile extends HBase
.withFilePath(f)
.withBloomType(StoreFile.BloomType.ROW)
.withMaxKeyCount(2000)
+ .withChecksumType(CKTYPE)
+ .withBytesPerChecksum(CKBYTES)
.build();
bloomWriteRead(writer, fs);
}
@@ -420,6 +426,8 @@ public class TestStoreFile extends HBase
fs, StoreFile.DEFAULT_BLOCKSIZE_SMALL)
.withFilePath(f)
.withMaxKeyCount(2000)
+ .withChecksumType(CKTYPE)
+ .withBytesPerChecksum(CKBYTES)
.build();
// add delete family
@@ -490,6 +498,8 @@ public class TestStoreFile extends HBase
.withFilePath(f)
.withBloomType(bt[x])
.withMaxKeyCount(expKeys[x])
+ .withChecksumType(CKTYPE)
+ .withBytesPerChecksum(CKBYTES)
.build();
long now = System.currentTimeMillis();
@@ -565,6 +575,8 @@ public class TestStoreFile extends HBase
.withFilePath(f)
.withBloomType(StoreFile.BloomType.ROW)
.withMaxKeyCount(2000)
+ .withChecksumType(CKTYPE)
+ .withBytesPerChecksum(CKBYTES)
.build();
assertFalse(writer.hasGeneralBloom());
writer.close();
@@ -592,6 +604,8 @@ public class TestStoreFile extends HBase
.withFilePath(f)
.withBloomType(StoreFile.BloomType.ROW)
.withMaxKeyCount(Integer.MAX_VALUE)
+ .withChecksumType(CKTYPE)
+ .withBytesPerChecksum(CKBYTES)
.build();
assertFalse(writer.hasGeneralBloom());
writer.close();
@@ -859,6 +873,8 @@ public class TestStoreFile extends HBase
blockSize)
.withFilePath(path)
.withMaxKeyCount(2000)
+ .withChecksumType(CKTYPE)
+ .withBytesPerChecksum(CKBYTES)
.build();
// We'll write N-1 KVs to ensure we don't write an extra block
kvs.remove(kvs.size()-1);
@@ -890,6 +906,8 @@ public class TestStoreFile extends HBase
.withFilePath(path)
.withDataBlockEncoder(dataBlockEncoder)
.withMaxKeyCount(2000)
+ .withChecksumType(CKTYPE)
+ .withBytesPerChecksum(CKBYTES)
.build();
writer.close();
Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestCloseRegionHandler.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestCloseRegionHandler.java?rev=1298641&r1=1298640&r2=1298641&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestCloseRegionHandler.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestCloseRegionHandler.java Thu Mar 8 22:55:49 2012
@@ -133,7 +133,8 @@ public class TestCloseRegionHandler {
@Test public void testZKClosingNodeVersionMismatch()
throws IOException, NodeExistsException, KeeperException {
final Server server = new MockServer(HTU);
- final RegionServerServices rss = new MockRegionServerServices();
+ final MockRegionServerServices rss = new MockRegionServerServices();
+ rss.setFileSystem(HTU.getTestFileSystem());
HTableDescriptor htd = TEST_HTD;
final HRegionInfo hri = TEST_HRI;
@@ -169,7 +170,8 @@ public class TestCloseRegionHandler {
@Test public void testCloseRegion()
throws IOException, NodeExistsException, KeeperException {
final Server server = new MockServer(HTU);
- final RegionServerServices rss = new MockRegionServerServices();
+ final MockRegionServerServices rss = new MockRegionServerServices();
+ rss.setFileSystem(HTU.getTestFileSystem());
HTableDescriptor htd = TEST_HTD;
HRegionInfo hri = TEST_HRI;
Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/util/MockRegionServerServices.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/util/MockRegionServerServices.java?rev=1298641&r1=1298640&r2=1298641&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/util/MockRegionServerServices.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/util/MockRegionServerServices.java Thu Mar 8 22:55:49 2012
@@ -24,8 +24,10 @@ import java.util.Map;
import java.util.concurrent.ConcurrentSkipListMap;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.catalog.CatalogTracker;
+import org.apache.hadoop.hbase.fs.HFileSystem;
import org.apache.hadoop.hbase.ipc.RpcServer;
import org.apache.hadoop.hbase.regionserver.CompactionRequestor;
import org.apache.hadoop.hbase.regionserver.FlushRequester;
@@ -45,6 +47,7 @@ public class MockRegionServerServices im
private boolean stopping = false;
private final ConcurrentSkipListMap<byte[], Boolean> rit =
new ConcurrentSkipListMap<byte[], Boolean>(Bytes.BYTES_COMPARATOR);
+ private HFileSystem hfs = null;
@Override
public boolean removeFromOnlineRegions(String encodedRegionName) {
@@ -147,5 +150,13 @@ public class MockRegionServerServices im
public boolean isAborted() {
return false;
}
-
+
+ @Override
+ public HFileSystem getFileSystem() {
+ return this.hfs;
+ }
+
+ public void setFileSystem(FileSystem hfs) {
+ this.hfs = (HFileSystem)hfs;
+ }
}