You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by mb...@apache.org on 2012/02/17 02:56:35 UTC
svn commit: r1245291 [7/7] - in /hbase/branches/0.89-fb/src:
main/java/org/apache/hadoop/hbase/ main/java/org/apache/hadoop/hbase/client/
main/java/org/apache/hadoop/hbase/io/
main/java/org/apache/hadoop/hbase/io/encoding/
main/java/org/apache/hadoop/h...
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/HFileReadWriteTest.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/HFileReadWriteTest.java?rev=1245291&r1=1245290&r2=1245291&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/HFileReadWriteTest.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/HFileReadWriteTest.java Fri Feb 17 01:56:33 2012
@@ -1,21 +1,18 @@
/*
- * Copyright 2011 The Apache Software Foundation
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
*/
package org.apache.hadoop.hbase.regionserver;
@@ -25,9 +22,7 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Random;
-import java.util.Set;
import java.util.SortedSet;
-import java.util.TreeSet;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentSkipListSet;
import java.util.concurrent.ExecutionException;
@@ -54,16 +49,19 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.io.hfile.BlockType;
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
+import org.apache.hadoop.hbase.io.hfile.BlockCache;
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hadoop.hbase.io.hfile.Compression;
import org.apache.hadoop.hbase.io.hfile.HFile;
-import org.apache.hadoop.hbase.io.hfile.HFileBlock;
+import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoder;
+import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoderImpl;
import org.apache.hadoop.hbase.io.hfile.HFilePrettyPrinter;
-import org.apache.hadoop.hbase.io.hfile.LruBlockCache;
-import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
+import org.apache.hadoop.hbase.io.hfile.NoOpDataBlockEncoder;
+import org.apache.hadoop.hbase.util.BloomFilterFactory;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.MD5Hash;
import org.apache.hadoop.util.StringUtils;
@@ -73,6 +71,8 @@ import org.apache.hadoop.util.StringUtil
*/
public class HFileReadWriteTest {
+ private static final String TABLE_NAME = "MyTable";
+
private static enum Workload {
MERGE("merge", "Merge the specified HFiles", 1, Integer.MAX_VALUE),
RANDOM_READS("read", "Perform a random read benchmark on the given HFile",
@@ -136,11 +136,19 @@ public class HFileReadWriteTest {
private Workload workload;
private FileSystem fs;
private Configuration conf;
+
+
+ private CacheConfig cacheConf;
private List<String> inputFileNames;
private Path outputDir;
private int numReadThreads;
private int durationSec;
+ private DataBlockEncoding dataBlockEncoding;
+ private boolean encodeInCacheOnly;
+ private HFileDataBlockEncoder dataBlockEncoder =
+ NoOpDataBlockEncoder.INSTANCE;
+
private StoreFile.BloomType bloomType = StoreFile.BloomType.NONE;
private int blockSize;
private Compression.Algorithm compression = Compression.Algorithm.NONE;
@@ -158,8 +166,21 @@ public class HFileReadWriteTest {
private SortedSet<String> keysRead = new ConcurrentSkipListSet<String>();
private List<StoreFile> inputStoreFiles;
+ public static final String OPT_DATA_BLOCK_ENCODING =
+ HColumnDescriptor.DATA_BLOCK_ENCODING.toLowerCase();
+ public static final String OPT_DATA_BLOCK_ENCODING_USAGE =
+ "Encoding algorithm (e.g. prefix "
+ + "compression) to use for data blocks in the test column family, "
+ + "one of " + Arrays.toString(DataBlockEncoding.values()) + ".";
+ public static final String OPT_ENCODE_IN_CACHE_ONLY =
+ "encode_in_cache_only";
+ public static final String OPT_ENCODE_IN_CACHE_ONLY_USAGE =
+ "If this is specified, data blocks will only be encoded in block " +
+ "cache but not on disk";
+
public HFileReadWriteTest() {
conf = HBaseConfiguration.create();
+ cacheConf = new CacheConfig(conf);
}
@SuppressWarnings("unchecked")
@@ -177,11 +198,15 @@ public class HFileReadWriteTest {
options.addOption(BLOCK_SIZE_OPTION, true, "HFile block size" +
Workload.MERGE.onlyUsedFor());
options.addOption(DURATION_OPTION, true, "The amount of time to run the " +
- "random read workload for" + Workload.RANDOM_READS.onlyUsedFor());
+ "random read workload for" + Workload.RANDOM_READS.onlyUsedFor());
options.addOption(NUM_THREADS_OPTION, true, "The number of random " +
"reader threads" + Workload.RANDOM_READS.onlyUsedFor());
options.addOption(NUM_THREADS_OPTION, true, "The number of random " +
"reader threads" + Workload.RANDOM_READS.onlyUsedFor());
+ options.addOption(OPT_DATA_BLOCK_ENCODING, true,
+ OPT_DATA_BLOCK_ENCODING_USAGE);
+ options.addOption(OPT_ENCODE_IN_CACHE_ONLY, false,
+ OPT_ENCODE_IN_CACHE_ONLY_USAGE);
options.addOptionGroup(Workload.getOptionGroup());
if (args.length == 0) {
@@ -233,6 +258,25 @@ public class HFileReadWriteTest {
BLOOM_FILTER_OPTION));
}
+ encodeInCacheOnly =
+ cmdLine.hasOption(OPT_ENCODE_IN_CACHE_ONLY);
+
+ if (cmdLine.hasOption(OPT_DATA_BLOCK_ENCODING)) {
+ dataBlockEncoding = DataBlockEncoding.valueOf(
+ cmdLine.getOptionValue(OPT_DATA_BLOCK_ENCODING));
+ // Optionally encode on disk, always encode in cache.
+ dataBlockEncoder = new HFileDataBlockEncoderImpl(
+ encodeInCacheOnly ? DataBlockEncoding.NONE : dataBlockEncoding,
+ dataBlockEncoding);
+ } else {
+ if (encodeInCacheOnly) {
+ LOG.error("The -" + OPT_ENCODE_IN_CACHE_ONLY +
+ " option does not make sense without -" +
+ OPT_DATA_BLOCK_ENCODING);
+ return false;
+ }
+ }
+
blockSize = conf.getInt("hfile.min.blocksize.size", 65536);
if (cmdLine.hasOption(BLOCK_SIZE_OPTION))
blockSize = Integer.valueOf(cmdLine.getOptionValue(BLOCK_SIZE_OPTION));
@@ -257,6 +301,7 @@ public class HFileReadWriteTest {
numReadThreads = Integer.parseInt(
cmdLine.getOptionValue(NUM_THREADS_OPTION));
}
+
Collections.sort(inputFileNames);
return true;
@@ -292,7 +337,7 @@ public class HFileReadWriteTest {
if (outputDir != null &&
(!fs.exists(outputDir) || !fs.getFileStatus(outputDir).isDir())) {
LOG.error(outputDir.toString() + " does not exist or is not a " +
- "directory");
+ "directory");
return false;
}
@@ -304,23 +349,27 @@ public class HFileReadWriteTest {
List<StoreFileScanner> scanners =
StoreFileScanner.getScannersForStoreFiles(inputStoreFiles, false,
- false, true);
+ false);
HColumnDescriptor columnDescriptor = new HColumnDescriptor(
HFileReadWriteTest.class.getSimpleName());
columnDescriptor.setBlocksize(blockSize);
columnDescriptor.setBloomFilterType(bloomType);
columnDescriptor.setCompressionType(compression);
+
+ columnDescriptor.setDataBlockEncoding(dataBlockEncoding);
HRegionInfo regionInfo = new HRegionInfo();
+ HTableDescriptor htd = new HTableDescriptor(TABLE_NAME);
HRegion region = new HRegion(outputDir, null, fs, conf, regionInfo, null);
Store store = new Store(outputDir, region, columnDescriptor, fs, conf);
- StoreFile.Writer writer = StoreFile.createWriter(fs, outputDir,
- blockSize,
- compression, KeyValue.COMPARATOR, this.conf, new CacheConfig(this.conf),
- bloomType, maxKeyCount);
+ StoreFile.Writer writer =
+ StoreFile.createWriter(fs, outputDir, blockSize, compression,
+ dataBlockEncoder, KeyValue.COMPARATOR, this.conf,
+ new CacheConfig(conf), bloomType, BloomFilterFactory.getErrorRate(conf),
+ maxKeyCount, null);
- StatisticsPrinter statsPrinter = new StatisticsPrinter(getHFileReaders());
+ StatisticsPrinter statsPrinter = new StatisticsPrinter();
statsPrinter.startThread();
try {
@@ -426,8 +475,8 @@ public class HFileReadWriteTest {
throws IOException {
// We are passing the ROWCOL Bloom filter type, but StoreFile will still
// use the Bloom filter type specified in the HFile.
- return new StoreFile(fs, filePath, conf, new CacheConfig(this.conf),
- StoreFile.BloomType.ROWCOL);
+ return new StoreFile(fs, filePath, conf, cacheConf,
+ StoreFile.BloomType.ROWCOL, dataBlockEncoder);
}
public static int charToHex(int c) {
@@ -590,12 +639,7 @@ public class HFileReadWriteTest {
private volatile boolean stopRequested;
private volatile Thread thread;
- private Set<String> fsBlockReadMetrics = new TreeSet<String>();
- private boolean isCompaction;
-
- public StatisticsPrinter(HFile.Reader reader) {
- this(new HFile.Reader[] { reader });
- }
+ private long totalSeekAndReads, totalPositionalReads;
/**
* Run the statistics collector in a separate thread without an executor.
@@ -613,19 +657,6 @@ public class HFileReadWriteTest {
}.start();
}
- public StatisticsPrinter(HFile.Reader[] readers) {
- isCompaction = workload == Workload.MERGE;
- for (HFile.Reader reader : readers) {
- fsBlockReadMetrics.add(
- SchemaMetrics.ALL_SCHEMA_METRICS.getBlockMetricName(
- BlockType.BlockCategory.ALL_CATEGORIES, isCompaction,
- SchemaMetrics.BlockMetricType.READ_COUNT));
- }
-
- LOG.info("Using the following metrics for the number of data blocks " +
- "read: " + fsBlockReadMetrics);
- }
-
@Override
public Boolean call() throws Exception {
LOG.info("Starting statistics printer");
@@ -658,13 +689,19 @@ public class HFileReadWriteTest {
double kvPerSec = kvCount / timeSec;
long bytes = totalBytes.get();
double bytesPerSec = bytes / timeSec;
- long blocks = getTotalBlocksRead();
- double blkReadPerSec = blocks / timeSec;
- double seekReadPerSec = HFileBlock.getNumSeekAndReadOperations() /
- timeSec;
- double preadPerSec = HFileBlock.getNumPositionalReadOperations() /
- timeSec;
+ // readOps and preadOps counters get reset on access, so we have to
+ // accumulate them here. HRegion metrics publishing thread should not
+ // be running in this tool, so no one else should be resetting these
+ // metrics.
+ totalSeekAndReads += HFile.getReadOps();
+ totalPositionalReads += HFile.getPreadOps();
+ long totalBlocksRead = totalSeekAndReads + totalPositionalReads;
+
+ double blkReadPerSec = totalBlocksRead / timeSec;
+
+ double seekReadPerSec = totalSeekAndReads / timeSec;
+ double preadPerSec = totalPositionalReads / timeSec;
boolean isRead = workload == Workload.RANDOM_READS;
@@ -677,7 +714,7 @@ public class HFileReadWriteTest {
sb.append(", blk/sec: " + (long) blkReadPerSec);
sb.append(", total KV: " + numKV);
sb.append(", total bytes: " + totalBytes);
- sb.append(", total blk: " + blocks);
+ sb.append(", total blk: " + totalBlocksRead);
sb.append(", seekRead/sec: " + (long) seekReadPerSec);
sb.append(", pread/sec: " + (long) preadPerSec);
@@ -688,13 +725,6 @@ public class HFileReadWriteTest {
LOG.info(sb.toString());
}
- private long getTotalBlocksRead() {
- long totalBlocksRead = 0;
- for (String metric : fsBlockReadMetrics)
- return HRegion.getNumericMetric(metric);
- return totalBlocksRead;
- }
-
public void requestStop() {
stopRequested = true;
if (thread != null)
@@ -750,7 +780,7 @@ public class HFileReadWriteTest {
boolean pread = true;
for (int i = 0; i < numReadThreads; ++i)
ecs.submit(new RandomReader(i, reader, pread));
- ecs.submit(new StatisticsPrinter(reader.getHFileReader()));
+ ecs.submit(new StatisticsPrinter());
Future<Boolean> result;
while (true) {
try {
@@ -780,9 +810,10 @@ public class HFileReadWriteTest {
storeFile.closeReader(true);
exec.shutdown();
- LruBlockCache c = (LruBlockCache) new CacheConfig(conf).getBlockCache();
- if (c != null)
+ BlockCache c = cacheConf.getBlockCache();
+ if (c != null) {
c.shutdown();
+ }
}
LOG.info("Worker threads completed: " + numCompleted);
LOG.info("Worker threads failed: " + numFailed);
@@ -824,4 +855,4 @@ public class HFileReadWriteTest {
}
}
-}
+}
\ No newline at end of file
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactSelection.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactSelection.java?rev=1245291&r1=1245290&r2=1245291&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactSelection.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactSelection.java Fri Feb 17 01:56:33 2012
@@ -39,6 +39,7 @@ import org.apache.hadoop.hbase.HConstant
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
+import org.apache.hadoop.hbase.io.hfile.NoOpDataBlockEncoder;
import org.apache.hadoop.hbase.regionserver.wal.HLog;
import org.apache.hadoop.hbase.util.Bytes;
@@ -68,7 +69,7 @@ public class TestCompactSelection extend
this.conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, 0);
this.conf.setInt("hbase.hstore.compaction.min", minFiles);
this.conf.setInt("hbase.hstore.compaction.max", maxFiles);
- this.conf.setLong("hbase.hregion.memstore.flush.size", minSize);
+ this.conf.setLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, minSize);
this.conf.setLong("hbase.hstore.compaction.max.size", maxSize);
this.conf.setFloat("hbase.hstore.compaction.ratio", 1.0F);
@@ -100,7 +101,8 @@ public class TestCompactSelection extend
MockStoreFile(long length, boolean isRef) throws IOException {
super(TEST_UTIL.getTestFileSystem(), TEST_FILE,
TEST_UTIL.getConfiguration(),
- new CacheConfig(TEST_UTIL.getConfiguration()), BloomType.NONE);
+ new CacheConfig(TEST_UTIL.getConfiguration()), BloomType.NONE,
+ NoOpDataBlockEncoder.INSTANCE);
this.length = length;
this.isRef = isRef;
}
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java?rev=1245291&r1=1245290&r2=1245291&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompaction.java Fri Feb 17 01:56:33 2012
@@ -19,12 +19,15 @@
*/
package org.apache.hadoop.hbase.regionserver;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.spy;
+
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Collection;
+import java.util.HashMap;
import java.util.List;
-
-import static org.junit.Assert.fail;
+import java.util.Map;
+import java.util.Map.Entry;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -36,21 +39,20 @@ import org.apache.hadoop.hbase.HBaseTest
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.regionserver.StoreFile;
-import org.apache.hadoop.hbase.regionserver.wal.HLog;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
+import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoder;
+import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoderImpl;
import org.apache.hadoop.hbase.io.hfile.HFileScanner;
+import org.apache.hadoop.hbase.regionserver.wal.HLog;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hdfs.MiniDFSCluster;
-
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
-import static org.mockito.Mockito.doAnswer;
-import static org.mockito.Mockito.spy;
/**
@@ -59,6 +61,7 @@ import static org.mockito.Mockito.spy;
public class TestCompaction extends HBaseTestCase {
static final Log LOG = LogFactory.getLog(TestCompaction.class.getName());
private HRegion r = null;
+ private HTableDescriptor htd = null;
private Path compactionDir = null;
private Path regionCompactionDir = null;
private static final byte [] COLUMN_FAMILY = fam1;
@@ -75,9 +78,9 @@ public class TestCompaction extends HBas
super();
// Set cache flush size to 1MB
- conf.setInt("hbase.hregion.memstore.flush.size", 1024*1024);
- conf.setInt("hbase.hregion.memstore.block.multiplier", 10);
- this.cluster = null;
+ conf.setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 1024*1024);
+ conf.setInt("hbase.hregion.memstore.block.multiplier", 100);
+
compactionThreshold = conf.getInt("hbase.hstore.compactionThreshold", 3);
firstRowBytes = START_KEY.getBytes(HConstants.UTF8_ENCODING);
@@ -98,7 +101,7 @@ public class TestCompaction extends HBas
this.conf.set(HConstants.HBASE_DIR,
this.cluster.getFileSystem().getHomeDirectory().toString());
super.setUp();
- HTableDescriptor htd = createTableDescriptor(getName());
+ this.htd = createTableDescriptor(getName());
this.r = createNewHRegion(htd, null, null);
}
@@ -153,6 +156,42 @@ public class TestCompaction extends HBas
* @throws Exception
*/
public void testMajorCompaction() throws Exception {
+ majorCompaction();
+ }
+
+ public void testDataBlockEncodingInCacheOnly() throws Exception {
+ majorCompactionWithDataBlockEncoding(true);
+ }
+
+ public void testDataBlockEncodingEverywhere() throws Exception {
+ majorCompactionWithDataBlockEncoding(false);
+ }
+
+ public void majorCompactionWithDataBlockEncoding(boolean inCacheOnly)
+ throws Exception {
+ Map<Store, HFileDataBlockEncoder> replaceBlockCache =
+ new HashMap<Store, HFileDataBlockEncoder>();
+ for (Entry<byte[], Store> pair : r.getStores().entrySet()) {
+ Store store = pair.getValue();
+ HFileDataBlockEncoder blockEncoder = store.getDataBlockEncoder();
+ replaceBlockCache.put(pair.getValue(), blockEncoder);
+ final DataBlockEncoding inCache = DataBlockEncoding.PREFIX;
+ final DataBlockEncoding onDisk = inCacheOnly ? DataBlockEncoding.NONE :
+ inCache;
+ store.setDataBlockEncoderInTest(new HFileDataBlockEncoderImpl(
+ onDisk, inCache));
+ }
+
+ majorCompaction();
+
+ // restore settings
+ for (Entry<Store, HFileDataBlockEncoder> entry :
+ replaceBlockCache.entrySet()) {
+ entry.getKey().setDataBlockEncoderInTest(entry.getValue());
+ }
+ }
+
+ private void majorCompaction() throws Exception {
createStoreFile(r);
for (int i = 0; i < compactionThreshold; i++) {
createStoreFile(r);
@@ -176,14 +215,20 @@ public class TestCompaction extends HBas
secondRowBytes[START_KEY_BYTES.length - 1]++;
// Always 3 versions if that is what max versions is.
- result = r.get(new Get(secondRowBytes).addFamily(COLUMN_FAMILY_TEXT).setMaxVersions(100), null);
- assertEquals(compactionThreshold, result.size());
+ result = r.get(new Get(secondRowBytes).addFamily(COLUMN_FAMILY_TEXT).
+ setMaxVersions(100), null);
+ LOG.debug("Row " + Bytes.toStringBinary(secondRowBytes) + " after " +
+ "initial compaction: " + result);
+ assertEquals("Invalid number of versions of row "
+ + Bytes.toStringBinary(secondRowBytes) + ".", compactionThreshold,
+ result.size());
// Now add deletes to memstore and then flush it.
// That will put us over
// the compaction threshold of 3 store files. Compacting these store files
// should result in a compacted store file that has no references to the
// deleted row.
+ LOG.debug("Adding deletes to memstore and flushing");
Delete delete = new Delete(secondRowBytes, System.currentTimeMillis(), null);
byte [][] famAndQf = {COLUMN_FAMILY, null};
delete.deleteFamily(famAndQf[0]);
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java?rev=1245291&r1=1245290&r2=1245291&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompoundBloomFilter.java Fri Feb 17 01:56:33 2012
@@ -43,6 +43,7 @@ import org.apache.hadoop.hbase.client.Sc
import org.apache.hadoop.hbase.io.hfile.BlockCache;
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hadoop.hbase.io.hfile.HFile;
+import org.apache.hadoop.hbase.io.hfile.NoOpDataBlockEncoder;
import org.apache.hadoop.hbase.io.hfile.TestHFileWriterV2;
import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType;
import org.apache.hadoop.hbase.util.BloomFilterFactory;
@@ -192,7 +193,8 @@ public class TestCompoundBloomFilter {
private void readStoreFile(int t, BloomType bt, List<KeyValue> kvs,
Path sfPath) throws IOException {
- StoreFile sf = new StoreFile(fs, sfPath, conf, cacheConf, bt);
+ StoreFile sf = new StoreFile(fs, sfPath, conf, cacheConf, bt,
+ NoOpDataBlockEncoder.INSTANCE);
StoreFile.Reader r = sf.createReader();
final boolean pread = true; // does not really matter
StoreFileScanner scanner = r.getStoreFileScanner(true, pread);
@@ -292,8 +294,8 @@ public class TestCompoundBloomFilter {
cacheConf = new CacheConfig(conf);
StoreFile.Writer w = StoreFile.createWriter(fs,
- HBaseTestingUtility.getTestDir(), BLOCK_SIZES[t], null, null, conf,
- cacheConf, bt, 0);
+ HBaseTestingUtility.getTestDir(), BLOCK_SIZES[t], null, null,
+ conf, cacheConf, bt, 0);
assertTrue(w.hasGeneralBloom());
assertTrue(w.getGeneralBloomWriter() instanceof CompoundBloomFilterWriter);
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java?rev=1245291&r1=1245290&r2=1245291&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestFSErrorsExposed.java Fri Feb 17 01:56:33 2012
@@ -46,6 +46,7 @@ import org.apache.hadoop.hbase.client.HB
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hadoop.hbase.io.hfile.HFileScanner;
+import org.apache.hadoop.hbase.io.hfile.NoOpDataBlockEncoder;
import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.Test;
@@ -77,7 +78,9 @@ public class TestFSErrorsExposed {
writer, Bytes.toBytes("cf"), Bytes.toBytes("qual"));
StoreFile sf = new StoreFile(fs, writer.getPath(),
- util.getConfiguration(), cacheConf, StoreFile.BloomType.NONE);
+ util.getConfiguration(), cacheConf, StoreFile.BloomType.NONE,
+ NoOpDataBlockEncoder.INSTANCE);
+
StoreFile.Reader reader = sf.createReader();
HFileScanner scanner = reader.getScanner(false, true);
@@ -120,7 +123,8 @@ public class TestFSErrorsExposed {
writer, Bytes.toBytes("cf"), Bytes.toBytes("qual"));
StoreFile sf = new StoreFile(fs, writer.getPath(), util.getConfiguration(),
- cacheConf, BloomType.NONE);
+ cacheConf, BloomType.NONE, NoOpDataBlockEncoder.INSTANCE);
+
List<StoreFileScanner> scanners = StoreFileScanner.getScannersForStoreFiles(
Collections.singletonList(sf), false, true, false);
KeyValueScanner scanner = scanners.get(0);
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestMultiColumnScanner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestMultiColumnScanner.java?rev=1245291&r1=1245290&r2=1245291&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestMultiColumnScanner.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestMultiColumnScanner.java Fri Feb 17 01:56:33 2012
@@ -19,7 +19,9 @@
*/
package org.apache.hadoop.hbase.regionserver;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.util.ArrayList;
@@ -33,6 +35,7 @@ import java.util.Random;
import java.util.Set;
import java.util.TreeSet;
+import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.HBaseTestingUtility;
@@ -44,8 +47,10 @@ import org.apache.hadoop.hbase.KeyValueT
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
import org.apache.hadoop.hbase.io.hfile.BlockType;
import org.apache.hadoop.hbase.io.hfile.Compression;
+import org.apache.hadoop.hbase.io.hfile.HFile;
import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType;
import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics.BlockMetricType;
@@ -109,8 +114,9 @@ public class TestMultiColumnScanner {
private final static HBaseTestingUtility TEST_UTIL =
new HBaseTestingUtility();
- private Compression.Algorithm comprAlgo;
- private StoreFile.BloomType bloomType;
+ private final Compression.Algorithm comprAlgo;
+ private final StoreFile.BloomType bloomType;
+ private final DataBlockEncoding dataBlockEncoding;
private long lastBlocksRead;
private long lastCacheHits;
@@ -133,13 +139,23 @@ public class TestMultiColumnScanner {
@Parameters
public static final Collection<Object[]> parameters() {
- return HBaseTestingUtility.BLOOM_AND_COMPRESSION_COMBINATIONS;
+ List<Object[]> parameters = new ArrayList<Object[]>();
+ for (Object[] bloomAndCompressionParams :
+ HBaseTestingUtility.BLOOM_AND_COMPRESSION_COMBINATIONS) {
+ for (boolean useDataBlockEncoding : new boolean[]{false, true}) {
+ parameters.add(ArrayUtils.add(bloomAndCompressionParams,
+ useDataBlockEncoding));
+ }
+ }
+ return parameters;
}
public TestMultiColumnScanner(Compression.Algorithm comprAlgo,
- StoreFile.BloomType bloomType) {
+ StoreFile.BloomType bloomType, boolean useDataBlockEncoding) {
this.comprAlgo = comprAlgo;
this.bloomType = bloomType;
+ this.dataBlockEncoding = useDataBlockEncoding ? DataBlockEncoding.PREFIX :
+ DataBlockEncoding.NONE;
}
private long getBlocksRead() {
@@ -177,8 +193,9 @@ public class TestMultiColumnScanner {
@Test
public void testMultiColumnScanner() throws IOException {
- HRegion region = createRegion(TABLE_NAME, comprAlgo, bloomType,
- MAX_VERSIONS);
+ HRegion region = TEST_UTIL.createTestRegion(TABLE_NAME, FAMILY, comprAlgo,
+ bloomType, MAX_VERSIONS, HFile.DEFAULT_BLOCKSIZE,
+ dataBlockEncoding, true);
List<String> rows = sequentialStrings("row", NUM_ROWS);
List<String> qualifiers = sequentialStrings("qual", NUM_COLUMNS);
List<KeyValue> kvs = new ArrayList<KeyValue>();
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanWithBloomError.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanWithBloomError.java?rev=1245291&r1=1245290&r2=1245291&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanWithBloomError.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestScanWithBloomError.java Fri Feb 17 01:56:33 2012
@@ -20,7 +20,9 @@ import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValueTestUtil;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
import org.apache.hadoop.hbase.io.hfile.Compression;
+import org.apache.hadoop.hbase.io.hfile.HFile;
import org.apache.hadoop.hbase.io.hfile.HFilePrettyPrinter;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.Before;
@@ -76,8 +78,10 @@ public class TestScanWithBloomError {
@Test
public void testThreeStoreFiles() throws IOException {
- region = createRegion(TABLE_NAME, Compression.Algorithm.GZ, bloomType,
- MAX_VERSIONS);
+ region = TEST_UTIL.createTestRegion(TABLE_NAME, FAMILY,
+ Compression.Algorithm.GZ, bloomType,
+ TestMultiColumnScanner.MAX_VERSIONS, HFile.DEFAULT_BLOCKSIZE,
+ DataBlockEncoding.NONE, true);
createStoreFile(new int[] {1, 2, 6});
createStoreFile(new int[] {1, 2, 3, 7});
createStoreFile(new int[] {1, 9});
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestSeekOptimizations.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestSeekOptimizations.java?rev=1245291&r1=1245290&r2=1245291&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestSeekOptimizations.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestSeekOptimizations.java Fri Feb 17 01:56:33 2012
@@ -43,7 +43,9 @@ import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
import org.apache.hadoop.hbase.io.hfile.Compression;
+import org.apache.hadoop.hbase.io.hfile.HFile;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.After;
import org.junit.Before;
@@ -112,6 +114,9 @@ public class TestSeekOptimizations {
private long totalSeekDiligent, totalSeekLazy;
+ private final static HBaseTestingUtility TEST_UTIL =
+ new HBaseTestingUtility();
+
@Parameters
public static final Collection<Object[]> parameters() {
return HBaseTestingUtility.BLOOM_AND_COMPRESSION_COMBINATIONS;
@@ -131,9 +136,9 @@ public class TestSeekOptimizations {
@Test
public void testMultipleTimestampRanges() throws IOException {
- region = TestMultiColumnScanner.createRegion(
- TestSeekOptimizations.class.getName(), comprAlgo, bloomType,
- Integer.MAX_VALUE);
+ region = TEST_UTIL.createTestRegion(TestSeekOptimizations.class.getName(),
+ FAMILY, comprAlgo, bloomType, Integer.MAX_VALUE,
+ HFile.DEFAULT_BLOCKSIZE, DataBlockEncoding.NONE, true);
// Delete the given timestamp and everything before.
final long latestDelTS = USE_MANY_STORE_FILES ? 1397 : -1;
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java?rev=1245291&r1=1245290&r2=1245291&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/TestStoreFile.java Fri Feb 17 01:56:33 2012
@@ -36,15 +36,21 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseTestCase;
import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.io.Reference.Range;
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
import org.apache.hadoop.hbase.io.hfile.BlockCache;
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hadoop.hbase.io.hfile.HFile;
+import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoder;
+import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoderImpl;
import org.apache.hadoop.hbase.io.hfile.HFileScanner;
import org.apache.hadoop.hbase.io.hfile.LruBlockCache.CacheStats;
+import org.apache.hadoop.hbase.io.hfile.NoOpDataBlockEncoder;
+import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType;
import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
import org.apache.hadoop.hbase.util.BloomFilterFactory;
import org.apache.hadoop.hbase.util.Bytes;
@@ -100,7 +106,7 @@ public class TestStoreFile extends HBase
conf, cacheConf);
writeStoreFile(writer);
checkHalfHFile(new StoreFile(this.fs, writer.getPath(), conf, cacheConf,
- StoreFile.BloomType.NONE));
+ StoreFile.BloomType.NONE, NoOpDataBlockEncoder.INSTANCE));
}
private void writeStoreFile(final StoreFile.Writer writer) throws IOException {
@@ -141,7 +147,7 @@ public class TestStoreFile extends HBase
conf, cacheConf);
writeStoreFile(writer);
StoreFile hsf = new StoreFile(this.fs, writer.getPath(), conf, cacheConf,
- StoreFile.BloomType.NONE);
+ StoreFile.BloomType.NONE, NoOpDataBlockEncoder.INSTANCE);
StoreFile.Reader reader = hsf.createReader();
// Split on a row, not in middle of row. Midkey returned by reader
// may be in middle of row. Create new one with empty column and
@@ -153,7 +159,7 @@ public class TestStoreFile extends HBase
// Make a reference
Path refPath = StoreFile.split(fs, dir, hsf, midRow, Range.top);
StoreFile refHsf = new StoreFile(this.fs, refPath, conf, cacheConf,
- StoreFile.BloomType.NONE);
+ StoreFile.BloomType.NONE, NoOpDataBlockEncoder.INSTANCE);
// Now confirm that I can read from the reference and that it only gets
// keys from top half of the file.
HFileScanner s = refHsf.createReader().getScanner(false, false);
@@ -189,10 +195,12 @@ public class TestStoreFile extends HBase
Path bottomPath = StoreFile.split(this.fs, bottomDir,
f, midRow, Range.bottom);
// Make readers on top and bottom.
- StoreFile.Reader top = new StoreFile(this.fs, topPath, conf, cacheConf,
- StoreFile.BloomType.NONE).createReader();
- StoreFile.Reader bottom = new StoreFile(this.fs, bottomPath, conf, cacheConf,
- StoreFile.BloomType.NONE).createReader();
+ StoreFile.Reader top =
+ new StoreFile(this.fs, topPath, conf, cacheConf, BloomType.NONE,
+ NoOpDataBlockEncoder.INSTANCE).createReader();
+ StoreFile.Reader bottom = new StoreFile(this.fs, bottomPath,
+ conf, cacheConf, BloomType.NONE,
+ NoOpDataBlockEncoder.INSTANCE).createReader();
ByteBuffer previous = null;
LOG.info("Midkey: " + midKV.toString());
ByteBuffer bbMidkeyBytes = ByteBuffer.wrap(midkey);
@@ -248,9 +256,11 @@ public class TestStoreFile extends HBase
bottomPath = StoreFile.split(this.fs, bottomDir, f, badmidkey,
Range.bottom);
top = new StoreFile(this.fs, topPath, conf, cacheConf,
- StoreFile.BloomType.NONE).createReader();
+ StoreFile.BloomType.NONE,
+ NoOpDataBlockEncoder.INSTANCE).createReader();
bottom = new StoreFile(this.fs, bottomPath, conf, cacheConf,
- StoreFile.BloomType.NONE).createReader();
+ StoreFile.BloomType.NONE,
+ NoOpDataBlockEncoder.INSTANCE).createReader();
bottomScanner = bottom.getScanner(false, false);
int count = 0;
while ((!bottomScanner.isSeeked() && bottomScanner.seekTo()) ||
@@ -293,9 +303,11 @@ public class TestStoreFile extends HBase
bottomPath = StoreFile.split(this.fs, bottomDir, f, badmidkey,
Range.bottom);
top = new StoreFile(this.fs, topPath, conf, cacheConf,
- StoreFile.BloomType.NONE).createReader();
+ StoreFile.BloomType.NONE,
+ NoOpDataBlockEncoder.INSTANCE).createReader();
bottom = new StoreFile(this.fs, bottomPath, conf, cacheConf,
- StoreFile.BloomType.NONE).createReader();
+ StoreFile.BloomType.NONE,
+ NoOpDataBlockEncoder.INSTANCE).createReader();
first = true;
bottomScanner = bottom.getScanner(false, false);
while ((!bottomScanner.isSeeked() && bottomScanner.seekTo()) ||
@@ -353,7 +365,8 @@ public class TestStoreFile extends HBase
}
writer.close();
- StoreFile.Reader reader = new StoreFile.Reader(fs, f, cacheConf);
+ StoreFile.Reader reader = new StoreFile.Reader(fs, f, cacheConf,
+ DataBlockEncoding.NONE);
reader.loadFileInfo();
reader.loadBloomfilter();
StoreFileScanner scanner = reader.getStoreFileScanner(false, false);
@@ -391,10 +404,10 @@ public class TestStoreFile extends HBase
// write the file
Path f = new Path(ROOT_DIR, getName());
- StoreFile.Writer writer = new StoreFile.Writer(fs, f,
- StoreFile.DEFAULT_BLOCKSIZE_SMALL, HFile.DEFAULT_COMPRESSION_ALGORITHM,
- conf, cacheConf, KeyValue.COMPARATOR, StoreFile.BloomType.ROW, 2000);
-
+ StoreFile.Writer writer =
+ new StoreFile.Writer(fs, f, StoreFile.DEFAULT_BLOCKSIZE_SMALL,
+ HFile.DEFAULT_COMPRESSION_ALGORITHM, conf, cacheConf,
+ KeyValue.COMPARATOR, StoreFile.BloomType.ROW, 2000);
bloomWriteRead(writer, fs);
}
@@ -410,7 +423,8 @@ public class TestStoreFile extends HBase
Path f = new Path(ROOT_DIR, getName());
StoreFile.Writer writer = new StoreFile.Writer(fs, f,
StoreFile.DEFAULT_BLOCKSIZE_SMALL, HFile.DEFAULT_COMPRESSION_ALGORITHM,
- conf, cacheConf, KeyValue.COMPARATOR, StoreFile.BloomType.NONE, 2000);
+ conf, cacheConf, KeyValue.COMPARATOR, StoreFile.BloomType.NONE,
+ 2000);
// add delete family
long now = System.currentTimeMillis();
@@ -422,8 +436,8 @@ public class TestStoreFile extends HBase
}
writer.close();
- StoreFile.Reader reader = new StoreFile.Reader(fs, writer.getPath(),
- cacheConf);
+ StoreFile.Reader reader = new StoreFile.Reader(fs, f, cacheConf,
+ DataBlockEncoding.NONE);
reader.loadFileInfo();
reader.loadBloomfilter();
@@ -495,7 +509,8 @@ public class TestStoreFile extends HBase
}
writer.close();
- StoreFile.Reader reader = new StoreFile.Reader(fs, f, cacheConf);
+ StoreFile.Reader reader = new StoreFile.Reader(fs, f, cacheConf,
+ DataBlockEncoding.NONE);
reader.loadFileInfo();
reader.loadBloomfilter();
StoreFileScanner scanner = reader.getStoreFileScanner(true, true);
@@ -549,7 +564,8 @@ public class TestStoreFile extends HBase
// this should not create a bloom because the max keys is too small
StoreFile.Writer writer = new StoreFile.Writer(fs, f,
StoreFile.DEFAULT_BLOCKSIZE_SMALL, HFile.DEFAULT_COMPRESSION_ALGORITHM,
- conf, cacheConf, KeyValue.COMPARATOR, StoreFile.BloomType.ROW, 2000);
+ conf, cacheConf, KeyValue.COMPARATOR, StoreFile.BloomType.ROW,
+ 2000);
assertFalse(writer.hasGeneralBloom());
writer.close();
fs.delete(f, true);
@@ -677,7 +693,7 @@ public class TestStoreFile extends HBase
writer.close();
StoreFile hsf = new StoreFile(this.fs, writer.getPath(), conf, cacheConf,
- StoreFile.BloomType.NONE);
+ StoreFile.BloomType.NONE, NoOpDataBlockEncoder.INSTANCE);
StoreFile.Reader reader = hsf.createReader();
StoreFileScanner scanner = reader.getStoreFileScanner(false, false);
TreeSet<byte[]> columns = new TreeSet<byte[]>();
@@ -721,7 +737,7 @@ public class TestStoreFile extends HBase
Path pathCowOff = new Path(baseDir, "123456789");
StoreFile.Writer writer = writeStoreFile(conf, cacheConf, pathCowOff, 3);
StoreFile hsf = new StoreFile(this.fs, writer.getPath(), conf, cacheConf,
- StoreFile.BloomType.NONE);
+ StoreFile.BloomType.NONE, NoOpDataBlockEncoder.INSTANCE);
LOG.debug(hsf.getPath().toString());
// Read this file, we should see 3 misses
@@ -743,7 +759,7 @@ public class TestStoreFile extends HBase
Path pathCowOn = new Path(baseDir, "123456788");
writer = writeStoreFile(conf, cacheConf, pathCowOn, 3);
hsf = new StoreFile(this.fs, writer.getPath(), conf, cacheConf,
- StoreFile.BloomType.NONE);
+ StoreFile.BloomType.NONE, NoOpDataBlockEncoder.INSTANCE);
// Read this file, we should see 3 hits
reader = hsf.createReader();
@@ -759,14 +775,14 @@ public class TestStoreFile extends HBase
// Let's read back the two files to ensure the blocks exactly match
hsf = new StoreFile(this.fs, pathCowOff, conf, cacheConf,
- StoreFile.BloomType.NONE);
+ StoreFile.BloomType.NONE, NoOpDataBlockEncoder.INSTANCE);
StoreFile.Reader readerOne = hsf.createReader();
readerOne.loadFileInfo();
StoreFileScanner scannerOne = readerOne.getStoreFileScanner(true, true,
false);
scannerOne.seek(KeyValue.LOWESTKEY);
hsf = new StoreFile(this.fs, pathCowOn, conf, cacheConf,
- StoreFile.BloomType.NONE);
+ StoreFile.BloomType.NONE, NoOpDataBlockEncoder.INSTANCE);
StoreFile.Reader readerTwo = hsf.createReader();
readerTwo.loadFileInfo();
StoreFileScanner scannerTwo = readerTwo.getStoreFileScanner(true, true,
@@ -798,7 +814,7 @@ public class TestStoreFile extends HBase
conf.setBoolean("hbase.rs.evictblocksonclose", true);
cacheConf = new CacheConfig(conf);
hsf = new StoreFile(this.fs, pathCowOff, conf, cacheConf,
- StoreFile.BloomType.NONE);
+ StoreFile.BloomType.NONE, NoOpDataBlockEncoder.INSTANCE);
reader = hsf.createReader();
reader.close(cacheConf.shouldEvictOnClose());
@@ -812,7 +828,7 @@ public class TestStoreFile extends HBase
conf.setBoolean("hbase.rs.evictblocksonclose", false);
cacheConf = new CacheConfig(conf);
hsf = new StoreFile(this.fs, pathCowOn, conf, cacheConf,
- StoreFile.BloomType.NONE);
+ StoreFile.BloomType.NONE, NoOpDataBlockEncoder.INSTANCE);
reader = hsf.createReader();
reader.close(cacheConf.shouldEvictOnClose());
@@ -839,7 +855,8 @@ public class TestStoreFile extends HBase
int blockSize = totalSize / numBlocks;
StoreFile.Writer writer = new StoreFile.Writer(fs, path, blockSize,
HFile.DEFAULT_COMPRESSION_ALGORITHM,
- conf, cacheConf, KeyValue.COMPARATOR, StoreFile.BloomType.NONE, 2000);
+ conf, cacheConf, KeyValue.COMPARATOR, StoreFile.BloomType.NONE,
+ 2000);
// We'll write N-1 KVs to ensure we don't write an extra block
kvs.remove(kvs.size()-1);
for (KeyValue kv : kvs) {
@@ -849,4 +866,40 @@ public class TestStoreFile extends HBase
writer.close();
return writer;
}
+
+ /**
+ * Check if data block encoding information is saved correctly in HFile's
+ * file info.
+ */
+ public void testDataBlockEncodingMetaData() throws IOException {
+ Path dir = new Path(new Path(this.testDir, "regionname"), "familyname");
+ Path path = new Path(dir, "1234567890");
+
+ DataBlockEncoding dataBlockEncoderAlgo =
+ DataBlockEncoding.FAST_DIFF;
+ HFileDataBlockEncoder dataBlockEncoder =
+ new HFileDataBlockEncoderImpl(
+ dataBlockEncoderAlgo,
+ dataBlockEncoderAlgo);
+ cacheConf = new CacheConfig(conf);
+ StoreFile.Writer writer = new StoreFile.Writer(fs,
+ path, HFile.DEFAULT_BLOCKSIZE,
+ HFile.DEFAULT_COMPRESSION_ALGORITHM,
+ dataBlockEncoder,
+ conf,
+ cacheConf,
+ KeyValue.COMPARATOR,
+ StoreFile.BloomType.NONE,
+ HColumnDescriptor.DEFAULT_BLOOMFILTER_ERROR_RATE, 2000, null);
+ writer.close();
+
+ StoreFile storeFile = new StoreFile(fs, writer.getPath(), conf,
+ cacheConf, BloomType.NONE, dataBlockEncoder);
+ StoreFile.Reader reader = storeFile.createReader();
+
+ Map<byte[], byte[]> fileInfo = reader.loadFileInfo();
+ byte[] value = fileInfo.get(StoreFile.DATA_BLOCK_ENCODING);
+
+ assertEquals(dataBlockEncoderAlgo.getNameInBytes(), value);
+ }
}
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java?rev=1245291&r1=1245290&r2=1245291&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestLogRolling.java Fri Feb 17 01:56:33 2012
@@ -108,7 +108,7 @@ public class TestLogRolling extends HBas
conf.setInt("hbase.hregion.memstore.optionalflushcount", 2);
// We flush the cache after every 8192 bytes
- conf.setInt("hbase.hregion.memstore.flush.size", 8192);
+ conf.setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 8192);
// Increase the amount of time between client retries
conf.setLong("hbase.client.pause", 15 * 1000);
Added: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/util/LoadTestKVGenerator.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/util/LoadTestKVGenerator.java?rev=1245291&view=auto
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/util/LoadTestKVGenerator.java (added)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/util/LoadTestKVGenerator.java Fri Feb 17 01:56:33 2012
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import java.util.Random;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.MD5Hash;
+
+/**
+ * A generator of random keys and values for load testing. Keys are generated
+ * by converting numeric indexes to strings and prefixing them with an MD5
+ * hash. Values are generated by selecting value size in the configured range
+ * and generating a pseudo-random sequence of bytes seeded by key, column
+ * qualifier, and value size.
+ * <p>
+ * Not thread-safe, so a separate instance is needed for every writer thread/
+ */
+public class LoadTestKVGenerator {
+
+ /** A random number generator for determining value size */
+ private Random randomForValueSize = new Random();
+
+ private final int minValueSize;
+ private final int maxValueSize;
+
+ public LoadTestKVGenerator(int minValueSize, int maxValueSize) {
+ if (minValueSize <= 0 || maxValueSize <= 0) {
+ throw new IllegalArgumentException("Invalid min/max value sizes: " +
+ minValueSize + ", " + maxValueSize);
+ }
+ this.minValueSize = minValueSize;
+ this.maxValueSize = maxValueSize;
+ }
+
+ /**
+ * Verifies that the given byte array is the same as what would be generated
+ * for the given row key and qualifier. We are assuming that the value size
+ * is correct, and only verify the actual bytes. However, if the min/max
+ * value sizes are set sufficiently high, an accidental match should be
+ * extremely improbable.
+ */
+ public static boolean verify(String rowKey, String qual, byte[] value) {
+ byte[] expectedData = getValueForRowColumn(rowKey, qual, value.length);
+ return Bytes.equals(expectedData, value);
+ }
+
+ /**
+ * Converts the given key to string, and prefixes it with the MD5 hash of
+ * the index's string representation.
+ */
+ public static String md5PrefixedKey(long key) {
+ String stringKey = Long.toString(key);
+ String md5hash = MD5Hash.getMD5AsHex(Bytes.toBytes(stringKey));
+
+ // flip the key to randomize
+ return md5hash + "-" + stringKey;
+ }
+
+ /**
+ * Generates a value for the given key index and column qualifier. Size is
+ * selected randomly in the configured range. The generated value depends
+ * only on the combination of the key, qualifier, and the selected value
+ * size. This allows to verify the actual value bytes when reading, as done
+ * in {@link #verify(String, String, byte[])}.
+ */
+ public byte[] generateRandomSizeValue(long key, String qual) {
+ String rowKey = md5PrefixedKey(key);
+ int dataSize = minValueSize + randomForValueSize.nextInt(
+ Math.abs(maxValueSize - minValueSize));
+ return getValueForRowColumn(rowKey, qual, dataSize);
+ }
+
+ /**
+ * Generates random bytes of the given size for the given row and column
+ * qualifier. The random seed is fully determined by these parameters.
+ */
+ private static byte[] getValueForRowColumn(String rowKey, String qual,
+ int dataSize) {
+ Random seededRandom = new Random(rowKey.hashCode() + qual.hashCode() +
+ dataSize);
+ byte[] randomBytes = new byte[dataSize];
+ seededRandom.nextBytes(randomBytes);
+ return randomBytes;
+ }
+
+ public static byte[] longToByteArrayKey(long rowKey) {
+ return md5PrefixedKey(rowKey).getBytes();
+ }
+
+}
Added: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/util/TestByteBufferUtils.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/util/TestByteBufferUtils.java?rev=1245291&view=auto
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/util/TestByteBufferUtils.java (added)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/util/TestByteBufferUtils.java Fri Feb 17 01:56:33 2012
@@ -0,0 +1,310 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.apache.hadoop.io.WritableUtils;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestByteBufferUtils {
+
+ private byte[] array;
+
+ /**
+ * Create an array with sample data.
+ */
+ @Before
+ public void setUp() {
+ array = new byte[8];
+ for (int i = 0; i < array.length; ++i) {
+ array[i] = (byte) ('a' + i);
+ }
+ }
+
+ private static final int MAX_VLONG_LENGTH = 9;
+ private static final Collection<Long> testNumbers;
+
+ private static void addNumber(Set<Long> a, long l) {
+ if (l != Long.MIN_VALUE) {
+ a.add(l - 1);
+ }
+ a.add(l);
+ if (l != Long.MAX_VALUE) {
+ a.add(l + 1);
+ }
+ for (long divisor = 3; divisor <= 10; ++divisor) {
+ for (long delta = -1; delta <= 1; ++delta) {
+ a.add(l / divisor + delta);
+ }
+ }
+ }
+
+ static {
+ SortedSet<Long> a = new TreeSet<Long>();
+ for (int i = 0; i <= 63; ++i) {
+ long v = (-1L) << i;
+ assertTrue(v < 0);
+ addNumber(a, v);
+ v = (1L << i) - 1;
+ assertTrue(v >= 0);
+ addNumber(a, v);
+ }
+
+ testNumbers = Collections.unmodifiableSet(a);
+ System.err.println("Testing variable-length long serialization using: "
+ + testNumbers + " (count: " + testNumbers.size() + ")");
+ assertEquals(1753, testNumbers.size());
+ assertEquals(Long.MIN_VALUE, a.first().longValue());
+ assertEquals(Long.MAX_VALUE, a.last().longValue());
+ }
+
+ @Test
+ public void testReadWriteVLong() {
+ for (long l : testNumbers) {
+ ByteBuffer b = ByteBuffer.allocate(MAX_VLONG_LENGTH);
+ ByteBufferUtils.writeVLong(b, l);
+ b.flip();
+ assertEquals(l, ByteBufferUtils.readVLong(b));
+ }
+ }
+
+ @Test
+ public void testConsistencyWithHadoopVLong() throws IOException {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutputStream dos = new DataOutputStream(baos);
+ for (long l : testNumbers) {
+ baos.reset();
+ ByteBuffer b = ByteBuffer.allocate(MAX_VLONG_LENGTH);
+ ByteBufferUtils.writeVLong(b, l);
+ String bufStr = Bytes.toStringBinary(b.array(),
+ b.arrayOffset(), b.position());
+ WritableUtils.writeVLong(dos, l);
+ String baosStr = Bytes.toStringBinary(baos.toByteArray());
+ assertEquals(baosStr, bufStr);
+ }
+ }
+
+ /**
+ * Test copying to stream from buffer.
+ */
+ @Test
+ public void testMoveBufferToStream() {
+ final int arrayOffset = 7;
+ final int initialPosition = 10;
+ final int endPadding = 5;
+ byte[] arrayWrapper =
+ new byte[arrayOffset + initialPosition + array.length + endPadding];
+ System.arraycopy(array, 0, arrayWrapper,
+ arrayOffset + initialPosition, array.length);
+ ByteBuffer buffer = ByteBuffer.wrap(arrayWrapper, arrayOffset,
+ initialPosition + array.length).slice();
+ assertEquals(initialPosition + array.length, buffer.limit());
+ assertEquals(0, buffer.position());
+ buffer.position(initialPosition);
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ try {
+ ByteBufferUtils.moveBufferToStream(bos, buffer, array.length);
+ } catch (IOException e) {
+ fail("IOException in testCopyToStream()");
+ }
+ assertArrayEquals(array, bos.toByteArray());
+ assertEquals(initialPosition + array.length, buffer.position());
+ }
+
+ /**
+ * Test copying to stream from buffer with offset.
+ * @throws IOException On test failure.
+ */
+ @Test
+ public void testCopyToStreamWithOffset() throws IOException {
+ ByteBuffer buffer = ByteBuffer.wrap(array);
+
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+
+ ByteBufferUtils.copyBufferToStream(bos, buffer, array.length / 2,
+ array.length / 2);
+
+ byte[] returnedArray = bos.toByteArray();
+ for (int i = 0; i < array.length / 2; ++i) {
+ int pos = array.length / 2 + i;
+ assertEquals(returnedArray[i], array[pos]);
+ }
+ }
+
+ /**
+ * Test copying data from stream.
+ * @throws IOException On test failure.
+ */
+ @Test
+ public void testCopyFromStream() throws IOException {
+ ByteBuffer buffer = ByteBuffer.allocate(array.length);
+ ByteArrayInputStream bis = new ByteArrayInputStream(array);
+ DataInputStream dis = new DataInputStream(bis);
+
+ ByteBufferUtils.copyFromStreamToBuffer(buffer, dis, array.length / 2);
+ ByteBufferUtils.copyFromStreamToBuffer(buffer, dis,
+ array.length - array.length / 2);
+ for (int i = 0; i < array.length; ++i) {
+ assertEquals(array[i], buffer.get(i));
+ }
+ }
+
+ /**
+ * Test copying from buffer.
+ */
+ @Test
+ public void testCopyFromBuffer() {
+ ByteBuffer srcBuffer = ByteBuffer.allocate(array.length);
+ ByteBuffer dstBuffer = ByteBuffer.allocate(array.length);
+ srcBuffer.put(array);
+
+ ByteBufferUtils.copyFromBufferToBuffer(dstBuffer, srcBuffer,
+ array.length / 2, array.length / 4);
+ for (int i = 0; i < array.length / 4; ++i) {
+ assertEquals(srcBuffer.get(i + array.length / 2),
+ dstBuffer.get(i));
+ }
+ }
+
+ /**
+ * Test 7-bit encoding of integers.
+ * @throws IOException On test failure.
+ */
+ @Test
+ public void testCompressedInt() throws IOException {
+ testCompressedInt(0);
+ testCompressedInt(Integer.MAX_VALUE);
+ testCompressedInt(Integer.MIN_VALUE);
+
+ for (int i = 0; i < 3; i++) {
+ testCompressedInt((128 << i) - 1);
+ }
+
+ for (int i = 0; i < 3; i++) {
+ testCompressedInt((128 << i));
+ }
+ }
+
+ /**
+ * Test how much bytes we need to store integer.
+ */
+ @Test
+ public void testIntFitsIn() {
+ assertEquals(1, ByteBufferUtils.intFitsIn(0));
+ assertEquals(1, ByteBufferUtils.intFitsIn(1));
+ assertEquals(2, ByteBufferUtils.intFitsIn(1 << 8));
+ assertEquals(3, ByteBufferUtils.intFitsIn(1 << 16));
+ assertEquals(4, ByteBufferUtils.intFitsIn(-1));
+ assertEquals(4, ByteBufferUtils.intFitsIn(Integer.MAX_VALUE));
+ assertEquals(4, ByteBufferUtils.intFitsIn(Integer.MIN_VALUE));
+ }
+
+ /**
+ * Test how much bytes we need to store long.
+ */
+ @Test
+ public void testLongFitsIn() {
+ assertEquals(1, ByteBufferUtils.longFitsIn(0));
+ assertEquals(1, ByteBufferUtils.longFitsIn(1));
+ assertEquals(3, ByteBufferUtils.longFitsIn(1l << 16));
+ assertEquals(5, ByteBufferUtils.longFitsIn(1l << 32));
+ assertEquals(8, ByteBufferUtils.longFitsIn(-1));
+ assertEquals(8, ByteBufferUtils.longFitsIn(Long.MIN_VALUE));
+ assertEquals(8, ByteBufferUtils.longFitsIn(Long.MAX_VALUE));
+ }
+
+ /**
+ * Test if we are comparing equal bytes.
+ */
+ @Test
+ public void testArePartEqual() {
+ byte[] array = new byte[] { 1, 2, 3, 4, 5, 1, 2, 3, 4 };
+ ByteBuffer buffer = ByteBuffer.wrap(array);
+ assertTrue(ByteBufferUtils.arePartsEqual(buffer, 0, 4, 5, 4));
+ assertTrue(ByteBufferUtils.arePartsEqual(buffer, 1, 2, 6, 2));
+ assertFalse(ByteBufferUtils.arePartsEqual(buffer, 1, 2, 6, 3));
+ assertFalse(ByteBufferUtils.arePartsEqual(buffer, 1, 3, 6, 2));
+ assertFalse(ByteBufferUtils.arePartsEqual(buffer, 0, 3, 6, 3));
+ }
+
+ /**
+ * Test serializing int to bytes
+ */
+ @Test
+ public void testPutInt() {
+ testPutInt(0);
+ testPutInt(Integer.MAX_VALUE);
+
+ for (int i = 0; i < 3; i++) {
+ testPutInt((128 << i) - 1);
+ }
+
+ for (int i = 0; i < 3; i++) {
+ testPutInt((128 << i));
+ }
+ }
+
+ // Utility methods invoked from test methods
+
+ private void testCompressedInt(int value) throws IOException {
+ int parsedValue = 0;
+
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ ByteBufferUtils.putCompressedInt(bos, value);
+
+ ByteArrayInputStream bis = new ByteArrayInputStream(
+ bos.toByteArray());
+ parsedValue = ByteBufferUtils.readCompressedInt(bis);
+
+ assertEquals(value, parsedValue);
+ }
+
+ private void testPutInt(int value) {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ try {
+ ByteBufferUtils.putInt(baos, value);
+ } catch (IOException e) {
+ throw new RuntimeException("Bug in putIn()", e);
+ }
+
+ ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
+ DataInputStream dis = new DataInputStream(bais);
+ try {
+ assertEquals(dis.readInt(), value);
+ } catch (IOException e) {
+ throw new RuntimeException("Bug in test!", e);
+ }
+ }
+}