You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by te...@apache.org on 2011/12/25 03:49:48 UTC
svn commit: r1223042 [2/3] - in /hbase/trunk/src:
main/java/org/apache/hadoop/hbase/ main/java/org/apache/hadoop/hbase/io/
main/java/org/apache/hadoop/hbase/io/encoding/
main/java/org/apache/hadoop/hbase/io/hfile/
main/java/org/apache/hadoop/hbase/mapr...
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java?rev=1223042&r1=1223041&r2=1223042&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java Sun Dec 25 02:49:46 2011
@@ -30,8 +30,6 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.io.encoding.DataBlockEncoder;
-import org.apache.hadoop.hbase.io.encoding.DataBlockEncodings;
import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory;
import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
import org.apache.hadoop.hbase.util.Bytes;
@@ -82,20 +80,17 @@ public class HFileReaderV2 extends Abstr
* @param size Length of the stream.
* @param closeIStream Whether to close the stream.
* @param cacheConf Cache configuration.
- * @param blockEncoder what kind of data block encoding will be used
* @throws IOException
*/
public HFileReaderV2(Path path, FixedFileTrailer trailer,
final FSDataInputStream fsdis, final long size,
- final boolean closeIStream, final CacheConfig cacheConf,
- final HFileDataBlockEncoder blockEncoder)
+ final boolean closeIStream, final CacheConfig cacheConf)
throws IOException {
- super(path, trailer, fsdis, size, closeIStream, cacheConf,
- blockEncoder);
+ super(path, trailer, fsdis, size, closeIStream, cacheConf);
+
trailer.expectVersion(2);
- HFileBlock.FSReaderV2 fsBlockReader = new HFileBlock.FSReaderV2(fsdis,
- compressAlgo, fileSize, blockEncoder);
- this.fsBlockReader = fsBlockReader;
+ fsBlockReader = new HFileBlock.FSReaderV2(fsdis, compressAlgo,
+ fileSize);
// Comparator class name is stored in the trailer in version 2.
comparator = trailer.createComparator();
@@ -128,10 +123,8 @@ public class HFileReaderV2 extends Abstr
avgKeyLen = Bytes.toInt(fileInfo.get(FileInfo.AVG_KEY_LEN));
avgValueLen = Bytes.toInt(fileInfo.get(FileInfo.AVG_VALUE_LEN));
byte [] keyValueFormatVersion = fileInfo.get(HFileWriterV2.KEY_VALUE_VERSION);
- includesMemstoreTS = keyValueFormatVersion != null &&
- Bytes.toInt(keyValueFormatVersion) ==
- HFileWriterV2.KEY_VALUE_VER_WITH_MEMSTORE;
- fsBlockReader.setIncludesMemstoreTS(includesMemstoreTS);
+ includesMemstoreTS = (keyValueFormatVersion != null &&
+ Bytes.toInt(keyValueFormatVersion) == HFileWriterV2.KEY_VALUE_VER_WITH_MEMSTORE);
// Store all other load-on-open blocks for further consumption.
HFileBlock b;
@@ -152,15 +145,9 @@ public class HFileReaderV2 extends Abstr
* @param isCompaction is scanner being used for a compaction?
* @return Scanner on this file.
*/
- @Override
- public HFileScanner getScanner(boolean cacheBlocks, final boolean pread,
+ @Override
+ public HFileScanner getScanner(boolean cacheBlocks, final boolean pread,
final boolean isCompaction) {
- // check if we want to use data block encoding in memory
- if (blockEncoder.useEncodedScanner(isCompaction)) {
- return new EncodedScannerV2(this, cacheBlocks, pread, isCompaction,
- includesMemstoreTS);
- }
-
return new ScannerV2(this, cacheBlocks, pread, isCompaction);
}
@@ -271,8 +258,6 @@ public class HFileReaderV2 extends Abstr
if (cacheConf.isBlockCacheEnabled()) {
HFileBlock cachedBlock =
(HFileBlock) cacheConf.getBlockCache().getBlock(cacheKey, cacheBlock);
- cachedBlock = blockEncoder.afterBlockCache(cachedBlock,
- isCompaction, shouldIncludeMemstoreTS());
if (cachedBlock != null) {
BlockCategory blockCategory =
cachedBlock.getBlockType().getCategory();
@@ -280,9 +265,8 @@ public class HFileReaderV2 extends Abstr
getSchemaMetrics().updateOnCacheHit(blockCategory, isCompaction);
- if (cachedBlock.getBlockType() == BlockType.DATA) {
+ if (cachedBlock.getBlockType() == BlockType.DATA)
HFile.dataBlockReadCnt.incrementAndGet();
- }
return cachedBlock;
}
// Carry on, please load.
@@ -308,8 +292,6 @@ public class HFileReaderV2 extends Abstr
// Cache the block
if (cacheBlock && cacheConf.shouldCacheBlockOnRead(
hfileBlock.getBlockType().getCategory())) {
- hfileBlock = blockEncoder.beforeBlockCache(
- hfileBlock, includesMemstoreTS);
cacheConf.getBlockCache().cacheBlock(cacheKey, hfileBlock,
cacheConf.isInMemory());
}
@@ -318,8 +300,7 @@ public class HFileReaderV2 extends Abstr
HFile.dataBlockReadCnt.incrementAndGet();
}
- return blockEncoder.afterReadFromDiskAndPuttingInCache(hfileBlock,
- isCompaction, includesMemstoreTS);
+ return hfileBlock;
} finally {
offsetLock.releaseLockEntry(lockEntry);
}
@@ -364,154 +345,25 @@ public class HFileReaderV2 extends Abstr
}
}
- protected abstract static class AbstractScannerV2
- extends AbstractHFileReader.Scanner {
- protected HFileBlock block;
-
- public AbstractScannerV2(HFileReaderV2 r, boolean cacheBlocks,
- final boolean pread, final boolean isCompaction) {
- super(r, cacheBlocks, pread, isCompaction);
- }
-
- /**
- * An internal API function. Seek to the given key, optionally rewinding to
- * the first key of the block before doing the seek.
- *
- * @param key key byte array
- * @param offset key offset in the key byte array
- * @param length key length
- * @param rewind whether to rewind to the first key of the block before
- * doing the seek. If this is false, we are assuming we never go
- * back, otherwise the result is undefined.
- * @return -1 if the key is earlier than the first key of the file,
- * 0 if we are at the given key, and 1 if we are past the given key
- * @throws IOException
- */
- protected int seekTo(byte[] key, int offset, int length, boolean rewind)
- throws IOException {
- HFileBlockIndex.BlockIndexReader indexReader =
- reader.getDataBlockIndexReader();
- HFileBlock seekToBlock = indexReader.seekToDataBlock(key, offset, length,
- block, cacheBlocks, pread, isCompaction);
- if (seekToBlock == null) {
- // This happens if the key e.g. falls before the beginning of the file.
- return -1;
- }
- return loadBlockAndSeekToKey(seekToBlock, rewind, key, offset, length,
- false);
- }
-
- protected abstract ByteBuffer getFirstKeyInBlock(HFileBlock curBlock);
-
- protected abstract int loadBlockAndSeekToKey(HFileBlock seekToBlock,
- boolean rewind, byte[] key, int offset, int length, boolean seekBefore)
- throws IOException;
-
- @Override
- public int seekTo(byte[] key, int offset, int length) throws IOException {
- // Always rewind to the first key of the block, because the given key
- // might be before or after the current key.
- return seekTo(key, offset, length, true);
- }
-
- @Override
- public int reseekTo(byte[] key, int offset, int length) throws IOException {
- if (isSeeked()) {
- ByteBuffer bb = getKey();
- int compared = reader.getComparator().compare(key, offset,
- length, bb.array(), bb.arrayOffset(), bb.limit());
- if (compared < 1) {
- // If the required key is less than or equal to current key, then
- // don't do anything.
- return compared;
- }
- }
-
- // Don't rewind on a reseek operation, because reseek implies that we are
- // always going forward in the file.
- return seekTo(key, offset, length, false);
- }
-
- @Override
- public boolean seekBefore(byte[] key, int offset, int length)
- throws IOException {
- HFileBlock seekToBlock =
- reader.getDataBlockIndexReader().seekToDataBlock(key, offset, length,
- block, cacheBlocks, pread, isCompaction);
- if (seekToBlock == null) {
- return false;
- }
- ByteBuffer firstKey = getFirstKeyInBlock(seekToBlock);
- if (reader.getComparator().compare(firstKey.array(),
- firstKey.arrayOffset(), firstKey.limit(), key, offset, length) == 0)
- {
- long previousBlockOffset = seekToBlock.getPrevBlockOffset();
- // The key we are interested in
- if (previousBlockOffset == -1) {
- // we have a 'problem', the key we want is the first of the file.
- return false;
- }
-
- // It is important that we compute and pass onDiskSize to the block
- // reader so that it does not have to read the header separately to
- // figure out the size.
- seekToBlock = reader.readBlock(previousBlockOffset,
- seekToBlock.getOffset() - previousBlockOffset, cacheBlocks,
- pread, isCompaction);
-
- // TODO shortcut: seek forward in this block to the last key of the
- // block.
- }
- loadBlockAndSeekToKey(seekToBlock, true, key, offset, length, true);
- return true;
- }
-
-
- /**
- * Scans blocks in the "scanned" section of the {@link HFile} until the next
- * data block is found.
- *
- * @return the next block, or null if there are no more data blocks
- * @throws IOException
- */
- protected HFileBlock readNextDataBlock() throws IOException {
- long lastDataBlockOffset = reader.getTrailer().getLastDataBlockOffset();
- if (block == null)
- return null;
-
- HFileBlock curBlock = block;
-
- do {
- if (curBlock.getOffset() >= lastDataBlockOffset)
- return null;
-
- if (curBlock.getOffset() < 0) {
- throw new IOException("Invalid block file offset: " + block);
- }
- curBlock = reader.readBlock(curBlock.getOffset()
- + curBlock.getOnDiskSizeWithHeader(),
- curBlock.getNextBlockOnDiskSizeWithHeader(), cacheBlocks, pread,
- isCompaction);
- } while (!(curBlock.getBlockType().equals(BlockType.DATA) ||
- curBlock.getBlockType().equals(BlockType.ENCODED_DATA)));
-
- return curBlock;
- }
- }
-
/**
* Implementation of {@link HFileScanner} interface.
*/
- protected static class ScannerV2 extends AbstractScannerV2 {
+ protected static class ScannerV2 extends AbstractHFileReader.Scanner {
+ private HFileBlock block;
private HFileReaderV2 reader;
public ScannerV2(HFileReaderV2 r, boolean cacheBlocks,
final boolean pread, final boolean isCompaction) {
- super(r, cacheBlocks, pread, isCompaction);
+ super(cacheBlocks, pread, isCompaction);
this.reader = r;
}
@Override
+ public HFileReaderV2 getReader() {
+ return reader;
+ }
+
+ @Override
public KeyValue getKeyValue() {
if (!isSeeked())
return null;
@@ -600,6 +452,36 @@ public class HFileReaderV2 extends Abstr
}
/**
+ * Scans blocks in the "scanned" section of the {@link HFile} until the next
+ * data block is found.
+ *
+ * @return the next block, or null if there are no more data blocks
+ * @throws IOException
+ */
+ private HFileBlock readNextDataBlock() throws IOException {
+ long lastDataBlockOffset = reader.getTrailer().getLastDataBlockOffset();
+ if (block == null)
+ return null;
+
+ HFileBlock curBlock = block;
+
+ do {
+ if (curBlock.getOffset() >= lastDataBlockOffset)
+ return null;
+
+ if (curBlock.getOffset() < 0) {
+ throw new IOException("Invalid block file offset: " + block);
+ }
+ curBlock = reader.readBlock(curBlock.getOffset()
+ + curBlock.getOnDiskSizeWithHeader(),
+ curBlock.getNextBlockOnDiskSizeWithHeader(), cacheBlocks, pread,
+ isCompaction);
+ } while (!curBlock.getBlockType().equals(BlockType.DATA));
+
+ return curBlock;
+ }
+
+ /**
* Positions this scanner at the start of the file.
*
* @return false if empty file; i.e. a call to next would return false and
@@ -635,7 +517,70 @@ public class HFileReaderV2 extends Abstr
}
@Override
- protected int loadBlockAndSeekToKey(HFileBlock seekToBlock, boolean rewind,
+ public int seekTo(byte[] key) throws IOException {
+ return seekTo(key, 0, key.length);
+ }
+
+ /**
+ * An internal API function. Seek to the given key, optionally rewinding to
+ * the first key of the block before doing the seek.
+ *
+ * @param key key byte array
+ * @param offset key offset in the key byte array
+ * @param length key length
+ * @param rewind whether to rewind to the first key of the block before
+ * doing the seek. If this is false, we are assuming we never go
+ * back, otherwise the result is undefined.
+ * @return -1 if the key is earlier than the first key of the file,
+ * 0 if we are at the given key, and 1 if we are past the given key
+ * @throws IOException
+ */
+ private int seekTo(byte[] key, int offset, int length, boolean rewind)
+ throws IOException {
+ HFileBlockIndex.BlockIndexReader indexReader =
+ reader.getDataBlockIndexReader();
+ HFileBlock seekToBlock = indexReader.seekToDataBlock(key, offset, length,
+ block, cacheBlocks, pread, isCompaction);
+
+ if (seekToBlock == null) {
+ // This happens if the key e.g. falls before the beginning of the file.
+ return -1;
+ }
+ return loadBlockAndSeekToKey(seekToBlock, rewind, key, offset, length,
+ false);
+ }
+
+ @Override
+ public int seekTo(byte[] key, int offset, int length) throws IOException {
+ // Always rewind to the first key of the block, because the given key
+ // might be before or after the current key.
+ return seekTo(key, offset, length, true);
+ }
+
+ @Override
+ public int reseekTo(byte[] key) throws IOException {
+ return reseekTo(key, 0, key.length);
+ }
+
+ @Override
+ public int reseekTo(byte[] key, int offset, int length) throws IOException {
+ if (isSeeked()) {
+ ByteBuffer bb = getKey();
+ int compared = reader.getComparator().compare(key, offset,
+ length, bb.array(), bb.arrayOffset(), bb.limit());
+ if (compared < 1) {
+ // If the required key is less than or equal to current key, then
+ // don't do anything.
+ return compared;
+ }
+ }
+
+ // Don't rewind on a reseek operation, because reseek implies that we are
+ // always going forward in the file.
+ return seekTo(key, offset, length, false);
+ }
+
+ private int loadBlockAndSeekToKey(HFileBlock seekToBlock, boolean rewind,
byte[] key, int offset, int length, boolean seekBefore)
throws IOException {
if (block == null || block.getOffset() != seekToBlock.getOffset()) {
@@ -654,13 +599,6 @@ public class HFileReaderV2 extends Abstr
*/
private void updateCurrBlock(HFileBlock newBlock) {
block = newBlock;
-
- // sanity check
- if (block.getBlockType() != BlockType.DATA) {
- throw new IllegalStateException(
- "ScannerV2 works only on data blocks");
- }
-
blockBuffer = block.getBufferWithoutHeader();
readKeyValueLen();
blockFetches++;
@@ -775,7 +713,11 @@ public class HFileReaderV2 extends Abstr
}
@Override
- protected ByteBuffer getFirstKeyInBlock(HFileBlock curBlock) {
+ public boolean seekBefore(byte[] key) throws IOException {
+ return seekBefore(key, 0, key.length);
+ }
+
+ private ByteBuffer getFirstKeyInBlock(HFileBlock curBlock) {
ByteBuffer buffer = curBlock.getBufferWithoutHeader();
// It is safe to manipulate this buffer because we own the buffer object.
buffer.rewind();
@@ -788,174 +730,53 @@ public class HFileReaderV2 extends Abstr
}
@Override
- public String getKeyString() {
- return Bytes.toStringBinary(blockBuffer.array(),
- blockBuffer.arrayOffset() + blockBuffer.position()
- + KEY_VALUE_LEN_SIZE, currKeyLen);
- }
-
- @Override
- public String getValueString() {
- return Bytes.toString(blockBuffer.array(), blockBuffer.arrayOffset()
- + blockBuffer.position() + KEY_VALUE_LEN_SIZE + currKeyLen,
- currValueLen);
- }
- }
-
- /**
- * ScannerV2 which operate on encoded data blocks.
- */
- protected static class EncodedScannerV2 extends AbstractScannerV2 {
- private DataBlockEncoder.EncodedSeeker seeker = null;
- private DataBlockEncoder dataBlockEncoder = null;
- private final boolean includesMemstoreTS;
-
- public EncodedScannerV2(HFileReaderV2 reader, boolean cacheBlocks,
- boolean pread, boolean isCompaction, boolean includesMemstoreTS) {
- super(reader, cacheBlocks, pread, isCompaction);
- this.includesMemstoreTS = includesMemstoreTS;
- }
-
- private void setDataBlockEncoder(DataBlockEncoder dataBlockEncoder) {
- this.dataBlockEncoder = dataBlockEncoder;
- seeker = dataBlockEncoder.createSeeker(reader.getComparator(),
- includesMemstoreTS);
- }
-
- /**
- * Updates the current block to be the given {@link HFileBlock}. Seeks to
- * the the first key/value pair.
- *
- * @param newBlock the block to make current
- */
- private void updateCurrentBlock(HFileBlock newBlock) {
- block = newBlock;
-
- // sanity checks
- if (block.getBlockType() != BlockType.ENCODED_DATA) {
- throw new IllegalStateException(
- "EncodedScannerV2 works only on encoded data blocks");
- }
-
- short dataBlockEncoderId = block.getDataBlockEncodingId();
- if (dataBlockEncoder == null
- || !DataBlockEncodings.isCorrectEncoder(dataBlockEncoder,
- dataBlockEncoderId)) {
- DataBlockEncoder encoder =
- DataBlockEncodings.getDataBlockEncoderFromId(dataBlockEncoderId);
- setDataBlockEncoder(encoder);
- }
-
- seeker.setCurrentBuffer(getEncodedBuffer(newBlock));
- blockFetches++;
- }
-
- private ByteBuffer getEncodedBuffer(HFileBlock newBlock) {
- ByteBuffer origBlock = newBlock.getBufferReadOnly();
- ByteBuffer encodedBlock = ByteBuffer.wrap(origBlock.array(),
- origBlock.arrayOffset() + HFileBlock.HEADER_SIZE +
- DataBlockEncodings.ID_SIZE,
- origBlock.limit() - HFileBlock.HEADER_SIZE -
- DataBlockEncodings.ID_SIZE).slice();
- return encodedBlock;
- }
-
- @Override
- public boolean seekTo() throws IOException {
- if (reader == null) {
- return false;
- }
-
- if (reader.getTrailer().getEntryCount() == 0) {
- // No data blocks.
+ public boolean seekBefore(byte[] key, int offset, int length)
+ throws IOException {
+ HFileBlock seekToBlock =
+ reader.getDataBlockIndexReader().seekToDataBlock(key, offset,
+ length, block, cacheBlocks, pread, isCompaction);
+ if (seekToBlock == null) {
return false;
}
-
- long firstDataBlockOffset =
- reader.getTrailer().getFirstDataBlockOffset();
- if (block != null && block.getOffset() == firstDataBlockOffset) {
- seeker.rewind();
- return true;
- }
-
- block = reader.readBlock(firstDataBlockOffset, -1, cacheBlocks, pread,
- isCompaction);
- if (block.getOffset() < 0) {
- throw new IOException("Invalid block offset: " + block.getOffset());
- }
- updateCurrentBlock(block);
- return true;
- }
-
- @Override
- public boolean next() throws IOException {
- boolean isValid = seeker.next();
- if (!isValid) {
- block = readNextDataBlock();
- isValid = block != null;
- if (isValid) {
- updateCurrentBlock(block);
+ ByteBuffer firstKey = getFirstKeyInBlock(seekToBlock);
+ if (reader.getComparator().compare(firstKey.array(),
+ firstKey.arrayOffset(), firstKey.limit(), key, offset, length) == 0)
+ {
+ long previousBlockOffset = seekToBlock.getPrevBlockOffset();
+ // The key we are interested in
+ if (previousBlockOffset == -1) {
+ // we have a 'problem', the key we want is the first of the file.
+ return false;
}
- }
- return isValid;
- }
- @Override
- public ByteBuffer getKey() {
- assertValidSeek();
- return seeker.getKey();
- }
-
- @Override
- public ByteBuffer getValue() {
- assertValidSeek();
- return seeker.getValue();
- }
+ // It is important that we compute and pass onDiskSize to the block
+ // reader so that it does not have to read the header separately to
+ // figure out the size.
+ seekToBlock = reader.readBlock(previousBlockOffset,
+ seekToBlock.getOffset() - previousBlockOffset, cacheBlocks,
+ pread, isCompaction);
- @Override
- public KeyValue getKeyValue() {
- if (block == null) {
- return null;
+ // TODO shortcut: seek forward in this block to the last key of the
+ // block.
}
- return seeker.getKeyValueObject();
+ loadBlockAndSeekToKey(seekToBlock, true, key, offset, length, true);
+ return true;
}
@Override
public String getKeyString() {
- ByteBuffer keyBuffer = getKey();
- return Bytes.toStringBinary(keyBuffer.array(),
- keyBuffer.arrayOffset(), keyBuffer.limit());
+ return Bytes.toStringBinary(blockBuffer.array(),
+ blockBuffer.arrayOffset() + blockBuffer.position()
+ + KEY_VALUE_LEN_SIZE, currKeyLen);
}
@Override
public String getValueString() {
- ByteBuffer valueBuffer = getValue();
- return Bytes.toStringBinary(valueBuffer.array(),
- valueBuffer.arrayOffset(), valueBuffer.limit());
- }
-
- private void assertValidSeek() {
- if (block == null) {
- throw new NotSeekedException();
- }
- }
-
- @Override
- protected ByteBuffer getFirstKeyInBlock(HFileBlock curBlock) {
- return dataBlockEncoder.getFirstKeyInBlock(getEncodedBuffer(curBlock));
+ return Bytes.toString(blockBuffer.array(), blockBuffer.arrayOffset()
+ + blockBuffer.position() + KEY_VALUE_LEN_SIZE + currKeyLen,
+ currValueLen);
}
- @Override
- protected int loadBlockAndSeekToKey(HFileBlock seekToBlock, boolean rewind,
- byte[] key, int offset, int length, boolean seekBefore)
- throws IOException {
- if (block == null || block.getOffset() != seekToBlock.getOffset()) {
- updateCurrentBlock(seekToBlock);
- } else if (rewind) {
- seeker.rewind();
- }
- return seeker.blockSeekTo(key, offset, length, seekBefore);
- }
}
/**
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java?rev=1223042&r1=1223041&r2=1223042&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java Sun Dec 25 02:49:46 2011
@@ -35,10 +35,8 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValue.KeyComparator;
-import org.apache.hadoop.hbase.io.hfile.Compression.Algorithm;
import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
import org.apache.hadoop.hbase.io.hfile.HFile.Writer;
-import org.apache.hadoop.hbase.regionserver.MemStore;
import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
import org.apache.hadoop.hbase.util.BloomFilterWriter;
import org.apache.hadoop.hbase.util.Bytes;
@@ -46,7 +44,7 @@ import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.compress.Compressor;
/**
- * Writes version 1 HFiles. Mainly used for testing backwards-compatibility.
+ * Writes version 1 HFiles. Mainly used for testing backwards-compatibilty.
*/
public class HFileWriterV1 extends AbstractHFileWriter {
@@ -93,17 +91,16 @@ public class HFileWriterV1 extends Abstr
@Override
public Writer createWriter(FileSystem fs, Path path, int blockSize,
- Algorithm compressAlgo, HFileDataBlockEncoder dataBlockEncoder,
- KeyComparator comparator)
+ Compression.Algorithm compressAlgo, final KeyComparator comparator)
throws IOException {
return new HFileWriterV1(conf, cacheConf, fs, path, blockSize,
- compressAlgo, dataBlockEncoder, comparator);
+ compressAlgo, comparator);
}
@Override
public Writer createWriter(FileSystem fs, Path path, int blockSize,
- String compressAlgoName, KeyComparator comparator)
- throws IOException {
+ String compressAlgoName,
+ final KeyComparator comparator) throws IOException {
return new HFileWriterV1(conf, cacheConf, fs, path, blockSize,
compressAlgoName, comparator);
}
@@ -120,8 +117,7 @@ public class HFileWriterV1 extends Abstr
public Writer createWriter(final FSDataOutputStream ostream,
final int blockSize, final Compression.Algorithm compress,
final KeyComparator c) throws IOException {
- return new HFileWriterV1(cacheConf, ostream, blockSize, compress,
- new NoOpDataBlockEncoder(), c);
+ return new HFileWriterV1(cacheConf, ostream, blockSize, compress, c);
}
}
@@ -131,7 +127,7 @@ public class HFileWriterV1 extends Abstr
throws IOException {
this(conf, cacheConf, fs, path, HFile.DEFAULT_BLOCKSIZE,
HFile.DEFAULT_COMPRESSION_ALGORITHM,
- new NoOpDataBlockEncoder(), null);
+ null);
}
/**
@@ -142,18 +138,15 @@ public class HFileWriterV1 extends Abstr
Path path, int blockSize, String compressAlgoName,
final KeyComparator comparator) throws IOException {
this(conf, cacheConf, fs, path, blockSize,
- compressionByName(compressAlgoName), new NoOpDataBlockEncoder(),
- comparator);
+ compressionByName(compressAlgoName), comparator);
}
/** Constructor that takes a path, creates and closes the output stream. */
- public HFileWriterV1(Configuration conf, CacheConfig cacheConf,
- FileSystem fs, Path path,
- int blockSize, Compression.Algorithm compress,
- HFileDataBlockEncoder blockEncoder,
+ public HFileWriterV1(Configuration conf, CacheConfig cacheConf, FileSystem fs,
+ Path path, int blockSize, Compression.Algorithm compress,
final KeyComparator comparator) throws IOException {
super(cacheConf, createOutputStream(conf, fs, path), path,
- blockSize, compress, blockEncoder, comparator);
+ blockSize, compress, comparator);
SchemaMetrics.configureGlobally(conf);
}
@@ -164,17 +157,15 @@ public class HFileWriterV1 extends Abstr
throws IOException {
this(cacheConf, outputStream, blockSize,
Compression.getCompressionAlgorithmByName(compressAlgoName),
- new NoOpDataBlockEncoder(), comparator);
+ comparator);
}
/** Constructor that takes a stream. */
public HFileWriterV1(CacheConfig cacheConf,
final FSDataOutputStream outputStream, final int blockSize,
- final Compression.Algorithm compress,
- HFileDataBlockEncoder blockEncoder, final KeyComparator comparator)
+ final Compression.Algorithm compress, final KeyComparator comparator)
throws IOException {
- super(cacheConf, outputStream, null, blockSize, compress,
- blockEncoder, comparator);
+ super(cacheConf, outputStream, null, blockSize, compress, comparator);
}
/**
@@ -211,17 +202,13 @@ public class HFileWriterV1 extends Abstr
if (cacheConf.shouldCacheDataOnWrite()) {
baosDos.flush();
- // we do not do dataBlockEncoding on disk HFile V2.
byte[] bytes = baos.toByteArray();
HFileBlock cBlock = new HFileBlock(BlockType.DATA,
(int) (outputStream.getPos() - blockBegin), bytes.length, -1,
- ByteBuffer.wrap(bytes, 0, bytes.length), HFileBlock.FILL_HEADER,
- blockBegin, MemStore.NO_PERSISTENT_TS);
- HFileBlock codedBlock = blockEncoder.beforeBlockCache(cBlock,
- false);
- passSchemaMetricsTo(codedBlock);
+ ByteBuffer.wrap(bytes, 0, bytes.length), true, blockBegin);
+ passSchemaMetricsTo(cBlock);
cacheConf.getBlockCache().cacheBlock(
- HFile.getBlockCacheKey(name, blockBegin), codedBlock);
+ HFile.getBlockCacheKey(name, blockBegin), cBlock);
baosDos.close();
}
blockNumber++;
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java?rev=1223042&r1=1223041&r2=1223042&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java Sun Dec 25 02:49:46 2011
@@ -49,13 +49,9 @@ public class HFileWriterV2 extends Abstr
static final Log LOG = LogFactory.getLog(HFileWriterV2.class);
/** Max memstore (mvcc) timestamp in FileInfo */
- public static final byte [] MAX_MEMSTORE_TS_KEY =
- Bytes.toBytes("MAX_MEMSTORE_TS_KEY");
-
+ public static final byte [] MAX_MEMSTORE_TS_KEY = Bytes.toBytes("MAX_MEMSTORE_TS_KEY");
/** KeyValue version in FileInfo */
- public static final byte [] KEY_VALUE_VERSION =
- Bytes.toBytes("KEY_VALUE_VERSION");
-
+ public static final byte [] KEY_VALUE_VERSION = Bytes.toBytes("KEY_VALUE_VERSION");
/** Version for KeyValue which includes memstore timestamp */
public static final int KEY_VALUE_VER_WITH_MEMSTORE = 1;
@@ -96,10 +92,10 @@ public class HFileWriterV2 extends Abstr
@Override
public Writer createWriter(FileSystem fs, Path path, int blockSize,
- Compression.Algorithm compress, HFileDataBlockEncoder blockEncoder,
+ Compression.Algorithm compress,
final KeyComparator comparator) throws IOException {
return new HFileWriterV2(conf, cacheConf, fs, path, blockSize,
- compress, blockEncoder, comparator);
+ compress, comparator);
}
@Override
@@ -132,7 +128,7 @@ public class HFileWriterV2 extends Abstr
FileSystem fs, Path path)
throws IOException {
this(conf, cacheConf, fs, path, HFile.DEFAULT_BLOCKSIZE,
- HFile.DEFAULT_COMPRESSION_ALGORITHM, null, null);
+ HFile.DEFAULT_COMPRESSION_ALGORITHM, null);
}
/**
@@ -143,16 +139,15 @@ public class HFileWriterV2 extends Abstr
Path path, int blockSize, String compressAlgoName,
final KeyComparator comparator) throws IOException {
this(conf, cacheConf, fs, path, blockSize,
- compressionByName(compressAlgoName), null, comparator);
+ compressionByName(compressAlgoName), comparator);
}
/** Constructor that takes a path, creates and closes the output stream. */
public HFileWriterV2(Configuration conf, CacheConfig cacheConf, FileSystem fs,
Path path, int blockSize, Compression.Algorithm compressAlgo,
- HFileDataBlockEncoder blockEncoder,
final KeyComparator comparator) throws IOException {
super(cacheConf, createOutputStream(conf, fs, path), path,
- blockSize, compressAlgo, blockEncoder, comparator);
+ blockSize, compressAlgo, comparator);
SchemaMetrics.configureGlobally(conf);
finishInit(conf);
}
@@ -172,8 +167,7 @@ public class HFileWriterV2 extends Abstr
final FSDataOutputStream outputStream, final int blockSize,
final Compression.Algorithm compress, final KeyComparator comparator)
throws IOException {
- super(cacheConf, outputStream, null, blockSize, compress, null,
- comparator);
+ super(cacheConf, outputStream, null, blockSize, compress, comparator);
finishInit(conf);
}
@@ -183,8 +177,7 @@ public class HFileWriterV2 extends Abstr
throw new IllegalStateException("finishInit called twice");
// HFile filesystem-level (non-caching) block writer
- fsBlockWriter = new HFileBlock.Writer(compressAlgo, blockEncoder,
- includeMemstoreTS);
+ fsBlockWriter = new HFileBlock.Writer(compressAlgo);
// Data block index writer
boolean cacheIndexesOnWrite = cacheConf.shouldCacheIndexesOnWrite();
@@ -232,9 +225,8 @@ public class HFileWriterV2 extends Abstr
long startTimeNs = System.nanoTime();
// Update the first data block offset for scanning.
- if (firstDataBlockOffset == -1) {
+ if (firstDataBlockOffset == -1)
firstDataBlockOffset = outputStream.getPos();
- }
// Update the last data block offset
lastDataBlockOffset = outputStream.getPos();
@@ -250,12 +242,10 @@ public class HFileWriterV2 extends Abstr
HFile.writeOps.incrementAndGet();
if (cacheConf.shouldCacheDataOnWrite()) {
- HFileBlock cBlock = fsBlockWriter.getBlockForCaching();
- HFileBlock codedBlock = blockEncoder.beforeBlockCache(cBlock,
- includeMemstoreTS);
- passSchemaMetricsTo(codedBlock);
+ HFileBlock blockForCaching = fsBlockWriter.getBlockForCaching();
+ passSchemaMetricsTo(blockForCaching);
cacheConf.getBlockCache().cacheBlock(
- HFile.getBlockCacheKey(name, lastDataBlockOffset), codedBlock);
+ HFile.getBlockCacheKey(name, lastDataBlockOffset), blockForCaching);
}
}
@@ -266,7 +256,7 @@ public class HFileWriterV2 extends Abstr
long offset = outputStream.getPos();
boolean cacheThisBlock = ibw.cacheOnWrite();
ibw.writeInlineBlock(fsBlockWriter.startWriting(
- ibw.getInlineBlockType()));
+ ibw.getInlineBlockType(), cacheThisBlock));
fsBlockWriter.writeHeaderAndData(outputStream);
ibw.blockWritten(offset, fsBlockWriter.getOnDiskSizeWithHeader(),
fsBlockWriter.getUncompressedSizeWithoutHeader());
@@ -275,11 +265,9 @@ public class HFileWriterV2 extends Abstr
if (cacheThisBlock) {
// Cache this block on write.
HFileBlock cBlock = fsBlockWriter.getBlockForCaching();
- HFileBlock codedBlock = blockEncoder.beforeBlockCache(cBlock,
- includeMemstoreTS);
- passSchemaMetricsTo(codedBlock);
+ passSchemaMetricsTo(cBlock);
cacheConf.getBlockCache().cacheBlock(
- HFile.getBlockCacheKey(name, offset), codedBlock);
+ HFile.getBlockCacheKey(name, offset), cBlock);
}
}
}
@@ -292,7 +280,8 @@ public class HFileWriterV2 extends Abstr
*/
private void newBlock() throws IOException {
// This is where the next block begins.
- fsBlockWriter.startWriting(BlockType.DATA);
+ fsBlockWriter.startWriting(BlockType.DATA,
+ cacheConf.shouldCacheDataOnWrite());
firstKeyInBlock = null;
}
@@ -424,7 +413,8 @@ public class HFileWriterV2 extends Abstr
// store the beginning offset
long offset = outputStream.getPos();
// write the metadata content
- DataOutputStream dos = fsBlockWriter.startWriting(BlockType.META);
+ DataOutputStream dos = fsBlockWriter.startWriting(BlockType.META,
+ cacheConf.shouldCacheDataOnWrite());
metaData.get(i).write(dos);
fsBlockWriter.writeHeaderAndData(outputStream);
@@ -450,7 +440,7 @@ public class HFileWriterV2 extends Abstr
// Meta block index.
metaBlockIndexWriter.writeSingleLevelIndex(fsBlockWriter.startWriting(
- BlockType.ROOT_INDEX), "meta");
+ BlockType.ROOT_INDEX, false), "meta");
fsBlockWriter.writeHeaderAndData(outputStream);
totalUncompressedBytes += fsBlockWriter.getUncompressedSizeWithHeader();
@@ -460,7 +450,8 @@ public class HFileWriterV2 extends Abstr
}
// File info
- writeFileInfo(trailer, fsBlockWriter.startWriting(BlockType.FILE_INFO));
+ writeFileInfo(trailer, fsBlockWriter.startWriting(BlockType.FILE_INFO,
+ false));
fsBlockWriter.writeHeaderAndData(outputStream);
totalUncompressedBytes += fsBlockWriter.getUncompressedSizeWithHeader();
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java?rev=1223042&r1=1223041&r2=1223042&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java Sun Dec 25 02:49:46 2011
@@ -722,10 +722,4 @@ public class LruBlockCache implements Bl
public void shutdown() {
this.scheduleThreadPool.shutdown();
}
-
- /** Clears the cache. Used in tests. */
- public void clearCache() {
- map.clear();
- }
-
}
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java?rev=1223042&r1=1223041&r2=1223042&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/mapreduce/LoadIncrementalHFiles.java Sun Dec 25 02:49:46 2011
@@ -65,8 +65,6 @@ import org.apache.hadoop.hbase.io.Refere
import org.apache.hadoop.hbase.io.Reference.Range;
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hadoop.hbase.io.hfile.Compression.Algorithm;
-import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoder;
-import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoderImpl;
import org.apache.hadoop.hbase.io.hfile.HFile;
import org.apache.hadoop.hbase.io.hfile.HFileScanner;
import org.apache.hadoop.hbase.regionserver.StoreFile;
@@ -530,13 +528,9 @@ public class LoadIncrementalHFiles exten
CacheConfig cacheConf = new CacheConfig(conf);
HalfStoreFileReader halfReader = null;
StoreFile.Writer halfWriter = null;
- HFileDataBlockEncoder dataBlockEncoder = new HFileDataBlockEncoderImpl(
- familyDescriptor.getDataBlockEncodingOnDisk(),
- familyDescriptor.getDataBlockEncodingInCache(),
- familyDescriptor.useEncodedDataBlockSeek());
try {
halfReader = new HalfStoreFileReader(fs, inFile, cacheConf,
- reference, dataBlockEncoder);
+ reference);
Map<byte[], byte[]> fileInfo = halfReader.loadFileInfo();
int blocksize = familyDescriptor.getBlocksize();
@@ -544,8 +538,7 @@ public class LoadIncrementalHFiles exten
BloomType bloomFilterType = familyDescriptor.getBloomFilterType();
halfWriter = new StoreFile.Writer(
- fs, outFile, blocksize, compression, dataBlockEncoder,
- conf, cacheConf,
+ fs, outFile, blocksize, compression, conf, cacheConf,
KeyValue.COMPARATOR, bloomFilterType, 0);
HFileScanner scanner = halfReader.getScanner(false, false, false);
scanner.seekTo();
@@ -645,6 +638,7 @@ public class LoadIncrementalHFiles exten
Path[] hfiles = FileUtil.stat2Paths(fs.listStatus(familyDir));
for (Path hfile : hfiles) {
if (hfile.getName().startsWith("_")) continue;
+
HFile.Reader reader = HFile.createReader(fs, hfile,
new CacheConfig(getConf()));
final byte[] first, last;
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java?rev=1223042&r1=1223041&r2=1223042&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java Sun Dec 25 02:49:46 2011
@@ -870,10 +870,6 @@ public class MemStore implements HeapSiz
ClassSize.COPYONWRITE_ARRAYSET + ClassSize.COPYONWRITE_ARRAYLIST +
(2 * ClassSize.CONCURRENT_SKIPLISTMAP));
- // Constants for whether to serialize memstore timestamp.
- public static final boolean NO_PERSISTENT_TS = false;
- public static final boolean PERSISTENT_TS = true;
-
/*
* Calculate how the MemStore size has changed. Includes overhead of the
* backing Map.
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java?rev=1223042&r1=1223041&r2=1223042&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java Sun Dec 25 02:49:46 2011
@@ -48,8 +48,6 @@ import org.apache.hadoop.hbase.client.Sc
import org.apache.hadoop.hbase.io.HeapSize;
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hadoop.hbase.io.hfile.Compression;
-import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoder;
-import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoderImpl;
import org.apache.hadoop.hbase.io.hfile.HFile;
import org.apache.hadoop.hbase.io.hfile.HFileScanner;
import org.apache.hadoop.hbase.io.hfile.InvalidHFileException;
@@ -146,7 +144,6 @@ public class Store extends SchemaConfigu
private final Compression.Algorithm compression;
/** Compression algorithm for major compaction */
private final Compression.Algorithm compactionCompression;
- private HFileDataBlockEncoder dataBlockEncoder;
// Comparing KeyValues
final KeyValue.KVComparator comparator;
@@ -184,12 +181,6 @@ public class Store extends SchemaConfigu
this.compactionCompression =
(family.getCompactionCompression() != Compression.Algorithm.NONE) ?
family.getCompactionCompression() : this.compression;
-
- this.dataBlockEncoder =
- new HFileDataBlockEncoderImpl(family.getDataBlockEncodingOnDisk(),
- family.getDataBlockEncodingInCache(),
- family.useEncodedDataBlockSeek());
-
this.comparator = info.getComparator();
// getTimeToLive returns ttl in seconds. Convert to milliseconds.
this.ttl = family.getTimeToLive();
@@ -279,21 +270,6 @@ public class Store extends SchemaConfigu
public Path getHomedir() {
return homedir;
}
-
- /**
- * @return the data block encoder
- */
- public HFileDataBlockEncoder getDataBlockEncoder() {
- return dataBlockEncoder;
- }
-
- /**
- * Should be used only in tests.
- * @param blockEncoder the block delta encoder to use
- */
- public void setDataBlockEncoderInTest(HFileDataBlockEncoder blockEncoder) {
- this.dataBlockEncoder = blockEncoder;
- }
/*
* Creates an unsorted list of StoreFile loaded from the given directory.
@@ -316,9 +292,8 @@ public class Store extends SchemaConfigu
continue;
}
StoreFile curfile = new StoreFile(fs, p, this.conf, this.cacheConf,
- this.family.getBloomFilterType(), this.dataBlockEncoder);
+ this.family.getBloomFilterType());
passSchemaMetricsTo(curfile);
-
curfile.createReader();
long length = curfile.getReader().length();
this.storeSize += length;
@@ -472,9 +447,8 @@ public class Store extends SchemaConfigu
StoreFile.rename(fs, srcPath, dstPath);
StoreFile sf = new StoreFile(fs, dstPath, this.conf, this.cacheConf,
- this.family.getBloomFilterType(), this.dataBlockEncoder);
+ this.family.getBloomFilterType());
passSchemaMetricsTo(sf);
-
sf.createReader();
LOG.info("Moved hfile " + srcPath + " into store directory " +
@@ -581,6 +555,7 @@ public class Store extends SchemaConfigu
MonitoredTask status)
throws IOException {
StoreFile.Writer writer;
+ String fileName;
// Find the smallest read point across all the Scanners.
long smallestReadPoint = region.getSmallestReadPoint();
long flushed = 0;
@@ -676,9 +651,8 @@ public class Store extends SchemaConfigu
status.setStatus("Flushing " + this + ": reopening flushed file");
StoreFile sf = new StoreFile(this.fs, dstPath, this.conf, this.cacheConf,
- this.family.getBloomFilterType(), this.dataBlockEncoder);
+ this.family.getBloomFilterType());
passSchemaMetricsTo(sf);
-
StoreFile.Reader r = sf.createReader();
this.storeSize += r.length();
this.totalUncompressedBytes += r.getTotalUncompressedBytes();
@@ -716,7 +690,7 @@ public class Store extends SchemaConfigu
Compression.Algorithm compression)
throws IOException {
StoreFile.Writer w = StoreFile.createWriter(fs, region.getTmpDir(),
- blocksize, compression, dataBlockEncoder, comparator, conf, cacheConf,
+ blocksize, compression, comparator, conf, cacheConf,
family.getBloomFilterType(), maxKeyCount);
// The store file writer's path does not include the CF name, so we need
// to configure the HFile writer directly.
@@ -1442,7 +1416,7 @@ public class Store extends SchemaConfigu
StoreFile storeFile = null;
try {
storeFile = new StoreFile(this.fs, path, this.conf,
- this.cacheConf, this.family.getBloomFilterType(), null);
+ this.cacheConf, this.family.getBloomFilterType());
passSchemaMetricsTo(storeFile);
storeFile.createReader();
} catch (IOException e) {
@@ -1494,7 +1468,7 @@ public class Store extends SchemaConfigu
" to " + destPath);
}
result = new StoreFile(this.fs, destPath, this.conf, this.cacheConf,
- this.family.getBloomFilterType(), this.dataBlockEncoder);
+ this.family.getBloomFilterType());
passSchemaMetricsTo(result);
result.createReader();
}
@@ -2082,8 +2056,8 @@ public class Store extends SchemaConfigu
}
public static final long FIXED_OVERHEAD =
- ClassSize.align(SchemaConfigured.SCHEMA_CONFIGURED_UNALIGNED_HEAP_SIZE +
- + (19 * ClassSize.REFERENCE) + (7 * Bytes.SIZEOF_LONG)
+ ClassSize.align(new SchemaConfigured().heapSize()
+ + (18 * ClassSize.REFERENCE) + (7 * Bytes.SIZEOF_LONG)
+ (5 * Bytes.SIZEOF_INT) + Bytes.SIZEOF_BOOLEAN);
public static final long DEEP_OVERHEAD = ClassSize.align(FIXED_OVERHEAD
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java?rev=1223042&r1=1223041&r2=1223042&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java Sun Dec 25 02:49:46 2011
@@ -56,8 +56,6 @@ import org.apache.hadoop.hbase.io.hfile.
import org.apache.hadoop.hbase.io.hfile.HFileWriterV2;
import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics;
import org.apache.hadoop.hbase.regionserver.metrics.SchemaConfigured;
-import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoder;
-import org.apache.hadoop.hbase.io.hfile.NoOpDataBlockEncoder;
import org.apache.hadoop.hbase.util.BloomFilter;
import org.apache.hadoop.hbase.util.BloomFilterFactory;
import org.apache.hadoop.hbase.util.BloomFilterWriter;
@@ -132,10 +130,6 @@ public class StoreFile extends SchemaCon
/** Key for timestamp of earliest-put in metadata*/
public static final byte[] EARLIEST_PUT_TS = Bytes.toBytes("EARLIEST_PUT_TS");
- /** Type of encoding used for data blocks in HFile. Stored in file info. */
- public static final byte[] DATA_BLOCK_ENCODING =
- Bytes.toBytes("DATA_BLOCK_ENCODING");
-
// Make default block size for StoreFiles 8k while testing. TODO: FIX!
// Need to make it 8k for testing.
public static final int DEFAULT_BLOCKSIZE_SMALL = 8 * 1024;
@@ -154,10 +148,7 @@ public class StoreFile extends SchemaCon
// Block cache configuration and reference.
private final CacheConfig cacheConf;
- // What kind of data block encoding will be used
- private HFileDataBlockEncoder dataBlockEncoder;
-
- // HDFS blocks distribution information
+ // HDFS blocks distribuion information
private HDFSBlocksDistribution hdfsBlocksDistribution;
// Keys for metadata stored in backing HFile.
@@ -216,23 +207,6 @@ public class StoreFile extends SchemaCon
private long modificationTimeStamp = 0L;
/**
- * Ignore bloom filters, don't use option inMemory
- * and dataBlockEncoding in memory.
- * @param fs The current file system to use
- * @param p The path of the file.
- * @param conf The current configuration.
- * @throws IOException When opening the reader fails.
- */
- StoreFile(final FileSystem fs,
- final Path p,
- final Configuration conf,
- final CacheConfig cacheConf)
- throws IOException {
- this(fs, p, conf, cacheConf, BloomType.NONE,
- new NoOpDataBlockEncoder());
- }
-
- /**
* Constructor, loads a reader and it's indices, etc. May allocate a
* substantial amount of ram depending on the underlying files (10-20MB?).
*
@@ -246,20 +220,17 @@ public class StoreFile extends SchemaCon
* as the Bloom filter type actually present in the HFile, because
* column family configuration might change. If this is
* {@link BloomType#NONE}, the existing Bloom filter is ignored.
- * @param dataBlockEncoder data block encoding algorithm.
* @throws IOException When opening the reader fails.
*/
StoreFile(final FileSystem fs,
final Path p,
final Configuration conf,
final CacheConfig cacheConf,
- final BloomType cfBloomType,
- final HFileDataBlockEncoder dataBlockEncoder)
+ final BloomType cfBloomType)
throws IOException {
this.fs = fs;
this.path = p;
this.cacheConf = cacheConf;
- this.dataBlockEncoder = dataBlockEncoder;
if (isReference(p)) {
this.reference = Reference.read(fs, p);
this.referencePath = getReferredToFile(this.path);
@@ -522,10 +493,9 @@ public class StoreFile extends SchemaCon
}
if (isReference()) {
this.reader = new HalfStoreFileReader(this.fs, this.referencePath,
- this.cacheConf, this.reference, this.dataBlockEncoder);
+ this.cacheConf, this.reference);
} else {
- this.reader = new Reader(this.fs, this.path, this.cacheConf,
- this.dataBlockEncoder);
+ this.reader = new Reader(this.fs, this.path, this.cacheConf);
}
if (isSchemaConfigured()) {
@@ -707,8 +677,8 @@ public class StoreFile extends SchemaCon
public static Writer createWriter(final FileSystem fs, final Path dir,
final int blocksize, Configuration conf, CacheConfig cacheConf)
throws IOException {
- return createWriter(fs, dir, blocksize, null, new NoOpDataBlockEncoder(),
- null, conf, cacheConf, BloomType.NONE, 0);
+ return createWriter(fs, dir, blocksize, null, null, conf, cacheConf,
+ BloomType.NONE, 0);
}
/**
@@ -719,7 +689,6 @@ public class StoreFile extends SchemaCon
* Creates a file with a unique name in this directory.
* @param blocksize
* @param algorithm Pass null to get default.
- * @param dataBlockEncoder Pass null to disable data block encoding.
* @param c Pass null to get default.
* @param conf HBase system configuration. used with bloom filters
* @param cacheConf Cache configuration and reference.
@@ -732,7 +701,6 @@ public class StoreFile extends SchemaCon
final Path dir,
final int blocksize,
final Compression.Algorithm algorithm,
- final HFileDataBlockEncoder dataBlockEncoder,
final KeyValue.KVComparator c,
final Configuration conf,
final CacheConfig cacheConf,
@@ -750,7 +718,7 @@ public class StoreFile extends SchemaCon
return new Writer(fs, path, blocksize,
algorithm == null? HFile.DEFAULT_COMPRESSION_ALGORITHM: algorithm,
- dataBlockEncoder, conf, cacheConf, c == null ? KeyValue.COMPARATOR : c, bloomType,
+ conf, cacheConf, c == null ? KeyValue.COMPARATOR: c, bloomType,
maxKeyCount);
}
@@ -846,8 +814,6 @@ public class StoreFile extends SchemaCon
private KeyValue lastDeleteFamilyKV = null;
private long deleteFamilyCnt = 0;
- protected HFileDataBlockEncoder dataBlockEncoder;
-
TimeRangeTracker timeRangeTracker = new TimeRangeTracker();
/* isTimeRangeTrackerSet keeps track if the timeRange has already been set
* When flushing a memstore, we set TimeRange and use this variable to
@@ -872,16 +838,13 @@ public class StoreFile extends SchemaCon
* @throws IOException problem writing to FS
*/
public Writer(FileSystem fs, Path path, int blocksize,
- Compression.Algorithm compress,
- HFileDataBlockEncoder dataBlockEncoder, final Configuration conf,
+ Compression.Algorithm compress, final Configuration conf,
CacheConfig cacheConf,
final KVComparator comparator, BloomType bloomType, long maxKeys)
throws IOException {
- this.dataBlockEncoder = dataBlockEncoder != null ?
- dataBlockEncoder : new NoOpDataBlockEncoder();
writer = HFile.getWriterFactory(conf, cacheConf).createWriter(
fs, path, blocksize,
- compress, this.dataBlockEncoder, comparator.getRawComparator());
+ compress, comparator.getRawComparator());
this.kvComparator = comparator;
@@ -1118,10 +1081,6 @@ public class StoreFile extends SchemaCon
}
public void close() throws IOException {
- // (optional) Add data block encoding used to save this file
- // It is mostly for statistics and debugging purpose.
- dataBlockEncoder.saveMetadata(this);
-
boolean hasGeneralBloom = this.closeGeneralBloomFilter();
boolean hasDeleteFamilyBloom = this.closeDeleteFamilyBloomFilter();
@@ -1160,12 +1119,10 @@ public class StoreFile extends SchemaCon
private byte[] lastBloomKey;
private long deleteFamilyCnt = -1;
- public Reader(FileSystem fs, Path path, CacheConfig cacheConf,
- HFileDataBlockEncoder dataBlockEncoder)
+ public Reader(FileSystem fs, Path path, CacheConfig cacheConf)
throws IOException {
super(path);
- reader = HFile.createReader(fs, path, cacheConf,
- dataBlockEncoder);
+ reader = HFile.createReader(fs, path, cacheConf);
bloomFilterType = BloomType.NONE;
}
@@ -1305,7 +1262,7 @@ public class StoreFile extends SchemaCon
default:
return true;
- }
+ }
}
public boolean passesDeleteFamilyBloomFilter(byte[] row, int rowOffset,
@@ -1355,7 +1312,7 @@ public class StoreFile extends SchemaCon
return true;
byte[] key;
- switch (bloomFilterType) {
+ switch (bloomFilterType) {
case ROW:
if (col != null) {
throw new RuntimeException("Row-only Bloom filter called with " +
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java?rev=1223042&r1=1223041&r2=1223042&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java Sun Dec 25 02:49:46 2011
@@ -177,7 +177,7 @@ class StoreFileScanner implements KeyVal
realSeekDone = true;
}
} catch (IOException ioe) {
- throw new IOException("Could not seek " + this + " " + key, ioe);
+ throw new IOException("Could not seek " + this, ioe);
}
}
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaConfigured.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaConfigured.java?rev=1223042&r1=1223041&r2=1223042&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaConfigured.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/metrics/SchemaConfigured.java Sun Dec 25 02:49:46 2011
@@ -83,15 +83,6 @@ public class SchemaConfigured implements
}
/**
- * Creates an instance corresponding to an unknown table and column family.
- * Used in unit tests.
- */
- public static SchemaConfigured createUnknown() {
- return new SchemaConfigured(null, SchemaMetrics.UNKNOWN,
- SchemaMetrics.UNKNOWN);
- }
-
- /**
* Default constructor. Only use when column/family name are not known at
* construction (i.e. for HFile blocks).
*/
Modified: hbase/trunk/src/main/ruby/hbase/admin.rb
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/ruby/hbase/admin.rb?rev=1223042&r1=1223041&r2=1223042&view=diff
==============================================================================
--- hbase/trunk/src/main/ruby/hbase/admin.rb (original)
+++ hbase/trunk/src/main/ruby/hbase/admin.rb Sun Dec 25 02:49:46 2011
@@ -532,9 +532,6 @@ module Hbase
family.setInMemory(JBoolean.valueOf(arg[org.apache.hadoop.hbase.HColumnDescriptor::IN_MEMORY])) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::IN_MEMORY)
family.setTimeToLive(JInteger.valueOf(arg[org.apache.hadoop.hbase.HColumnDescriptor::TTL])) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::TTL)
family.setCompressionType(org.apache.hadoop.hbase.io.hfile.Compression::Algorithm.valueOf(arg[org.apache.hadoop.hbase.HColumnDescriptor::COMPRESSION])) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::COMPRESSION)
- family.setDataBlockEncodingOnDisk(org.apache.hadoop.hbase.io.encoding.DataBlockEncodingAlgorithms::Algorithm.valueOf(arg[org.apache.hadoop.hbase.HColumnDescriptor::DATA_BLOCK_ENCODING_ON_DISK])) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::DATA_BLOCK_ENCODING_ON_DISK)
- family.setDataBlockEncodingInCache(org.apache.hadoop.hbase.io.encoding.DataBlockEncodingAlgorithms::Algorithm.valueOf(arg[org.apache.hadoop.hbase.HColumnDescriptor::DATA_BLOCK_ENCODING_IN_CACHE])) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::DATA_BLOCK_ENCODING_IN_CACHE)
- family.setEncodedDataBlockSeek(JBoolean.valueOf(arg[org.apache.hadoop.hbase.HColumnDescriptor::ENCODED_DATA_BLOCK_SEEK])) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::ENCODED_DATA_BLOCK_SEEK)
family.setBlocksize(JInteger.valueOf(arg[org.apache.hadoop.hbase.HColumnDescriptor::BLOCKSIZE])) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::BLOCKSIZE)
family.setMaxVersions(JInteger.valueOf(arg[org.apache.hadoop.hbase.HColumnDescriptor::VERSIONS])) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::VERSIONS)
family.setMinVersions(JInteger.valueOf(arg[org.apache.hadoop.hbase.HColumnDescriptor::MIN_VERSIONS])) if arg.include?(org.apache.hadoop.hbase.HColumnDescriptor::MIN_VERSIONS)
Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java?rev=1223042&r1=1223041&r2=1223042&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java Sun Dec 25 02:49:46 2011
@@ -221,33 +221,18 @@ public abstract class HBaseTestCase exte
final int minVersions, final int versions, final int ttl, boolean keepDeleted) {
HTableDescriptor htd = new HTableDescriptor(name);
htd.addFamily(new HColumnDescriptor(fam1, minVersions, versions,
- keepDeleted,
- HColumnDescriptor.DEFAULT_COMPRESSION,
- HColumnDescriptor.DEFAULT_DATA_BLOCK_ENCODING_ON_DISK,
- HColumnDescriptor.DEFAULT_DATA_BLOCK_ENCODING_IN_CACHE,
- HColumnDescriptor.DEFAULT_ENCODED_DATA_BLOCK_SEEK,
- false, false,
- HColumnDescriptor.DEFAULT_BLOCKSIZE, ttl,
- HColumnDescriptor.DEFAULT_BLOOMFILTER,
- HConstants.REPLICATION_SCOPE_LOCAL));
+ keepDeleted, HColumnDescriptor.DEFAULT_COMPRESSION, false, false,
+ HColumnDescriptor.DEFAULT_BLOCKSIZE, ttl,
+ HColumnDescriptor.DEFAULT_BLOOMFILTER,
+ HConstants.REPLICATION_SCOPE_LOCAL));
htd.addFamily(new HColumnDescriptor(fam2, minVersions, versions,
- keepDeleted,
- HColumnDescriptor.DEFAULT_COMPRESSION,
- HColumnDescriptor.DEFAULT_DATA_BLOCK_ENCODING_ON_DISK,
- HColumnDescriptor.DEFAULT_DATA_BLOCK_ENCODING_IN_CACHE,
- HColumnDescriptor.DEFAULT_ENCODED_DATA_BLOCK_SEEK,
- false, false,
+ keepDeleted, HColumnDescriptor.DEFAULT_COMPRESSION, false, false,
HColumnDescriptor.DEFAULT_BLOCKSIZE, ttl,
HColumnDescriptor.DEFAULT_BLOOMFILTER,
HConstants.REPLICATION_SCOPE_LOCAL));
htd.addFamily(new HColumnDescriptor(fam3, minVersions, versions,
- keepDeleted,
- HColumnDescriptor.DEFAULT_COMPRESSION,
- HColumnDescriptor.DEFAULT_DATA_BLOCK_ENCODING_ON_DISK,
- HColumnDescriptor.DEFAULT_DATA_BLOCK_ENCODING_IN_CACHE,
- HColumnDescriptor.DEFAULT_ENCODED_DATA_BLOCK_SEEK,
- false, false,
- HColumnDescriptor.DEFAULT_BLOCKSIZE, ttl,
+ keepDeleted, HColumnDescriptor.DEFAULT_COMPRESSION, false, false,
+ HColumnDescriptor.DEFAULT_BLOCKSIZE, ttl,
HColumnDescriptor.DEFAULT_BLOOMFILTER,
HConstants.REPLICATION_SCOPE_LOCAL));
return htd;
Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java?rev=1223042&r1=1223041&r2=1223042&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/HFilePerformanceEvaluation.java Sun Dec 25 02:49:46 2011
@@ -191,8 +191,7 @@ public class HFilePerformanceEvaluation
void setUp() throws Exception {
writer =
HFile.getWriterFactoryNoCache(conf).createWriter(this.fs,
- this.mf, RFILE_BLOCKSIZE, (Compression.Algorithm) null, null,
- null);
+ this.mf, RFILE_BLOCKSIZE, (Compression.Algorithm) null, null);
}
@Override
@@ -366,4 +365,4 @@ public class HFilePerformanceEvaluation
public static void main(String[] args) throws Exception {
new HFilePerformanceEvaluation().runBenchmarks();
}
-}
+}
\ No newline at end of file
Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java?rev=1223042&r1=1223041&r2=1223042&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java Sun Dec 25 02:49:46 2011
@@ -143,9 +143,6 @@ public class TestFromClientSide {
HColumnDescriptor.DEFAULT_VERSIONS,
true,
HColumnDescriptor.DEFAULT_COMPRESSION,
- HColumnDescriptor.DEFAULT_DATA_BLOCK_ENCODING_ON_DISK,
- HColumnDescriptor.DEFAULT_DATA_BLOCK_ENCODING_IN_CACHE,
- HColumnDescriptor.DEFAULT_ENCODED_DATA_BLOCK_SEEK,
HColumnDescriptor.DEFAULT_IN_MEMORY,
HColumnDescriptor.DEFAULT_BLOCKCACHE,
HColumnDescriptor.DEFAULT_BLOCKSIZE,
Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/TestHalfStoreFileReader.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/TestHalfStoreFileReader.java?rev=1223042&r1=1223041&r2=1223042&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/TestHalfStoreFileReader.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/TestHalfStoreFileReader.java Sun Dec 25 02:49:46 2011
@@ -31,7 +31,6 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.*;
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
-import org.apache.hadoop.hbase.io.encoding.DataBlockEncodings;
import org.apache.hadoop.hbase.io.hfile.HFile;
import org.apache.hadoop.hbase.io.hfile.HFileScanner;
import org.apache.hadoop.hbase.util.Bytes;
@@ -99,7 +98,7 @@ public class TestHalfStoreFileReader {
CacheConfig cacheConf)
throws IOException {
final HalfStoreFileReader halfreader =
- new HalfStoreFileReader(fs, p, cacheConf, bottom, null);
+ new HalfStoreFileReader(fs, p, cacheConf, bottom);
halfreader.loadFileInfo();
final HFileScanner scanner = halfreader.getScanner(false, false);
Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java?rev=1223042&r1=1223041&r2=1223042&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java Sun Dec 25 02:49:46 2011
@@ -295,14 +295,7 @@ public class TestHeapSize extends TestCa
assertEquals(expected, actual);
}
- // SchemaConfigured
- LOG.debug("Heap size for: " + SchemaConfigured.class.getName());
- SchemaConfigured sc = new SchemaConfigured(null, "myTable", "myCF");
- assertEquals(ClassSize.estimateBase(SchemaConfigured.class, true),
- sc.heapSize());
-
// Store Overhead
- LOG.debug("Heap size for: " + Store.class.getName());
cl = Store.class;
actual = Store.FIXED_OVERHEAD;
expected = ClassSize.estimateBase(cl, false);
@@ -326,6 +319,10 @@ public class TestHeapSize extends TestCa
// accounted for. But we have satisfied our two core requirements.
// Sizing is quite accurate now, and our tests will throw errors if
// any of these classes are modified without updating overhead sizes.
+
+ SchemaConfigured sc = new SchemaConfigured(null, "myTable", "myCF");
+ assertEquals(ClassSize.estimateBase(SchemaConfigured.class, true),
+ sc.heapSize());
}
@org.junit.Rule
Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/CacheTestUtils.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/CacheTestUtils.java?rev=1223042&r1=1223041&r2=1223042&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/CacheTestUtils.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/CacheTestUtils.java Sun Dec 25 02:49:46 2011
@@ -41,15 +41,9 @@ import org.apache.hadoop.hbase.regionser
public class CacheTestUtils {
- private static final boolean includesMemstoreTS = true;
+ /*Just checks if heapsize grows when something is cached, and gets smaller when the same object is evicted*/
- /**
- * Just checks if heapsize grows when something is cached, and gets smaller
- * when the same object is evicted
- */
-
- public static void testHeapSizeChanges(final BlockCache toBeTested,
- final int blockSize) {
+ public static void testHeapSizeChanges(final BlockCache toBeTested, final int blockSize){
HFileBlockPair[] blocks = generateHFileBlocks(blockSize, 1);
long heapSize = ((HeapSize) toBeTested).heapSize();
toBeTested.cacheBlock(blocks[0].blockName, blocks[0].block);
@@ -322,8 +316,7 @@ public class CacheTestUtils {
HFileBlock generated = new HFileBlock(BlockType.DATA,
onDiskSizeWithoutHeader, uncompressedSizeWithoutHeader,
- prevBlockOffset, cachedBuffer, HFileBlock.DONT_FILL_HEADER,
- blockSize, includesMemstoreTS);
+ prevBlockOffset, cachedBuffer, false, blockSize);
String strKey;
/* No conflicting keys */
Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java?rev=1223042&r1=1223041&r2=1223042&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java Sun Dec 25 02:49:46 2011
@@ -33,7 +33,6 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.*;
-import org.apache.hadoop.hbase.io.encoding.DataBlockEncodings;
import org.apache.hadoop.hbase.regionserver.StoreFile;
import org.apache.hadoop.hbase.util.BloomFilterFactory;
import org.junit.After;
@@ -43,7 +42,6 @@ import org.junit.experimental.categories
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
-
import static org.junit.Assert.*;
/**
@@ -63,12 +61,10 @@ public class TestCacheOnWrite {
private FileSystem fs;
private Random rand = new Random(12983177L);
private Path storeFilePath;
+ private Compression.Algorithm compress;
+ private CacheOnWriteType cowType;
private BlockCache blockCache;
- private String testDescription;
-
- private final CacheOnWriteType cowType;
- private final Compression.Algorithm compress;
- private final BlockEncoderTestType encoderType;
+ private String testName;
private static final int DATA_BLOCK_SIZE = 2048;
private static final int NUM_KV = 25000;
@@ -80,90 +76,49 @@ public class TestCacheOnWrite {
KeyValue.Type.values().length - 2;
private static enum CacheOnWriteType {
- DATA_BLOCKS(CacheConfig.CACHE_BLOCKS_ON_WRITE_KEY,
- BlockType.DATA, BlockType.ENCODED_DATA),
- BLOOM_BLOCKS(CacheConfig.CACHE_BLOOM_BLOCKS_ON_WRITE_KEY,
- BlockType.BLOOM_CHUNK),
- INDEX_BLOCKS(CacheConfig.CACHE_INDEX_BLOCKS_ON_WRITE_KEY,
- BlockType.LEAF_INDEX, BlockType.INTERMEDIATE_INDEX);
+ DATA_BLOCKS(BlockType.DATA, CacheConfig.CACHE_BLOCKS_ON_WRITE_KEY),
+ BLOOM_BLOCKS(BlockType.BLOOM_CHUNK,
+ CacheConfig.CACHE_BLOOM_BLOCKS_ON_WRITE_KEY),
+ INDEX_BLOCKS(BlockType.LEAF_INDEX,
+ CacheConfig.CACHE_INDEX_BLOCKS_ON_WRITE_KEY);
private final String confKey;
- private final BlockType blockType1;
- private final BlockType blockType2;
-
- private CacheOnWriteType(String confKey, BlockType blockType) {
- this(confKey, blockType, blockType);
- }
+ private final BlockType inlineBlockType;
- private CacheOnWriteType(String confKey, BlockType blockType1,
- BlockType blockType2) {
- this.blockType1 = blockType1;
- this.blockType2 = blockType2;
+ private CacheOnWriteType(BlockType inlineBlockType, String confKey) {
+ this.inlineBlockType = inlineBlockType;
this.confKey = confKey;
}
public boolean shouldBeCached(BlockType blockType) {
- return blockType == blockType1 || blockType == blockType2;
+ return blockType == inlineBlockType
+ || blockType == BlockType.INTERMEDIATE_INDEX
+ && inlineBlockType == BlockType.LEAF_INDEX;
}
public void modifyConf(Configuration conf) {
- for (CacheOnWriteType cowType : CacheOnWriteType.values()) {
+ for (CacheOnWriteType cowType : CacheOnWriteType.values())
conf.setBoolean(cowType.confKey, cowType == this);
- }
}
}
- private static final DataBlockEncodings.Algorithm ENCODING_ALGO =
- DataBlockEncodings.Algorithm.PREFIX;
-
- /** Provides fancy names for four combinations of two booleans */
- private static enum BlockEncoderTestType {
- NO_BLOCK_ENCODING(false, false),
- BLOCK_ENCODING_IN_CACHE_ONLY(false, true),
- BLOCK_ENCODING_ON_DISK_ONLY(true, false),
- BLOCK_ENCODING_EVERYWHERE(true, true);
-
- private final boolean encodeOnDisk;
- private final boolean encodeInCache;
-
- BlockEncoderTestType(boolean encodeOnDisk, boolean encodeInCache) {
- this.encodeOnDisk = encodeOnDisk;
- this.encodeInCache = encodeInCache;
- }
-
- public HFileDataBlockEncoder getEncoder() {
- // We always use an encoded seeker. It should not have effect if there
- // is no encoding in cache.
- return new HFileDataBlockEncoderImpl(
- encodeOnDisk ? ENCODING_ALGO : DataBlockEncodings.Algorithm.NONE,
- encodeInCache ? ENCODING_ALGO : DataBlockEncodings.Algorithm.NONE,
- true);
- }
- }
-
public TestCacheOnWrite(CacheOnWriteType cowType,
- Compression.Algorithm compress, BlockEncoderTestType encoderType) {
+ Compression.Algorithm compress) {
this.cowType = cowType;
this.compress = compress;
- this.encoderType = encoderType;
- testDescription = "[cacheOnWrite=" + cowType + ", compress=" + compress +
- ", encoderType=" + encoderType + "]";
- System.out.println(testDescription);
+ testName = "[cacheOnWrite=" + cowType + ", compress=" + compress + "]";
+ System.out.println(testName);
}
@Parameters
public static Collection<Object[]> getParameters() {
List<Object[]> cowTypes = new ArrayList<Object[]>();
- for (CacheOnWriteType cowType : CacheOnWriteType.values()) {
+ for (CacheOnWriteType cowType : CacheOnWriteType.values())
for (Compression.Algorithm compress :
HBaseTestingUtility.COMPRESSION_ALGORITHMS) {
- for (BlockEncoderTestType encoderType :
- BlockEncoderTestType.values()) {
- cowTypes.add(new Object[] { cowType, compress, encoderType });
- }
+ cowTypes.add(new Object[] { cowType, compress });
}
- }
return cowTypes;
}
@@ -201,10 +156,10 @@ public class TestCacheOnWrite {
private void readStoreFile() throws IOException {
HFileReaderV2 reader = (HFileReaderV2) HFile.createReader(fs,
- storeFilePath, cacheConf, encoderType.getEncoder());
+ storeFilePath, cacheConf);
LOG.info("HFile information: " + reader);
HFileScanner scanner = reader.getScanner(false, false);
- assertTrue(testDescription, scanner.seekTo());
+ assertTrue(testName, scanner.seekTo());
long offset = 0;
HFileBlock prevBlock = null;
@@ -219,11 +174,10 @@ public class TestCacheOnWrite {
// Flags: don't cache the block, use pread, this is not a compaction.
HFileBlock block = reader.readBlock(offset, onDiskSize, false, true,
false);
- BlockCacheKey blockCacheKey = HFile.getBlockCacheKey(reader.getName(),
- offset);
+ BlockCacheKey blockCacheKey = HFile.getBlockCacheKey(reader.getName(), offset);
boolean isCached = blockCache.getBlock(blockCacheKey, true) != null;
boolean shouldBeCached = cowType.shouldBeCached(block.getBlockType());
- assertEquals(testDescription + " " + block, shouldBeCached, isCached);
+ assertEquals(testName + " " + block, shouldBeCached, isCached);
prevBlock = block;
offset += block.getOnDiskSizeWithHeader();
BlockType bt = block.getBlockType();
@@ -233,10 +187,8 @@ public class TestCacheOnWrite {
LOG.info("Block count by type: " + blockCountByType);
String countByType = blockCountByType.toString();
- BlockType cachedDataBlockType =
- encoderType.encodeInCache ? BlockType.ENCODED_DATA : BlockType.DATA;
- assertEquals("{" + cachedDataBlockType
- + "=1379, LEAF_INDEX=173, BLOOM_CHUNK=9, INTERMEDIATE_INDEX=24}",
+ assertEquals(
+ "{DATA=1379, LEAF_INDEX=173, BLOOM_CHUNK=9, INTERMEDIATE_INDEX=24}",
countByType);
reader.close();
@@ -262,9 +214,8 @@ public class TestCacheOnWrite {
Path storeFileParentDir = new Path(TEST_UTIL.getDataTestDir(),
"test_cache_on_write");
StoreFile.Writer sfw = StoreFile.createWriter(fs, storeFileParentDir,
- DATA_BLOCK_SIZE, compress, encoderType.getEncoder(),
- KeyValue.COMPARATOR, conf, cacheConf, StoreFile.BloomType.ROWCOL,
- NUM_KV);
+ DATA_BLOCK_SIZE, compress, KeyValue.COMPARATOR, conf,
+ cacheConf, StoreFile.BloomType.ROWCOL, NUM_KV);
final int rowLen = 32;
for (int i = 0; i < NUM_KV; ++i) {
@@ -285,6 +236,7 @@ public class TestCacheOnWrite {
storeFilePath = sfw.getPath();
}
+
@org.junit.Rule
public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();