You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by mb...@apache.org on 2012/03/28 08:40:30 UTC
svn commit: r1306182 - in /hbase/branches/0.89-fb/src:
main/java/org/apache/hadoop/hbase/io/encoding/
main/java/org/apache/hadoop/hbase/io/hfile/
test/java/org/apache/hadoop/hbase/io/encoding/
test/java/org/apache/hadoop/hbase/io/hfile/ test/java/org/a...
Author: mbautin
Date: Wed Mar 28 06:40:29 2012
New Revision: 1306182
URL: http://svn.apache.org/viewvc?rev=1306182&view=rev
Log:
[jira] [HBASE-5469] [89-fb] Add baseline compression efficiency to DataBlockEncodingTool
Summary: Fixing DataBlockEncodingTool. This is the 89-fb version of
https://reviews.facebook.net/D2409.
Test Plan:
Run unit tests
Run DataBlockEncodingTool on a few files
Reviewers: kannan, aaiyer, liyintang, kranganathan, pkhemani
Reviewed By: kannan
Differential Revision: https://reviews.facebook.net/D2463
Modified:
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/CopyKeyDataBlockEncoder.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/DataBlockEncoder.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/EncodedDataBlock.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/Compression.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoderImpl.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/DataBlockEncodingTool.java
hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/EncodedSeekPerformanceTest.java
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java?rev=1306182&r1=1306181&r2=1306182&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java Wed Mar 28 06:40:29 2012
@@ -21,8 +21,8 @@ import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
-import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValue.SamePrefixComparator;
+import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.util.ByteBufferUtils;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.RawComparator;
@@ -36,9 +36,9 @@ abstract class BufferedDataBlockEncoder
private static int INITIAL_KEY_BUFFER_SIZE = 512;
@Override
- public ByteBuffer uncompressKeyValues(DataInputStream source,
+ public ByteBuffer decodeKeyValues(DataInputStream source,
boolean includesMemstoreTS) throws IOException {
- return uncompressKeyValues(source, 0, 0, includesMemstoreTS);
+ return decodeKeyValues(source, 0, 0, includesMemstoreTS);
}
protected static class SeekerState {
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/CopyKeyDataBlockEncoder.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/CopyKeyDataBlockEncoder.java?rev=1306182&r1=1306181&r2=1306182&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/CopyKeyDataBlockEncoder.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/CopyKeyDataBlockEncoder.java Wed Mar 28 06:40:29 2012
@@ -31,7 +31,7 @@ import org.apache.hadoop.io.RawComparato
*/
public class CopyKeyDataBlockEncoder extends BufferedDataBlockEncoder {
@Override
- public void compressKeyValues(DataOutputStream out,
+ public void encodeKeyValues(DataOutputStream out,
ByteBuffer in, boolean includesMemstoreTS) throws IOException {
in.rewind();
ByteBufferUtils.putInt(out, in.limit());
@@ -39,7 +39,7 @@ public class CopyKeyDataBlockEncoder ext
}
@Override
- public ByteBuffer uncompressKeyValues(DataInputStream source,
+ public ByteBuffer decodeKeyValues(DataInputStream source,
int preserveHeaderLength, int skipLastBytes, boolean includesMemstoreTS)
throws IOException {
int decompressedSize = source.readInt();
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/DataBlockEncoder.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/DataBlockEncoder.java?rev=1306182&r1=1306181&r2=1306182&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/DataBlockEncoder.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/DataBlockEncoder.java Wed Mar 28 06:40:29 2012
@@ -32,42 +32,42 @@ import org.apache.hadoop.io.RawComparato
* <li>the values are always iterated forward from beginning of block</li>
* <li>knowledge of Key Value format</li>
* </ul>
- * It is designed to work fast enough to be feasible as in memory compression.
+ * It is designed to work fast enough to be feasible as in memory encoding.
*/
public interface DataBlockEncoder {
/**
- * Compress KeyValues and write them to output buffer.
- * @param out Where to write compressed data.
- * @param in Source of KeyValue for compression.
+ * Encode KeyValues and write them to output buffer.
+ * @param out Where to write encoded data.
+ * @param in Source of KeyValue for encoding.
* @param includesMemstoreTS true if including memstore timestamp after every
* key-value pair
* @throws IOException If there is an error writing to output stream.
*/
- public void compressKeyValues(DataOutputStream out,
+ public void encodeKeyValues(DataOutputStream out,
ByteBuffer in, boolean includesMemstoreTS) throws IOException;
/**
- * Uncompress.
- * @param source Compressed stream of KeyValues.
+ * Decode.
+ * @param source Encoded stream of KeyValues.
* @param includesMemstoreTS true if including memstore timestamp after every
* key-value pair
- * @return Uncompressed block of KeyValues.
+ * @return decoded block of KeyValues.
* @throws IOException If there is an error in source.
*/
- public ByteBuffer uncompressKeyValues(DataInputStream source,
+ public ByteBuffer decodeKeyValues(DataInputStream source,
boolean includesMemstoreTS) throws IOException;
/**
- * Uncompress.
- * @param source Compressed stream of KeyValues.
+ * Decode.
+ * @param source encoded stream of KeyValues.
* @param allocateHeaderLength allocate this many bytes for the header.
* @param skipLastBytes Do not copy n last bytes.
* @param includesMemstoreTS true if including memstore timestamp after every
* key-value pair
- * @return Uncompressed block of KeyValues.
+ * @return decoded block of KeyValues.
* @throws IOException If there is an error in source.
*/
- public ByteBuffer uncompressKeyValues(DataInputStream source,
+ public ByteBuffer decodeKeyValues(DataInputStream source,
int allocateHeaderLength, int skipLastBytes, boolean includesMemstoreTS)
throws IOException;
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java?rev=1306182&r1=1306181&r2=1306182&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java Wed Mar 28 06:40:29 2012
@@ -314,7 +314,7 @@ public class DiffKeyDeltaEncoder extends
}
@Override
- public void compressKeyValues(DataOutputStream out,
+ public void encodeKeyValues(DataOutputStream out,
ByteBuffer in, boolean includesMemstoreTS) throws IOException {
in.rewind();
ByteBufferUtils.putInt(out, in.limit());
@@ -333,7 +333,7 @@ public class DiffKeyDeltaEncoder extends
}
@Override
- public ByteBuffer uncompressKeyValues(DataInputStream source,
+ public ByteBuffer decodeKeyValues(DataInputStream source,
int allocHeaderLength, int skipLastBytes, boolean includesMemstoreTS)
throws IOException {
int decompressedSize = source.readInt();
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/EncodedDataBlock.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/EncodedDataBlock.java?rev=1306182&r1=1306181&r2=1306182&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/EncodedDataBlock.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/EncodedDataBlock.java Wed Mar 28 06:40:29 2012
@@ -21,23 +21,30 @@ import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
+import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.util.Iterator;
import org.apache.commons.lang.NotImplementedException;
import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.io.hfile.Compression.Algorithm;
+import org.apache.hadoop.hbase.io.hfile.HFileBlock;
+import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.compress.Compressor;
+import com.google.common.base.Preconditions;
+import com.google.common.io.NullOutputStream;
+
/**
* Encapsulates a data block compressed using a particular encoding algorithm.
* Useful for testing and benchmarking.
*/
public class EncodedDataBlock {
- private static final int BUFFER_SIZE = 4 * 1024;
- protected DataBlockEncoder dataBlockEncoder;
- ByteArrayOutputStream uncompressedOutputStream;
- ByteBuffer uncompressedBuffer;
- private byte[] cacheCompressData;
+ private byte[] rawKVs;
+ private ByteBuffer rawBuffer;
+ private DataBlockEncoder dataBlockEncoder;
+
+ private byte[] cachedEncodedData;
private ByteArrayOutputStream compressedStream = new ByteArrayOutputStream();
private boolean includesMemstoreTS;
@@ -46,19 +53,11 @@ public class EncodedDataBlock {
* @param dataBlockEncoder Algorithm used for compression.
*/
public EncodedDataBlock(DataBlockEncoder dataBlockEncoder,
- boolean includesMemstoreTS) {
+ boolean includesMemstoreTS, DataBlockEncoding encoding, byte[] rawKVs) {
+ Preconditions.checkNotNull(encoding,
+ "Cannot create encoded data block with null encoder");
this.dataBlockEncoder = dataBlockEncoder;
- uncompressedOutputStream = new ByteArrayOutputStream(BUFFER_SIZE);
- }
-
- /**
- * Add KeyValue and compress it.
- * @param kv Item to be added and compressed.
- */
- public void addKv(KeyValue kv) {
- cacheCompressData = null;
- uncompressedOutputStream.write(
- kv.getBuffer(), kv.getOffset(), kv.getLength());
+ this.rawKVs = rawKVs;
}
/**
@@ -66,19 +65,20 @@ public class EncodedDataBlock {
* @return Forwards sequential iterator.
*/
public Iterator<KeyValue> getIterator() {
- final int uncompressedSize = uncompressedOutputStream.size();
- final ByteArrayInputStream bais = new ByteArrayInputStream(
- getCompressedData());
+ final int rawSize = rawKVs.length;
+ byte[] encodedDataWithHeader = getEncodedData();
+ int bytesToSkip = 0;
+ ByteArrayInputStream bais = new ByteArrayInputStream(encodedDataWithHeader,
+ bytesToSkip, encodedDataWithHeader.length - bytesToSkip);
final DataInputStream dis = new DataInputStream(bais);
-
return new Iterator<KeyValue>() {
private ByteBuffer decompressedData = null;
@Override
public boolean hasNext() {
if (decompressedData == null) {
- return uncompressedSize > 0;
+ return rawSize > 0;
}
return decompressedData.hasRemaining();
}
@@ -87,7 +87,7 @@ public class EncodedDataBlock {
public KeyValue next() {
if (decompressedData == null) {
try {
- decompressedData = dataBlockEncoder.uncompressKeyValues(
+ decompressedData = dataBlockEncoder.decodeKeyValues(
dis, includesMemstoreTS);
} catch (IOException e) {
throw new RuntimeException("Problem with data block encoder, " +
@@ -121,81 +121,75 @@ public class EncodedDataBlock {
* @return Size in bytes of compressed data.
*/
public int getSize() {
- return getCompressedData().length;
+ return getEncodedData().length;
}
/**
* Find the size of compressed data assuming that buffer will be compressed
* using given algorithm.
- * @param compressor Algorithm used for compression.
- * @param buffer Array to be compressed.
+ * @param algo compression algorithm
+ * @param compressor compressor already requested from codec
+ * @param inputBuffer Array to be compressed.
* @param offset Offset to beginning of the data.
* @param length Length to be compressed.
* @return Size of compressed data in bytes.
*/
- public static int checkCompressedSize(Compressor compressor, byte[] buffer,
- int offset, int length) {
- byte[] compressedBuffer = new byte[buffer.length];
- // in fact the buffer could be of any positive size
- compressor.setInput(buffer, offset, length);
- compressor.finish();
- int currentPos = 0;
- while (!compressor.finished()) {
- try {
- // we don't care about compressed data,
- // we just want to callculate number of bytes
- currentPos += compressor.compress(compressedBuffer, 0,
- compressedBuffer.length);
- } catch (IOException e) {
- throw new RuntimeException(
- "For some reason compressor couldn't read data. " +
- "It is likely a problem with " +
- compressor.getClass().getName(), e);
- }
+ public static int getCompressedSize(Algorithm algo, Compressor compressor,
+ byte[] inputBuffer, int offset, int length) throws IOException {
+ DataOutputStream compressedStream = new DataOutputStream(
+ new NullOutputStream());
+ if (compressor != null) {
+ compressor.reset();
}
- return currentPos;
+ OutputStream compressingStream = algo.createCompressionStream(
+ compressedStream, compressor, 0);
+
+ compressingStream.write(inputBuffer, offset, length);
+ compressingStream.flush();
+ compressingStream.close();
+
+ return compressedStream.size();
}
/**
* Estimate size after second stage of compression (e.g. LZO).
- * @param compressor Algorithm which will be used for compressions.
+ * @param comprAlgo compression algorithm to be used for compression
+ * @param compressor compressor corresponding to the given compression
+ * algorithm
* @return Size after second stage of compression.
*/
- public int checkCompressedSize(Compressor compressor) {
- // compress
- byte[] compressedBytes = getCompressedData();
- return checkCompressedSize(compressor, compressedBytes, 0,
+ public int getEncodedCompressedSize(Algorithm comprAlgo,
+ Compressor compressor) throws IOException {
+ byte[] compressedBytes = getEncodedData();
+ return getCompressedSize(comprAlgo, compressor, compressedBytes, 0,
compressedBytes.length);
}
- private byte[] getCompressedData() {
- // is cached
- if (cacheCompressData != null) {
- return cacheCompressData;
+ /** @return encoded data with header */
+ private byte[] getEncodedData() {
+ if (cachedEncodedData != null) {
+ return cachedEncodedData;
}
- cacheCompressData = doCompressData();
-
- return cacheCompressData;
+ cachedEncodedData = encodeData();
+ return cachedEncodedData;
}
private ByteBuffer getUncompressedBuffer() {
- if (uncompressedBuffer == null ||
- uncompressedBuffer.limit() < uncompressedOutputStream.size()) {
- uncompressedBuffer = ByteBuffer.wrap(
- uncompressedOutputStream.toByteArray());
+ if (rawBuffer == null || rawBuffer.limit() < rawKVs.length) {
+ rawBuffer = ByteBuffer.wrap(rawKVs);
}
- return uncompressedBuffer;
+ return rawBuffer;
}
/**
- * Do the compression.
- * @return Compressed byte buffer.
+ * Do the encoding, but do not cache the encoded data.
+ * @return encoded data block with header and checksum
*/
- public byte[] doCompressData() {
+ public byte[] encodeData() {
compressedStream.reset();
DataOutputStream dataOut = new DataOutputStream(compressedStream);
try {
- this.dataBlockEncoder.compressKeyValues(
+ this.dataBlockEncoder.encodeKeyValues(
dataOut, getUncompressedBuffer(), includesMemstoreTS);
} catch (IOException e) {
throw new RuntimeException(String.format(
@@ -210,12 +204,4 @@ public class EncodedDataBlock {
public String toString() {
return dataBlockEncoder.toString();
}
-
- /**
- * Get uncompressed buffer.
- * @return The buffer.
- */
- public byte[] getRawKeyValues() {
- return uncompressedOutputStream.toByteArray();
- }
}
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java?rev=1306182&r1=1306181&r2=1306182&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java Wed Mar 28 06:40:29 2012
@@ -341,7 +341,7 @@ public class FastDiffDeltaEncoder extend
}
@Override
- public void compressKeyValues(DataOutputStream out,
+ public void encodeKeyValues(DataOutputStream out,
ByteBuffer in, boolean includesMemstoreTS) throws IOException {
in.rewind();
ByteBufferUtils.putInt(out, in.limit());
@@ -360,7 +360,7 @@ public class FastDiffDeltaEncoder extend
}
@Override
- public ByteBuffer uncompressKeyValues(DataInputStream source,
+ public ByteBuffer decodeKeyValues(DataInputStream source,
int allocHeaderLength, int skipLastBytes, boolean includesMemstoreTS)
throws IOException {
int decompressedSize = source.readInt();
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java?rev=1306182&r1=1306181&r2=1306182&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java Wed Mar 28 06:40:29 2012
@@ -16,10 +16,14 @@
*/
package org.apache.hadoop.hbase.io.encoding;
+import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
+import java.io.FilterOutputStream;
import java.io.IOException;
+import java.lang.reflect.Field;
import java.nio.ByteBuffer;
+import java.util.Arrays;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.util.ByteBufferUtils;
@@ -73,7 +77,7 @@ public class PrefixKeyDeltaEncoder exten
}
@Override
- public void compressKeyValues(DataOutputStream writeHere,
+ public void encodeKeyValues(DataOutputStream writeHere,
ByteBuffer in, boolean includesMemstoreTS) throws IOException {
in.rewind();
ByteBufferUtils.putInt(writeHere, in.limit());
@@ -89,7 +93,7 @@ public class PrefixKeyDeltaEncoder exten
}
@Override
- public ByteBuffer uncompressKeyValues(DataInputStream source,
+ public ByteBuffer decodeKeyValues(DataInputStream source,
int allocHeaderLength, int skipLastBytes, boolean includesMemstoreTS)
throws IOException {
int decompressedSize = source.readInt();
@@ -99,7 +103,7 @@ public class PrefixKeyDeltaEncoder exten
int prevKeyOffset = 0;
while (source.available() > skipLastBytes) {
- prevKeyOffset = uncompressKeyValue(source, buffer, prevKeyOffset);
+ prevKeyOffset = decodeKeyValue(source, buffer, prevKeyOffset);
afterDecodingKeyValue(source, buffer, includesMemstoreTS);
}
@@ -111,7 +115,7 @@ public class PrefixKeyDeltaEncoder exten
return buffer;
}
- private int uncompressKeyValue(DataInputStream source, ByteBuffer buffer,
+ private int decodeKeyValue(DataInputStream source, ByteBuffer buffer,
int prevKeyOffset)
throws IOException, EncoderBufferTooSmallException {
int keyLength = ByteBufferUtils.readCompressedInt(source);
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/Compression.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/Compression.java?rev=1306182&r1=1306181&r2=1306182&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/Compression.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/Compression.java Wed Mar 28 06:40:29 2012
@@ -193,7 +193,7 @@ public final class Compression {
* Creates a compression stream without any additional wrapping into
* buffering streams.
*/
- CompressionOutputStream createPlainCompressionStream(
+ public CompressionOutputStream createPlainCompressionStream(
OutputStream downStream, Compressor compressor) throws IOException {
CompressionCodec codec = getCodec(conf);
((Configurable)codec).getConf().setInt("io.file.buffer.size", 32 * 1024);
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java?rev=1306182&r1=1306181&r2=1306182&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java Wed Mar 28 06:40:29 2012
@@ -35,6 +35,7 @@ import java.util.concurrent.atomic.Atomi
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
import org.apache.hadoop.hbase.io.hfile.Compression.Algorithm;
import org.apache.hadoop.hbase.regionserver.MemStore;
@@ -1531,4 +1532,3 @@ public class HFileBlock extends SchemaCo
}
}
-
Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoderImpl.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoderImpl.java?rev=1306182&r1=1306181&r2=1306182&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoderImpl.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoderImpl.java Wed Mar 28 06:40:29 2012
@@ -180,7 +180,7 @@ public class HFileDataBlockEncoderImpl i
try {
encodedStream.write(HFileBlock.DUMMY_HEADER);
algo.writeIdInBytes(dataOut);
- encoder.compressKeyValues(dataOut, in,
+ encoder.encodeKeyValues(dataOut, in,
includesMemstoreTS);
} catch (IOException e) {
throw new RuntimeException(String.format("Bug in data block encoder " +
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java?rev=1306182&r1=1306181&r2=1306182&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java Wed Mar 28 06:40:29 2012
@@ -31,13 +31,13 @@ import java.util.List;
import java.util.Random;
import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValue.Type;
+import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.Test;
import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
+import org.junit.runners.Parameterized;
/**
* Test all of the data block encoding algorithms for correctness.
@@ -67,13 +67,13 @@ public class TestDataBlockEncoders {
// encode
ByteArrayOutputStream baos = new ByteArrayOutputStream();
DataOutputStream dataOut = new DataOutputStream(baos);
- encoder.compressKeyValues(dataOut, dataset, includesMemstoreTS);
+ encoder.encodeKeyValues(dataOut, dataset, includesMemstoreTS);
// decode
ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
DataInputStream dis = new DataInputStream(bais);
ByteBuffer actualDataset;
- actualDataset = encoder.uncompressKeyValues(dis, includesMemstoreTS);
+ actualDataset = encoder.decodeKeyValues(dis, includesMemstoreTS);
dataset.rewind();
actualDataset.rewind();
@@ -148,7 +148,7 @@ public class TestDataBlockEncoders {
for (DataBlockEncoder encoder : dataBlockEncoders) {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
DataOutputStream dataOut = new DataOutputStream(baos);
- encoder.compressKeyValues(dataOut, originalBuffer, includesMemstoreTS);
+ encoder.encodeKeyValues(dataOut, originalBuffer, includesMemstoreTS);
ByteBuffer encodedBuffer = ByteBuffer.wrap(baos.toByteArray());
DataBlockEncoder.EncodedSeeker seeker =
encoder.createSeeker(KeyValue.KEY_COMPARATOR, includesMemstoreTS);
@@ -199,7 +199,7 @@ public class TestDataBlockEncoders {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
DataOutputStream dataOut = new DataOutputStream(baos);
try {
- encoder.compressKeyValues(dataOut, originalBuffer, includesMemstoreTS);
+ encoder.encodeKeyValues(dataOut, originalBuffer, includesMemstoreTS);
} catch (IOException e) {
throw new RuntimeException(String.format(
"Bug while encoding using '%s'", encoder.toString()), e);
@@ -259,7 +259,7 @@ public class TestDataBlockEncoders {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
DataOutputStream dataOut = new DataOutputStream(baos);
try {
- encoder.compressKeyValues(dataOut, originalBuffer, includesMemstoreTS);
+ encoder.encodeKeyValues(dataOut, originalBuffer, includesMemstoreTS);
} catch (IOException e) {
throw new RuntimeException(String.format(
"Bug while encoding using '%s'", encoder.toString()), e);
@@ -340,4 +340,4 @@ public class TestDataBlockEncoders {
assertEquals("Input of two methods is changed", onDataset, dataset);
}
}
-}
+}
\ No newline at end of file
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java?rev=1306182&r1=1306181&r2=1306182&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java Wed Mar 28 06:40:29 2012
@@ -60,8 +60,8 @@ import org.apache.hadoop.io.compress.Com
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
+import org.junit.runners.Parameterized;
@RunWith(Parameterized.class)
public class TestHFileBlock {
@@ -420,7 +420,7 @@ public class TestHFileBlock {
encodedBuf = rawBuf;
} else {
ByteArrayOutputStream encodedOut = new ByteArrayOutputStream();
- encoding.getEncoder().compressKeyValues(
+ encoding.getEncoder().encodeKeyValues(
new DataOutputStream(encodedOut),
rawBuf.duplicate(), includesMemstoreTS);
// We need to account for the two-byte encoding algorithm ID that
@@ -730,4 +730,4 @@ public class TestHFileBlock {
}
}
-}
+}
\ No newline at end of file
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/DataBlockEncodingTool.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/DataBlockEncodingTool.java?rev=1306182&r1=1306181&r2=1306182&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/DataBlockEncodingTool.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/DataBlockEncodingTool.java Wed Mar 28 06:40:29 2012
@@ -20,7 +20,7 @@ import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
-import java.io.OutputStream;
+import java.text.DecimalFormat;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
@@ -38,14 +38,15 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.io.encoding.EncodedDataBlock;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoder;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
+import org.apache.hadoop.hbase.io.encoding.EncodedDataBlock;
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
+import org.apache.hadoop.hbase.io.hfile.Compression.Algorithm;
import org.apache.hadoop.hbase.io.hfile.Compression;
import org.apache.hadoop.hbase.io.hfile.NoOpDataBlockEncoder;
-import org.apache.hadoop.hbase.io.hfile.Compression.Algorithm;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.io.compress.CompressionOutputStream;
import org.apache.hadoop.io.compress.Compressor;
import org.apache.hadoop.io.compress.Decompressor;
@@ -60,33 +61,85 @@ public class DataBlockEncodingTool {
private static final boolean includesMemstoreTS = true;
/**
- * How many times should benchmark run.
- * More times means better data in terms of statistics.
- * It has to be larger than BENCHMARK_N_OMIT.
+ * How many times to run the benchmark. More times means better data in terms
+ * of statistics but slower execution. Has to be strictly larger than
+ * {@link DEFAULT_BENCHMARK_N_OMIT}.
*/
- public static int BENCHMARK_N_TIMES = 12;
+ private static final int DEFAULT_BENCHMARK_N_TIMES = 12;
/**
- * How many first runs should omit benchmark.
- * Usually it is one in order to exclude setup cost.
- * Has to be 0 or larger.
+ * How many first runs should not be included in the benchmark. Done in order
+ * to exclude setup cost.
*/
- public static int BENCHMARK_N_OMIT = 2;
+ private static final int DEFAULT_BENCHMARK_N_OMIT = 2;
+
+ /** HFile name to be used in benchmark */
+ private static final String OPT_HFILE_NAME = "f";
+
+ /** Maximum number of key/value pairs to process in a single benchmark run */
+ private static final String OPT_KV_LIMIT = "n";
+
+ /** Whether to run a benchmark to measure read throughput */
+ private static final String OPT_MEASURE_THROUGHPUT = "b";
+
+ /** If this is specified, no correctness testing will be done */
+ private static final String OPT_OMIT_CORRECTNESS_TEST = "c";
+
+ /** What encoding algorithm to test */
+ private static final String OPT_ENCODING_ALGORITHM = "a";
+
+ /** Number of times to run each benchmark */
+ private static final String OPT_BENCHMARK_N_TIMES = "t";
+
+ /** Number of first runs of every benchmark to omit from statistics */
+ private static final String OPT_BENCHMARK_N_OMIT = "omit";
/** Compression algorithm to use if not specified on the command line */
private static final Algorithm DEFAULT_COMPRESSION =
Compression.Algorithm.GZ;
+ private static final DecimalFormat DELIMITED_DECIMAL_FORMAT =
+ new DecimalFormat();
+
+ static {
+ DELIMITED_DECIMAL_FORMAT.setGroupingSize(3);
+ }
+
+ private static final String PCT_FORMAT = "%.2f %%";
+ private static final String INT_FORMAT = "%d";
+
+ private static int benchmarkNTimes = DEFAULT_BENCHMARK_N_TIMES;
+ private static int benchmarkNOmit = DEFAULT_BENCHMARK_N_OMIT;
+
private List<EncodedDataBlock> codecs = new ArrayList<EncodedDataBlock>();
- private int totalPrefixLength = 0;
- private int totalKeyLength = 0;
- private int totalValueLength = 0;
- private int totalKeyRedundancyLength = 0;
-
- final private String compressionAlgorithmName;
- final private Algorithm compressionAlgorithm;
- final private Compressor compressor;
- final private Decompressor decompressor;
+ private long totalPrefixLength = 0;
+ private long totalKeyLength = 0;
+ private long totalValueLength = 0;
+ private long totalKeyRedundancyLength = 0;
+ private long totalCFLength = 0;
+
+ private byte[] rawKVs;
+
+ private final String compressionAlgorithmName;
+ private final Algorithm compressionAlgorithm;
+ private final Compressor compressor;
+ private final Decompressor decompressor;
+
+ private static enum Manipulation {
+ ENCODING,
+ DECODING,
+ COMPRESSION,
+ DECOMPRESSION;
+
+ @Override
+ public String toString() {
+ String s = super.toString();
+ StringBuilder sb = new StringBuilder();
+ sb.append(s.charAt(0));
+ sb.append(s.substring(1).toLowerCase());
+ return sb.toString();
+ }
+ }
/**
* @param compressionAlgorithmName What kind of algorithm should be used
@@ -110,23 +163,21 @@ public class DataBlockEncodingTool {
throws IOException {
scanner.seek(KeyValue.LOWESTKEY);
- KeyValue currentKv;
+ KeyValue currentKV;
byte[] previousKey = null;
byte[] currentKey;
- List<DataBlockEncoder> dataBlockEncoders =
- DataBlockEncoding.getAllEncoders();
+ DataBlockEncoding[] encodings = DataBlockEncoding.values();
- for (DataBlockEncoder d : dataBlockEncoders) {
- codecs.add(new EncodedDataBlock(d, includesMemstoreTS));
- }
+ ByteArrayOutputStream uncompressedOutputStream =
+ new ByteArrayOutputStream();
int j = 0;
- while ((currentKv = scanner.next()) != null && j < kvLimit) {
+ while ((currentKV = scanner.next()) != null && j < kvLimit) {
// Iterates through key/value pairs
j++;
- currentKey = currentKv.getKey();
+ currentKey = currentKV.getKey();
if (previousKey != null) {
for (int i = 0; i < previousKey.length && i < currentKey.length &&
previousKey[i] == currentKey[i]; ++i) {
@@ -134,16 +185,30 @@ public class DataBlockEncodingTool {
}
}
- for (EncodedDataBlock codec : codecs) {
- codec.addKv(currentKv);
- }
+ uncompressedOutputStream.write(currentKV.getBuffer(),
+ currentKV.getOffset(), currentKV.getLength());
previousKey = currentKey;
- totalPrefixLength += currentKv.getLength() - currentKv.getKeyLength() -
- currentKv.getValueLength();
- totalKeyLength += currentKv.getKeyLength();
- totalValueLength += currentKv.getValueLength();
+ int kLen = currentKV.getKeyLength();
+ int vLen = currentKV.getValueLength();
+ int cfLen = currentKV.getFamilyLength(currentKV.getFamilyOffset());
+ int restLen = currentKV.getLength() - kLen - vLen;
+
+ totalKeyLength += kLen;
+ totalValueLength += vLen;
+ totalPrefixLength += restLen;
+ totalCFLength += cfLen;
+ }
+
+ rawKVs = uncompressedOutputStream.toByteArray();
+
+ for (DataBlockEncoding encoding : encodings) {
+ if (encoding == DataBlockEncoding.NONE) {
+ continue;
+ }
+ DataBlockEncoder d = encoding.getEncoder();
+ codecs.add(new EncodedDataBlock(d, includesMemstoreTS, encoding, rawKVs));
}
}
@@ -227,15 +292,14 @@ public class DataBlockEncodingTool {
/**
* Benchmark codec's speed.
*/
- public void benchmarkCodecs() {
+ public void benchmarkCodecs() throws IOException {
+ LOG.info("Starting a throughput benchmark for data block encoding codecs");
int prevTotalSize = -1;
for (EncodedDataBlock codec : codecs) {
prevTotalSize = benchmarkEncoder(prevTotalSize, codec);
}
- byte[] buffer = codecs.get(0).getRawKeyValues();
-
- benchmarkDefaultCompression(prevTotalSize, buffer);
+ benchmarkDefaultCompression(prevTotalSize, rawKVs);
}
/**
@@ -251,7 +315,7 @@ public class DataBlockEncodingTool {
// decompression time
List<Long> durations = new ArrayList<Long>();
- for (int itTime = 0; itTime < BENCHMARK_N_TIMES; ++itTime) {
+ for (int itTime = 0; itTime < benchmarkNTimes; ++itTime) {
totalSize = 0;
Iterator<KeyValue> it;
@@ -265,7 +329,7 @@ public class DataBlockEncodingTool {
totalSize += it.next().getLength();
}
final long finishTime = System.nanoTime();
- if (itTime >= BENCHMARK_N_OMIT) {
+ if (itTime >= benchmarkNOmit) {
durations.add(finishTime - startTime);
}
@@ -276,26 +340,27 @@ public class DataBlockEncodingTool {
prevTotalSize = totalSize;
}
- // compression time
- List<Long> compressDurations = new ArrayList<Long>();
- for (int itTime = 0; itTime < BENCHMARK_N_TIMES; ++itTime) {
+ List<Long> encodingDurations = new ArrayList<Long>();
+ for (int itTime = 0; itTime < benchmarkNTimes; ++itTime) {
final long startTime = System.nanoTime();
- codec.doCompressData();
+ codec.encodeData();
final long finishTime = System.nanoTime();
- if (itTime >= BENCHMARK_N_OMIT) {
- compressDurations.add(finishTime - startTime);
+ if (itTime >= benchmarkNOmit) {
+ encodingDurations.add(finishTime - startTime);
}
}
System.out.println(codec.toString() + ":");
- printBenchmarkResult(totalSize, compressDurations, false);
- printBenchmarkResult(totalSize, durations, true);
+ printBenchmarkResult(totalSize, encodingDurations, Manipulation.ENCODING);
+ printBenchmarkResult(totalSize, durations, Manipulation.DECODING);
+ System.out.println();
return prevTotalSize;
}
- private void benchmarkDefaultCompression(int totalSize, byte[] rawBuffer) {
- benchmarkAlgorithm(compressionAlgorithm, compressor, decompressor,
+ private void benchmarkDefaultCompression(int totalSize, byte[] rawBuffer)
+ throws IOException {
+ benchmarkAlgorithm(compressionAlgorithm,
compressionAlgorithmName.toUpperCase(), rawBuffer, 0, totalSize);
}
@@ -308,24 +373,22 @@ public class DataBlockEncodingTool {
* @param buffer Buffer to be compressed.
* @param offset Position of the beginning of the data.
* @param length Length of data in buffer.
+ * @throws IOException
*/
- public static void benchmarkAlgorithm(
- Compression.Algorithm algorithm,
- Compressor compressorCodec,
- Decompressor decompressorCodec,
- String name,
- byte[] buffer, int offset, int length) {
+ public void benchmarkAlgorithm(Compression.Algorithm algorithm, String name,
+ byte[] buffer, int offset, int length) throws IOException {
System.out.println(name + ":");
// compress it
List<Long> compressDurations = new ArrayList<Long>();
ByteArrayOutputStream compressedStream = new ByteArrayOutputStream();
- OutputStream compressingStream;
+ CompressionOutputStream compressingStream =
+ algorithm.createPlainCompressionStream(compressedStream, compressor);
try {
- for (int itTime = 0; itTime < BENCHMARK_N_TIMES; ++itTime) {
+ for (int itTime = 0; itTime < benchmarkNTimes; ++itTime) {
final long startTime = System.nanoTime();
- compressingStream = algorithm.createCompressionStream(
- compressedStream, compressorCodec, 0);
+ compressingStream.resetState();
+ compressedStream.reset();
compressingStream.write(buffer, offset, length);
compressingStream.flush();
compressedStream.toByteArray();
@@ -333,36 +396,31 @@ public class DataBlockEncodingTool {
final long finishTime = System.nanoTime();
// add time record
- if (itTime >= BENCHMARK_N_OMIT) {
+ if (itTime >= benchmarkNOmit) {
compressDurations.add(finishTime - startTime);
}
-
- if (itTime + 1 < BENCHMARK_N_TIMES) { // not the last one
- compressedStream.reset();
- }
}
} catch (IOException e) {
throw new RuntimeException(String.format(
"Benchmark, or encoding algorithm '%s' cause some stream problems",
name), e);
}
- printBenchmarkResult(length, compressDurations, false);
-
+ compressingStream.close();
+ printBenchmarkResult(length, compressDurations, Manipulation.COMPRESSION);
byte[] compBuffer = compressedStream.toByteArray();
// uncompress it several times and measure performance
List<Long> durations = new ArrayList<Long>();
- for (int itTime = 0; itTime < BENCHMARK_N_TIMES; ++itTime) {
+ for (int itTime = 0; itTime < benchmarkNTimes; ++itTime) {
final long startTime = System.nanoTime();
byte[] newBuf = new byte[length + 1];
try {
-
ByteArrayInputStream downStream = new ByteArrayInputStream(compBuffer,
0, compBuffer.length);
InputStream decompressedStream = algorithm.createDecompressionStream(
- downStream, decompressorCodec, 0);
+ downStream, decompressor, 0);
int destOffset = 0;
int nextChunk;
@@ -371,7 +429,7 @@ public class DataBlockEncodingTool {
}
decompressedStream.close();
- // iterate over KeyValue
+ // iterate over KeyValues
KeyValue kv;
for (int pos = 0; pos < length; pos += kv.getLength()) {
kv = new KeyValue(newBuf, pos);
@@ -397,91 +455,116 @@ public class DataBlockEncodingTool {
}
// add time record
- if (itTime >= BENCHMARK_N_OMIT) {
+ if (itTime >= benchmarkNOmit) {
durations.add(finishTime - startTime);
}
}
- printBenchmarkResult(length, durations, true);
+ printBenchmarkResult(length, durations, Manipulation.DECOMPRESSION);
+ System.out.println();
}
+ private static final double BYTES_IN_MB = 1024 * 1024.0;
+ private static final double NS_IN_SEC = 1000.0 * 1000.0 * 1000.0;
+ private static final double MB_SEC_COEF = NS_IN_SEC / BYTES_IN_MB;
+
private static void printBenchmarkResult(int totalSize,
- List<Long> durationsInNanoSed, boolean isDecompression) {
+ List<Long> durationsInNanoSec, Manipulation manipulation) {
+ final int n = durationsInNanoSec.size();
long meanTime = 0;
- for (long time : durationsInNanoSed) {
+ for (long time : durationsInNanoSec) {
meanTime += time;
}
- meanTime /= durationsInNanoSed.size();
+ meanTime /= n;
+
+ double meanMBPerSec = totalSize * MB_SEC_COEF / meanTime;
+ double mbPerSecSTD = 0;
+ if (n > 0) {
+ for (long time : durationsInNanoSec) {
+ double mbPerSec = totalSize * MB_SEC_COEF / time;
+ double dev = mbPerSec - meanMBPerSec;
+ mbPerSecSTD += dev * dev;
+ }
+ mbPerSecSTD = Math.sqrt(mbPerSecSTD / n);
+ }
- long standardDev = 0;
- for (long time : durationsInNanoSed) {
- standardDev += (time - meanTime) * (time - meanTime);
- }
- standardDev = (long) Math.sqrt(standardDev / durationsInNanoSed.size());
-
- final double million = 1000.0 * 1000.0 * 1000.0;
- double mbPerSec = (totalSize * million) / (1024.0 * 1024.0 * meanTime);
- double mbPerSecDev = (totalSize * million) /
- (1024.0 * 1024.0 * (meanTime - standardDev));
-
- System.out.println(String.format(
- " %s performance:%s %6.2f MB/s (+/- %.2f MB/s)",
- isDecompression ? "Decompression" : "Compression",
- isDecompression ? "" : " ",
- mbPerSec, mbPerSecDev - mbPerSec));
+ outputTuple(manipulation + " performance", "%6.2f MB/s (+/- %.2f MB/s)",
+ meanMBPerSec, mbPerSecSTD);
+ }
+
+ private static void outputTuple(String caption, String format,
+ Object... values) {
+ if (format.startsWith(INT_FORMAT)) {
+ format = "%s" + format.substring(INT_FORMAT.length());
+ values[0] = DELIMITED_DECIMAL_FORMAT.format(values[0]);
+ }
+
+ StringBuilder sb = new StringBuilder();
+ sb.append(" ");
+ sb.append(caption);
+ sb.append(":");
+
+ String v = String.format(format, values);
+ int padding = 60 - sb.length() - v.length();
+ for (int i = 0; i < padding; ++i) {
+ sb.append(' ');
+ }
+ sb.append(v);
+ System.out.println(sb);
}
/**
* Display statistics of different compression algorithms.
+ * @throws IOException
*/
- public void displayStatistics() {
- int totalLength = totalPrefixLength + totalKeyLength + totalValueLength;
- if (compressor != null) { // might be null e.g. for pure-Java GZIP
- compressor.reset();
- }
+ public void displayStatistics() throws IOException {
+ final String comprAlgo = compressionAlgorithmName.toUpperCase();
+ long rawBytes = totalKeyLength + totalPrefixLength + totalValueLength;
+
+ System.out.println("Raw data size:");
+ outputTuple("Raw bytes", INT_FORMAT, rawBytes);
+ outputTuplePct("Key bytes", totalKeyLength);
+ outputTuplePct("Value bytes", totalValueLength);
+ outputTuplePct("KV infrastructure", totalPrefixLength);
+ outputTuplePct("CF overhead", totalCFLength);
+ outputTuplePct("Total key redundancy", totalKeyRedundancyLength);
+
+ int compressedSize = EncodedDataBlock.getCompressedSize(
+ compressionAlgorithm, compressor, rawKVs, 0, rawKVs.length);
+ outputTuple(comprAlgo + " only size", INT_FORMAT,
+ compressedSize);
+ outputSavings(comprAlgo + " only", compressedSize, rawBytes);
+ System.out.println();
- for(EncodedDataBlock codec : codecs) {
+ for (EncodedDataBlock codec : codecs) {
System.out.println(codec.toString());
- int saved = totalKeyLength + totalPrefixLength + totalValueLength
- - codec.getSize();
- System.out.println(
- String.format(" Saved bytes: %8d", saved));
- double keyRatio = (saved * 100.0) / (totalPrefixLength + totalKeyLength);
- double allRatio = (saved * 100.0) / totalLength;
- System.out.println(
- String.format(" Key compression ratio: %.2f %%", keyRatio));
- System.out.println(
- String.format(" All compression ratio: %.2f %%", allRatio));
-
- String compressedSizeCaption =
- String.format(" %s compressed size: ",
- compressionAlgorithmName.toUpperCase());
- String compressOnlyRatioCaption =
- String.format(" %s compression ratio: ",
- compressionAlgorithmName.toUpperCase());
-
- if (compressor != null) {
- int compressedSize = codec.checkCompressedSize(compressor);
- System.out.println(compressedSizeCaption +
- String.format("%8d", compressedSize));
- double compressOnlyRatio =
- 100.0 * (1.0 - compressedSize / (0.0 + totalLength));
- System.out.println(compressOnlyRatioCaption
- + String.format("%.2f %%", compressOnlyRatio));
- } else {
- System.out.println(compressedSizeCaption + "N/A");
- System.out.println(compressOnlyRatioCaption + "N/A");
- }
+ long encodedBytes = codec.getSize();
+ outputTuple("Encoded bytes", INT_FORMAT, encodedBytes);
+ outputSavings("Key encoding", encodedBytes - totalValueLength,
+ rawBytes - totalValueLength);
+ outputSavings("Total encoding", encodedBytes, rawBytes);
+
+ int encodedCompressedSize = codec.getEncodedCompressedSize(
+ compressionAlgorithm, compressor);
+ outputTuple("Encoding + " + comprAlgo + " size", INT_FORMAT,
+ encodedCompressedSize);
+ outputSavings("Encoding + " + comprAlgo, encodedCompressedSize, rawBytes);
+ outputSavings("Encoding with " + comprAlgo, encodedCompressedSize,
+ compressedSize);
+
+ System.out.println();
}
+ }
- System.out.println(
- String.format("Total KV prefix length: %8d", totalPrefixLength));
- System.out.println(
- String.format("Total key length: %8d", totalKeyLength));
- System.out.println(
- String.format("Total key redundancy: %8d",
- totalKeyRedundancyLength));
- System.out.println(
- String.format("Total value length: %8d", totalValueLength));
+ private void outputTuplePct(String caption, long size) {
+ outputTuple(caption, INT_FORMAT + " (" + PCT_FORMAT + ")",
+ size, size * 100.0 / rawKVs.length);
+ }
+
+ private void outputSavings(String caption, long part, long whole) {
+ double pct = 100.0 * (1 - 1.0 * part / whole);
+ double times = whole * 1.0 / part;
+ outputTuple(caption + " savings", PCT_FORMAT + " (%.2f x)",
+ pct, times);
}
/**
@@ -541,22 +624,35 @@ public class DataBlockEncodingTool {
}
/**
- * A command line interface to benchmarks.
+ * A command line interface to benchmarks. Parses command-line arguments and
+ * runs the appropriate benchmarks.
* @param args Should have length at least 1 and holds the file path to HFile.
* @throws IOException If you specified the wrong file.
*/
public static void main(final String[] args) throws IOException {
// set up user arguments
Options options = new Options();
- options.addOption("f", true, "HFile to analyse (REQUIRED)");
- options.getOption("f").setArgName("FILENAME");
- options.addOption("n", true,
- "Limit number of KeyValue which will be analysed");
- options.getOption("n").setArgName("NUMBER");
- options.addOption("b", false, "Measure read throughput");
- options.addOption("c", false, "Omit corectness tests.");
- options.addOption("a", true,
+ options.addOption(OPT_HFILE_NAME, true, "HFile to analyse (REQUIRED)");
+ options.getOption(OPT_HFILE_NAME).setArgName("FILENAME");
+ options.addOption(OPT_KV_LIMIT, true,
+ "Maximum number of KeyValues to process. A benchmark stops running " +
+ "after iterating over this many KV pairs.");
+ options.getOption(OPT_KV_LIMIT).setArgName("NUMBER");
+ options.addOption(OPT_MEASURE_THROUGHPUT, false,
+ "Measure read throughput");
+ options.addOption(OPT_OMIT_CORRECTNESS_TEST, false,
+ "Omit corectness tests.");
+ options.addOption(OPT_ENCODING_ALGORITHM, true,
"What kind of compression algorithm use for comparison.");
+ options.addOption(OPT_BENCHMARK_N_TIMES,
+ true, "Number of times to run each benchmark. Default value: " +
+ DEFAULT_BENCHMARK_N_TIMES);
+ options.addOption(OPT_BENCHMARK_N_OMIT, true,
+ "Number of first runs of every benchmark to exclude from "
+ + "statistics (" + DEFAULT_BENCHMARK_N_OMIT
+ + " by default, so that " + "only the last "
+ + (DEFAULT_BENCHMARK_N_TIMES - DEFAULT_BENCHMARK_N_OMIT)
+ + " times are included in statistics.)");
// parse arguments
CommandLineParser parser = new PosixParser();
@@ -570,24 +666,44 @@ public class DataBlockEncodingTool {
}
int kvLimit = Integer.MAX_VALUE;
- if (cmd.hasOption("n")) {
- kvLimit = Integer.parseInt(cmd.getOptionValue("n"));
+ if (cmd.hasOption(OPT_KV_LIMIT)) {
+ kvLimit = Integer.parseInt(cmd.getOptionValue(OPT_KV_LIMIT));
}
// basic argument sanity checks
- if (!cmd.hasOption("f")) {
- System.err.println("ERROR: Filename is required!");
+ if (!cmd.hasOption(OPT_HFILE_NAME)) {
+ LOG.error("Please specify HFile name using the " + OPT_HFILE_NAME
+ + " option");
printUsage(options);
System.exit(-1);
}
- String pathName = cmd.getOptionValue("f");
+ String pathName = cmd.getOptionValue(OPT_HFILE_NAME);
String compressionName = DEFAULT_COMPRESSION.getName();
- if (cmd.hasOption("a")) {
- compressionName = cmd.getOptionValue("a").toLowerCase();
+ if (cmd.hasOption(OPT_ENCODING_ALGORITHM)) {
+ compressionName =
+ cmd.getOptionValue(OPT_ENCODING_ALGORITHM).toLowerCase();
+ }
+ boolean doBenchmark = cmd.hasOption(OPT_MEASURE_THROUGHPUT);
+ boolean doVerify = !cmd.hasOption(OPT_OMIT_CORRECTNESS_TEST);
+
+ if (cmd.hasOption(OPT_BENCHMARK_N_TIMES)) {
+ benchmarkNTimes = Integer.valueOf(cmd.getOptionValue(
+ OPT_BENCHMARK_N_TIMES));
+ }
+ if (cmd.hasOption(OPT_BENCHMARK_N_OMIT)) {
+ benchmarkNOmit =
+ Integer.valueOf(cmd.getOptionValue(OPT_BENCHMARK_N_OMIT));
+ }
+ if (benchmarkNTimes < benchmarkNOmit) {
+ LOG.error("The number of times to run each benchmark ("
+ + benchmarkNTimes
+ + ") must be greater than the number of benchmark runs to exclude "
+ + "from statistics (" + benchmarkNOmit + ")");
+ System.exit(1);
}
- boolean doBenchmark = cmd.hasOption("b");
- boolean doVerify = !cmd.hasOption("c");
+ LOG.info("Running benchmark " + benchmarkNTimes + " times. " +
+ "Excluding the first " + benchmarkNOmit + " times from statistics.");
final Configuration conf = HBaseConfiguration.create();
try {
@@ -598,4 +714,4 @@ public class DataBlockEncodingTool {
}
}
-}
+}
\ No newline at end of file
Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/EncodedSeekPerformanceTest.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/EncodedSeekPerformanceTest.java?rev=1306182&r1=1306181&r2=1306182&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/EncodedSeekPerformanceTest.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/EncodedSeekPerformanceTest.java Wed Mar 28 06:40:29 2012
@@ -173,7 +173,6 @@ public class EncodedSeekPerformanceTest
List<HFileDataBlockEncoder> encoders =
new ArrayList<HFileDataBlockEncoder>();
- encoders.add(new HFileDataBlockEncoderImpl(DataBlockEncoding.NONE));
for (DataBlockEncoding encodingAlgo : DataBlockEncoding.values()) {
encoders.add(new HFileDataBlockEncoderImpl(DataBlockEncoding.NONE,
encodingAlgo));