You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by mb...@apache.org on 2012/03/28 08:40:30 UTC

svn commit: r1306182 - in /hbase/branches/0.89-fb/src: main/java/org/apache/hadoop/hbase/io/encoding/ main/java/org/apache/hadoop/hbase/io/hfile/ test/java/org/apache/hadoop/hbase/io/encoding/ test/java/org/apache/hadoop/hbase/io/hfile/ test/java/org/a...

Author: mbautin
Date: Wed Mar 28 06:40:29 2012
New Revision: 1306182

URL: http://svn.apache.org/viewvc?rev=1306182&view=rev
Log:
[jira] [HBASE-5469] [89-fb] Add baseline compression efficiency to DataBlockEncodingTool

Summary: Fixing DataBlockEncodingTool. This is the 89-fb version of
https://reviews.facebook.net/D2409.

Test Plan:
Run unit tests
Run DataBlockEncodingTool on a few files

Reviewers: kannan, aaiyer, liyintang, kranganathan, pkhemani

Reviewed By: kannan

Differential Revision: https://reviews.facebook.net/D2463

Modified:
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/CopyKeyDataBlockEncoder.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/DataBlockEncoder.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/EncodedDataBlock.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/Compression.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoderImpl.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/DataBlockEncodingTool.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/EncodedSeekPerformanceTest.java

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java?rev=1306182&r1=1306181&r2=1306182&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java Wed Mar 28 06:40:29 2012
@@ -21,8 +21,8 @@ import java.io.DataOutputStream;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 
-import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.KeyValue.SamePrefixComparator;
+import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.util.ByteBufferUtils;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.io.RawComparator;
@@ -36,9 +36,9 @@ abstract class BufferedDataBlockEncoder 
   private static int INITIAL_KEY_BUFFER_SIZE = 512;
 
   @Override
-  public ByteBuffer uncompressKeyValues(DataInputStream source,
+  public ByteBuffer decodeKeyValues(DataInputStream source,
       boolean includesMemstoreTS) throws IOException {
-    return uncompressKeyValues(source, 0, 0, includesMemstoreTS);
+    return decodeKeyValues(source, 0, 0, includesMemstoreTS);
   }
 
   protected static class SeekerState {

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/CopyKeyDataBlockEncoder.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/CopyKeyDataBlockEncoder.java?rev=1306182&r1=1306181&r2=1306182&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/CopyKeyDataBlockEncoder.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/CopyKeyDataBlockEncoder.java Wed Mar 28 06:40:29 2012
@@ -31,7 +31,7 @@ import org.apache.hadoop.io.RawComparato
  */
 public class CopyKeyDataBlockEncoder extends BufferedDataBlockEncoder {
   @Override
-  public void compressKeyValues(DataOutputStream out,
+  public void encodeKeyValues(DataOutputStream out,
       ByteBuffer in, boolean includesMemstoreTS) throws IOException {
     in.rewind();
     ByteBufferUtils.putInt(out, in.limit());
@@ -39,7 +39,7 @@ public class CopyKeyDataBlockEncoder ext
   }
 
   @Override
-  public ByteBuffer uncompressKeyValues(DataInputStream source,
+  public ByteBuffer decodeKeyValues(DataInputStream source,
       int preserveHeaderLength, int skipLastBytes, boolean includesMemstoreTS)
       throws IOException {
     int decompressedSize = source.readInt();

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/DataBlockEncoder.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/DataBlockEncoder.java?rev=1306182&r1=1306181&r2=1306182&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/DataBlockEncoder.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/DataBlockEncoder.java Wed Mar 28 06:40:29 2012
@@ -32,42 +32,42 @@ import org.apache.hadoop.io.RawComparato
  * <li>the values are always iterated forward from beginning of block</li>
  * <li>knowledge of Key Value format</li>
  * </ul>
- * It is designed to work fast enough to be feasible as in memory compression.
+ * It is designed to work fast enough to be feasible as in memory encoding.
  */
 public interface DataBlockEncoder {
   /**
-   * Compress KeyValues and write them to output buffer.
-   * @param out Where to write compressed data.
-   * @param in Source of KeyValue for compression.
+   * Encode KeyValues and write them to output buffer.
+   * @param out Where to write encoded data.
+   * @param in Source of KeyValue for encoding.
    * @param includesMemstoreTS true if including memstore timestamp after every
    *          key-value pair
    * @throws IOException If there is an error writing to output stream.
    */
-  public void compressKeyValues(DataOutputStream out,
+  public void encodeKeyValues(DataOutputStream out,
       ByteBuffer in, boolean includesMemstoreTS) throws IOException;
 
   /**
-   * Uncompress.
-   * @param source Compressed stream of KeyValues.
+   * Decode.
+   * @param source Encoded stream of KeyValues.
    * @param includesMemstoreTS true if including memstore timestamp after every
    *          key-value pair
-   * @return Uncompressed block of KeyValues.
+   * @return decoded block of KeyValues.
    * @throws IOException If there is an error in source.
    */
-  public ByteBuffer uncompressKeyValues(DataInputStream source,
+  public ByteBuffer decodeKeyValues(DataInputStream source,
       boolean includesMemstoreTS) throws IOException;
 
   /**
-   * Uncompress.
-   * @param source Compressed stream of KeyValues.
+   * Decode.
+   * @param source encoded stream of KeyValues.
    * @param allocateHeaderLength allocate this many bytes for the header.
    * @param skipLastBytes Do not copy n last bytes.
    * @param includesMemstoreTS true if including memstore timestamp after every
    *          key-value pair
-   * @return Uncompressed block of KeyValues.
+   * @return decoded block of KeyValues.
    * @throws IOException If there is an error in source.
    */
-  public ByteBuffer uncompressKeyValues(DataInputStream source,
+  public ByteBuffer decodeKeyValues(DataInputStream source,
       int allocateHeaderLength, int skipLastBytes, boolean includesMemstoreTS)
       throws IOException;
 

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java?rev=1306182&r1=1306181&r2=1306182&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java Wed Mar 28 06:40:29 2012
@@ -314,7 +314,7 @@ public class DiffKeyDeltaEncoder extends
   }
 
   @Override
-  public void compressKeyValues(DataOutputStream out,
+  public void encodeKeyValues(DataOutputStream out,
       ByteBuffer in, boolean includesMemstoreTS) throws IOException {
     in.rewind();
     ByteBufferUtils.putInt(out, in.limit());
@@ -333,7 +333,7 @@ public class DiffKeyDeltaEncoder extends
   }
 
   @Override
-  public ByteBuffer uncompressKeyValues(DataInputStream source,
+  public ByteBuffer decodeKeyValues(DataInputStream source,
       int allocHeaderLength, int skipLastBytes, boolean includesMemstoreTS)
       throws IOException {
     int decompressedSize = source.readInt();

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/EncodedDataBlock.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/EncodedDataBlock.java?rev=1306182&r1=1306181&r2=1306182&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/EncodedDataBlock.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/EncodedDataBlock.java Wed Mar 28 06:40:29 2012
@@ -21,23 +21,30 @@ import java.io.ByteArrayOutputStream;
 import java.io.DataInputStream;
 import java.io.DataOutputStream;
 import java.io.IOException;
+import java.io.OutputStream;
 import java.nio.ByteBuffer;
 import java.util.Iterator;
 
 import org.apache.commons.lang.NotImplementedException;
 import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.io.hfile.Compression.Algorithm;
+import org.apache.hadoop.hbase.io.hfile.HFileBlock;
+import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.io.compress.Compressor;
 
+import com.google.common.base.Preconditions;
+import com.google.common.io.NullOutputStream;
+
 /**
  * Encapsulates a data block compressed using a particular encoding algorithm.
  * Useful for testing and benchmarking.
  */
 public class EncodedDataBlock {
-  private static final int BUFFER_SIZE = 4 * 1024;
-  protected DataBlockEncoder dataBlockEncoder;
-  ByteArrayOutputStream uncompressedOutputStream;
-  ByteBuffer uncompressedBuffer;
-  private byte[] cacheCompressData;
+  private byte[] rawKVs;
+  private ByteBuffer rawBuffer;
+  private DataBlockEncoder dataBlockEncoder;
+
+  private byte[] cachedEncodedData;
   private ByteArrayOutputStream compressedStream = new ByteArrayOutputStream();
   private boolean includesMemstoreTS;
 
@@ -46,19 +53,11 @@ public class EncodedDataBlock {
    * @param dataBlockEncoder Algorithm used for compression.
    */
   public EncodedDataBlock(DataBlockEncoder dataBlockEncoder,
-      boolean includesMemstoreTS) {
+      boolean includesMemstoreTS, DataBlockEncoding encoding, byte[] rawKVs) {
+    Preconditions.checkNotNull(encoding,
+        "Cannot create encoded data block with null encoder");
     this.dataBlockEncoder = dataBlockEncoder;
-    uncompressedOutputStream = new ByteArrayOutputStream(BUFFER_SIZE);
-  }
-
-  /**
-   * Add KeyValue and compress it.
-   * @param kv Item to be added and compressed.
-   */
-  public void addKv(KeyValue kv) {
-    cacheCompressData = null;
-    uncompressedOutputStream.write(
-        kv.getBuffer(), kv.getOffset(), kv.getLength());
+    this.rawKVs = rawKVs;
   }
 
   /**
@@ -66,19 +65,20 @@ public class EncodedDataBlock {
    * @return Forwards sequential iterator.
    */
   public Iterator<KeyValue> getIterator() {
-    final int uncompressedSize = uncompressedOutputStream.size();
-    final ByteArrayInputStream bais = new ByteArrayInputStream(
-        getCompressedData());
+    final int rawSize = rawKVs.length;
+    byte[] encodedDataWithHeader = getEncodedData();
+    int bytesToSkip = 0;
+    ByteArrayInputStream bais = new ByteArrayInputStream(encodedDataWithHeader,
+        bytesToSkip, encodedDataWithHeader.length - bytesToSkip);
     final DataInputStream dis = new DataInputStream(bais);
 
-
     return new Iterator<KeyValue>() {
       private ByteBuffer decompressedData = null;
 
       @Override
       public boolean hasNext() {
         if (decompressedData == null) {
-          return uncompressedSize > 0;
+          return rawSize > 0;
         }
         return decompressedData.hasRemaining();
       }
@@ -87,7 +87,7 @@ public class EncodedDataBlock {
       public KeyValue next() {
         if (decompressedData == null) {
           try {
-            decompressedData = dataBlockEncoder.uncompressKeyValues(
+            decompressedData = dataBlockEncoder.decodeKeyValues(
                 dis, includesMemstoreTS);
           } catch (IOException e) {
             throw new RuntimeException("Problem with data block encoder, " +
@@ -121,81 +121,75 @@ public class EncodedDataBlock {
    * @return Size in bytes of compressed data.
    */
   public int getSize() {
-    return getCompressedData().length;
+    return getEncodedData().length;
   }
 
   /**
    * Find the size of compressed data assuming that buffer will be compressed
    * using given algorithm.
-   * @param compressor Algorithm used for compression.
-   * @param buffer Array to be compressed.
+   * @param algo compression algorithm
+   * @param compressor compressor already requested from codec
+   * @param inputBuffer Array to be compressed.
    * @param offset Offset to beginning of the data.
    * @param length Length to be compressed.
    * @return Size of compressed data in bytes.
    */
-  public static int checkCompressedSize(Compressor compressor, byte[] buffer,
-      int offset, int length) {
-    byte[] compressedBuffer = new byte[buffer.length];
-    // in fact the buffer could be of any positive size
-    compressor.setInput(buffer, offset, length);
-    compressor.finish();
-    int currentPos = 0;
-    while (!compressor.finished()) {
-      try {
-        // we don't care about compressed data,
-        // we just want to callculate number of bytes
-        currentPos += compressor.compress(compressedBuffer, 0,
-            compressedBuffer.length);
-      } catch (IOException e) {
-        throw new RuntimeException(
-            "For some reason compressor couldn't read data. " +
-            "It is likely a problem with " +
-            compressor.getClass().getName(), e);
-      }
+  public static int getCompressedSize(Algorithm algo, Compressor compressor,
+      byte[] inputBuffer, int offset, int length) throws IOException {
+    DataOutputStream compressedStream = new DataOutputStream(
+        new NullOutputStream());
+    if (compressor != null) {
+      compressor.reset();
     }
-    return currentPos;
+    OutputStream compressingStream = algo.createCompressionStream(
+        compressedStream, compressor, 0);
+
+    compressingStream.write(inputBuffer, offset, length);
+    compressingStream.flush();
+    compressingStream.close();
+
+    return compressedStream.size();
   }
 
   /**
    * Estimate size after second stage of compression (e.g. LZO).
-   * @param compressor Algorithm which will be used for compressions.
+   * @param comprAlgo compression algorithm to be used for compression
+   * @param compressor compressor corresponding to the given compression
+   *          algorithm
    * @return Size after second stage of compression.
    */
-  public int checkCompressedSize(Compressor compressor) {
-    // compress
-    byte[] compressedBytes = getCompressedData();
-    return checkCompressedSize(compressor, compressedBytes, 0,
+  public int getEncodedCompressedSize(Algorithm comprAlgo,
+      Compressor compressor) throws IOException {
+    byte[] compressedBytes = getEncodedData();
+    return getCompressedSize(comprAlgo, compressor, compressedBytes, 0,
         compressedBytes.length);
   }
 
-  private byte[] getCompressedData() {
-    // is cached
-    if (cacheCompressData != null) {
-      return cacheCompressData;
+  /** @return encoded data with header */
+  private byte[] getEncodedData() {
+    if (cachedEncodedData != null) {
+      return cachedEncodedData;
     }
-    cacheCompressData = doCompressData();
-
-    return cacheCompressData;
+    cachedEncodedData = encodeData();
+    return cachedEncodedData;
   }
 
   private ByteBuffer getUncompressedBuffer() {
-    if (uncompressedBuffer == null ||
-        uncompressedBuffer.limit() < uncompressedOutputStream.size()) {
-      uncompressedBuffer = ByteBuffer.wrap(
-          uncompressedOutputStream.toByteArray());
+    if (rawBuffer == null || rawBuffer.limit() < rawKVs.length) {
+      rawBuffer = ByteBuffer.wrap(rawKVs);
     }
-    return uncompressedBuffer;
+    return rawBuffer;
   }
 
   /**
-   * Do the compression.
-   * @return Compressed byte buffer.
+   * Do the encoding, but do not cache the encoded data.
+   * @return encoded data block with header and checksum
    */
-  public byte[] doCompressData() {
+  public byte[] encodeData() {
     compressedStream.reset();
     DataOutputStream dataOut = new DataOutputStream(compressedStream);
     try {
-      this.dataBlockEncoder.compressKeyValues(
+      this.dataBlockEncoder.encodeKeyValues(
           dataOut, getUncompressedBuffer(), includesMemstoreTS);
     } catch (IOException e) {
       throw new RuntimeException(String.format(
@@ -210,12 +204,4 @@ public class EncodedDataBlock {
   public String toString() {
     return dataBlockEncoder.toString();
   }
-
-  /**
-   * Get uncompressed buffer.
-   * @return The buffer.
-   */
-  public byte[] getRawKeyValues() {
-    return uncompressedOutputStream.toByteArray();
-  }
 }

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java?rev=1306182&r1=1306181&r2=1306182&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java Wed Mar 28 06:40:29 2012
@@ -341,7 +341,7 @@ public class FastDiffDeltaEncoder extend
   }
 
   @Override
-  public void compressKeyValues(DataOutputStream out,
+  public void encodeKeyValues(DataOutputStream out,
       ByteBuffer in, boolean includesMemstoreTS) throws IOException {
     in.rewind();
     ByteBufferUtils.putInt(out, in.limit());
@@ -360,7 +360,7 @@ public class FastDiffDeltaEncoder extend
   }
 
   @Override
-  public ByteBuffer uncompressKeyValues(DataInputStream source,
+  public ByteBuffer decodeKeyValues(DataInputStream source,
       int allocHeaderLength, int skipLastBytes, boolean includesMemstoreTS)
           throws IOException {
     int decompressedSize = source.readInt();

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java?rev=1306182&r1=1306181&r2=1306182&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java Wed Mar 28 06:40:29 2012
@@ -16,10 +16,14 @@
  */
 package org.apache.hadoop.hbase.io.encoding;
 
+import java.io.ByteArrayOutputStream;
 import java.io.DataInputStream;
 import java.io.DataOutputStream;
+import java.io.FilterOutputStream;
 import java.io.IOException;
+import java.lang.reflect.Field;
 import java.nio.ByteBuffer;
+import java.util.Arrays;
 
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.util.ByteBufferUtils;
@@ -73,7 +77,7 @@ public class PrefixKeyDeltaEncoder exten
   }
 
   @Override
-  public void compressKeyValues(DataOutputStream writeHere,
+  public void encodeKeyValues(DataOutputStream writeHere,
       ByteBuffer in, boolean includesMemstoreTS) throws IOException {
     in.rewind();
     ByteBufferUtils.putInt(writeHere, in.limit());
@@ -89,7 +93,7 @@ public class PrefixKeyDeltaEncoder exten
   }
 
   @Override
-  public ByteBuffer uncompressKeyValues(DataInputStream source,
+  public ByteBuffer decodeKeyValues(DataInputStream source,
       int allocHeaderLength, int skipLastBytes, boolean includesMemstoreTS)
           throws IOException {
     int decompressedSize = source.readInt();
@@ -99,7 +103,7 @@ public class PrefixKeyDeltaEncoder exten
     int prevKeyOffset = 0;
 
     while (source.available() > skipLastBytes) {
-      prevKeyOffset = uncompressKeyValue(source, buffer, prevKeyOffset);
+      prevKeyOffset = decodeKeyValue(source, buffer, prevKeyOffset);
       afterDecodingKeyValue(source, buffer, includesMemstoreTS);
     }
 
@@ -111,7 +115,7 @@ public class PrefixKeyDeltaEncoder exten
     return buffer;
   }
 
-  private int uncompressKeyValue(DataInputStream source, ByteBuffer buffer,
+  private int decodeKeyValue(DataInputStream source, ByteBuffer buffer,
       int prevKeyOffset)
           throws IOException, EncoderBufferTooSmallException {
     int keyLength = ByteBufferUtils.readCompressedInt(source);

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/Compression.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/Compression.java?rev=1306182&r1=1306181&r2=1306182&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/Compression.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/Compression.java Wed Mar 28 06:40:29 2012
@@ -193,7 +193,7 @@ public final class Compression {
      * Creates a compression stream without any additional wrapping into
      * buffering streams.
      */
-    CompressionOutputStream createPlainCompressionStream(
+    public CompressionOutputStream createPlainCompressionStream(
         OutputStream downStream, Compressor compressor) throws IOException {
       CompressionCodec codec = getCodec(conf);
       ((Configurable)codec).getConf().setInt("io.file.buffer.size", 32 * 1024);

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java?rev=1306182&r1=1306181&r2=1306182&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java Wed Mar 28 06:40:29 2012
@@ -35,6 +35,7 @@ import java.util.concurrent.atomic.Atomi
 
 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
 import org.apache.hadoop.hbase.io.hfile.Compression.Algorithm;
 import org.apache.hadoop.hbase.regionserver.MemStore;
@@ -1531,4 +1532,3 @@ public class HFileBlock extends SchemaCo
   }
 
 }
-

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoderImpl.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoderImpl.java?rev=1306182&r1=1306181&r2=1306182&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoderImpl.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileDataBlockEncoderImpl.java Wed Mar 28 06:40:29 2012
@@ -180,7 +180,7 @@ public class HFileDataBlockEncoderImpl i
     try {
       encodedStream.write(HFileBlock.DUMMY_HEADER);
       algo.writeIdInBytes(dataOut);
-      encoder.compressKeyValues(dataOut, in,
+      encoder.encodeKeyValues(dataOut, in,
           includesMemstoreTS);
     } catch (IOException e) {
       throw new RuntimeException(String.format("Bug in data block encoder " +

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java?rev=1306182&r1=1306181&r2=1306182&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java Wed Mar 28 06:40:29 2012
@@ -31,13 +31,13 @@ import java.util.List;
 import java.util.Random;
 
 import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.KeyValue.Type;
+import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.junit.Test;
 import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
 import org.junit.runners.Parameterized.Parameters;
+import org.junit.runners.Parameterized;
 
 /**
  * Test all of the data block encoding algorithms for correctness.
@@ -67,13 +67,13 @@ public class TestDataBlockEncoders {
     // encode
     ByteArrayOutputStream baos = new ByteArrayOutputStream();
     DataOutputStream dataOut = new DataOutputStream(baos);
-    encoder.compressKeyValues(dataOut, dataset, includesMemstoreTS);
+    encoder.encodeKeyValues(dataOut, dataset, includesMemstoreTS);
 
     // decode
     ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
     DataInputStream dis = new DataInputStream(bais);
     ByteBuffer actualDataset;
-    actualDataset = encoder.uncompressKeyValues(dis, includesMemstoreTS);
+    actualDataset = encoder.decodeKeyValues(dis, includesMemstoreTS);
 
     dataset.rewind();
     actualDataset.rewind();
@@ -148,7 +148,7 @@ public class TestDataBlockEncoders {
     for (DataBlockEncoder encoder : dataBlockEncoders) {
       ByteArrayOutputStream baos = new ByteArrayOutputStream();
       DataOutputStream dataOut = new DataOutputStream(baos);
-      encoder.compressKeyValues(dataOut, originalBuffer, includesMemstoreTS);
+      encoder.encodeKeyValues(dataOut, originalBuffer, includesMemstoreTS);
       ByteBuffer encodedBuffer = ByteBuffer.wrap(baos.toByteArray());
       DataBlockEncoder.EncodedSeeker seeker =
           encoder.createSeeker(KeyValue.KEY_COMPARATOR, includesMemstoreTS);
@@ -199,7 +199,7 @@ public class TestDataBlockEncoders {
       ByteArrayOutputStream baos = new ByteArrayOutputStream();
       DataOutputStream dataOut = new DataOutputStream(baos);
       try {
-        encoder.compressKeyValues(dataOut, originalBuffer, includesMemstoreTS);
+        encoder.encodeKeyValues(dataOut, originalBuffer, includesMemstoreTS);
       } catch (IOException e) {
         throw new RuntimeException(String.format(
             "Bug while encoding using '%s'", encoder.toString()), e);
@@ -259,7 +259,7 @@ public class TestDataBlockEncoders {
       ByteArrayOutputStream baos = new ByteArrayOutputStream();
       DataOutputStream dataOut = new DataOutputStream(baos);
       try {
-        encoder.compressKeyValues(dataOut, originalBuffer, includesMemstoreTS);
+        encoder.encodeKeyValues(dataOut, originalBuffer, includesMemstoreTS);
       } catch (IOException e) {
         throw new RuntimeException(String.format(
             "Bug while encoding using '%s'", encoder.toString()), e);
@@ -340,4 +340,4 @@ public class TestDataBlockEncoders {
       assertEquals("Input of two methods is changed", onDataset, dataset);
     }
   }
-}
+}
\ No newline at end of file

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java?rev=1306182&r1=1306181&r2=1306182&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java Wed Mar 28 06:40:29 2012
@@ -60,8 +60,8 @@ import org.apache.hadoop.io.compress.Com
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
 import org.junit.runners.Parameterized.Parameters;
+import org.junit.runners.Parameterized;
 
 @RunWith(Parameterized.class)
 public class TestHFileBlock {
@@ -420,7 +420,7 @@ public class TestHFileBlock {
       encodedBuf = rawBuf;
     } else {
       ByteArrayOutputStream encodedOut = new ByteArrayOutputStream();
-      encoding.getEncoder().compressKeyValues(
+      encoding.getEncoder().encodeKeyValues(
           new DataOutputStream(encodedOut),
           rawBuf.duplicate(), includesMemstoreTS);
       // We need to account for the two-byte encoding algorithm ID that
@@ -730,4 +730,4 @@ public class TestHFileBlock {
     }
   }
 
-}
+}
\ No newline at end of file

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/DataBlockEncodingTool.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/DataBlockEncodingTool.java?rev=1306182&r1=1306181&r2=1306182&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/DataBlockEncodingTool.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/DataBlockEncodingTool.java Wed Mar 28 06:40:29 2012
@@ -20,7 +20,7 @@ import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
-import java.io.OutputStream;
+import java.text.DecimalFormat;
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
@@ -38,14 +38,15 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.io.encoding.EncodedDataBlock;
 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoder;
 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
+import org.apache.hadoop.hbase.io.encoding.EncodedDataBlock;
 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
+import org.apache.hadoop.hbase.io.hfile.Compression.Algorithm;
 import org.apache.hadoop.hbase.io.hfile.Compression;
 import org.apache.hadoop.hbase.io.hfile.NoOpDataBlockEncoder;
-import org.apache.hadoop.hbase.io.hfile.Compression.Algorithm;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.io.compress.CompressionOutputStream;
 import org.apache.hadoop.io.compress.Compressor;
 import org.apache.hadoop.io.compress.Decompressor;
 
@@ -60,33 +61,85 @@ public class DataBlockEncodingTool {
   private static final boolean includesMemstoreTS = true;
 
   /**
-   * How many times should benchmark run.
-   * More times means better data in terms of statistics.
-   * It has to be larger than BENCHMARK_N_OMIT.
+   * How many times to run the benchmark. More times means better data in terms
+   * of statistics but slower execution. Has to be strictly larger than
+   * {@link DEFAULT_BENCHMARK_N_OMIT}.
    */
-  public static int BENCHMARK_N_TIMES = 12;
+  private static final int DEFAULT_BENCHMARK_N_TIMES = 12;
 
   /**
-   * How many first runs should omit benchmark.
-   * Usually it is one in order to exclude setup cost.
-   * Has to be 0 or larger.
+   * How many first runs should not be included in the benchmark. Done in order
+   * to exclude setup cost.
    */
-  public static int BENCHMARK_N_OMIT = 2;
+  private static final int DEFAULT_BENCHMARK_N_OMIT = 2;
+
+  /** HFile name to be used in benchmark */
+  private static final String OPT_HFILE_NAME = "f";
+
+  /** Maximum number of key/value pairs to process in a single benchmark run */
+  private static final String OPT_KV_LIMIT = "n";
+
+  /** Whether to run a benchmark to measure read throughput */
+  private static final String OPT_MEASURE_THROUGHPUT = "b";
+
+  /** If this is specified, no correctness testing will be done */
+  private static final String OPT_OMIT_CORRECTNESS_TEST = "c";
+
+  /** What encoding algorithm to test */
+  private static final String OPT_ENCODING_ALGORITHM = "a";
+
+  /** Number of times to run each benchmark */
+  private static final String OPT_BENCHMARK_N_TIMES = "t";
+
+  /** Number of first runs of every benchmark to omit from statistics */
+  private static final String OPT_BENCHMARK_N_OMIT = "omit";
 
   /** Compression algorithm to use if not specified on the command line */
   private static final Algorithm DEFAULT_COMPRESSION =
       Compression.Algorithm.GZ;
 
+  private static final DecimalFormat DELIMITED_DECIMAL_FORMAT =
+      new DecimalFormat();
+
+  static {
+    DELIMITED_DECIMAL_FORMAT.setGroupingSize(3);
+  }
+
+  private static final String PCT_FORMAT = "%.2f %%";
+  private static final String INT_FORMAT = "%d";
+
+  private static int benchmarkNTimes = DEFAULT_BENCHMARK_N_TIMES;
+  private static int benchmarkNOmit = DEFAULT_BENCHMARK_N_OMIT;
+
   private List<EncodedDataBlock> codecs = new ArrayList<EncodedDataBlock>();
-  private int totalPrefixLength = 0;
-  private int totalKeyLength = 0;
-  private int totalValueLength = 0;
-  private int totalKeyRedundancyLength = 0;
-
-  final private String compressionAlgorithmName;
-  final private Algorithm compressionAlgorithm;
-  final private Compressor compressor;
-  final private Decompressor decompressor;
+  private long totalPrefixLength = 0;
+  private long totalKeyLength = 0;
+  private long totalValueLength = 0;
+  private long totalKeyRedundancyLength = 0;
+  private long totalCFLength = 0;
+
+  private byte[] rawKVs;
+
+  private final String compressionAlgorithmName;
+  private final Algorithm compressionAlgorithm;
+  private final Compressor compressor;
+  private final Decompressor decompressor;
+
+  private static enum Manipulation {
+    ENCODING,
+    DECODING,
+    COMPRESSION,
+    DECOMPRESSION;
+
+    @Override
+    public String toString() {
+      String s = super.toString();
+      StringBuilder sb = new StringBuilder();
+      sb.append(s.charAt(0));
+      sb.append(s.substring(1).toLowerCase());
+      return sb.toString();
+    }
+  }
 
   /**
    * @param compressionAlgorithmName What kind of algorithm should be used
@@ -110,23 +163,21 @@ public class DataBlockEncodingTool {
       throws IOException {
     scanner.seek(KeyValue.LOWESTKEY);
 
-    KeyValue currentKv;
+    KeyValue currentKV;
 
     byte[] previousKey = null;
     byte[] currentKey;
 
-    List<DataBlockEncoder> dataBlockEncoders =
-        DataBlockEncoding.getAllEncoders();
+    DataBlockEncoding[] encodings = DataBlockEncoding.values();
 
-    for (DataBlockEncoder d : dataBlockEncoders) {
-      codecs.add(new EncodedDataBlock(d, includesMemstoreTS));
-    }
+    ByteArrayOutputStream uncompressedOutputStream =
+        new ByteArrayOutputStream();
 
     int j = 0;
-    while ((currentKv = scanner.next()) != null && j < kvLimit) {
+    while ((currentKV = scanner.next()) != null && j < kvLimit) {
       // Iterates through key/value pairs
       j++;
-      currentKey = currentKv.getKey();
+      currentKey = currentKV.getKey();
       if (previousKey != null) {
         for (int i = 0; i < previousKey.length && i < currentKey.length &&
             previousKey[i] == currentKey[i]; ++i) {
@@ -134,16 +185,30 @@ public class DataBlockEncodingTool {
         }
       }
 
-      for (EncodedDataBlock codec : codecs) {
-        codec.addKv(currentKv);
-      }
+      uncompressedOutputStream.write(currentKV.getBuffer(),
+          currentKV.getOffset(), currentKV.getLength());
 
       previousKey = currentKey;
 
-      totalPrefixLength += currentKv.getLength() - currentKv.getKeyLength() -
-          currentKv.getValueLength();
-      totalKeyLength += currentKv.getKeyLength();
-      totalValueLength += currentKv.getValueLength();
+      int kLen = currentKV.getKeyLength();
+      int vLen = currentKV.getValueLength();
+      int cfLen = currentKV.getFamilyLength(currentKV.getFamilyOffset());
+      int restLen = currentKV.getLength() - kLen - vLen;
+
+      totalKeyLength += kLen;
+      totalValueLength += vLen;
+      totalPrefixLength += restLen;
+      totalCFLength += cfLen;
+    }
+
+    rawKVs = uncompressedOutputStream.toByteArray();
+
+    for (DataBlockEncoding encoding : encodings) {
+      if (encoding == DataBlockEncoding.NONE) {
+        continue;
+      }
+      DataBlockEncoder d = encoding.getEncoder();
+      codecs.add(new EncodedDataBlock(d, includesMemstoreTS, encoding, rawKVs));
     }
   }
 
@@ -227,15 +292,14 @@ public class DataBlockEncodingTool {
   /**
    * Benchmark codec's speed.
    */
-  public void benchmarkCodecs() {
+  public void benchmarkCodecs() throws IOException {
+    LOG.info("Starting a throughput benchmark for data block encoding codecs");
     int prevTotalSize = -1;
     for (EncodedDataBlock codec : codecs) {
       prevTotalSize = benchmarkEncoder(prevTotalSize, codec);
     }
 
-    byte[] buffer = codecs.get(0).getRawKeyValues();
-
-    benchmarkDefaultCompression(prevTotalSize, buffer);
+    benchmarkDefaultCompression(prevTotalSize, rawKVs);
   }
 
   /**
@@ -251,7 +315,7 @@ public class DataBlockEncodingTool {
 
     // decompression time
     List<Long> durations = new ArrayList<Long>();
-    for (int itTime = 0; itTime < BENCHMARK_N_TIMES; ++itTime) {
+    for (int itTime = 0; itTime < benchmarkNTimes; ++itTime) {
       totalSize = 0;
 
       Iterator<KeyValue> it;
@@ -265,7 +329,7 @@ public class DataBlockEncodingTool {
         totalSize += it.next().getLength();
       }
       final long finishTime = System.nanoTime();
-      if (itTime >= BENCHMARK_N_OMIT) {
+      if (itTime >= benchmarkNOmit) {
         durations.add(finishTime - startTime);
       }
 
@@ -276,26 +340,27 @@ public class DataBlockEncodingTool {
       prevTotalSize = totalSize;
     }
 
-    // compression time
-    List<Long> compressDurations = new ArrayList<Long>();
-    for (int itTime = 0; itTime < BENCHMARK_N_TIMES; ++itTime) {
+    List<Long> encodingDurations = new ArrayList<Long>();
+    for (int itTime = 0; itTime < benchmarkNTimes; ++itTime) {
       final long startTime = System.nanoTime();
-      codec.doCompressData();
+      codec.encodeData();
       final long finishTime = System.nanoTime();
-      if (itTime >= BENCHMARK_N_OMIT) {
-        compressDurations.add(finishTime - startTime);
+      if (itTime >= benchmarkNOmit) {
+        encodingDurations.add(finishTime - startTime);
       }
     }
 
     System.out.println(codec.toString() + ":");
-    printBenchmarkResult(totalSize, compressDurations, false);
-    printBenchmarkResult(totalSize, durations, true);
+    printBenchmarkResult(totalSize, encodingDurations, Manipulation.ENCODING);
+    printBenchmarkResult(totalSize, durations, Manipulation.DECODING);
+    System.out.println();
 
     return prevTotalSize;
   }
 
-  private void benchmarkDefaultCompression(int totalSize, byte[] rawBuffer) {
-    benchmarkAlgorithm(compressionAlgorithm, compressor, decompressor,
+  private void benchmarkDefaultCompression(int totalSize, byte[] rawBuffer)
+      throws IOException {
+    benchmarkAlgorithm(compressionAlgorithm,
         compressionAlgorithmName.toUpperCase(), rawBuffer, 0, totalSize);
   }
 
@@ -308,24 +373,22 @@ public class DataBlockEncodingTool {
    * @param buffer Buffer to be compressed.
    * @param offset Position of the beginning of the data.
    * @param length Length of data in buffer.
+   * @throws IOException
    */
-  public static void benchmarkAlgorithm(
-      Compression.Algorithm algorithm,
-      Compressor compressorCodec,
-      Decompressor decompressorCodec,
-      String name,
-      byte[] buffer, int offset, int length) {
+  public void benchmarkAlgorithm(Compression.Algorithm algorithm, String name,
+      byte[] buffer, int offset, int length) throws IOException {
     System.out.println(name + ":");
 
     // compress it
     List<Long> compressDurations = new ArrayList<Long>();
     ByteArrayOutputStream compressedStream = new ByteArrayOutputStream();
-    OutputStream compressingStream;
+    CompressionOutputStream compressingStream =
+        algorithm.createPlainCompressionStream(compressedStream, compressor);
     try {
-      for (int itTime = 0; itTime < BENCHMARK_N_TIMES; ++itTime) {
+      for (int itTime = 0; itTime < benchmarkNTimes; ++itTime) {
         final long startTime = System.nanoTime();
-        compressingStream = algorithm.createCompressionStream(
-            compressedStream, compressorCodec, 0);
+        compressingStream.resetState();
+        compressedStream.reset();
         compressingStream.write(buffer, offset, length);
         compressingStream.flush();
         compressedStream.toByteArray();
@@ -333,36 +396,31 @@ public class DataBlockEncodingTool {
         final long finishTime = System.nanoTime();
 
         // add time record
-        if (itTime >= BENCHMARK_N_OMIT) {
+        if (itTime >= benchmarkNOmit) {
           compressDurations.add(finishTime - startTime);
         }
-
-        if (itTime + 1 < BENCHMARK_N_TIMES) { // not the last one
-          compressedStream.reset();
-        }
       }
     } catch (IOException e) {
       throw new RuntimeException(String.format(
           "Benchmark, or encoding algorithm '%s' cause some stream problems",
           name), e);
     }
-    printBenchmarkResult(length, compressDurations, false);
-
+    compressingStream.close();
+    printBenchmarkResult(length, compressDurations, Manipulation.COMPRESSION);
 
     byte[] compBuffer = compressedStream.toByteArray();
 
     // uncompress it several times and measure performance
     List<Long> durations = new ArrayList<Long>();
-    for (int itTime = 0; itTime < BENCHMARK_N_TIMES; ++itTime) {
+    for (int itTime = 0; itTime < benchmarkNTimes; ++itTime) {
       final long startTime = System.nanoTime();
       byte[] newBuf = new byte[length + 1];
 
       try {
-
         ByteArrayInputStream downStream = new ByteArrayInputStream(compBuffer,
             0, compBuffer.length);
         InputStream decompressedStream = algorithm.createDecompressionStream(
-            downStream, decompressorCodec, 0);
+            downStream, decompressor, 0);
 
         int destOffset = 0;
         int nextChunk;
@@ -371,7 +429,7 @@ public class DataBlockEncodingTool {
         }
         decompressedStream.close();
 
-        // iterate over KeyValue
+        // iterate over KeyValues
         KeyValue kv;
         for (int pos = 0; pos < length; pos += kv.getLength()) {
           kv = new KeyValue(newBuf, pos);
@@ -397,91 +455,116 @@ public class DataBlockEncodingTool {
       }
 
       // add time record
-      if (itTime >= BENCHMARK_N_OMIT) {
+      if (itTime >= benchmarkNOmit) {
         durations.add(finishTime - startTime);
       }
     }
-    printBenchmarkResult(length, durations, true);
+    printBenchmarkResult(length, durations, Manipulation.DECOMPRESSION);
+    System.out.println();
   }
 
+  private static final double BYTES_IN_MB = 1024 * 1024.0;
+  private static final double NS_IN_SEC = 1000.0 * 1000.0 * 1000.0;
+  private static final double MB_SEC_COEF = NS_IN_SEC / BYTES_IN_MB;
+
   private static void printBenchmarkResult(int totalSize,
-      List<Long> durationsInNanoSed, boolean isDecompression) {
+      List<Long> durationsInNanoSec, Manipulation manipulation) {
+    final int n = durationsInNanoSec.size();
     long meanTime = 0;
-    for (long time : durationsInNanoSed) {
+    for (long time : durationsInNanoSec) {
       meanTime += time;
     }
-    meanTime /= durationsInNanoSed.size();
+    meanTime /= n;
+
+    double meanMBPerSec = totalSize * MB_SEC_COEF / meanTime;
+    double mbPerSecSTD = 0;
+    if (n > 0) {
+      for (long time : durationsInNanoSec) {
+        double mbPerSec = totalSize * MB_SEC_COEF / time;
+        double dev = mbPerSec - meanMBPerSec;
+        mbPerSecSTD += dev * dev;
+      }
+      mbPerSecSTD = Math.sqrt(mbPerSecSTD / n);
+    }
 
-    long standardDev = 0;
-    for (long time : durationsInNanoSed) {
-      standardDev += (time - meanTime) * (time - meanTime);
-    }
-    standardDev = (long) Math.sqrt(standardDev / durationsInNanoSed.size());
-
-    final double million = 1000.0 * 1000.0 * 1000.0;
-    double mbPerSec = (totalSize * million) / (1024.0 * 1024.0 * meanTime);
-    double mbPerSecDev = (totalSize * million) /
-        (1024.0 * 1024.0 * (meanTime - standardDev));
-
-    System.out.println(String.format(
-        "  %s performance:%s %6.2f MB/s (+/- %.2f MB/s)",
-        isDecompression ? "Decompression" : "Compression",
-        isDecompression ? "" : "  ",
-        mbPerSec, mbPerSecDev - mbPerSec));
+    outputTuple(manipulation + " performance", "%6.2f MB/s (+/- %.2f MB/s)",
+         meanMBPerSec, mbPerSecSTD);
+  }
+
+  private static void outputTuple(String caption, String format,
+      Object... values) {
+    if (format.startsWith(INT_FORMAT)) {
+      format = "%s" + format.substring(INT_FORMAT.length());
+      values[0] = DELIMITED_DECIMAL_FORMAT.format(values[0]);
+    }
+
+    StringBuilder sb = new StringBuilder();
+    sb.append("  ");
+    sb.append(caption);
+    sb.append(":");
+
+    String v = String.format(format, values);
+    int padding = 60 - sb.length() - v.length();
+    for (int i = 0; i < padding; ++i) {
+      sb.append(' ');
+    }
+    sb.append(v);
+    System.out.println(sb);
   }
 
   /**
    * Display statistics of different compression algorithms.
+   * @throws IOException
    */
-  public void displayStatistics() {
-    int totalLength = totalPrefixLength + totalKeyLength + totalValueLength;
-    if (compressor != null) {  // might be null e.g. for pure-Java GZIP
-      compressor.reset();
-    }
+  public void displayStatistics() throws IOException {
+    final String comprAlgo = compressionAlgorithmName.toUpperCase();
+    long rawBytes = totalKeyLength + totalPrefixLength + totalValueLength;
+
+    System.out.println("Raw data size:");
+    outputTuple("Raw bytes", INT_FORMAT, rawBytes);
+    outputTuplePct("Key bytes", totalKeyLength);
+    outputTuplePct("Value bytes", totalValueLength);
+    outputTuplePct("KV infrastructure", totalPrefixLength);
+    outputTuplePct("CF overhead", totalCFLength);
+    outputTuplePct("Total key redundancy", totalKeyRedundancyLength);
+
+    int compressedSize = EncodedDataBlock.getCompressedSize(
+        compressionAlgorithm, compressor, rawKVs, 0, rawKVs.length);
+    outputTuple(comprAlgo + " only size", INT_FORMAT,
+        compressedSize);
+    outputSavings(comprAlgo + " only", compressedSize, rawBytes);
+    System.out.println();
 
-    for(EncodedDataBlock codec : codecs) {
+    for (EncodedDataBlock codec : codecs) {
       System.out.println(codec.toString());
-      int saved = totalKeyLength + totalPrefixLength + totalValueLength
-          - codec.getSize();
-      System.out.println(
-          String.format("  Saved bytes:                 %8d", saved));
-      double keyRatio = (saved * 100.0) / (totalPrefixLength + totalKeyLength);
-      double allRatio = (saved * 100.0) / totalLength;
-      System.out.println(
-          String.format("  Key compression ratio:        %.2f %%", keyRatio));
-      System.out.println(
-          String.format("  All compression ratio:        %.2f %%", allRatio));
-
-      String compressedSizeCaption =
-          String.format("  %s compressed size:         ",
-              compressionAlgorithmName.toUpperCase());
-      String compressOnlyRatioCaption =
-          String.format("  %s compression ratio:        ",
-              compressionAlgorithmName.toUpperCase());
-
-      if (compressor != null) {
-        int compressedSize = codec.checkCompressedSize(compressor);
-        System.out.println(compressedSizeCaption +
-            String.format("%8d", compressedSize));
-        double compressOnlyRatio =
-            100.0 * (1.0 - compressedSize / (0.0 + totalLength));
-        System.out.println(compressOnlyRatioCaption
-            + String.format("%.2f %%", compressOnlyRatio));
-      } else {
-        System.out.println(compressedSizeCaption + "N/A");
-        System.out.println(compressOnlyRatioCaption + "N/A");
-      }
+      long encodedBytes = codec.getSize();
+      outputTuple("Encoded bytes", INT_FORMAT, encodedBytes);
+      outputSavings("Key encoding", encodedBytes - totalValueLength,
+          rawBytes - totalValueLength);
+      outputSavings("Total encoding", encodedBytes, rawBytes);
+
+      int encodedCompressedSize = codec.getEncodedCompressedSize(
+          compressionAlgorithm, compressor);
+      outputTuple("Encoding + " + comprAlgo + " size", INT_FORMAT,
+          encodedCompressedSize);
+      outputSavings("Encoding + " + comprAlgo, encodedCompressedSize, rawBytes);
+      outputSavings("Encoding with " + comprAlgo, encodedCompressedSize,
+          compressedSize);
+
+      System.out.println();
     }
+  }
 
-    System.out.println(
-        String.format("Total KV prefix length:   %8d", totalPrefixLength));
-    System.out.println(
-        String.format("Total key length:         %8d", totalKeyLength));
-    System.out.println(
-        String.format("Total key redundancy:     %8d",
-            totalKeyRedundancyLength));
-    System.out.println(
-        String.format("Total value length:       %8d", totalValueLength));
+  private void outputTuplePct(String caption, long size) {
+    outputTuple(caption, INT_FORMAT + " (" + PCT_FORMAT + ")",
+        size, size * 100.0 / rawKVs.length);
+  }
+
+  private void outputSavings(String caption, long part, long whole) {
+    double pct = 100.0 * (1 - 1.0 * part / whole);
+    double times = whole * 1.0 / part;
+    outputTuple(caption + " savings", PCT_FORMAT + " (%.2f x)",
+        pct, times);
   }
 
   /**
@@ -541,22 +624,35 @@ public class DataBlockEncodingTool {
   }
 
   /**
-   * A command line interface to benchmarks.
+   * A command line interface to benchmarks. Parses command-line arguments and
+   * runs the appropriate benchmarks.
    * @param args Should have length at least 1 and holds the file path to HFile.
    * @throws IOException If you specified the wrong file.
    */
   public static void main(final String[] args) throws IOException {
     // set up user arguments
     Options options = new Options();
-    options.addOption("f", true, "HFile to analyse (REQUIRED)");
-    options.getOption("f").setArgName("FILENAME");
-    options.addOption("n", true,
-        "Limit number of KeyValue which will be analysed");
-    options.getOption("n").setArgName("NUMBER");
-    options.addOption("b", false, "Measure read throughput");
-    options.addOption("c", false, "Omit corectness tests.");
-    options.addOption("a", true,
+    options.addOption(OPT_HFILE_NAME, true, "HFile to analyse (REQUIRED)");
+    options.getOption(OPT_HFILE_NAME).setArgName("FILENAME");
+    options.addOption(OPT_KV_LIMIT, true,
+        "Maximum number of KeyValues to process. A benchmark stops running " +
+        "after iterating over this many KV pairs.");
+    options.getOption(OPT_KV_LIMIT).setArgName("NUMBER");
+    options.addOption(OPT_MEASURE_THROUGHPUT, false,
+        "Measure read throughput");
+    options.addOption(OPT_OMIT_CORRECTNESS_TEST, false,
+        "Omit corectness tests.");
+    options.addOption(OPT_ENCODING_ALGORITHM, true,
         "What kind of compression algorithm use for comparison.");
+    options.addOption(OPT_BENCHMARK_N_TIMES,
+        true, "Number of times to run each benchmark. Default value: " +
+            DEFAULT_BENCHMARK_N_TIMES);
+    options.addOption(OPT_BENCHMARK_N_OMIT, true,
+        "Number of first runs of every benchmark to exclude from "
+            + "statistics (" + DEFAULT_BENCHMARK_N_OMIT
+            + " by default, so that " + "only the last "
+            + (DEFAULT_BENCHMARK_N_TIMES - DEFAULT_BENCHMARK_N_OMIT)
+            + " times are included in statistics.)");
 
     // parse arguments
     CommandLineParser parser = new PosixParser();
@@ -570,24 +666,44 @@ public class DataBlockEncodingTool {
     }
 
     int kvLimit = Integer.MAX_VALUE;
-    if (cmd.hasOption("n")) {
-      kvLimit = Integer.parseInt(cmd.getOptionValue("n"));
+    if (cmd.hasOption(OPT_KV_LIMIT)) {
+      kvLimit = Integer.parseInt(cmd.getOptionValue(OPT_KV_LIMIT));
     }
 
     // basic argument sanity checks
-    if (!cmd.hasOption("f")) {
-      System.err.println("ERROR: Filename is required!");
+    if (!cmd.hasOption(OPT_HFILE_NAME)) {
+      LOG.error("Please specify HFile name using the " + OPT_HFILE_NAME
+          + " option");
       printUsage(options);
       System.exit(-1);
     }
 
-    String pathName = cmd.getOptionValue("f");
+    String pathName = cmd.getOptionValue(OPT_HFILE_NAME);
     String compressionName = DEFAULT_COMPRESSION.getName();
-    if (cmd.hasOption("a")) {
-      compressionName = cmd.getOptionValue("a").toLowerCase();
+    if (cmd.hasOption(OPT_ENCODING_ALGORITHM)) {
+      compressionName =
+          cmd.getOptionValue(OPT_ENCODING_ALGORITHM).toLowerCase();
+    }
+    boolean doBenchmark = cmd.hasOption(OPT_MEASURE_THROUGHPUT);
+    boolean doVerify = !cmd.hasOption(OPT_OMIT_CORRECTNESS_TEST);
+
+    if (cmd.hasOption(OPT_BENCHMARK_N_TIMES)) {
+      benchmarkNTimes = Integer.valueOf(cmd.getOptionValue(
+          OPT_BENCHMARK_N_TIMES));
+    }
+    if (cmd.hasOption(OPT_BENCHMARK_N_OMIT)) {
+      benchmarkNOmit =
+          Integer.valueOf(cmd.getOptionValue(OPT_BENCHMARK_N_OMIT));
+    }
+    if (benchmarkNTimes < benchmarkNOmit) {
+      LOG.error("The number of times to run each benchmark ("
+          + benchmarkNTimes
+          + ") must be greater than the number of benchmark runs to exclude "
+          + "from statistics (" + benchmarkNOmit + ")");
+      System.exit(1);
     }
-    boolean doBenchmark = cmd.hasOption("b");
-    boolean doVerify = !cmd.hasOption("c");
+    LOG.info("Running benchmark " + benchmarkNTimes + " times. " +
+        "Excluding the first " + benchmarkNOmit + " times from statistics.");
 
     final Configuration conf = HBaseConfiguration.create();
     try {
@@ -598,4 +714,4 @@ public class DataBlockEncodingTool {
     }
   }
 
-}
+}
\ No newline at end of file

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/EncodedSeekPerformanceTest.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/EncodedSeekPerformanceTest.java?rev=1306182&r1=1306181&r2=1306182&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/EncodedSeekPerformanceTest.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/regionserver/EncodedSeekPerformanceTest.java Wed Mar 28 06:40:29 2012
@@ -173,7 +173,6 @@ public class EncodedSeekPerformanceTest 
     List<HFileDataBlockEncoder> encoders =
         new ArrayList<HFileDataBlockEncoder>();
 
-    encoders.add(new HFileDataBlockEncoderImpl(DataBlockEncoding.NONE));
     for (DataBlockEncoding encodingAlgo : DataBlockEncoding.values()) {
       encoders.add(new HFileDataBlockEncoderImpl(DataBlockEncoding.NONE,
           encodingAlgo));