You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2011/05/20 01:00:47 UTC
svn commit: r1125163 - in /hbase/trunk: ./ src/docbkx/
src/main/java/org/apache/hadoop/hbase/
src/main/java/org/apache/hadoop/hbase/io/hfile/
src/main/java/org/apache/hadoop/hbase/util/
src/test/java/org/apache/hadoop/hbase/io/hfile/ src/test/java/org/...
Author: stack
Date: Thu May 19 23:00:47 2011
New Revision: 1125163
URL: http://svn.apache.org/viewvc?rev=1125163&view=rev
Log:
HBASE-3691 Add compressor support for 'snappy', google's compressor
Modified:
hbase/trunk/CHANGES.txt
hbase/trunk/src/docbkx/book.xml
hbase/trunk/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/Compression.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/CompressionTest.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFilePerformance.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java
Modified: hbase/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hbase/trunk/CHANGES.txt?rev=1125163&r1=1125162&r2=1125163&view=diff
==============================================================================
--- hbase/trunk/CHANGES.txt (original)
+++ hbase/trunk/CHANGES.txt Thu May 19 23:00:47 2011
@@ -257,6 +257,8 @@ Release 0.91.0 - Unreleased
HBASE-3837 Show regions in transition on the master web page (todd)
HBASE-3839 Add monitoring of currently running tasks to the master and
RS web UIs
+ HBASE-3691 Add compressor support for 'snappy', google's compressor
+ (Nichole Treadway and Nicholas Telford)
Release 0.90.4 - Unreleased
Modified: hbase/trunk/src/docbkx/book.xml
URL: http://svn.apache.org/viewvc/hbase/trunk/src/docbkx/book.xml?rev=1125163&r1=1125162&r2=1125163&view=diff
==============================================================================
--- hbase/trunk/src/docbkx/book.xml (original)
+++ hbase/trunk/src/docbkx/book.xml Thu May 19 23:00:47 2011
@@ -1373,6 +1373,45 @@ false
reports in your logs; see <xref linkend="brand.new.compressor" />).
</para>
</section>
+ <section xml:id="snappy.compression">
+ <title>
+ SNAPPY
+ </title>
+ <para>
+ To set snappy compression on a column family, do as following:
+ <orderedlist>
+ <listitem>
+ <para>
+ Install hadoop-snappy using these instructions: http://code.google.com/p/hadoop-snappy/
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ You need to ensure the hadoop-snappy libs (incl. the native libs) are in the HBase classpath. One way to do this is
+ to just symlink the libs from <filename>HADOOP_HOME/lib</filename> to <filename>HBASE_HOME/lib</filename>.
+ This needs to be done on all HBase nodes, as with LZO.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ Use CompressionTest to verify snappy support is enabled and the libs can be loaded:
+ <programlisting>$ hbase org.apache.hadoop.hbase.util.CompressionTest hdfs://host/path/to/hbase snappy</programlisting>
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ Create a column family with snappy compression and verify it in the hbase shell:
+ <programlisting>$ hbase> create 't1', { NAME => 'cf1', COMPRESSION => 'snappy' }
+hbase> describe 't1'</programlisting>
+ In the output of the "describe" command, you need to ensure it lists "COMPRESSION => 'snappy'"
+ </para>
+ </listitem>
+
+ </orderedlist>
+
+ </para>
+ </section>
+
</appendix>
<appendix xml:id="faq">
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java?rev=1125163&r1=1125162&r2=1125163&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/HColumnDescriptor.java Thu May 19 23:00:47 2011
@@ -432,6 +432,7 @@ public class HColumnDescriptor implement
switch (type) {
case LZO: compressionType = "LZO"; break;
case GZ: compressionType = "GZ"; break;
+ case SNAPPY: compressionType = "SNAPPY"; break;
default: compressionType = "NONE"; break;
}
setValue(COMPRESSION, compressionType);
@@ -456,6 +457,7 @@ public class HColumnDescriptor implement
switch (type) {
case LZO: compressionType = "LZO"; break;
case GZ: compressionType = "GZ"; break;
+ case SNAPPY: compressionType = "SNAPPY"; break;
default: compressionType = "NONE"; break;
}
setValue(COMPRESSION_COMPACT, compressionType);
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/Compression.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/Compression.java?rev=1125163&r1=1125162&r2=1125163&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/Compression.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/Compression.java Thu May 19 23:00:47 2011
@@ -140,6 +140,25 @@ public final class Compression {
return downStream;
}
+ },
+ SNAPPY("snappy") {
+ // Use base type to avoid compile-time dependencies.
+ private transient CompressionCodec snappyCodec;
+
+ @Override
+ CompressionCodec getCodec(Configuration conf) {
+ if (snappyCodec == null) {
+ try {
+ Class<?> externalCodec =
+ ClassLoader.getSystemClassLoader().loadClass("org.apache.hadoop.io.compress.SnappyCodec");
+ snappyCodec = (CompressionCodec) ReflectionUtils.newInstance(externalCodec,
+ conf);
+ } catch (ClassNotFoundException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ return snappyCodec;
+ }
};
private final Configuration conf;
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/CompressionTest.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/CompressionTest.java?rev=1125163&r1=1125162&r2=1125163&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/CompressionTest.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/util/CompressionTest.java Thu May 19 23:00:47 2011
@@ -94,7 +94,7 @@ public class CompressionTest {
public static void usage() {
System.err.println(
- "Usage: CompressionTest <path> none|gz|lzo\n" +
+ "Usage: CompressionTest <path> none|gz|lzo|snappy\n" +
"\n" +
"For example:\n" +
" hbase " + CompressionTest.class + " file:///tmp/testfile gz\n");
Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFilePerformance.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFilePerformance.java?rev=1125163&r1=1125162&r2=1125163&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFilePerformance.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFilePerformance.java Thu May 19 23:00:47 2011
@@ -129,7 +129,7 @@ public class TestHFilePerformance extend
* @param fileType "HFile" or "SequenceFile"
* @param keyLength
* @param valueLength
- * @param codecName "none", "lzo", "gz"
+ * @param codecName "none", "lzo", "gz", "snappy"
* @param rows number of rows to be written.
* @param writeMethod used for HFile only.
* @param minBlockSize used for HFile only.
Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java?rev=1125163&r1=1125162&r2=1125163&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java Thu May 19 23:00:47 2011
@@ -281,7 +281,7 @@ public class TestHFileSeek extends TestC
private Options buildOptions() {
Option compress =
- OptionBuilder.withLongOpt("compress").withArgName("[none|lzo|gz]")
+ OptionBuilder.withLongOpt("compress").withArgName("[none|lzo|gz|snappy]")
.hasArg().withDescription("compression scheme").create('c');
Option fileSize =
@@ -446,7 +446,7 @@ public class TestHFileSeek extends TestC
private void validateOptions() throws ParseException {
if (!compress.equals("none") && !compress.equals("lzo")
- && !compress.equals("gz")) {
+ && !compress.equals("gz") && !compress.equals("snappy")) {
throw new ParseException("Unknown compression scheme: " + compress);
}
Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java?rev=1125163&r1=1125162&r2=1125163&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java Thu May 19 23:00:47 2011
@@ -438,6 +438,9 @@ public class TestHFileOutputFormat {
familyToCompression.put("Family1!@#!@#&", Compression.Algorithm.LZO);
}
if (numCfs-- > 0) {
+ familyToCompression.put("Family2=asdads&!AASD", Compression.Algorithm.SNAPPY);
+ }
+ if (numCfs-- > 0) {
familyToCompression.put("Family2=asdads&!AASD", Compression.Algorithm.GZ);
}
if (numCfs-- > 0) {
Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java?rev=1125163&r1=1125162&r2=1125163&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java Thu May 19 23:00:47 2011
@@ -54,5 +54,6 @@ public class TestCompressionTest {
assertFalse(CompressionTest.testCompression("LZO"));
assertTrue(CompressionTest.testCompression("NONE"));
assertTrue(CompressionTest.testCompression("GZ"));
+ assertTrue(CompressionTest.testCompression("SNAPPY"));
}
}