You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2011/06/07 01:16:10 UTC
svn commit: r1132833 - in /hbase/trunk: CHANGES.txt pom.xml
src/assembly/all.xml src/docbkx/book.xml src/docbkx/build.xml
src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java
Author: stack
Date: Mon Jun 6 23:16:09 2011
New Revision: 1132833
URL: http://svn.apache.org/viewvc?rev=1132833&view=rev
Log:
HBASE-3873 Mavenize Hadoop Snappy JAR/SOs project dependencies
Modified:
hbase/trunk/CHANGES.txt
hbase/trunk/pom.xml
hbase/trunk/src/assembly/all.xml
hbase/trunk/src/docbkx/book.xml
hbase/trunk/src/docbkx/build.xml
hbase/trunk/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java
Modified: hbase/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hbase/trunk/CHANGES.txt?rev=1132833&r1=1132832&r2=1132833&view=diff
==============================================================================
--- hbase/trunk/CHANGES.txt (original)
+++ hbase/trunk/CHANGES.txt Mon Jun 6 23:16:09 2011
@@ -241,6 +241,8 @@ Release 0.91.0 - Unreleased
HBASE-3592 Guava snuck back in as a dependency via hbase-3777
HBASE-3808 Implement Executor.toString for master handlers at least
(Brock Noland)
+ HBASE-3873 Mavenize Hadoop Snappy JAR/SOs project dependencies
+ (Alejandro Abdelnur)
TASKS
HBASE-3559 Move report of split to master OFF the heartbeat channel
Modified: hbase/trunk/pom.xml
URL: http://svn.apache.org/viewvc/hbase/trunk/pom.xml?rev=1132833&r1=1132832&r2=1132833&view=diff
==============================================================================
--- hbase/trunk/pom.xml (original)
+++ hbase/trunk/pom.xml Mon Jun 6 23:16:09 2011
@@ -416,10 +416,15 @@
<exclude>**/*$*</exclude>
<exclude>${test.exclude.pattern}</exclude>
</excludes>
+ <environmentVariables>
+ <LD_LIBRARY_PATH>${env.LD_LIBRARY_PATH}:${project.build.directory}/nativelib</LD_LIBRARY_PATH>
+ <DYLD_LIBRARY_PATH>${env.DYLD_LIBRARY_PATH}:${project.build.directory}/nativelib</DYLD_LIBRARY_PATH>
+ </environmentVariables>
</configuration>
</plugin>
<plugin>
<artifactId>maven-antrun-plugin</artifactId>
+ <version>1.6</version>
<executions>
<execution>
<id>generate</id>
@@ -472,12 +477,50 @@
<tasks>
<replace file="${project.build.outputDirectory}/hbase-default.xml"
token="@@@VERSION@@@" value="${project.version}" />
+ <mkdir dir="${project.build.directory}/nativelib"/>
+ <exec executable="tar" dir="${project.build.directory}/nativelib" failonerror="false">
+ <arg value="xf"/>
+ <arg value="hadoop-snappy-nativelibs.tar"/>
+ </exec>
</tasks>
</configuration>
<goals>
<goal>run</goal>
</goals>
</execution>
+ <execution>
+ <id>package</id>
+ <phase>package</phase>
+ <configuration>
+ <target>
+ <!-- Complements the assembly -->
+
+ <mkdir dir="${project.build.directory}/${project.artifactId}-${project.version}/${project.artifactId}-${project.version}/lib/native/${build.platform}"/>
+
+ <!-- Using Unix cp to preserve symlinks, using script to handle wildcards -->
+ <echo file="${project.build.directory}/copynativelibs.sh">
+ if [ `ls ${project.build.directory}/nativelib | wc -l` -ne 0 ]; then
+ cp -PR ${project.build.directory}/nativelib/lib* ${project.build.directory}/${project.artifactId}-${project.version}/${project.artifactId}-${project.version}/lib/native/${build.platform}
+ fi
+ </echo>
+ <exec executable="sh" dir="${project.build.directory}" failonerror="true">
+ <arg line="./copynativelibs.sh"/>
+ </exec>
+
+ <!-- Using Unix tar to preserve symlinks -->
+ <exec executable="tar" failonerror="yes"
+ dir="${project.build.directory}/${project.artifactId}-${project.version}">
+ <arg value="czf"/>
+ <arg value="${project.build.directory}/${project.artifactId}-${project.version}.tar.gz"/>
+ <arg value="."/>
+ </exec>
+
+ </target>
+ </configuration>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ </execution>
</executions>
</plugin>
<plugin>
@@ -587,6 +630,7 @@
<stax-api.version>1.0.1</stax-api.version>
<thrift.version>0.6.1</thrift.version>
<zookeeper.version>3.3.3</zookeeper.version>
+ <hadoop-snappy.version>0.0.1-cdh3u1-SNAPSHOT</hadoop-snappy.version>
<package.prefix>/usr</package.prefix>
<package.conf.dir>/etc/hbase</package.conf.dir>
@@ -978,6 +1022,78 @@
</plugins>
</build>
</profile>
+
+ <profile>
+ <id>os.linux</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ <os>
+ <family>Linux</family>
+ </os>
+ </activation>
+ <properties>
+ <build.platform>${os.name}-${os.arch}-${sun.arch.data.model}</build.platform>
+ </properties>
+ </profile>
+ <profile>
+ <id>os.mac</id>
+ <activation>
+ <os>
+ <family>Mac</family>
+ </os>
+ </activation>
+ <properties>
+ <build.platform>Mac_OS_X-${sun.arch.data.model}</build.platform>
+ </properties>
+ </profile>
+
+ <profile>
+ <id>hadoop-snappy</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ <property>
+ <name>snappy</name>
+ </property>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-snappy</artifactId>
+ <version>${hadoop-snappy.version}</version>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>get-hadoop-snappy-native</id>
+ <phase>generate-resources</phase>
+ <goals>
+ <goal>copy</goal>
+ </goals>
+ <configuration>
+ <artifactItems>
+ <artifactItem>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-snappy</artifactId>
+ <version>${hadoop-snappy.version}</version>
+ <classifier>${build.platform}</classifier>
+ <type>tar</type>
+ <overWrite>false</overWrite>
+ <outputDirectory>${project.build.directory}/nativelib</outputDirectory>
+ <destFileName>hadoop-snappy-nativelibs.tar</destFileName>
+ </artifactItem>
+ </artifactItems>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
</profiles>
<!-- See http://jira.codehaus.org/browse/MSITE-443 why the settings need to be here and not in pluginManagement. -->
Modified: hbase/trunk/src/assembly/all.xml
URL: http://svn.apache.org/viewvc/hbase/trunk/src/assembly/all.xml?rev=1132833&r1=1132832&r2=1132833&view=diff
==============================================================================
--- hbase/trunk/src/assembly/all.xml (original)
+++ hbase/trunk/src/assembly/all.xml Mon Jun 6 23:16:09 2011
@@ -7,7 +7,7 @@
-->
<id>all</id>
<formats>
- <format>tar.gz</format>
+ <format>dir</format>
</formats>
<fileSets>
<fileSet>
Modified: hbase/trunk/src/docbkx/book.xml
URL: http://svn.apache.org/viewvc/hbase/trunk/src/docbkx/book.xml?rev=1132833&r1=1132832&r2=1132833&view=diff
==============================================================================
--- hbase/trunk/src/docbkx/book.xml (original)
+++ hbase/trunk/src/docbkx/book.xml Mon Jun 6 23:16:09 2011
@@ -1408,23 +1408,19 @@ false
SNAPPY
</title>
<para>
- To set SNAPPY compression on a column family, do as following:
+ If snappy is installed, HBase can make use of it (courtesy of
+ <link xlink:href="http://code.google.com/p/hadoop-snappy/">hadoop-snappy</link>).
+
<orderedlist>
<listitem>
<para>
- Install hadoop-snappy using these instructions: http://code.google.com/p/hadoop-snappy/
- </para>
- </listitem>
- <listitem>
- <para>
- You need to ensure the hadoop-snappy libs (incl. the native libs) are in the HBase classpath. One way to do this is
- to just symlink the libs from <filename>HADOOP_HOME/lib</filename> to <filename>HBASE_HOME/lib</filename>.
- This needs to be done on all HBase nodes, as with LZO.
+ Build and install <link xlink:href="http://code.google.com/p/snappy/">snappy</link> on all nodes
+ of your cluster.
</para>
</listitem>
<listitem>
<para>
- Use CompressionTest to verify snappy support is enabled and the libs can be loaded:
+ Use CompressionTest to verify snappy support is enabled and the libs can be loaded ON ALL NODES of your cluster:
<programlisting>$ hbase org.apache.hadoop.hbase.util.CompressionTest hdfs://host/path/to/hbase snappy</programlisting>
</para>
</listitem>
Modified: hbase/trunk/src/docbkx/build.xml
URL: http://svn.apache.org/viewvc/hbase/trunk/src/docbkx/build.xml?rev=1132833&r1=1132832&r2=1132833&view=diff
==============================================================================
--- hbase/trunk/src/docbkx/build.xml (original)
+++ hbase/trunk/src/docbkx/build.xml Mon Jun 6 23:16:09 2011
@@ -10,6 +10,12 @@
<title>Building HBase</title>
<para>This chapter will be of interest only to those building HBase from source.
</para>
+ <section xml:id="build.snappy">
+ <title>Building in snappy compression support</title>
+ <p>Pass <code>-Dsnappy</code> to trigger the snappy maven profile for building
+ snappy native libs into hbase.</p>
+ </section>
+
<section xml:id="mvn_repo">
<title>Adding an HBase release to Apache's Maven Repository</title>
<para>Follow the instructions at
Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java?rev=1132833&r1=1132832&r2=1132833&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/util/TestCompressionTest.java Mon Jun 6 23:16:09 2011
@@ -54,6 +54,23 @@ public class TestCompressionTest {
assertFalse(CompressionTest.testCompression("LZO"));
assertTrue(CompressionTest.testCompression("NONE"));
assertTrue(CompressionTest.testCompression("GZ"));
- assertFalse(CompressionTest.testCompression("SNAPPY"));
+
+ if (isCompressionAvailable("org.apache.hadoop.io.compress.SnappyCodec")) {
+ assertTrue(CompressionTest.testCompression("SNAPPY"));
+ }
+ else {
+ assertFalse(CompressionTest.testCompression("SNAPPY"));
+ }
}
+
+ private boolean isCompressionAvailable(String codecClassName) {
+ try {
+ Thread.currentThread().getContextClassLoader().loadClass(codecClassName);
+ return true;
+ }
+ catch (Exception ex) {
+ return false;
+ }
+ }
+
}