You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ec...@apache.org on 2014/08/01 00:07:42 UTC
[06/50] [abbrv] git commit: [master] Pull native libs from internal
jars.
[master] Pull native libs from internal jars.
Summary: .so's are now published internally. We should pull from there and un-jar at runtime.
Test Plan: deploy on dev038 and see that lzo works.
Reviewers: oozie, jjmaestro, adela, pwhite
Reviewed By: pwhite
Subscribers: hbase-prodeng@, hbase-eng@
Differential Revision: https://phabricator.fb.com/D1357483
Tasks: 4427997
git-svn-id: svn+ssh://tubbs/svnhive/hadoop/branches/titan/VENDOR.hbase/hbase-trunk@42651 e7acf4d4-3532-417f-9e73-7a9ae25a1f51
Conflicts:
.gitignore
lib/native/Linux-amd64-64/libhadoop.a
lib/native/Linux-amd64-64/libhadoop.la
lib/native/Linux-amd64-64/libhadoop.so
lib/native/Linux-amd64-64/libhadoop.so.1
lib/native/Linux-amd64-64/libhadoop.so.1.0.0
lib/native/Linux-amd64-64/libhadoopsnappy.a
lib/native/Linux-amd64-64/libhadoopsnappy.la
lib/native/Linux-amd64-64/libhadoopsnappy.lai
lib/native/Linux-amd64-64/libhadoopsnappy.so
lib/native/Linux-amd64-64/libhadoopsnappy.so.0
lib/native/Linux-amd64-64/libhadoopsnappy.so.0.0.1
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a404f16c
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a404f16c
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a404f16c
Branch: refs/heads/0.89-fb
Commit: a404f16c9c592c5581e86b6f4eba9b2f128a30ef
Parents: 4f7a7ca
Author: elliott <el...@e7acf4d4-3532-417f-9e73-7a9ae25a1f51>
Authored: Tue Jun 3 21:29:46 2014 +0000
Committer: Elliott Clark <el...@fb.com>
Committed: Thu Jul 31 14:44:02 2014 -0700
----------------------------------------------------------------------
.gitignore | 3 +-
bin/hbase | 30 ++++++++++++++
lib/native/Linux-amd64-64/libhadoop.a | Bin 110792 -> 0 bytes
lib/native/Linux-amd64-64/libhadoop.la | 35 ----------------
lib/native/Linux-amd64-64/libhadoop.so | Bin 71019 -> 0 bytes
lib/native/Linux-amd64-64/libhadoop.so.1 | Bin 71019 -> 0 bytes
lib/native/Linux-amd64-64/libhadoop.so.1.0.0 | Bin 71019 -> 0 bytes
lib/native/Linux-amd64-64/libhadoopsnappy.a | Bin 86232 -> 0 bytes
lib/native/Linux-amd64-64/libhadoopsnappy.la | 35 ----------------
lib/native/Linux-amd64-64/libhadoopsnappy.lai | 35 ----------------
lib/native/Linux-amd64-64/libhadoopsnappy.so | Bin 55233 -> 0 bytes
lib/native/Linux-amd64-64/libhadoopsnappy.so.0 | Bin 55233 -> 0 bytes
.../Linux-amd64-64/libhadoopsnappy.so.0.0.1 | Bin 55233 -> 0 bytes
lib/native/Linux-amd64-64/libsnappy.a | Bin 310360 -> 0 bytes
lib/native/Linux-amd64-64/libsnappy.la | 41 -------------------
lib/native/Linux-amd64-64/libsnappy.so | Bin 151171 -> 0 bytes
lib/native/Linux-amd64-64/libsnappy.so.1 | Bin 151171 -> 0 bytes
lib/native/Linux-amd64-64/libsnappy.so.1.1.3 | Bin 151171 -> 0 bytes
pom.xml | 35 +++++++++++++++-
19 files changed, 66 insertions(+), 148 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/a404f16c/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 42f6eaa..3c4afc6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,7 +4,8 @@
/.project
/.settings
/build
-/lib/native/Linux-amd64-64/libgpl*
+/lib/native/
+/lib/hadoop-lzo-0.4.1.jar
/.idea/
/logs
/target
http://git-wip-us.apache.org/repos/asf/hbase/blob/a404f16c/bin/hbase
----------------------------------------------------------------------
diff --git a/bin/hbase b/bin/hbase
index 43e2472..2834729 100755
--- a/bin/hbase
+++ b/bin/hbase
@@ -152,6 +152,34 @@ add_maven_test_classes_to_classpath() {
fi
}
+expand_native_jars() {
+ native_dir=$HBASE_HOME/lib/native/
+ if [ -d ${native_dir} ]; then
+ return;
+ fi
+
+ mkdir -p "${native_dir}"
+
+ while read -d: jarname; do
+ if [[ ${jarname} == *-native*.jar ]] && [ -f ${jarname} ]; then
+ expand_single_native_jar ${jarname} ${native_dir}
+ fi
+ done <<< $1
+}
+
+expand_single_native_jar() {
+ jar=$1
+ target=$2
+ TEMPDIR=`mktemp --directory -t hbase-native.XXXXXXX 2>/dev/null || mktemp -d -t 'hbase-native'`
+ pushd $TEMPDIR
+ unzip $jar
+ for nd in $(find . -name "native" -type d); do
+ cp -r ${nd}/* ${target}
+ done
+ popd
+ rm -rf $TEMPDIR
+}
+
# Add maven target directory
if $in_sources_dir; then
add_maven_deps_to_classpath
@@ -227,6 +255,8 @@ fi
# restore ordinary behaviour
unset IFS
+expand_native_jars $CLASSPATH
+
# figure out which class to run
if [ "$COMMAND" = "shell" ] ; then
CLASS="org.jruby.Main -X+O ${HBASE_HOME}/bin/hirb.rb"
http://git-wip-us.apache.org/repos/asf/hbase/blob/a404f16c/lib/native/Linux-amd64-64/libhadoop.a
----------------------------------------------------------------------
diff --git a/lib/native/Linux-amd64-64/libhadoop.a b/lib/native/Linux-amd64-64/libhadoop.a
deleted file mode 100644
index 8ab6256..0000000
Binary files a/lib/native/Linux-amd64-64/libhadoop.a and /dev/null differ
http://git-wip-us.apache.org/repos/asf/hbase/blob/a404f16c/lib/native/Linux-amd64-64/libhadoop.la
----------------------------------------------------------------------
diff --git a/lib/native/Linux-amd64-64/libhadoop.la b/lib/native/Linux-amd64-64/libhadoop.la
deleted file mode 100644
index 49d8da2..0000000
--- a/lib/native/Linux-amd64-64/libhadoop.la
+++ /dev/null
@@ -1,35 +0,0 @@
-# libhadoop.la - a libtool library file
-# Generated by ltmain.sh - GNU libtool 1.5.22 (1.1220.2.365 2005/12/18 22:14:06)
-#
-# Please DO NOT delete this file!
-# It is necessary for linking the library.
-
-# The name that we can dlopen(3).
-dlname='libhadoop.so.1'
-
-# Names of this library.
-library_names='libhadoop.so.1.0.0 libhadoop.so.1 libhadoop.so'
-
-# The name of the static archive.
-old_library='libhadoop.a'
-
-# Libraries that this one depends upon.
-dependency_libs=' -L/usr/local/jdk-6u14-64/jre/lib/amd64/server -ljvm -ldl'
-
-# Version information for libhadoop.
-current=1
-age=0
-revision=0
-
-# Is this an already installed library?
-installed=yes
-
-# Should we warn about portability when linking against -modules?
-shouldnotlink=no
-
-# Files to dlopen/dlpreopen
-dlopen=''
-dlpreopen=''
-
-# Directory that this library needs to be installed in:
-libdir='/usr/local/lib'
http://git-wip-us.apache.org/repos/asf/hbase/blob/a404f16c/lib/native/Linux-amd64-64/libhadoop.so
----------------------------------------------------------------------
diff --git a/lib/native/Linux-amd64-64/libhadoop.so b/lib/native/Linux-amd64-64/libhadoop.so
deleted file mode 100644
index 4994011..0000000
Binary files a/lib/native/Linux-amd64-64/libhadoop.so and /dev/null differ
http://git-wip-us.apache.org/repos/asf/hbase/blob/a404f16c/lib/native/Linux-amd64-64/libhadoop.so.1
----------------------------------------------------------------------
diff --git a/lib/native/Linux-amd64-64/libhadoop.so.1 b/lib/native/Linux-amd64-64/libhadoop.so.1
deleted file mode 100644
index 4994011..0000000
Binary files a/lib/native/Linux-amd64-64/libhadoop.so.1 and /dev/null differ
http://git-wip-us.apache.org/repos/asf/hbase/blob/a404f16c/lib/native/Linux-amd64-64/libhadoop.so.1.0.0
----------------------------------------------------------------------
diff --git a/lib/native/Linux-amd64-64/libhadoop.so.1.0.0 b/lib/native/Linux-amd64-64/libhadoop.so.1.0.0
deleted file mode 100644
index 4994011..0000000
Binary files a/lib/native/Linux-amd64-64/libhadoop.so.1.0.0 and /dev/null differ
http://git-wip-us.apache.org/repos/asf/hbase/blob/a404f16c/lib/native/Linux-amd64-64/libhadoopsnappy.a
----------------------------------------------------------------------
diff --git a/lib/native/Linux-amd64-64/libhadoopsnappy.a b/lib/native/Linux-amd64-64/libhadoopsnappy.a
deleted file mode 100644
index 1d6e9b6..0000000
Binary files a/lib/native/Linux-amd64-64/libhadoopsnappy.a and /dev/null differ
http://git-wip-us.apache.org/repos/asf/hbase/blob/a404f16c/lib/native/Linux-amd64-64/libhadoopsnappy.la
----------------------------------------------------------------------
diff --git a/lib/native/Linux-amd64-64/libhadoopsnappy.la b/lib/native/Linux-amd64-64/libhadoopsnappy.la
deleted file mode 100644
index 1c81f2e..0000000
--- a/lib/native/Linux-amd64-64/libhadoopsnappy.la
+++ /dev/null
@@ -1,35 +0,0 @@
-# libhadoopsnappy.la - a libtool library file
-# Generated by ltmain.sh - GNU libtool 1.5.22 (1.1220.2.365 2005/12/18 22:14:06)
-#
-# Please DO NOT delete this file!
-# It is necessary for linking the library.
-
-# The name that we can dlopen(3).
-dlname='libhadoopsnappy.so.0'
-
-# Names of this library.
-library_names='libhadoopsnappy.so.0.0.1 libhadoopsnappy.so.0 libhadoopsnappy.so'
-
-# The name of the static archive.
-old_library='libhadoopsnappy.a'
-
-# Libraries that this one depends upon.
-dependency_libs=' -L/usr/local/jdk-6u14-64/jre/lib/amd64/server -L/data/users/feifei/titan-git-svn/VENDOR/hadoop-0.20/src/native/lib/snappy/lib -ljvm -ldl'
-
-# Version information for libhadoopsnappy.
-current=0
-age=0
-revision=1
-
-# Is this an already installed library?
-installed=no
-
-# Should we warn about portability when linking against -modules?
-shouldnotlink=no
-
-# Files to dlopen/dlpreopen
-dlopen=''
-dlpreopen=''
-
-# Directory that this library needs to be installed in:
-libdir='/usr/local/lib'
http://git-wip-us.apache.org/repos/asf/hbase/blob/a404f16c/lib/native/Linux-amd64-64/libhadoopsnappy.lai
----------------------------------------------------------------------
diff --git a/lib/native/Linux-amd64-64/libhadoopsnappy.lai b/lib/native/Linux-amd64-64/libhadoopsnappy.lai
deleted file mode 100644
index 6d622cb..0000000
--- a/lib/native/Linux-amd64-64/libhadoopsnappy.lai
+++ /dev/null
@@ -1,35 +0,0 @@
-# libhadoopsnappy.la - a libtool library file
-# Generated by ltmain.sh - GNU libtool 1.5.22 (1.1220.2.365 2005/12/18 22:14:06)
-#
-# Please DO NOT delete this file!
-# It is necessary for linking the library.
-
-# The name that we can dlopen(3).
-dlname='libhadoopsnappy.so.0'
-
-# Names of this library.
-library_names='libhadoopsnappy.so.0.0.1 libhadoopsnappy.so.0 libhadoopsnappy.so'
-
-# The name of the static archive.
-old_library='libhadoopsnappy.a'
-
-# Libraries that this one depends upon.
-dependency_libs=' -L/usr/local/jdk-6u14-64/jre/lib/amd64/server -L/data/users/feifei/titan-git-svn/VENDOR/hadoop-0.20/src/native/lib/snappy/lib -ljvm -ldl'
-
-# Version information for libhadoopsnappy.
-current=0
-age=0
-revision=1
-
-# Is this an already installed library?
-installed=yes
-
-# Should we warn about portability when linking against -modules?
-shouldnotlink=no
-
-# Files to dlopen/dlpreopen
-dlopen=''
-dlpreopen=''
-
-# Directory that this library needs to be installed in:
-libdir='/usr/local/lib'
http://git-wip-us.apache.org/repos/asf/hbase/blob/a404f16c/lib/native/Linux-amd64-64/libhadoopsnappy.so
----------------------------------------------------------------------
diff --git a/lib/native/Linux-amd64-64/libhadoopsnappy.so b/lib/native/Linux-amd64-64/libhadoopsnappy.so
deleted file mode 100755
index 971cd1c..0000000
Binary files a/lib/native/Linux-amd64-64/libhadoopsnappy.so and /dev/null differ
http://git-wip-us.apache.org/repos/asf/hbase/blob/a404f16c/lib/native/Linux-amd64-64/libhadoopsnappy.so.0
----------------------------------------------------------------------
diff --git a/lib/native/Linux-amd64-64/libhadoopsnappy.so.0 b/lib/native/Linux-amd64-64/libhadoopsnappy.so.0
deleted file mode 100755
index 971cd1c..0000000
Binary files a/lib/native/Linux-amd64-64/libhadoopsnappy.so.0 and /dev/null differ
http://git-wip-us.apache.org/repos/asf/hbase/blob/a404f16c/lib/native/Linux-amd64-64/libhadoopsnappy.so.0.0.1
----------------------------------------------------------------------
diff --git a/lib/native/Linux-amd64-64/libhadoopsnappy.so.0.0.1 b/lib/native/Linux-amd64-64/libhadoopsnappy.so.0.0.1
deleted file mode 100755
index 971cd1c..0000000
Binary files a/lib/native/Linux-amd64-64/libhadoopsnappy.so.0.0.1 and /dev/null differ
http://git-wip-us.apache.org/repos/asf/hbase/blob/a404f16c/lib/native/Linux-amd64-64/libsnappy.a
----------------------------------------------------------------------
diff --git a/lib/native/Linux-amd64-64/libsnappy.a b/lib/native/Linux-amd64-64/libsnappy.a
deleted file mode 100644
index 16cfca9..0000000
Binary files a/lib/native/Linux-amd64-64/libsnappy.a and /dev/null differ
http://git-wip-us.apache.org/repos/asf/hbase/blob/a404f16c/lib/native/Linux-amd64-64/libsnappy.la
----------------------------------------------------------------------
diff --git a/lib/native/Linux-amd64-64/libsnappy.la b/lib/native/Linux-amd64-64/libsnappy.la
deleted file mode 100755
index d45a9b5..0000000
--- a/lib/native/Linux-amd64-64/libsnappy.la
+++ /dev/null
@@ -1,41 +0,0 @@
-# libsnappy.la - a libtool library file
-# Generated by ltmain.sh (GNU libtool) 2.2.6b Debian-2.2.6b-2ubuntu1
-#
-# Please DO NOT delete this file!
-# It is necessary for linking the library.
-
-# The name that we can dlopen(3).
-dlname='libsnappy.so.1'
-
-# Names of this library.
-library_names='libsnappy.so.1.1.3 libsnappy.so.1 libsnappy.so'
-
-# The name of the static archive.
-old_library='libsnappy.a'
-
-# Linker flags that can not go in dependency_libs.
-inherited_linker_flags=''
-
-# Libraries that this one depends upon.
-dependency_libs=''
-
-# Names of additional weak libraries provided by this library
-weak_library_names=''
-
-# Version information for libsnappy.
-current=2
-age=1
-revision=3
-
-# Is this an already installed library?
-installed=yes
-
-# Should we warn about portability when linking against -modules?
-shouldnotlink=no
-
-# Files to dlopen/dlpreopen
-dlopen=''
-dlpreopen=''
-
-# Directory that this library needs to be installed in:
-libdir='/usr/local/lib'
http://git-wip-us.apache.org/repos/asf/hbase/blob/a404f16c/lib/native/Linux-amd64-64/libsnappy.so
----------------------------------------------------------------------
diff --git a/lib/native/Linux-amd64-64/libsnappy.so b/lib/native/Linux-amd64-64/libsnappy.so
deleted file mode 100755
index 1e5e0b1..0000000
Binary files a/lib/native/Linux-amd64-64/libsnappy.so and /dev/null differ
http://git-wip-us.apache.org/repos/asf/hbase/blob/a404f16c/lib/native/Linux-amd64-64/libsnappy.so.1
----------------------------------------------------------------------
diff --git a/lib/native/Linux-amd64-64/libsnappy.so.1 b/lib/native/Linux-amd64-64/libsnappy.so.1
deleted file mode 100755
index 1e5e0b1..0000000
Binary files a/lib/native/Linux-amd64-64/libsnappy.so.1 and /dev/null differ
http://git-wip-us.apache.org/repos/asf/hbase/blob/a404f16c/lib/native/Linux-amd64-64/libsnappy.so.1.1.3
----------------------------------------------------------------------
diff --git a/lib/native/Linux-amd64-64/libsnappy.so.1.1.3 b/lib/native/Linux-amd64-64/libsnappy.so.1.1.3
deleted file mode 100755
index 1e5e0b1..0000000
Binary files a/lib/native/Linux-amd64-64/libsnappy.so.1.1.3 and /dev/null differ
http://git-wip-us.apache.org/repos/asf/hbase/blob/a404f16c/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 70fbee7..02e8ea3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -658,7 +658,7 @@
<compileSource>1.7</compileSource>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<hbase.version>0.21.0-SNAPSHOT</hbase.version>
- <hadoop.version>4.42343</hadoop.version>
+ <hadoop.version>4.42426</hadoop.version>
<swift.version>0.12.0</swift.version>
<commons-cli.version>1.2</commons-cli.version>
<commons-io.version>2.1</commons-io.version>
@@ -722,6 +722,22 @@
</dependency>
<dependency>
<groupId>com.facebook.hadoop</groupId>
+ <artifactId>hadoop-lzo</artifactId>
+ <classifier>native</classifier>
+ <version>${hadoop.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.facebook.hadoop</groupId>
+ <artifactId>hadoop-snappy-native</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.facebook.hadoop</groupId>
+ <artifactId>hadoop-native</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.facebook.hadoop</groupId>
<artifactId>hadoop-core</artifactId>
<version>${hadoop.version}</version>
<exclusions>
@@ -1010,6 +1026,23 @@
<scope>runtime</scope>
<optional>true</optional>
</dependency>
+ <dependency>
+ <groupId>com.facebook.hadoop</groupId>
+ <artifactId>hadoop-lzo</artifactId>
+ <classifier>native</classifier>
+ <scope>runtime</scope>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>com.facebook.hadoop</groupId>
+ <artifactId>hadoop-snappy-native</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.facebook.hadoop</groupId>
+ <artifactId>hadoop-native</artifactId>
+ <scope>runtime</scope>
+ <optional>true</optional>
+ </dependency>
<dependency>
<groupId>org.apache.zookeeper</groupId>