You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by bu...@apache.org on 2018/06/18 18:39:38 UTC
[4/5] hbase git commit: HBASE-20615 emphasize shaded artifacts in
client tarball.
HBASE-20615 emphasize shaded artifacts in client tarball.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/10cc1da7
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/10cc1da7
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/10cc1da7
Branch: refs/heads/master
Commit: 10cc1da76a926684bae080e508e6a79b4e8fb1fd
Parents: 8710825
Author: Sean Busbey <bu...@apache.org>
Authored: Fri May 18 11:11:42 2018 -0500
Committer: Sean Busbey <bu...@apache.org>
Committed: Mon Jun 18 11:31:13 2018 -0700
----------------------------------------------------------------------
bin/hbase | 226 ++++++++++++++++---
bin/hbase-config.sh | 10 +
hbase-assembly/pom.xml | 21 +-
.../src/main/assembly/client-components.xml | 43 +++-
hbase-assembly/src/main/assembly/client.xml | 131 ++++++-----
hbase-assembly/src/main/assembly/components.xml | 3 +-
.../src/main/assembly/hadoop-two-compat.xml | 80 ++++++-
pom.xml | 6 +
8 files changed, 417 insertions(+), 103 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/10cc1da7/bin/hbase
----------------------------------------------------------------------
diff --git a/bin/hbase b/bin/hbase
index 4f1c854..559a02e 100755
--- a/bin/hbase
+++ b/bin/hbase
@@ -71,11 +71,18 @@ if [ -d "${HBASE_HOME}/target" ]; then
in_dev_env=true
fi
+# Detect if we are in the omnibus tarball
+in_omnibus_tarball="false"
+if [ -f "${HBASE_HOME}/bin/hbase-daemons.sh" ]; then
+ in_omnibus_tarball="true"
+fi
+
read -d '' options_string << EOF
Options:
- --config DIR Configuration direction to use. Default: ./conf
- --hosts HOSTS Override the list in 'regionservers' file
- --auth-as-server Authenticate to ZooKeeper using servers configuration
+ --config DIR Configuration direction to use. Default: ./conf
+ --hosts HOSTS Override the list in 'regionservers' file
+ --auth-as-server Authenticate to ZooKeeper using servers configuration
+ --internal-classpath Skip attempting to use client facing jars (WARNING: unstable results between versions)
EOF
# if no args specified, show usage
if [ $# = 0 ]; then
@@ -87,16 +94,18 @@ if [ $# = 0 ]; then
echo " shell Run the HBase shell"
echo " hbck Run the hbase 'fsck' tool"
echo " snapshot Tool for managing snapshots"
- echo " wal Write-ahead-log analyzer"
- echo " hfile Store file analyzer"
- echo " zkcli Run the ZooKeeper shell"
- echo " master Run an HBase HMaster node"
- echo " regionserver Run an HBase HRegionServer node"
- echo " zookeeper Run a ZooKeeper server"
- echo " rest Run an HBase REST server"
- echo " thrift Run the HBase Thrift server"
- echo " thrift2 Run the HBase Thrift2 server"
- echo " clean Run the HBase clean up script"
+ if [ "${in_omnibus_tarball}" = "true" ]; then
+ echo " wal Write-ahead-log analyzer"
+ echo " hfile Store file analyzer"
+ echo " zkcli Run the ZooKeeper shell"
+ echo " master Run an HBase HMaster node"
+ echo " regionserver Run an HBase HRegionServer node"
+ echo " zookeeper Run a ZooKeeper server"
+ echo " rest Run an HBase REST server"
+ echo " thrift Run the HBase Thrift server"
+ echo " thrift2 Run the HBase Thrift2 server"
+ echo " clean Run the HBase clean up script"
+ fi
echo " classpath Dump hbase CLASSPATH"
echo " mapredcp Dump CLASSPATH entries required by mapreduce"
echo " pe Run PerformanceEvaluation"
@@ -186,9 +195,99 @@ for f in $HBASE_HOME/hbase-jars/hbase*.jar; do
fi
done
+#If avail, add Hadoop to the CLASSPATH and to the JAVA_LIBRARY_PATH
+# Allow this functionality to be disabled
+if [ "$HBASE_DISABLE_HADOOP_CLASSPATH_LOOKUP" != "true" ] ; then
+ HADOOP_IN_PATH=$(PATH="${HADOOP_HOME:-${HADOOP_PREFIX}}/bin:$PATH" which hadoop 2>/dev/null)
+fi
+
# Add libs to CLASSPATH
-for f in $HBASE_HOME/lib/*.jar; do
- CLASSPATH=${CLASSPATH}:$f;
+declare shaded_jar
+
+if [ "${INTERNAL_CLASSPATH}" != "true" ]; then
+ # find our shaded jars
+ declare shaded_client
+ declare shaded_client_byo_hadoop
+ declare shaded_mapreduce
+ for f in "${HBASE_HOME}"/lib/shaded-clients/hbase-shaded-client*.jar; do
+ if [[ "${f}" =~ byo-hadoop ]]; then
+ shaded_client_byo_hadoop="${f}"
+ else
+ shaded_client="${f}"
+ fi
+ done
+ for f in "${HBASE_HOME}"/lib/shaded-clients/hbase-shaded-mapreduce*.jar; do
+ shaded_mapreduce="${f}"
+ done
+
+ # If command can use our shaded client, use it
+ declare -a commands_in_client_jar=("classpath" "version")
+ for c in "${commands_in_client_jar[@]}"; do
+ if [ "${COMMAND}" = "${c}" ]; then
+ if [ -n "${HADOOP_IN_PATH}" ] && [ -f "${HADOOP_IN_PATH}" ]; then
+ # If we didn't find a jar above, this will just be blank and the
+ # check below will then default back to the internal classpath.
+ shaded_jar="${shaded_client_byo_hadoop}"
+ else
+ # If we didn't find a jar above, this will just be blank and the
+ # check below will then default back to the internal classpath.
+ shaded_jar="${shaded_client}"
+ fi
+ break
+ fi
+ done
+
+ # If command needs our shaded mapreduce, use it
+ # N.B "mapredcp" is not included here because in the shaded case it skips our built classpath
+ declare -a commands_in_mr_jar=("hbck" "snapshot" "canary" "regionsplitter" "pre-upgrade")
+ for c in "${commands_in_mr_jar[@]}"; do
+ if [ "${COMMAND}" = "${c}" ]; then
+ # If we didn't find a jar above, this will just be blank and the
+ # check below will then default back to the internal classpath.
+ shaded_jar="${shaded_mapreduce}"
+ break
+ fi
+ done
+
+ # Some commands specifically only can use shaded mapreduce when we'll get a full hadoop classpath at runtime
+ if [ -n "${HADOOP_IN_PATH}" ] && [ -f "${HADOOP_IN_PATH}" ]; then
+ declare -a commands_in_mr_need_hadoop=("backup" "restore" "rowcounter" "cellcounter")
+ for c in "${commands_in_mr_need_hadoop[@]}"; do
+ if [ "${COMMAND}" = "${c}" ]; then
+ # If we didn't find a jar above, this will just be blank and the
+ # check below will then default back to the internal classpath.
+ shaded_jar="${shaded_mapreduce}"
+ break
+ fi
+ done
+ fi
+fi
+
+
+if [ -n "${shaded_jar}" ] && [ -f "${shaded_jar}" ]; then
+ CLASSPATH="${CLASSPATH}:${shaded_jar}"
+# fall through to grabbing all the lib jars and hope we're in the omnibus tarball
+#
+# N.B. shell specifically can't rely on the shaded artifacts because RSGroups is only
+# available as non-shaded
+#
+# N.B. pe and ltt can't easily rely on shaded artifacts because they live in hbase-mapreduce:test-jar
+# and need some other jars that haven't been relocated. Currently enumerating that list
+# is too hard to be worth it.
+#
+else
+ for f in $HBASE_HOME/lib/*.jar; do
+ CLASSPATH=${CLASSPATH}:$f;
+ done
+ # make it easier to check for shaded/not later on.
+ shaded_jar=""
+fi
+for f in "${HBASE_HOME}"/lib/client-facing-thirdparty/*.jar; do
+ if [[ ! "${f}" =~ ^.*/htrace-core-3.*\.jar$ ]] && \
+ [ "${f}" != "htrace-core.jar$" ] && \
+ [[ ! "${f}" =~ ^.*/slf4j-log4j.*$ ]]; then
+ CLASSPATH="${CLASSPATH}:${f}"
+ fi
done
# default log directory & file
@@ -201,9 +300,9 @@ fi
function append_path() {
if [ -z "$1" ]; then
- echo $2
+ echo "$2"
else
- echo $1:$2
+ echo "$1:$2"
fi
}
@@ -214,18 +313,34 @@ if [ "$HBASE_LIBRARY_PATH" != "" ]; then
JAVA_LIBRARY_PATH=$(append_path "$JAVA_LIBRARY_PATH" "$HBASE_LIBRARY_PATH")
fi
-#If avail, add Hadoop to the CLASSPATH and to the JAVA_LIBRARY_PATH
-# Allow this functionality to be disabled
-if [ "$HBASE_DISABLE_HADOOP_CLASSPATH_LOOKUP" != "true" ] ; then
- HADOOP_IN_PATH=$(PATH="${HADOOP_HOME:-${HADOOP_PREFIX}}/bin:$PATH" which hadoop 2>/dev/null)
- if [ -f ${HADOOP_IN_PATH} ]; then
- HADOOP_JAVA_LIBRARY_PATH=$(HADOOP_CLASSPATH="$CLASSPATH" ${HADOOP_IN_PATH} \
- org.apache.hadoop.hbase.util.GetJavaProperty java.library.path 2>/dev/null)
- if [ -n "$HADOOP_JAVA_LIBRARY_PATH" ]; then
- JAVA_LIBRARY_PATH=$(append_path "${JAVA_LIBRARY_PATH}" "$HADOOP_JAVA_LIBRARY_PATH")
- fi
- CLASSPATH=$(append_path "${CLASSPATH}" `${HADOOP_IN_PATH} classpath 2>/dev/null`)
+#If configured and available, add Hadoop to the CLASSPATH and to the JAVA_LIBRARY_PATH
+if [ -n "${HADOOP_IN_PATH}" ] && [ -f "${HADOOP_IN_PATH}" ]; then
+ HADOOP_JAVA_LIBRARY_PATH=$(HADOOP_CLASSPATH="$CLASSPATH" "${HADOOP_IN_PATH}" \
+ org.apache.hadoop.hbase.util.GetJavaProperty java.library.path)
+ if [ -n "$HADOOP_JAVA_LIBRARY_PATH" ]; then
+ JAVA_LIBRARY_PATH=$(append_path "${JAVA_LIBRARY_PATH}" "$HADOOP_JAVA_LIBRARY_PATH")
fi
+ CLASSPATH=$(append_path "${CLASSPATH}" "$(${HADOOP_IN_PATH} classpath 2>/dev/null)")
+else
+ # Otherwise, if we're providing Hadoop we should include htrace 3 if we were built with a version that needs it.
+ for f in "${HBASE_HOME}"/lib/client-facing-thirdparty/htrace-core-3*.jar "${HBASE_HOME}"/lib/client-facing-thirdparty/htrace-core.jar; do
+ if [ -f "${f}" ]; then
+ CLASSPATH="${CLASSPATH}:${f}"
+ break
+ fi
+ done
+ # Some commands require special handling when using shaded jars. For these cases, we rely on hbase-shaded-mapreduce
+ # instead of hbase-shaded-client* because we make use of some IA.Private classes that aren't in the latter. However,
+ # we don't invoke them using the "hadoop jar" command so we need to ensure there are some Hadoop classes available
+ # when we're not doing runtime hadoop classpath lookup.
+ #
+ # luckily the set of classes we need are those packaged in the shaded-client.
+ for c in "${commands_in_mr_jar[@]}"; do
+ if [ "${COMMAND}" = "${c}" ] && [ -n "${shaded_jar}" ]; then
+ CLASSPATH="${CLASSPATH}:${shaded_client:?We couldn\'t find the shaded client jar even though we did find the shaded MR jar. for command ${COMMAND} we need both. please use --internal-classpath as a workaround.}"
+ break
+ fi
+ done
fi
# Add user-specified CLASSPATH last
@@ -250,11 +365,11 @@ if [ -d "${HBASE_HOME}/build/native" -o -d "${HBASE_HOME}/lib/native" ]; then
JAVA_PLATFORM=`CLASSPATH=${CLASSPATH} ${JAVA} org.apache.hadoop.util.PlatformName | sed -e "s/ /_/g"`
fi
if [ -d "$HBASE_HOME/build/native" ]; then
- JAVA_LIBRARY_PATH=$(append_path "$JAVA_LIBRARY_PATH" ${HBASE_HOME}/build/native/${JAVA_PLATFORM}/lib)
+ JAVA_LIBRARY_PATH=$(append_path "$JAVA_LIBRARY_PATH" "${HBASE_HOME}/build/native/${JAVA_PLATFORM}/lib")
fi
if [ -d "${HBASE_HOME}/lib/native" ]; then
- JAVA_LIBRARY_PATH=$(append_path "$JAVA_LIBRARY_PATH" ${HBASE_HOME}/lib/native/${JAVA_PLATFORM})
+ JAVA_LIBRARY_PATH=$(append_path "$JAVA_LIBRARY_PATH" "${HBASE_HOME}/lib/native/${JAVA_PLATFORM}")
fi
fi
@@ -379,8 +494,30 @@ elif [ "$COMMAND" = "zkcli" ] ; then
done
elif [ "$COMMAND" = "backup" ] ; then
CLASS='org.apache.hadoop.hbase.backup.BackupDriver'
+ if [ -n "${shaded_jar}" ] ; then
+ for f in "${HBASE_HOME}"/lib/hbase-backup*.jar; do
+ if [ -f "${f}" ]; then
+ CLASSPATH="${CLASSPATH}:${f}"
+ break
+ fi
+ done
+ fi
elif [ "$COMMAND" = "restore" ] ; then
CLASS='org.apache.hadoop.hbase.backup.RestoreDriver'
+ if [ -n "${shaded_jar}" ] ; then
+ for f in "${HBASE_HOME}"/lib/hbase-backup*.jar; do
+ if [ -f "${f}" ]; then
+ CLASSPATH="${CLASSPATH}:${f}"
+ break
+ fi
+ done
+ for f in "${HBASE_HOME}"/lib/commons-lang3*.jar; do
+ if [ -f "${f}" ]; then
+ CLASSPATH="${CLASSPATH}:${f}"
+ break
+ fi
+ done
+ fi
elif [ "$COMMAND" = "upgrade" ] ; then
echo "This command was used to upgrade to HBase 0.96, it was removed in HBase 2.0.0."
echo "Please follow the documentation at http://hbase.apache.org/book.html#upgrading."
@@ -451,9 +588,24 @@ elif [ "$COMMAND" = "clean" ] ; then
"$bin"/hbase-cleanup.sh --config ${HBASE_CONF_DIR} $@
exit $?
elif [ "$COMMAND" = "mapredcp" ] ; then
+ # If we didn't find a jar above, this will just be blank and the
+ # check below will then default back to the internal classpath.
+ shaded_jar="${shaded_mapreduce}"
+ if [ "${INTERNAL_CLASSPATH}" != "true" ] && [ -f "${shaded_jar}" ]; then
+ echo -n "${shaded_jar}"
+ for f in "${HBASE_HOME}"/lib/client-facing-thirdparty/*.jar; do
+ if [[ ! "${f}" =~ ^.*/htrace-core-3.*\.jar$ ]] && \
+ [ "${f}" != "htrace-core.jar$" ] && \
+ [[ ! "${f}" =~ ^.*/slf4j-log4j.*$ ]]; then
+ echo -n ":${f}"
+ fi
+ done
+ echo ""
+ exit 0
+ fi
CLASS='org.apache.hadoop.hbase.util.MapreduceDependencyClasspathTool'
elif [ "$COMMAND" = "classpath" ] ; then
- echo $CLASSPATH
+ echo "$CLASSPATH"
exit 0
elif [ "$COMMAND" = "pe" ] ; then
CLASS='org.apache.hadoop.hbase.PerformanceEvaluation'
@@ -500,8 +652,20 @@ else
fi
HEAP_SETTINGS="$JAVA_HEAP_MAX $JAVA_OFFHEAP_MAX"
+# by now if we're running a command it means we need logging
+for f in ${HBASE_HOME}/lib/client-facing-thirdparty/slf4j-log4j*.jar; do
+ if [ -f "${f}" ]; then
+ CLASSPATH="${CLASSPATH}:${f}"
+ break
+ fi
+done
+
# Exec unless HBASE_NOEXEC is set.
export CLASSPATH
+if [ "${DEBUG}" = "true" ]; then
+ echo "classpath=${CLASSPATH}" >&2
+ HBASE_OPTS="${HBASE_OPTS} -Xdiag"
+fi
if [ "${HBASE_NOEXEC}" != "" ]; then
"$JAVA" -Dproc_$COMMAND -XX:OnOutOfMemoryError="kill -9 %p" $HEAP_SETTINGS $HBASE_OPTS $CLASS "$@"
http://git-wip-us.apache.org/repos/asf/hbase/blob/10cc1da7/bin/hbase-config.sh
----------------------------------------------------------------------
diff --git a/bin/hbase-config.sh b/bin/hbase-config.sh
index 2e95ae7..1054751 100644
--- a/bin/hbase-config.sh
+++ b/bin/hbase-config.sh
@@ -84,6 +84,16 @@ do
exit 1
fi
shift
+ elif [ "--internal-classpath" = "$1" ]
+ then
+ shift
+ # shellcheck disable=SC2034
+ INTERNAL_CLASSPATH="true"
+ elif [ "--debug" = "$1" ]
+ then
+ shift
+ # shellcheck disable=SC2034
+ DEBUG="true"
else
# Presume we are at end of options and break
break
http://git-wip-us.apache.org/repos/asf/hbase/blob/10cc1da7/hbase-assembly/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-assembly/pom.xml b/hbase-assembly/pom.xml
index c0e577c..eeeff4a 100644
--- a/hbase-assembly/pom.xml
+++ b/hbase-assembly/pom.xml
@@ -190,6 +190,19 @@
</plugins>
</build>
<dependencies>
+ <!-- client artifacts for downstream use -->
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-shaded-client</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-shaded-client-byo-hadoop</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-shaded-mapreduce</artifactId>
+ </dependency>
<!-- Intra-project dependencies -->
<dependency>
<groupId>org.apache.hbase</groupId>
@@ -324,14 +337,6 @@
<groupId>jline</groupId>
<artifactId>jline</artifactId>
</dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-shaded-client</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-shaded-mapreduce</artifactId>
- </dependency>
</dependencies>
<profiles>
<profile>
http://git-wip-us.apache.org/repos/asf/hbase/blob/10cc1da7/hbase-assembly/src/main/assembly/client-components.xml
----------------------------------------------------------------------
diff --git a/hbase-assembly/src/main/assembly/client-components.xml b/hbase-assembly/src/main/assembly/client-components.xml
index 2fd1b57..2369f28 100644
--- a/hbase-assembly/src/main/assembly/client-components.xml
+++ b/hbase-assembly/src/main/assembly/client-components.xml
@@ -32,7 +32,8 @@
<directory>${project.basedir}/..</directory>
<outputDirectory>.</outputDirectory>
<includes>
- <include>CHANGES.txt</include>
+ <include>CHANGES.md</include>
+ <include>RELEASENOTES.md</include>
<include>README.txt</include>
</includes>
<fileMode>0644</fileMode>
@@ -56,7 +57,6 @@
<include>hbase-config.sh</include>
<include>hbase-jruby</include>
<include>hirb.rb</include>
- <include></include>
</includes>
<fileMode>0755</fileMode>
<directoryMode>0755</directoryMode>
@@ -88,5 +88,44 @@
<include>*.dylib</include>
</includes>
</fileSet>
+ <!-- This is only necessary until maven fixes the intra-project dependency bug
+ in maven 3.0. Until then, we have to include the test jars for sub-projects. When
+ fixed, the below dependencySet stuff is sufficient for pulling in the test jars as
+ well, as long as they are added as dependencies in this project. Right now, we only
+ have 1 submodule to accumulate, but we can copy/paste as necessary until maven is
+ fixed. -->
+ <!-- Used by PE and ltt -->
+ <fileSet>
+ <directory>${project.basedir}/../hbase-server/target/</directory>
+ <outputDirectory>lib</outputDirectory>
+ <includes>
+ <include>${server.test.jar}</include>
+ </includes>
+ <fileMode>0644</fileMode>
+ </fileSet>
+ <fileSet>
+ <directory>${project.basedir}/../hbase-mapreduce/target/</directory>
+ <outputDirectory>lib</outputDirectory>
+ <includes>
+ <include>${mapreduce.test.jar}</include>
+ </includes>
+ <fileMode>0644</fileMode>
+ </fileSet>
+ <fileSet>
+ <directory>${project.basedir}/../hbase-common/target/</directory>
+ <outputDirectory>lib</outputDirectory>
+ <includes>
+ <include>${common.test.jar}</include>
+ </includes>
+ <fileMode>0644</fileMode>
+ </fileSet>
+ <fileSet>
+ <directory>${project.basedir}/../hbase-zookeeper/target/</directory>
+ <outputDirectory>lib</outputDirectory>
+ <includes>
+ <include>${zookeeper.test.jar}</include>
+ </includes>
+ <fileMode>0644</fileMode>
+ </fileSet>
</fileSets>
</component>
http://git-wip-us.apache.org/repos/asf/hbase/blob/10cc1da7/hbase-assembly/src/main/assembly/client.xml
----------------------------------------------------------------------
diff --git a/hbase-assembly/src/main/assembly/client.xml b/hbase-assembly/src/main/assembly/client.xml
index 7951961..c875b95 100644
--- a/hbase-assembly/src/main/assembly/client.xml
+++ b/hbase-assembly/src/main/assembly/client.xml
@@ -31,73 +31,37 @@
<componentDescriptor>src/main/assembly/client-components.xml</componentDescriptor>
</componentDescriptors>
<moduleSets>
+ <!-- include regular jars so the shell can use them -->
<moduleSet>
<useAllReactorProjects>true</useAllReactorProjects>
<includes>
- <!-- Keep this list sorted by name -->
- <include>org.apache.hbase:hbase-annotations</include>
- <include>org.apache.hbase:hbase-client</include>
- <include>org.apache.hbase:hbase-common</include>
- <include>org.apache.hbase:hbase-hadoop-compat</include>
- <include>org.apache.hbase:hbase-hadoop2-compat</include>
- <include>org.apache.hbase:hbase-mapreduce</include>
- <include>org.apache.hbase:hbase-metrics</include>
- <include>org.apache.hbase:hbase-metrics-api</include>
- <include>org.apache.hbase:hbase-procedure</include>
- <include>org.apache.hbase:hbase-protocol</include>
- <include>org.apache.hbase:hbase-protocol-shaded</include>
- <include>org.apache.hbase:hbase-server</include>
- <include>org.apache.hbase:hbase-zookeeper</include>
+ <include>org.apache.hbase:hbase-shell</include>
</includes>
- <!-- Binaries for the dependencies also go in the hbase-jars directory -->
<binaries>
- <outputDirectory>lib</outputDirectory>
<unpack>false</unpack>
- <dependencySets>
- <!-- Exclude jruby-complete from hbase_home/lib -->
- <dependencySet>
+ <outputDirectory>lib</outputDirectory>
+ <dependencySets>
+ <dependencySet>
+ <!-- Exclude libraries that we put in their own dirs under lib/ -->
<excludes>
+ <exclude>org.jruby:jruby-complete</exclude>
<exclude>com.sun.jersey:*</exclude>
<exclude>com.sun.jersey.contribs:*</exclude>
- <!-- Exclude jars which typical clients don't need -->
- <exclude>org.apache.hbase:hbase-external-blockcache</exclude>
- <exclude>org.apache.hbase:hbase-http</exclude>
- <exclude>org.apache.hbase:hbase-replication</exclude>
- <exclude>org.apache.hbase:hbase-rest</exclude>
- <exclude>org.apache.hbase:hbase-rsgroup</exclude>
- <exclude>org.apache.hbase:hbase-shaded-client</exclude>
- <exclude>org.apache.hbase:hbase-shaded-mapreduce</exclude>
- <!-- At present, hbase-shell doesn't actually contain
- any Java code we need to include. Ruby files are
- copied elsewhere in this descriptor. -->
- <exclude>org.apache.hbase:hbase-shell</exclude>
- <exclude>org.apache.hbase:hbase-thrift</exclude>
- <exclude>org.jruby:jruby-complete</exclude>
+ <exclude>jline:jline</exclude>
+ <exclude>com.github.stephenc.findbugs:findbugs-annotations</exclude>
+ <exclude>commons-logging:commons-logging</exclude>
+ <exclude>log4j:log4j</exclude>
+ <exclude>org.apache.hbase:hbase-shaded-client</exclude>
+ <exclude>org.apache.hbase:hbase-shaded-client-byo-hadoop</exclude>
+ <exclude>org.apache.hbase:hbase-shaded-mapreduce</exclude>
+ <exclude>org.apache.htrace:htrace-core4</exclude>
+ <exclude>org.apache.htrace:htrace-core</exclude>
+ <exclude>org.apache.yetus:audience-annotations</exclude>
+ <exclude>org.slf4j:slf4j-api</exclude>
+ <exclude>org.slf4j:slf4j-log4j12</exclude>
</excludes>
- </dependencySet>
- </dependencySets>
- </binaries>
- </moduleSet>
- <moduleSet>
- <useAllReactorProjects>true</useAllReactorProjects>
- <includes>
- <include>org.apache.hbase:hbase-shaded-client</include>
- <include>org.apache.hbase:hbase-shaded-mapreduce</include>
- </includes>
- <!-- Binaries for the dependencies also go in the hbase-jars directory -->
- <binaries>
- <outputDirectory>shaded-lib</outputDirectory>
- <unpack>false</unpack>
- <dependencySets>
- <dependencySet>
- <includes>
- <!-- Keep this list sorted by name -->
- <include>org.apache.hbase:hbase-shaded-client</include>
- <include>org.apache.hbase:hbase-shaded-mapreduce</include>
- </includes>
- <useTransitiveDependencies>false</useTransitiveDependencies>
- </dependencySet>
- </dependencySets>
+ </dependencySet>
+ </dependencySets>
</binaries>
</moduleSet>
</moduleSets>
@@ -123,15 +87,66 @@
</file>
</files>
+ <dependencySets>
+ <dependencySet>
+ <outputDirectory>lib/shaded-clients</outputDirectory>
+ <includes>
+ <include>org.apache.hbase:hbase-shaded-client</include>
+ <include>org.apache.hbase:hbase-shaded-mapreduce</include>
+ <include>org.apache.hbase:hbase-shaded-client-byo-hadoop</include>
+ </includes>
+ </dependencySet>
<!-- Add jruby-complete to hbase_home/lib/ruby.
Update JRUBY_PACKAGED_WITH_HBASE in bin/hbase and hbase.cmd if you would like to update outputDirectory below -->
- <dependencySets>
<dependencySet>
<outputDirectory>lib/ruby</outputDirectory>
<includes>
<include>org.jruby:jruby-complete</include>
</includes>
</dependencySet>
+ <!-- Include third party dependencies the shaded clients expose in the lib directory
+ N.B. this will conflict with the omnibus tarball but these should be precisely
+ the same artifacts so blind overwrite of either should be fine.
+ -->
+ <dependencySet>
+ <outputDirectory>lib/client-facing-thirdparty</outputDirectory>
+ <useTransitiveDependencies>true</useTransitiveDependencies>
+ <!-- Unfortunately, we have to whitelist these because Maven
+ currently doesn't use the dependency-reduced-pom after
+ the shaded module has done its thing. That means if we
+ did this as "transitives of the shaded modules" we'd
+ get a duplication of all the jars we already have in our
+ shaded artifacts. See MNG-5899.
+
+ Check that things match by listing files and making
+ sure the runtime scoped things are all present in the
+ tarball. e.g.:
+
+ for module in hbase-shaded-mapreduce hbase-shaded-client; do
+ mvn dependency:list -f hbase-shaded/${module}/dependency-reduced-pom.xml
+ done | \
+ grep -E "runtime|compile" | \
+ grep -v -E "junit|(optional)" | \
+ cut -d ' ' -f 3- | \
+ sort -u
+
+ TODO we should check this in nightly
+
+ Alternatively, we could
+ stop waiting for Maven to fix this and build the client
+ tarball in a different build.
+ -->
+ <includes>
+ <include>com.github.stephenc.findbugs:findbugs-annotations</include>
+ <include>commons-logging:commons-logging</include>
+ <include>log4j:log4j</include>
+ <include>org.apache.htrace:htrace-core4</include>
+ <include>org.apache.htrace:htrace-core</include>
+ <include>org.apache.yetus:audience-annotations</include>
+ <include>org.slf4j:slf4j-api</include>
+ <include>org.slf4j:slf4j-log4j12</include>
+ </includes>
+ </dependencySet>
</dependencySets>
</assembly>
http://git-wip-us.apache.org/repos/asf/hbase/blob/10cc1da7/hbase-assembly/src/main/assembly/components.xml
----------------------------------------------------------------------
diff --git a/hbase-assembly/src/main/assembly/components.xml b/hbase-assembly/src/main/assembly/components.xml
index 59502d6..2eb16e7 100644
--- a/hbase-assembly/src/main/assembly/components.xml
+++ b/hbase-assembly/src/main/assembly/components.xml
@@ -37,7 +37,8 @@
<directory>${project.basedir}/..</directory>
<outputDirectory>.</outputDirectory>
<includes>
- <include>CHANGES.txt</include>
+ <include>CHANGES.md</include>
+ <include>RELEASENOTES.md</include>
<include>README.txt</include>
</includes>
<fileMode>0644</fileMode>
http://git-wip-us.apache.org/repos/asf/hbase/blob/10cc1da7/hbase-assembly/src/main/assembly/hadoop-two-compat.xml
----------------------------------------------------------------------
diff --git a/hbase-assembly/src/main/assembly/hadoop-two-compat.xml b/hbase-assembly/src/main/assembly/hadoop-two-compat.xml
index 69a800b..7243783 100644
--- a/hbase-assembly/src/main/assembly/hadoop-two-compat.xml
+++ b/hbase-assembly/src/main/assembly/hadoop-two-compat.xml
@@ -66,18 +66,51 @@
<outputDirectory>lib</outputDirectory>
<unpack>false</unpack>
<dependencySets>
- <!-- Exclude jruby-complete from hbase_home/lib -->
<dependencySet>
+ <!-- Exclude libraries that we put in their own dirs under lib/ -->
<excludes>
<exclude>org.jruby:jruby-complete</exclude>
<exclude>com.sun.jersey:*</exclude>
<exclude>com.sun.jersey.contribs:*</exclude>
<exclude>jline:jline</exclude>
+ <exclude>org.apache.hbase:hbase-shaded-client</exclude>
+ <exclude>org.apache.hbase:hbase-shaded-client-byo-hadoop</exclude>
+ <exclude>org.apache.hbase:hbase-shaded-mapreduce</exclude>
+ <exclude>com.github.stephenc.findbugs:findbugs-annotations</exclude>
+ <exclude>commons-logging:commons-logging</exclude>
+ <exclude>log4j:log4j</exclude>
+ <exclude>org.apache.htrace:htrace-core4</exclude>
+ <exclude>org.apache.htrace:htrace-core</exclude>
+ <exclude>org.apache.yetus:audience-annotations</exclude>
+ <exclude>org.slf4j:slf4j-api</exclude>
+ <exclude>org.slf4j:slf4j-log4j12</exclude>
</excludes>
</dependencySet>
</dependencySets>
</binaries>
</moduleSet>
+ <!-- Include shaded clients in their own directory -->
+ <moduleSet>
+ <useAllReactorProjects>true</useAllReactorProjects>
+ <includes>
+ <include>org.apache.hbase:hbase-shaded-client</include>
+ <include>org.apache.hbase:hbase-shaded-mapreduce</include>
+ <include>org.apache.hbase:hbase-shaded-client-byo-hadoop</include>
+ </includes>
+ <binaries>
+ <outputDirectory>lib/shaded-clients</outputDirectory>
+ <unpack>false</unpack>
+ <dependencySets>
+ <dependencySet>
+ <includes>
+ <include>org.apache.hbase:hbase-shaded-client</include>
+ <include>org.apache.hbase:hbase-shaded-mapreduce</include>
+ <include>org.apache.hbase:hbase-shaded-client-byo-hadoop</include>
+ </includes>
+ </dependencySet>
+ </dependencySets>
+ </binaries>
+ </moduleSet>
</moduleSets>
<!-- Include the generated LICENSE and NOTICE files -->
<files>
@@ -101,15 +134,56 @@
</file>
</files>
- <!-- Add jruby-complete to hbase_home/lib/ruby.
- Update JRUBY_PACKAGED_WITH_HBASE in bin/hbase and hbase.cmd if you would like to update outputDirectory below -->
<dependencySets>
+ <!-- Add jruby-complete to hbase_home/lib/ruby.
+ Update JRUBY_PACKAGED_WITH_HBASE in bin/hbase and hbase.cmd if you would like to update outputDirectory below -->
<dependencySet>
<outputDirectory>lib/ruby</outputDirectory>
<includes>
<include>org.jruby:jruby-complete</include>
</includes>
</dependencySet>
+ <!-- Include third party dependencies the shaded clients expose in the lib directory
+ -->
+ <dependencySet>
+ <outputDirectory>lib/client-facing-thirdparty</outputDirectory>
+ <useTransitiveDependencies>true</useTransitiveDependencies>
+ <!-- Unfortunately, we have to whitelist these because Maven
+ currently doesn't use the dependency-reduced-pom after
+ the shaded module has done its thing. That means if we
+ did this as "transitives of the shaded modules" we'd
+ get a duplication of all the jars we already have in our
+ shaded artifacts. See MNG-5899.
+
+ Check that things match by listing files and making
+ sure the runtime scoped things are all present in the
+ tarball. e.g.:
+
+ for module in hbase-shaded-mapreduce hbase-shaded-client; do
+ mvn dependency:list -f hbase-shaded/${module}/dependency-reduced-pom.xml
+ done | \
+ grep -E "runtime|compile" | \
+ grep -v -E "junit|(optional)" | \
+ cut -d ' ' -f 3- | \
+ sort -u
+
+ TODO we should check this in nightly
+
+ Alternatively, we could
+ stop waiting for Maven to fix this and build the client
+ tarball in a different build.
+ -->
+ <includes>
+ <include>com.github.stephenc.findbugs:findbugs-annotations</include>
+ <include>commons-logging:commons-logging</include>
+ <include>log4j:log4j</include>
+ <include>org.apache.htrace:htrace-core4</include>
+ <include>org.apache.htrace:htrace-core</include>
+ <include>org.apache.yetus:audience-annotations</include>
+ <include>org.slf4j:slf4j-api</include>
+ <include>org.slf4j:slf4j-log4j12</include>
+ </includes>
+ </dependencySet>
<dependencySet>
<outputDirectory>lib/zkcli</outputDirectory>
<includes>
http://git-wip-us.apache.org/repos/asf/hbase/blob/10cc1da7/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index b9a1b77..ea02fe8 100755
--- a/pom.xml
+++ b/pom.xml
@@ -1555,6 +1555,7 @@
<annotations.test.jar>hbase-annotations-${project.version}-tests.jar</annotations.test.jar>
<rsgroup.test.jar>hbase-rsgroup-${project.version}-tests.jar</rsgroup.test.jar>
<mapreduce.test.jar>hbase-mapreduce-${project.version}-tests.jar</mapreduce.test.jar>
+ <zookeeper.test.jar>hbase-zookeeper-${project.version}-tests.jar</zookeeper.test.jar>
<shell-executable>bash</shell-executable>
<surefire.version>2.21.0</surefire.version>
<surefire.provider>surefire-junit47</surefire.provider>
@@ -1860,6 +1861,11 @@
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-shaded-client-byo-hadoop</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
<artifactId>hbase-shaded-mapreduce</artifactId>
<version>${project.version}</version>
</dependency>