You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by st...@apache.org on 2020/07/31 05:00:03 UTC
[phoenix] branch master updated: PHOENIX-6028 Change default HBase
profile to 2.1 in 5.x
This is an automated email from the ASF dual-hosted git repository.
stoty pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/master by this push:
new 82c85a1 PHOENIX-6028 Change default HBase profile to 2.1 in 5.x
82c85a1 is described below
commit 82c85a1dccc84796e02d1eeb1157f2430ae69679
Author: Istvan Toth <st...@apache.org>
AuthorDate: Thu Jul 23 08:31:55 2020 +0200
PHOENIX-6028 Change default HBase profile to 2.1 in 5.x
* change default HBase profile
* bump the default HBase version to 2.2.5 for the 2.2 profile
* update BUILDING.md and add instuctions for PHOENIX-5593
* add dev/cache-apache-project-artifact.sh
* add dev/rebuild_hbase.sh (for CI jobs)
---
BUILDING.md | 35 +++++++--
dev/cache-apache-project-artifact.sh | 139 +++++++++++++++++++++++++++++++++++
dev/rebuild_hbase.sh | 29 ++++++++
pom.xml | 52 ++++++-------
4 files changed, 219 insertions(+), 36 deletions(-)
diff --git a/BUILDING.md b/BUILDING.md
index dc75d21..0bad82f 100644
--- a/BUILDING.md
+++ b/BUILDING.md
@@ -27,6 +27,27 @@ Building Apache Phoenix
Phoenix uses Maven (3.X) to build all its necessary resources.
+HBase 2 and Hadoop 3
+--------------------
+Phoenix 5.x requires Hadoop 3. While HBase 2.x is compatible with Hadoop 3, the public Maven Hbase
+artifacts are built with Hadoop 2, and are not.
+
+For this reason, when building Phoenix with an Hbase version later than 2.2.4, you need to rebuild
+HBase with Hadoop 3, and install it to the local maven repo of the build host.
+
+`$ wget https://downloads.apache.org/hbase/2.2.5/hbase-2.2.5-src.tar.gz`
+`$ tar xfvz hbase-2.2.5-src.tar.gz`
+`$ cd hbase-2.2.5`
+`$ mvn install -Dhadoop.profile=3.0 -DskipTests`
+
+Replace 2.2.5 with the actual Hbase version you are using in the Phoenix build.
+
+You can find the exact HBase version each phoenix HBase profile uses by checking <hbase.version>
+in the corresponding profile section at the end of phoenix/pom.xml, or you can specify the HBase
+version to build Phoenix with explicitly (see below)
+
+See https://issues.apache.org/jira/browse/PHOENIX-5993 for more information.
+
Building from source
--------------------
@@ -50,14 +71,17 @@ As Phoenix uses *limited public* HBase APIs, which sometimes change even within
Phoenix may not build or work with older releases of HBase, or ones that were released after
Phoenix, even within the same HBase minor release.
-By default, Phoenix will be built for the latest supported HBase 2.x release. You can specify the
-targeted HBase minor release by setting the `hbase.profile` system property for maven.
+By default, Phoenix will be built for the latest known patch level of the earliest HBase 2.x
+minor release that Phoenix supports.
+
+You can specify the targeted HBase minor release by setting the `hbase.profile` system property for
+maven.
You can also specify the exact HBase release to build Phoenix with by additionally
setting the `hbase.version` system property.
- * `mvn clean install` will build the for the latest known supported HBase 2.x relese
- * `mvn clean install -Dhbase.profile=2.1` will use the latest known supported HBase 2.1 release
+ * `mvn clean install` will use the latest known patch release of the the earliest supported HBase 2 minor relese
+ * `mvn clean install -Dhbase.profile=2.1` will use the latest known patch release of HBase 2.1
* `mvn clean install -Dhbase.profile=2.1 -Dhbase.version=2.1.7` will build with HBase 2.1.7
Phoenix verifies the specified `hbase.profile` and `hbase.version` properties, and will reject
@@ -65,9 +89,6 @@ combinations that are known not to work. You may disable this verification by ad
`-Denforcer.skip=true` to the maven command line. (In case you are using an HBase package that
modifies the canonical version number in a way that Phoenix cannot parse)
-*Note that the above reflects the intended behaviour for the 5.1 release. In the current
-development releases, we default to building with the old HBase 2.0.1 release.*
-
Importing into eclipse
----------------------
diff --git a/dev/cache-apache-project-artifact.sh b/dev/cache-apache-project-artifact.sh
new file mode 100755
index 0000000..8539350
--- /dev/null
+++ b/dev/cache-apache-project-artifact.sh
@@ -0,0 +1,139 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# This was lovingly copied from Apache HBase
+
+set -e
+function usage {
+ echo "Usage: ${0} [options] /path/to/download/file.tar.gz download/fragment/eg/project/subdir/some-artifact-version.tar.gz"
+ echo ""
+ echo " --force for a redownload even if /path/to/download/file.tar.gz exists."
+ echo " --working-dir /path/to/use Path for writing tempfiles. must exist."
+ echo " defaults to making a directory via mktemp that we clean."
+ echo " --keys url://to/project/KEYS where to get KEYS. needed to check signature on download."
+ echo ""
+ exit 1
+}
+# if no args specified, show usage
+if [ $# -lt 2 ]; then
+ usage
+fi
+
+
+# Get arguments
+declare done_if_cached="true"
+declare working_dir
+declare cleanup="true"
+declare keys
+while [ $# -gt 0 ]
+do
+ case "$1" in
+ --force) shift; done_if_cached="false";;
+ --working-dir) shift; working_dir=$1; cleanup="false"; shift;;
+ --keys) shift; keys=$1; shift;;
+ --) shift; break;;
+ -*) usage ;;
+ *) break;; # terminate while loop
+ esac
+done
+
+# should still have required args
+if [ $# -lt 2 ]; then
+ usage
+fi
+
+target="$1"
+artifact="$2"
+
+if [ -f "${target}" ] && [ "true" = "${done_if_cached}" ]; then
+ echo "Reusing existing download of '${artifact}'."
+ exit 0
+fi
+
+if [ -z "${working_dir}" ]; then
+ if ! working_dir="$(mktemp -d -t hbase-download-apache-artifact)" ; then
+ echo "Failed to create temporary working directory. Please specify via --working-dir" >&2
+ exit 1
+ fi
+else
+ # absolutes please
+ working_dir="$(cd "$(dirname "${working_dir}")"; pwd)/$(basename "${working_dir}")"
+ if [ ! -d "${working_dir}" ]; then
+ echo "passed working directory '${working_dir}' must already exist." >&2
+ exit 1
+ fi
+fi
+
+function cleanup {
+ if [ -n "${keys}" ]; then
+ echo "Stopping gpg agent daemon"
+ gpgconf --homedir "${working_dir}/.gpg" --kill gpg-agent
+ echo "Stopped gpg agent daemon"
+ fi
+
+ if [ "true" = "${cleanup}" ]; then
+ echo "cleaning up temp space."
+ rm -rf "${working_dir}"
+ fi
+}
+trap cleanup EXIT SIGQUIT
+
+echo "New download of '${artifact}'"
+
+# N.B. this comes first so that if gpg falls over we skip the expensive download.
+if [ -n "${keys}" ]; then
+ if [ ! -d "${working_dir}/.gpg" ]; then
+ rm -rf "${working_dir}/.gpg"
+ mkdir -p "${working_dir}/.gpg"
+ chmod -R 700 "${working_dir}/.gpg"
+ fi
+
+ echo "installing project KEYS"
+ curl -L --fail -o "${working_dir}/KEYS" "${keys}"
+ if ! gpg --homedir "${working_dir}/.gpg" --import "${working_dir}/KEYS" ; then
+ echo "ERROR importing the keys via gpg failed. If the output above mentions this error:" >&2
+ echo " gpg: can't connect to the agent: File name too long" >&2
+ # we mean to give them the command to run, not to run it.
+ #shellcheck disable=SC2016
+ echo 'then you prolly need to create /var/run/user/$(id -u)' >&2
+ echo "see this thread on gnupg-users: https://s.apache.org/uI7x" >&2
+ exit 2
+ fi
+
+ echo "downloading signature"
+ curl -L --fail -o "${working_dir}/artifact.asc" "https://archive.apache.org/dist/${artifact}.asc"
+fi
+
+echo "downloading artifact"
+if ! curl --dump-header "${working_dir}/artifact_download_headers.txt" -L --fail -o "${working_dir}/artifact" "https://www.apache.org/dyn/closer.lua?filename=${artifact}&action=download" ; then
+ echo "Artifact wasn't in mirror system. falling back to archive.a.o."
+ curl --dump-header "${working_dir}/artifact_fallback_headers.txt" -L --fail -o "${working_dir}/artifact" "http://archive.apache.org/dist/${artifact}"
+fi
+
+if [ -n "${keys}" ]; then
+ echo "verifying artifact signature"
+ gpg --homedir "${working_dir}/.gpg" --verify "${working_dir}/artifact.asc"
+ echo "signature good."
+fi
+
+echo "moving artifact into place at '${target}'"
+# ensure we're on the same filesystem
+mv "${working_dir}/artifact" "${target}.copying"
+# attempt atomic move
+mv "${target}.copying" "${target}"
+echo "all done!"
\ No newline at end of file
diff --git a/dev/rebuild_hbase.sh b/dev/rebuild_hbase.sh
new file mode 100755
index 0000000..3f39e9e
--- /dev/null
+++ b/dev/rebuild_hbase.sh
@@ -0,0 +1,29 @@
+# The name of the Apache Hbase source file
+HBASE_SOURCE_NAME="hbase-$1-src.tar.gz"
+# The relative path on the ASF mirrors for the Hbase source file
+HBASE_SOURCE_MIRROR_NAME="hbase/$1/$HBASE_SOURCE_NAME"
+
+# Downloads the specified HBase version source, extracts it,
+# then rebuilds and installs the maven artifacts locally with -Dhadoop.profile=3.0
+
+if [ $# -ne 1 ]
+ then
+ echo "Supply the Hbase version as paramater i.e.: rebuild_hbase.sh 2.2.6 "
+fi
+
+DEV_SUPPORT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+ARTIFACTS_DIR="$DEV_SUPPORT/artifacts"
+WORK_DIR="$DEV_SUPPORT/work"
+
+mkdir "$ARTIFACTS_DIR"
+mkdir "$WORK_DIR"
+
+$DEV_SUPPORT/cache-apache-project-artifact.sh --keys https://downloads.apache.org/hbase/KEYS \
+ --working-dir "$WORK_DIR" "$ARTIFACTS_DIR/$HBASE_SOURCE_NAME" "$HBASE_SOURCE_MIRROR_NAME"
+
+STARTDIR=$PWD
+cd $ARTIFACTS_DIR
+tar xfvz hbase-$1-src.tar.gz
+cd hbase-$1
+mvn clean install -Dhadoop.profile=3.0 -DskipTests
+cd ${STARTDIR}
diff --git a/pom.xml b/pom.xml
index 63d1def..33383c4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1466,44 +1466,36 @@
<!-- The dependencies should be defined only in phoenix-core
,but maven doesn't seem to support that -->
<profile>
- <!-- must be identical to phoenix-hbase-compat-2.2.1 profile -->
- <id>phoenix-hbase-compat-2.2.1-default</id>
+ <!-- must be identical to phoenix-hbase-compat-2.1.6 profile -->
+ <id>phoenix-hbase-compat-2.1.6-default</id>
<activation>
<property>
<name>!hbase.profile</name>
</property>
</activation>
<properties>
- <hbase.profile>2.2</hbase.profile>
- <hbase.compat.version>2.2.1</hbase.compat.version>
- <hbase.version>2.2.4</hbase.version>
- <!-- PHOENIX-5993 We should use this, if we could,
- as 2.2.4 is not the latest version, Hadoop 3.0.3 is unsupported with Hbase 2.2.x
- <hbase.version>2.2.5</hbase.version>
- <hadoop.version>3.1.2</hadoop.version>
- -->
+ <hbase.profile>2.1</hbase.profile>
+ <hbase.compat.version>2.1.6</hbase.compat.version>
+ <hbase.version>2.1.9</hbase.version>
</properties>
</profile>
<profile>
- <!-- PHOENIX-5993 This won't work with the public HBase artifacts -->
- <id>phoenix-hbase-compat-2.3.0</id>
+ <id>phoenix-hbase-compat-2.1.6</id>
<activation>
+ <activeByDefault>true</activeByDefault>
<property>
<name>hbase.profile</name>
- <value>2.3</value>
+ <value>2.1</value>
</property>
</activation>
<properties>
- <hbase.profile>2.3</hbase.profile>
- <hbase.compat.version>2.3.0</hbase.compat.version>
- <hbase.version>2.3.0</hbase.version>
- <!-- PHOENIX-6010 3.1.3 doesn't work because of Guava rebase -->
- <hadoop.version>3.1.2</hadoop.version>
- <hbase.thirdparty.version>3.3.0</hbase.thirdparty.version>
- <zookeeper.version>3.5.7</zookeeper.version>
+ <hbase.profile>2.1</hbase.profile>
+ <hbase.compat.version>2.1.6</hbase.compat.version>
+ <hbase.version>2.1.9</hbase.version>
</properties>
</profile>
<profile>
+ <!-- PHOENIX-5993 This won't work with the public HBase artifacts -->
<id>phoenix-hbase-compat-2.2.1</id>
<activation>
<property>
@@ -1514,26 +1506,28 @@
<properties>
<hbase.profile>2.2</hbase.profile>
<hbase.compat.version>2.2.1</hbase.compat.version>
- <hbase.version>2.2.4</hbase.version>
- <!-- PHOENIX-5993 We should use this, if we could
<hbase.version>2.2.5</hbase.version>
+ <!-- PHOENIX-6010 3.1.3 doesn't work because of Guava rebase -->
<hadoop.version>3.1.2</hadoop.version>
- -->
</properties>
</profile>
<profile>
- <id>phoenix-hbase-compat-2.1.6</id>
+ <!-- PHOENIX-5993 This won't work with the public HBase artifacts -->
+ <id>phoenix-hbase-compat-2.3.0</id>
<activation>
- <activeByDefault>true</activeByDefault>
<property>
<name>hbase.profile</name>
- <value>2.1</value>
+ <value>2.3</value>
</property>
</activation>
<properties>
- <hbase.profile>2.1</hbase.profile>
- <hbase.compat.version>2.1.6</hbase.compat.version>
- <hbase.version>2.1.9</hbase.version>
+ <hbase.profile>2.3</hbase.profile>
+ <hbase.compat.version>2.3.0</hbase.compat.version>
+ <hbase.version>2.3.0</hbase.version>
+ <!-- PHOENIX-6010 3.1.3 doesn't work because of Guava rebase -->
+ <hadoop.version>3.1.2</hadoop.version>
+ <hbase.thirdparty.version>3.3.0</hbase.thirdparty.version>
+ <zookeeper.version>3.5.7</zookeeper.version>
</properties>
</profile>
<profile>