You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by tm...@apache.org on 2019/06/03 21:34:27 UTC
[impala] branch master updated: IMPALA-8503: allow the Hive
Metastore to start with kudu-hive plugin
This is an automated email from the ASF dual-hosted git repository.
tmarshall pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
The following commit(s) were added to refs/heads/master by this push:
new ab3bc22 IMPALA-8503: allow the Hive Metastore to start with kudu-hive plugin
ab3bc22 is described below
commit ab3bc22534ba4dabbf8d4ef71b7faeef50dfed99
Author: Hao Hao <ha...@cloudera.com>
AuthorDate: Thu May 30 14:53:11 2019 -0700
IMPALA-8503: allow the Hive Metastore to start with kudu-hive plugin
This patch allows to start the Hive Metasotre with Kudu plugin which is
required for enabling Kudu's integration with the HMS. The Kudu plugin
is downloaded and extracted from native-toolchain S3 bucket.
Change-Id: I4bd1488ced51840ec986d29ed371e26168abcc76
Reviewed-on: http://gerrit.cloudera.org:8080/13319
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Reviewed-by: Thomas Marshall <tm...@cloudera.com>
---
bin/bootstrap_toolchain.py | 11 +++++++++--
bin/impala-config.sh | 3 ++-
fe/src/test/resources/hive-site.xml.py | 2 +-
impala-parent/pom.xml | 2 +-
testdata/bin/run-hive-server.sh | 9 +++++++++
5 files changed, 22 insertions(+), 5 deletions(-)
diff --git a/bin/bootstrap_toolchain.py b/bin/bootstrap_toolchain.py
index 34547fe..1f9b28c 100755
--- a/bin/bootstrap_toolchain.py
+++ b/bin/bootstrap_toolchain.py
@@ -414,7 +414,11 @@ def download_cdh_components(toolchain_root, cdh_components, url_prefix):
def download(component):
try:
- pkg_directory = package_directory(cdh_components_home, component.name,
+ # Download and unpack Kudu's java package
+ component_name = component.name
+ if component.name == "kudu-java":
+ component_name = "kudu"
+ pkg_directory = package_directory(cdh_components_home, component_name,
component.version)
if os.path.isdir(pkg_directory):
return
@@ -425,7 +429,7 @@ def download_cdh_components(toolchain_root, cdh_components, url_prefix):
platform_label = "-%s" % get_platform_release_label().cdh
# Download the package if it doesn't exist
file_name = "{0}-{1}{2}.tar.gz".format(
- component.name, component.version, platform_label)
+ component_name, component.version, platform_label)
if component.url is None:
download_path = url_prefix + file_name
@@ -564,6 +568,9 @@ if __name__ == "__main__":
"to use the toolchain Kudu.")
sys.exit(1)
cdh_components += [Package("kudu")]
+ # Always download Kudu's jars regardless of USE_CDH_KUDU since they
+ # aren't platform dependent and aren't packaged by the toolchain.
+ cdh_components += [Package("kudu-java")]
download_path_prefix = \
"https://{0}/build/cdh_components/{1}/tarballs/".format(toolchain_host,
cdh_build_number)
diff --git a/bin/impala-config.sh b/bin/impala-config.sh
index f4b739b..316cff3 100755
--- a/bin/impala-config.sh
+++ b/bin/impala-config.sh
@@ -171,7 +171,7 @@ export IMPALA_PARQUET_VERSION=1.9.0-cdh6.x-SNAPSHOT
export IMPALA_AVRO_JAVA_VERSION=1.8.2-cdh6.x-SNAPSHOT
export IMPALA_LLAMA_MINIKDC_VERSION=1.0.0
export IMPALA_KITE_VERSION=1.0.0-cdh6.x-SNAPSHOT
-export KUDU_JAVA_VERSION=1.10.0-cdh6.x-SNAPSHOT
+export IMPALA_KUDU_JAVA_VERSION=1.10.0-cdh6.x-SNAPSHOT
export CDH_HIVE_VERSION=2.1.1-cdh6.x-SNAPSHOT
export CDP_HIVE_VERSION=3.1.0.6.0.99.0-147
@@ -772,6 +772,7 @@ echo "IMPALA_HIVE_VERSION = $IMPALA_HIVE_VERSION"
echo "IMPALA_HBASE_VERSION = $IMPALA_HBASE_VERSION"
echo "IMPALA_SENTRY_VERSION = $IMPALA_SENTRY_VERSION"
echo "IMPALA_KUDU_VERSION = $IMPALA_KUDU_VERSION"
+echo "IMPALA_KUDU_JAVA_VERSION= $IMPALA_KUDU_JAVA_VERSION"
echo "IMPALA_RANGER_VERSION = $IMPALA_RANGER_VERSION"
# Kerberos things. If the cluster exists and is kerberized, source
diff --git a/fe/src/test/resources/hive-site.xml.py b/fe/src/test/resources/hive-site.xml.py
index 3e752c8..e46adc3 100644
--- a/fe/src/test/resources/hive-site.xml.py
+++ b/fe/src/test/resources/hive-site.xml.py
@@ -124,7 +124,7 @@ else:
# These are for enabling notification between Hive and Sentry as well as
# metastore event processing in Impala (see IMPALA-7954)
CONFIG.update({
- 'hive.metastore.transactional.event.listeners': 'org.apache.hive.hcatalog.listener.DbNotificationListener',
+ 'hive.metastore.transactional.event.listeners': 'org.apache.hive.hcatalog.listener.DbNotificationListener,org.apache.kudu.hive.metastore.KuduMetastorePlugin',
'hcatalog.message.factory.impl.json': 'org.apache.sentry.binding.metastore.messaging.json.SentryJSONMessageFactory',
'hive.metastore.dml.events': 'true',
})
diff --git a/impala-parent/pom.xml b/impala-parent/pom.xml
index 36bd281..72a4586 100644
--- a/impala-parent/pom.xml
+++ b/impala-parent/pom.xml
@@ -43,7 +43,7 @@ under the License.
<thrift.version>0.9.3</thrift.version>
<impala.extdatasrc.api.version>1.0-SNAPSHOT</impala.extdatasrc.api.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- <kudu.version>${env.KUDU_JAVA_VERSION}</kudu.version>
+ <kudu.version>${env.IMPALA_KUDU_JAVA_VERSION}</kudu.version>
<commons-io.version>2.6</commons-io.version>
<slf4j.version>1.7.25</slf4j.version>
<junit.version>4.12</junit.version>
diff --git a/testdata/bin/run-hive-server.sh b/testdata/bin/run-hive-server.sh
index 1718467..daa7bad 100755
--- a/testdata/bin/run-hive-server.sh
+++ b/testdata/bin/run-hive-server.sh
@@ -100,6 +100,15 @@ if ${USE_CDP_HIVE} ; then
done
fi
+# Add kudu-hive.jar to the Hive Metastore classpath, so that Kudu's HMS
+# plugin can be loaded.
+FILE_NAME="${CDH_COMPONENTS_HOME}/kudu-${IMPALA_KUDU_JAVA_VERSION}/\
+kudu-hive-${IMPALA_KUDU_JAVA_VERSION}.jar"
+export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}:${FILE_NAME}
+# Default to skip validation on Kudu tables if KUDU_SKIP_HMS_PLUGIN_VALIDATION
+# is unset.
+export KUDU_SKIP_HMS_PLUGIN_VALIDATION=${KUDU_SKIP_HMS_PLUGIN_VALIDATION:-1}
+
# Starts a Hive Metastore Server on the specified port.
# To debug log4j2 loading issues, add to HADOOP_CLIENT_OPTS:
# -Dorg.apache.logging.log4j.simplelog.StatusLogger.level=TRACE