You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by bh...@apache.org on 2019/02/21 23:50:52 UTC
[impala] 01/04: IMPALA-8168: Disable Sentry HDFS sync on S3
This is an automated email from the ASF dual-hosted git repository.
bharathv pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit b7df73d3392e485fb7c2ed335a49f8c42a254563
Author: Fredy Wijaya <fw...@cloudera.com>
AuthorDate: Wed Feb 6 09:58:18 2019 -0800
IMPALA-8168: Disable Sentry HDFS sync on S3
This patch updates the build script to disable Sentry HDFS sync plugin
on S3 since Sentry HDFS sync is not supported and it may cause issues,
such as making the tests flaky in a non-supported environment.
This patch also updates the CDH_BUILD_NUMBER to 909265 to test the new
Sentry changes for batch insert of HMS paths in full snapshot. It also
bumps Kudu version to 1.10.0.
Testing:
- Ran core tests on both HDFS and S3.
Change-Id: Ifa12f163a32bb6e597323149fbe565a6f66f5e69
Reviewed-on: http://gerrit.cloudera.org:8080/12482
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
bin/impala-config.sh | 12 +++++++++---
fe/src/test/resources/sentry-site_no_oo.xml.template | 2 +-
fe/src/test/resources/sentry-site_oo.xml.template | 2 +-
fe/src/test/resources/sentry-site_oo_nogrant.xml.template | 2 +-
4 files changed, 12 insertions(+), 6 deletions(-)
diff --git a/bin/impala-config.sh b/bin/impala-config.sh
index d619f1a..01bd918 100755
--- a/bin/impala-config.sh
+++ b/bin/impala-config.sh
@@ -160,7 +160,7 @@ fi
: ${CDH_DOWNLOAD_HOST:=native-toolchain.s3.amazonaws.com}
export CDH_DOWNLOAD_HOST
export CDH_MAJOR_VERSION=6
-export CDH_BUILD_NUMBER=742221
+export CDH_BUILD_NUMBER=909265
export IMPALA_HADOOP_VERSION=3.0.0-cdh6.x-SNAPSHOT
export IMPALA_HBASE_VERSION=2.1.0-cdh6.x-SNAPSHOT
export IMPALA_HIVE_VERSION=2.1.1-cdh6.x-SNAPSHOT
@@ -170,7 +170,7 @@ export IMPALA_PARQUET_VERSION=1.9.0-cdh6.x-SNAPSHOT
export IMPALA_AVRO_JAVA_VERSION=1.8.2-cdh6.x-SNAPSHOT
export IMPALA_LLAMA_MINIKDC_VERSION=1.0.0
export IMPALA_KITE_VERSION=1.0.0-cdh6.x-SNAPSHOT
-export KUDU_JAVA_VERSION=1.9.0-cdh6.x-SNAPSHOT
+export KUDU_JAVA_VERSION=1.10.0-cdh6.x-SNAPSHOT
# When IMPALA_(CDH_COMPONENT)_URL are overridden, they may contain '$(platform_label)'
# which will be substituted for the CDH platform label in bootstrap_toolchain.py
@@ -282,6 +282,12 @@ export LOCAL_FS="file:${WAREHOUSE_LOCATION_PREFIX}"
ESCAPED_IMPALA_HOME=$(sed "s/[^0-9a-zA-Z]/_/g" <<< "$IMPALA_HOME")
export METASTORE_DB=${METASTORE_DB-$(cut -c-63 <<< HMS$ESCAPED_IMPALA_HOME)}
export SENTRY_POLICY_DB=${SENTRY_POLICY_DB-$(cut -c-63 <<< SP$ESCAPED_IMPALA_HOME)}
+if [[ "${TARGET_FILESYSTEM}" == "s3" ]]; then
+ # On S3, disable Sentry HDFS sync plugin.
+ export SENTRY_PROCESSOR_FACTORIES="org.apache.sentry.api.service.thrift.SentryPolicyStoreProcessorFactory"
+else
+ export SENTRY_PROCESSOR_FACTORIES="org.apache.sentry.api.service.thrift.SentryPolicyStoreProcessorFactory,org.apache.sentry.hdfs.SentryHDFSServiceProcessorFactory"
+fi
RANGER_POLICY_DB=${RANGER_POLICY_DB-$(cut -c-63 <<< ranger$ESCAPED_IMPALA_HOME)}
# The DB script in Ranger expects the database name to be in lower case.
export RANGER_POLICY_DB=$(echo ${RANGER_POLICY_DB} | tr '[:upper:]' '[:lower:]')
@@ -615,7 +621,7 @@ fi
export KUDU_IS_SUPPORTED
if $USE_CDH_KUDU; then
- export IMPALA_KUDU_VERSION=${IMPALA_KUDU_VERSION-"1.9.0-cdh6.x-SNAPSHOT"}
+ export IMPALA_KUDU_VERSION=${IMPALA_KUDU_VERSION-"1.10.0-cdh6.x-SNAPSHOT"}
export IMPALA_KUDU_HOME=${CDH_COMPONENTS_HOME}/kudu-$IMPALA_KUDU_VERSION
else
export IMPALA_KUDU_VERSION=${IMPALA_KUDU_VERSION-"5211897"}
diff --git a/fe/src/test/resources/sentry-site_no_oo.xml.template b/fe/src/test/resources/sentry-site_no_oo.xml.template
index 3c29b5d..8ad8be6 100644
--- a/fe/src/test/resources/sentry-site_no_oo.xml.template
+++ b/fe/src/test/resources/sentry-site_no_oo.xml.template
@@ -81,7 +81,7 @@
<!-- These properties enable HMS follower. -->
<property>
<name>sentry.service.processor.factories</name>
- <value>org.apache.sentry.api.service.thrift.SentryPolicyStoreProcessorFactory,org.apache.sentry.hdfs.SentryHDFSServiceProcessorFactory</value>
+ <value>${SENTRY_PROCESSOR_FACTORIES}</value>
</property>
<property>
<name>sentry.policy.store.plugins</name>
diff --git a/fe/src/test/resources/sentry-site_oo.xml.template b/fe/src/test/resources/sentry-site_oo.xml.template
index bccb1c5..2a3fdce 100644
--- a/fe/src/test/resources/sentry-site_oo.xml.template
+++ b/fe/src/test/resources/sentry-site_oo.xml.template
@@ -81,7 +81,7 @@
<!-- These properties enable HMS follower. -->
<property>
<name>sentry.service.processor.factories</name>
- <value>org.apache.sentry.api.service.thrift.SentryPolicyStoreProcessorFactory,org.apache.sentry.hdfs.SentryHDFSServiceProcessorFactory</value>
+ <value>${SENTRY_PROCESSOR_FACTORIES}</value>
</property>
<property>
<name>sentry.policy.store.plugins</name>
diff --git a/fe/src/test/resources/sentry-site_oo_nogrant.xml.template b/fe/src/test/resources/sentry-site_oo_nogrant.xml.template
index 6e0076a..444eb9f 100644
--- a/fe/src/test/resources/sentry-site_oo_nogrant.xml.template
+++ b/fe/src/test/resources/sentry-site_oo_nogrant.xml.template
@@ -81,7 +81,7 @@
<!-- These properties enable HMS follower. -->
<property>
<name>sentry.service.processor.factories</name>
- <value>org.apache.sentry.api.service.thrift.SentryPolicyStoreProcessorFactory,org.apache.sentry.hdfs.SentryHDFSServiceProcessorFactory</value>
+ <value>${SENTRY_PROCESSOR_FACTORIES}</value>
</property>
<property>
<name>sentry.policy.store.plugins</name>