You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2018/03/23 21:54:32 UTC
[1/4] impala git commit: IMPALA-6722: include fs prefix for udf test
Repository: impala
Updated Branches:
refs/heads/master 08b60a15c -> 783de170c
IMPALA-6722: include fs prefix for udf test
test_native_functions_race failed tests since it did not
include a path prefix. The fix uses get_fs_path to include
the fs prefix.
Change-Id: I314d8c32e4bc3857aefd244b524fb6718d234f30
Reviewed-on: http://gerrit.cloudera.org:8080/9782
Reviewed-by: Alex Behm <al...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/466d1266
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/466d1266
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/466d1266
Branch: refs/heads/master
Commit: 466d1266fde76534affe0a0e6ff2b5f2c538ab3e
Parents: 08b60a1
Author: Vuk Ercegovac <ve...@cloudera.com>
Authored: Thu Mar 22 23:40:10 2018 -0700
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Fri Mar 23 19:50:23 2018 +0000
----------------------------------------------------------------------
tests/query_test/test_udfs.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/466d1266/tests/query_test/test_udfs.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_udfs.py b/tests/query_test/test_udfs.py
index 17890a4..644d80f 100644
--- a/tests/query_test/test_udfs.py
+++ b/tests/query_test/test_udfs.py
@@ -322,9 +322,10 @@ class TestUdfExecution(TestUdfBase):
create_another_fn = """create function if not exists {0}.other(float)
returns float location '{1}' symbol='Identity'"""
drop_another_fn = """drop function if exists {0}.other(float)"""
+ udf_path = get_fs_path('/test-warehouse/libTestUdfs.so')
setup_client = self.create_impala_client()
- setup_query = create_fn_to_use.format(unique_database, '/test-warehouse/libTestUdfs.so')
+ setup_query = create_fn_to_use.format(unique_database, udf_path)
try:
setup_client.execute(setup_query)
except Exception as e:
@@ -345,7 +346,7 @@ class TestUdfExecution(TestUdfBase):
time.sleep(1 + random.random())
client = self.create_impala_client()
drop = drop_another_fn.format(unique_database)
- create = create_another_fn.format(unique_database, '/test-warehouse/libTestUdfs.so')
+ create = create_another_fn.format(unique_database, udf_path)
try:
client.execute(drop)
client.execute(create)
[2/4] impala git commit: IMPALA-4277: Support multiple versions of
Hadoop ecosystem
Posted by ta...@apache.org.
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/testdata/cluster/node_templates/cdh5/etc/init.d/kudu-common
----------------------------------------------------------------------
diff --git a/testdata/cluster/node_templates/cdh5/etc/init.d/kudu-common b/testdata/cluster/node_templates/cdh5/etc/init.d/kudu-common
deleted file mode 100644
index a8d2d4f..0000000
--- a/testdata/cluster/node_templates/cdh5/etc/init.d/kudu-common
+++ /dev/null
@@ -1,51 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# This file is intended to be sourced. After sourcing the KUDU_COMMON_ARGS will be set.
-# KUDU_COMMON_ARGS should be passed to both the master and tservers on startup.
-#
-# KUDU_BUILD_DIR or USE_KUDU_DEBUG_BUILD must be set before this is sourced (default
-# values are set in impala-config.sh). If KUDU_BUILD_DIR is set then KUDU_HOME should
-# also be set otherwise the web pages may not function (but that should be the only
-# problem).
-
-if [[ -n "$KUDU_BUILD_DIR" ]]; then
- KUDU_BIN_DIR="$KUDU_BUILD_DIR/bin"
- KUDU_WWW_DIR="$KUDU_HOME/www"
-else
- KUDU_BIN_DIR="$IMPALA_TOOLCHAIN/kudu-$IMPALA_KUDU_VERSION"
- if $USE_KUDU_DEBUG_BUILD; then
- KUDU_BIN_DIR+=/debug/bin
- else
- KUDU_BIN_DIR+=/release/bin
- fi
- KUDU_WWW_DIR="$KUDU_BIN_DIR/../lib/kudu/www"
-fi
-
-KUDU_COMMON_ARGS=("-webserver_doc_root=$KUDU_WWW_DIR")
-
-# Add a dummy flag with value IBelongToTheMiniCluster so the process is easily
-# identifiable in case the pid file is removed.
-KUDU_COMMON_ARGS+=("-vmodule=IBelongToTheMiniCluster")
-KUDU_COMMON_ARGS+=("-memory_limit_hard_bytes=2147483648") # 2gb
-KUDU_COMMON_ARGS+=("-block_cache_capacity_mb=256")
-
-HOLE_PUNCH_TEST_FILE=${NODE_DIR}/var/lib/kudu/hole_punch_test
-if ! fallocate -p -o 1 -l 1 "$HOLE_PUNCH_TEST_FILE" &>/dev/null; then
- KUDU_COMMON_ARGS+=("-block_manager=file")
-fi
-rm -f "$HOLE_PUNCH_TEST_FILE"
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/testdata/cluster/node_templates/cdh5/etc/init.d/kudu-master
----------------------------------------------------------------------
diff --git a/testdata/cluster/node_templates/cdh5/etc/init.d/kudu-master b/testdata/cluster/node_templates/cdh5/etc/init.d/kudu-master
deleted file mode 100755
index a0b4234..0000000
--- a/testdata/cluster/node_templates/cdh5/etc/init.d/kudu-master
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-set -e
-
-DIR=$(dirname $0)
-
-. "$DIR/common"
-. "$DIR/kudu-common" # Sets KUDU_COMMON_ARGS
-
-function start {
- do_start "$KUDU_BIN_DIR"/kudu-master \
- -flagfile "$NODE_DIR"/etc/kudu/master.conf \
- "${KUDU_COMMON_ARGS[@]}"
-}
-
-$1
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/testdata/cluster/node_templates/cdh5/etc/init.d/kudu-tserver
----------------------------------------------------------------------
diff --git a/testdata/cluster/node_templates/cdh5/etc/init.d/kudu-tserver b/testdata/cluster/node_templates/cdh5/etc/init.d/kudu-tserver
deleted file mode 100755
index 60e49e6..0000000
--- a/testdata/cluster/node_templates/cdh5/etc/init.d/kudu-tserver
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-set -e
-
-DIR=$(dirname $0)
-
-. "$DIR/common"
-. "$DIR/kudu-common" # Sets KUDU_COMMON_ARGS
-
-function start {
- do_start "$KUDU_BIN_DIR"/kudu-tserver \
- -flagfile "$NODE_DIR"/etc/kudu/tserver.conf \
- "${KUDU_COMMON_ARGS[@]}"
-}
-
-$1
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/testdata/cluster/node_templates/cdh5/etc/kudu/master.conf.tmpl
----------------------------------------------------------------------
diff --git a/testdata/cluster/node_templates/cdh5/etc/kudu/master.conf.tmpl b/testdata/cluster/node_templates/cdh5/etc/kudu/master.conf.tmpl
deleted file mode 100644
index f272eb7..0000000
--- a/testdata/cluster/node_templates/cdh5/etc/kudu/master.conf.tmpl
+++ /dev/null
@@ -1,9 +0,0 @@
--fs_wal_dir=${NODE_DIR}/var/lib/kudu/master/wal
--fs_data_dirs=${NODE_DIR}/var/lib/kudu/master/data
--log_dir=${NODE_DIR}/var/log/kudu/master
-
-# fsync is disabled for additional speed. Sometimes operations are slow on EC2/GCE test
-# machines. Some data loss could occur if the system crashes before the OS has a chance
-# to flush data to disk but that is acceptable for development purposes.
--unlock_unsafe_flags
--never_fsync
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/testdata/cluster/node_templates/cdh5/etc/kudu/tserver.conf.tmpl
----------------------------------------------------------------------
diff --git a/testdata/cluster/node_templates/cdh5/etc/kudu/tserver.conf.tmpl b/testdata/cluster/node_templates/cdh5/etc/kudu/tserver.conf.tmpl
deleted file mode 100644
index c84bd91..0000000
--- a/testdata/cluster/node_templates/cdh5/etc/kudu/tserver.conf.tmpl
+++ /dev/null
@@ -1,11 +0,0 @@
--rpc_bind_addresses=0.0.0.0:${KUDU_TS_RPC_PORT}
--webserver_port=${KUDU_TS_WEBUI_PORT}
--fs_wal_dir=${NODE_DIR}/var/lib/kudu/ts/wal
--fs_data_dirs=${NODE_DIR}/var/lib/kudu/ts/data
--log_dir=${NODE_DIR}/var/log/kudu/ts
-
-# fsync is disabled for additional speed. Sometimes operations are slow on EC2/GCE test
-# machines. Some data loss could occur if the system crashes before the OS has a chance
-# to flush data to disk but that is acceptable for development purposes.
--unlock_unsafe_flags
--never_fsync
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/testdata/cluster/node_templates/cdh6/etc/init.d/kms
----------------------------------------------------------------------
diff --git a/testdata/cluster/node_templates/cdh6/etc/init.d/kms b/testdata/cluster/node_templates/cdh6/etc/init.d/kms
new file mode 100755
index 0000000..334d522
--- /dev/null
+++ b/testdata/cluster/node_templates/cdh6/etc/init.d/kms
@@ -0,0 +1,39 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+DIR=$(dirname $0)
+
+. "$DIR/common"
+
+# KMS_WEBUI_PORT from impala config
+export KMS_HTTP_PORT=$KMS_WEBUI_PORT
+
+# Path to the file where the catalina server writes the PID
+export CATALINA_PID=$PID_DIR/kms
+
+function start {
+
+ if [[ -e "$CATALINA_PID" ]]; then
+ rm "$CATALINA_PID"
+ fi
+
+ do_start $HADOOP_HOME/bin/hadoop kms &> "$LOG_DIR/$(basename $0).out"
+}
+
+$1
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/testdata/cluster/node_templates/common/etc/hadoop/conf/kms-acls.xml.tmpl
----------------------------------------------------------------------
diff --git a/testdata/cluster/node_templates/common/etc/hadoop/conf/kms-acls.xml.tmpl b/testdata/cluster/node_templates/common/etc/hadoop/conf/kms-acls.xml.tmpl
new file mode 100644
index 0000000..0eefdb4
--- /dev/null
+++ b/testdata/cluster/node_templates/common/etc/hadoop/conf/kms-acls.xml.tmpl
@@ -0,0 +1,68 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<configuration>
+ <property>
+ <name>hadoop.kms.acl.CREATE</name>
+ <value>*</value>
+ </property>
+
+ <property>
+ <name>hadoop.kms.acl.DELETE</name>
+ <value>*</value>
+ </property>
+
+ <property>
+ <name>hadoop.kms.acl.ROLLOVER</name>
+ <value>*</value>
+ </property>
+
+ <property>
+ <name>hadoop.kms.acl.GET</name>
+ <value>*</value>
+ </property>
+
+ <property>
+ <name>hadoop.kms.acl.GET_KEYS</name>
+ <value>*</value>
+ </property>
+
+ <property>
+ <name>hadoop.kms.acl.GET_METADATA</name>
+ <value>*</value>
+ </property>
+
+ <property>
+ <name>hadoop.kms.acl.SET_KEY_MATERIAL</name>
+ <value>*</value>
+ </property>
+
+ <property>
+ <name>hadoop.kms.acl.GENERATE_EEK</name>
+ <value>*</value>
+ </property>
+
+ <property>
+ <name>hadoop.kms.acl.DECRYPT_EEK</name>
+ <value>*</value>
+ </property>
+
+ <property>
+ <name>default.key.acl.MANAGEMENT</name>
+ <value>*</value>
+ </property>
+
+ <property>
+ <name>default.key.acl.GENERATE_EEK</name>
+ <value>*</value>
+ </property>
+
+ <property>
+ <name>default.key.acl.DECRYPT_EEK</name>
+ <value>*</value>
+ </property>
+
+ <property>
+ <name>default.key.acl.READ</name>
+ <value>*</value>
+ </property>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/testdata/cluster/node_templates/common/etc/hadoop/conf/kms-site.xml.tmpl
----------------------------------------------------------------------
diff --git a/testdata/cluster/node_templates/common/etc/hadoop/conf/kms-site.xml.tmpl b/testdata/cluster/node_templates/common/etc/hadoop/conf/kms-site.xml.tmpl
new file mode 100644
index 0000000..4027e32
--- /dev/null
+++ b/testdata/cluster/node_templates/common/etc/hadoop/conf/kms-site.xml.tmpl
@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<configuration>
+ <property>
+ <name>hadoop.kms.key.provider.uri</name>
+ <value>jceks://file@/${NODE_DIR}/data/kms.keystore</value>
+ </property>
+
+ <property>
+ <name>hadoop.kms.authentication.type</name>
+ <value>simple</value>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/testdata/cluster/node_templates/common/etc/hadoop/conf/yarn-site.xml.tmpl
----------------------------------------------------------------------
diff --git a/testdata/cluster/node_templates/common/etc/hadoop/conf/yarn-site.xml.tmpl b/testdata/cluster/node_templates/common/etc/hadoop/conf/yarn-site.xml.tmpl
index 001d750..0dd7bec 100644
--- a/testdata/cluster/node_templates/common/etc/hadoop/conf/yarn-site.xml.tmpl
+++ b/testdata/cluster/node_templates/common/etc/hadoop/conf/yarn-site.xml.tmpl
@@ -1,21 +1,46 @@
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!-- TODO: Remove any Llama-related configuration. Can this file be removed entirely? -->
<configuration>
<property>
<name>yarn.resourcemanager.webapp.address</name>
<value>0.0.0.0:${YARN_WEBUI_PORT}</value>
</property>
- <!-- Needed as long as multiple nodes are running on the same adderess -->
<property>
- <name>yarn.scheduler.include-port-in-node-name</name>
+ <name>yarn.nodemanager.address</name>
+ <value>127.0.0.1:${NODEMANAGER_PORT}</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.delete.debug-delay-sec</name>
+ <value>600</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.resource.memory-mb</name>
+ <value>16384</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.resource.cpu-vcores</name>
+ <value>16</value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.nodemanagers.heartbeat-interval-ms</name>
+ <value>100</value>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.fair.continuous-scheduling-enabled</name>
<value>true</value>
</property>
<property>
- <name>yarn.nodemanager.address</name>
- <value>127.0.0.1:${NODEMANAGER_PORT}</value>
+ <name>yarn.scheduler.fair.assignmultiple</name>
+ <value>true</value>
</property>
<property>
@@ -34,6 +59,16 @@
</property>
<property>
+ <name>yarn.nodemanager.local-dirs</name>
+ <value>${NODE_DIR}/data/yarn/local</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.log-dirs</name>
+ <value>${NODE_DIR}/data/yarn/logs</value>
+ </property>
+
+ <property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
@@ -70,12 +105,50 @@
<description>Classpath for typical applications.</description>
<name>yarn.application.classpath</name>
<value>
- $HADOOP_CONF_DIR,
- $HADOOP_HOME/share/hadoop/common/*,$HADOOP_HOME/share/hadoop/common/lib/*,
- $HADOOP_HOME/share/hadoop/hdfs/*,$HADOOP_HOME/share/hdfs/common/lib/*,
- $HADOOP_HOME/share/hadoop/mapred/*,$HADOOP_HOME/share/hadoop/mapred/lib/*,
- $HADOOP_HOME/share/hadoop/yarn/*,$HADOOP_HOME/share/hadoop/yarn/lib/*
- $HADOOP_HOME/share/hadoop/tools/*,$HADOOP_HOME/share/hadoop/tools/lib/*
+ ${HADOOP_CONF_DIR},
+ ${HADOOP_HOME}/share/hadoop/tools/lib/*,
+ ${HADOOP_HOME}/share/hadoop/common/*,
+ ${HADOOP_HOME}/share/hadoop/common/lib/*,
+ ${HADOOP_HOME}/share/hadoop/hdfs/*,
+ ${HADOOP_HOME}/share/hdfs/common/lib/*,
+ ${HADOOP_HOME}/share/hadoop/mapreduce/*,
+ ${HADOOP_HOME}/share/hadoop/mapreduce/lib/*,
+ ${HADOOP_HOME}/share/hadoop/yarn/*,
+ ${HADOOP_HOME}/share/hadoop/yarn/lib/*,
+ ${LZO_JAR_PATH}
</value>
</property>
+
+ <!-- BEGIN Kerberos settings -->
+
+ <!-- KERBEROS TODO: Add these to yarn.application.classpath.
+ ${IMPALA_FE_DIR}/target/*,${HADOOP_LZO}/build/*,
+ ${IMPALA_FE_DIR}/target/dependency/* -->
+
+ <!-- ResourceManager security configs -->
+ <property>
+ <name>yarn.resourcemanager.keytab</name>
+ <value>${KRB5_KTNAME}</value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.principal</name>
+ <value>${MINIKDC_PRINC_USER}</value>
+ <!-- Sort of horrible: instead of the yarn principle, we'll use ${USER}
+ so that we don't have a problem with file system permissions. -->
+ </property>
+
+ <!-- NodeManager security configs -->
+ <property>
+ <name>yarn.nodemanager.keytab</name>
+ <value>${KRB5_KTNAME}</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.principal</name>
+ <value>${MINIKDC_PRINC_USER}</value>
+ <!-- Also sort of horrible as per above -->
+ </property>
+ <!-- END Kerberos settings -->
+
</configuration>
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/testdata/cluster/node_templates/common/etc/init.d/common.tmpl
----------------------------------------------------------------------
diff --git a/testdata/cluster/node_templates/common/etc/init.d/common.tmpl b/testdata/cluster/node_templates/common/etc/init.d/common.tmpl
index c6f6204..8cde0b6 100644
--- a/testdata/cluster/node_templates/common/etc/init.d/common.tmpl
+++ b/testdata/cluster/node_templates/common/etc/init.d/common.tmpl
@@ -19,13 +19,20 @@ NODE_DIR="${NODE_DIR}"
PID_DIR="$NODE_DIR/var/run"
LOG_DIR="$NODE_DIR/var/log"
+if [[ $IMPALA_MINICLUSTER_PROFILE == 3 ]]; then
+ export HADOOP_PID_DIR=$NODE_DIR/var/run
+fi
export HADOOP_CONF_DIR="$NODE_DIR/etc/hadoop/conf"
-export YARN_CONF_DIR="$HADOOP_CONF_DIR"
+if [[ $IMPALA_MINICLUSTER_PROFILE == 2 ]]; then
+ export YARN_CONF_DIR="$HADOOP_CONF_DIR"
+fi
# Mark each process so they can be killed if needed. This is a safety mechanism for
# stopping the processes if the pid file has been removed for whatever reason.
export HADOOP_OPTS+=" -D${KILL_CLUSTER_MARKER}"
-export YARN_OPTS+=" -D${KILL_CLUSTER_MARKER}"
+if [[ $IMPALA_MINICLUSTER_PROFILE == 3 ]]; then
+ export YARN_OPTS+=" -D${KILL_CLUSTER_MARKER}"
+fi
# This is for KMS.
export CATALINA_OPTS+=" -D${KILL_CLUSTER_MARKER}"
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/testdata/cluster/node_templates/common/etc/init.d/kudu-common
----------------------------------------------------------------------
diff --git a/testdata/cluster/node_templates/common/etc/init.d/kudu-common b/testdata/cluster/node_templates/common/etc/init.d/kudu-common
new file mode 100644
index 0000000..a8d2d4f
--- /dev/null
+++ b/testdata/cluster/node_templates/common/etc/init.d/kudu-common
@@ -0,0 +1,51 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# This file is intended to be sourced. After sourcing the KUDU_COMMON_ARGS will be set.
+# KUDU_COMMON_ARGS should be passed to both the master and tservers on startup.
+#
+# KUDU_BUILD_DIR or USE_KUDU_DEBUG_BUILD must be set before this is sourced (default
+# values are set in impala-config.sh). If KUDU_BUILD_DIR is set then KUDU_HOME should
+# also be set otherwise the web pages may not function (but that should be the only
+# problem).
+
+if [[ -n "$KUDU_BUILD_DIR" ]]; then
+ KUDU_BIN_DIR="$KUDU_BUILD_DIR/bin"
+ KUDU_WWW_DIR="$KUDU_HOME/www"
+else
+ KUDU_BIN_DIR="$IMPALA_TOOLCHAIN/kudu-$IMPALA_KUDU_VERSION"
+ if $USE_KUDU_DEBUG_BUILD; then
+ KUDU_BIN_DIR+=/debug/bin
+ else
+ KUDU_BIN_DIR+=/release/bin
+ fi
+ KUDU_WWW_DIR="$KUDU_BIN_DIR/../lib/kudu/www"
+fi
+
+KUDU_COMMON_ARGS=("-webserver_doc_root=$KUDU_WWW_DIR")
+
+# Add a dummy flag with value IBelongToTheMiniCluster so the process is easily
+# identifiable in case the pid file is removed.
+KUDU_COMMON_ARGS+=("-vmodule=IBelongToTheMiniCluster")
+KUDU_COMMON_ARGS+=("-memory_limit_hard_bytes=2147483648") # 2gb
+KUDU_COMMON_ARGS+=("-block_cache_capacity_mb=256")
+
+HOLE_PUNCH_TEST_FILE=${NODE_DIR}/var/lib/kudu/hole_punch_test
+if ! fallocate -p -o 1 -l 1 "$HOLE_PUNCH_TEST_FILE" &>/dev/null; then
+ KUDU_COMMON_ARGS+=("-block_manager=file")
+fi
+rm -f "$HOLE_PUNCH_TEST_FILE"
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/testdata/cluster/node_templates/common/etc/init.d/kudu-master
----------------------------------------------------------------------
diff --git a/testdata/cluster/node_templates/common/etc/init.d/kudu-master b/testdata/cluster/node_templates/common/etc/init.d/kudu-master
new file mode 100755
index 0000000..a0b4234
--- /dev/null
+++ b/testdata/cluster/node_templates/common/etc/init.d/kudu-master
@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+
+DIR=$(dirname $0)
+
+. "$DIR/common"
+. "$DIR/kudu-common" # Sets KUDU_COMMON_ARGS
+
+function start {
+ do_start "$KUDU_BIN_DIR"/kudu-master \
+ -flagfile "$NODE_DIR"/etc/kudu/master.conf \
+ "${KUDU_COMMON_ARGS[@]}"
+}
+
+$1
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/testdata/cluster/node_templates/common/etc/init.d/kudu-tserver
----------------------------------------------------------------------
diff --git a/testdata/cluster/node_templates/common/etc/init.d/kudu-tserver b/testdata/cluster/node_templates/common/etc/init.d/kudu-tserver
new file mode 100755
index 0000000..60e49e6
--- /dev/null
+++ b/testdata/cluster/node_templates/common/etc/init.d/kudu-tserver
@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+
+DIR=$(dirname $0)
+
+. "$DIR/common"
+. "$DIR/kudu-common" # Sets KUDU_COMMON_ARGS
+
+function start {
+ do_start "$KUDU_BIN_DIR"/kudu-tserver \
+ -flagfile "$NODE_DIR"/etc/kudu/tserver.conf \
+ "${KUDU_COMMON_ARGS[@]}"
+}
+
+$1
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/testdata/cluster/node_templates/common/etc/init.d/yarn-common
----------------------------------------------------------------------
diff --git a/testdata/cluster/node_templates/common/etc/init.d/yarn-common b/testdata/cluster/node_templates/common/etc/init.d/yarn-common
index 84d0199..dc60971 100644
--- a/testdata/cluster/node_templates/common/etc/init.d/yarn-common
+++ b/testdata/cluster/node_templates/common/etc/init.d/yarn-common
@@ -15,7 +15,14 @@
# specific language governing permissions and limitations
# under the License.
-export YARN_LOG_DIR="$LOG_DIR/hadoop-yarn"
-export YARN_ROOT_LOGGER="${YARN_ROOT_LOGGER:-DEBUG,RFA}"
+if [[ $IMPALA_MINICLUSTER_PROFILE == 2 ]]; then
+ export YARN_LOG_DIR="$LOG_DIR/hadoop-yarn"
+ export YARN_ROOT_LOGGER="${YARN_ROOT_LOGGER:-DEBUG,RFA}"
-export YARN_LOGFILE=$(basename $0).log
+ export YARN_LOGFILE=$(basename $0).log
+elif [[ $IMPALA_MINICLUSTER_PROFILE == 3 ]]; then
+ export HADOOP_LOG_DIR="$LOG_DIR/hadoop-yarn"
+ export HADOOP_ROOT_LOGGER="${HADOOP_ROOT_LOGGER:-DEBUG,RFA}"
+
+ export HADOOP_LOGFILE=$(basename $0).log
+fi
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/testdata/cluster/node_templates/common/etc/kudu/master.conf.tmpl
----------------------------------------------------------------------
diff --git a/testdata/cluster/node_templates/common/etc/kudu/master.conf.tmpl b/testdata/cluster/node_templates/common/etc/kudu/master.conf.tmpl
new file mode 100644
index 0000000..f272eb7
--- /dev/null
+++ b/testdata/cluster/node_templates/common/etc/kudu/master.conf.tmpl
@@ -0,0 +1,9 @@
+-fs_wal_dir=${NODE_DIR}/var/lib/kudu/master/wal
+-fs_data_dirs=${NODE_DIR}/var/lib/kudu/master/data
+-log_dir=${NODE_DIR}/var/log/kudu/master
+
+# fsync is disabled for additional speed. Sometimes operations are slow on EC2/GCE test
+# machines. Some data loss could occur if the system crashes before the OS has a chance
+# to flush data to disk but that is acceptable for development purposes.
+-unlock_unsafe_flags
+-never_fsync
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/testdata/cluster/node_templates/common/etc/kudu/tserver.conf.tmpl
----------------------------------------------------------------------
diff --git a/testdata/cluster/node_templates/common/etc/kudu/tserver.conf.tmpl b/testdata/cluster/node_templates/common/etc/kudu/tserver.conf.tmpl
new file mode 100644
index 0000000..c84bd91
--- /dev/null
+++ b/testdata/cluster/node_templates/common/etc/kudu/tserver.conf.tmpl
@@ -0,0 +1,11 @@
+-rpc_bind_addresses=0.0.0.0:${KUDU_TS_RPC_PORT}
+-webserver_port=${KUDU_TS_WEBUI_PORT}
+-fs_wal_dir=${NODE_DIR}/var/lib/kudu/ts/wal
+-fs_data_dirs=${NODE_DIR}/var/lib/kudu/ts/data
+-log_dir=${NODE_DIR}/var/log/kudu/ts
+
+# fsync is disabled for additional speed. Sometimes operations are slow on EC2/GCE test
+# machines. Some data loss could occur if the system crashes before the OS has a chance
+# to flush data to disk but that is acceptable for development purposes.
+-unlock_unsafe_flags
+-never_fsync
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/testdata/pom.xml
----------------------------------------------------------------------
diff --git a/testdata/pom.xml b/testdata/pom.xml
index f00da3b..bf001c8 100644
--- a/testdata/pom.xml
+++ b/testdata/pom.xml
@@ -136,6 +136,27 @@ under the License.
<redirectTestOutputToFile>true</redirectTestOutputToFile>
</configuration>
</plugin>
+
+ <!-- Support different src dirs for different minicluster profiles. -->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <version>1.5</version>
+ <executions>
+ <execution>
+ <id>add-source</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>add-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>${project.basedir}/src/compat-minicluster-profile-${env.IMPALA_MINICLUSTER_PROFILE}/java</source>
+ </sources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/testdata/src/compat-minicluster-profile-2/java/org/apache/impala/datagenerator/HBaseTestDataRegionAssigment.java
----------------------------------------------------------------------
diff --git a/testdata/src/compat-minicluster-profile-2/java/org/apache/impala/datagenerator/HBaseTestDataRegionAssigment.java b/testdata/src/compat-minicluster-profile-2/java/org/apache/impala/datagenerator/HBaseTestDataRegionAssigment.java
new file mode 100644
index 0000000..a5799a1
--- /dev/null
+++ b/testdata/src/compat-minicluster-profile-2/java/org/apache/impala/datagenerator/HBaseTestDataRegionAssigment.java
@@ -0,0 +1,320 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.impala.datagenerator;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableMap;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.ClusterStatus;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.util.PairOfSameType;
+import org.apache.hadoop.hbase.util.Threads;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Maps;
+
+/**
+ * Splits HBase tables into regions and deterministically assigns regions to region
+ * servers.
+ */
+class HBaseTestDataRegionAssigment {
+ public class TableNotFoundException extends Exception {
+ public TableNotFoundException(String s) {
+ super(s);
+ }
+ }
+
+ private final static Logger LOG = LoggerFactory.getLogger(
+ HBaseTestDataRegionAssigment.class);
+ private final Configuration conf;
+ private final HBaseAdmin hbaseAdmin;
+ private final List<ServerName> sortedRS; // sorted list of region server name
+ private final String[] splitPoints = { "1", "3", "5", "7", "9"};
+
+ // Number of times to retry a series region-split/wait-for-split calls.
+ private final static int MAX_SPLIT_ATTEMPTS = 10;
+
+ // Maximum time in ms to wait for a region to be split.
+ private final static int WAIT_FOR_SPLIT_TIMEOUT = 10000;
+
+ public HBaseTestDataRegionAssigment() throws IOException {
+ conf = new Configuration();
+ hbaseAdmin = new HBaseAdmin(conf);
+ ClusterStatus clusterStatus = hbaseAdmin.getClusterStatus();
+ Collection<ServerName> regionServerNames = clusterStatus.getServers();
+ sortedRS = new ArrayList<ServerName>(regionServerNames);
+ Collections.sort(sortedRS);
+ }
+
+ public void close() throws IOException {
+ hbaseAdmin.close();
+ }
+
+ /**
+ * Split the table regions according to splitPoints and pair up adjacent regions to the
+ * same server. Each region pair in ([unbound:1,1:3], [3:5,5:7], [7:9,9:unbound])
+ * will be on the same server.
+ * The table must have data loaded and only a single region.
+ */
+ public void performAssigment(String tableName) throws IOException,
+ InterruptedException, TableNotFoundException {
+ HTableDescriptor[] desc = hbaseAdmin.listTables(tableName);
+ if (desc == null || desc.length == 0) {
+ throw new TableNotFoundException("Table " + tableName + " not found.");
+ }
+
+ if (hbaseAdmin.getTableRegions(tableName.getBytes()).size() == 1) {
+ // Split into regions
+ // The table has one region only to begin with. The logic of
+ // blockUntilRegionSplit requires that the input regionName has performed a split.
+ // If the table has already been split (i.e. regions count > 1), the same split
+ // call will be a no-op and this will cause blockUntilRegionSplit to break.
+ for (int i = 0; i < splitPoints.length; ++i) {
+ hbaseAdmin.majorCompact(tableName);
+ List<HRegionInfo> regions = hbaseAdmin.getTableRegions(tableName.getBytes());
+ HRegionInfo splitRegion = regions.get(regions.size() - 1);
+ int attempt = 1;
+ boolean done = false;
+ while (!done && attempt < MAX_SPLIT_ATTEMPTS) {
+ // HBase seems to not always properly receive/process this split RPC,
+ // so we need to retry the split/block several times.
+ hbaseAdmin.split(splitRegion.getRegionNameAsString(), splitPoints[i]);
+ done = blockUntilRegionSplit(conf, WAIT_FOR_SPLIT_TIMEOUT,
+ splitRegion.getRegionName(), true);
+ Thread.sleep(100);
+ ++attempt;
+ }
+ if (!done) {
+ throw new IllegalStateException(
+ String.format("Failed to split region '%s' after %s attempts.",
+ splitRegion.getRegionNameAsString(), WAIT_FOR_SPLIT_TIMEOUT));
+ }
+ LOG.info(String.format("Split region '%s' after %s attempts.",
+ splitRegion.getRegionNameAsString(), attempt));
+ }
+ }
+
+ // Sort the region by start key
+ List<HRegionInfo> regions = hbaseAdmin.getTableRegions(tableName.getBytes());
+ Preconditions.checkArgument(regions.size() == splitPoints.length + 1);
+ Collections.sort(regions);
+
+ // Pair up two adjacent regions to the same region server. That is,
+ // region server 1 <- regions (unbound:1), (1:3)
+ // region server 2 <- regions (3:5), (5:7)
+ // region server 3 <- regions (7:9), (9:unbound)
+ NavigableMap<HRegionInfo, ServerName> expectedLocs = Maps.newTreeMap();
+ for (int i = 0; i < regions.size(); ++i) {
+ HRegionInfo regionInfo = regions.get(i);
+ int rsIdx = (i / 2) % sortedRS.size();
+ ServerName regionServerName = sortedRS.get(rsIdx);
+ hbaseAdmin.move(regionInfo.getEncodedNameAsBytes(),
+ regionServerName.getServerName().getBytes());
+ expectedLocs.put(regionInfo, regionServerName);
+ }
+
+ // hbaseAdmin.move() is an asynchronous operation. HBase tests use sleep to wait for
+ // the move to complete. It should be done in 10sec.
+ int sleepCnt = 0;
+ HTable hbaseTable = new HTable(conf, tableName);
+ try {
+ while(!expectedLocs.equals(hbaseTable.getRegionLocations()) &&
+ sleepCnt < 100) {
+ Thread.sleep(100);
+ ++sleepCnt;
+ }
+ NavigableMap<HRegionInfo, ServerName> actualLocs = hbaseTable.getRegionLocations();
+ Preconditions.checkArgument(expectedLocs.equals(actualLocs));
+
+ // Log the actual region location map
+ for (Map.Entry<HRegionInfo, ServerName> entry: actualLocs.entrySet()) {
+ LOG.info(printKey(entry.getKey().getStartKey()) + " -> " +
+ entry.getValue().getHostAndPort());
+ }
+
+ // Force a major compaction such that the HBase table is backed by deterministic
+ // physical artifacts (files, WAL, etc.). Our #rows estimate relies on the sizes of
+ // these physical artifacts.
+ LOG.info("Major compacting HBase table: " + tableName);
+ hbaseAdmin.majorCompact(tableName);
+ } finally {
+ IOUtils.closeQuietly(hbaseTable);
+ }
+ }
+
+ /**
+ * Returns non-printable characters in escaped octal, otherwise returns the characters.
+ */
+ public static String printKey(byte[] key) {
+ StringBuilder result = new StringBuilder();
+ for (int i = 0; i < key.length; ++i) {
+ if (!Character.isISOControl(key[i])) {
+ result.append((char) key[i]);
+ } else {
+ result.append("\\");
+ result.append(Integer.toOctalString(key[i]));
+ }
+ }
+ return result.toString();
+ }
+
+ /**
+ * The following static methods blockUntilRegionSplit, getRegionRow,
+ * blockUntilRegionIsOpened and blockUntilRegionIsInMeta are copied from
+ * org.apache.hadoop.hbase.regionserver.TestEndToEndSplitTransaction
+ * to help block until a region split is completed.
+ *
+ * The original code was modified to return a true/false in case of success/failure.
+ *
+ * Blocks until the region split is complete in META and region server opens the
+ * daughters
+ */
+ private static boolean blockUntilRegionSplit(Configuration conf, long timeout,
+ final byte[] regionName, boolean waitForDaughters)
+ throws IOException, InterruptedException {
+ long start = System.currentTimeMillis();
+ HRegionInfo daughterA = null, daughterB = null;
+ HTable metaTable = new HTable(conf, TableName.META_TABLE_NAME);
+
+ try {
+ while (System.currentTimeMillis() - start < timeout) {
+ Result result = getRegionRow(metaTable, regionName);
+ if (result == null) {
+ break;
+ }
+
+ HRegionInfo region = HRegionInfo.getHRegionInfo(result);
+ if(region.isSplitParent()) {
+ PairOfSameType<HRegionInfo> pair = HRegionInfo.getDaughterRegions(result);
+ daughterA = pair.getFirst();
+ daughterB = pair.getSecond();
+ break;
+ }
+ Threads.sleep(100);
+ }
+ if (daughterA == null || daughterB == null) return false;
+
+ //if we are here, this means the region split is complete or timed out
+ if (waitForDaughters) {
+ long rem = timeout - (System.currentTimeMillis() - start);
+ blockUntilRegionIsInMeta(metaTable, rem, daughterA);
+
+ rem = timeout - (System.currentTimeMillis() - start);
+ blockUntilRegionIsInMeta(metaTable, rem, daughterB);
+
+ rem = timeout - (System.currentTimeMillis() - start);
+ blockUntilRegionIsOpened(conf, rem, daughterA);
+
+ rem = timeout - (System.currentTimeMillis() - start);
+ blockUntilRegionIsOpened(conf, rem, daughterB);
+ }
+ } finally {
+ IOUtils.closeQuietly(metaTable);
+ }
+ return true;
+ }
+
+ private static Result getRegionRow(HTable metaTable, byte[] regionName)
+ throws IOException {
+ Get get = new Get(regionName);
+ return metaTable.get(get);
+ }
+
+ private static void blockUntilRegionIsInMeta(HTable metaTable, long timeout,
+ HRegionInfo hri) throws IOException, InterruptedException {
+ long start = System.currentTimeMillis();
+ while (System.currentTimeMillis() - start < timeout) {
+ Result result = getRegionRow(metaTable, hri.getRegionName());
+ if (result != null) {
+ HRegionInfo info = HRegionInfo.getHRegionInfo(result);
+ if (info != null && !info.isOffline()) {
+ break;
+ }
+ }
+ Threads.sleep(10);
+ }
+ }
+
+ /**
+ * Starting with HBase 0.95.2 the Get class' c'tor no longer accepts
+ * empty key strings leading to the rather undesirable behavior that this method
+ * is not guaranteed to succeed. This method repeatedly attempts to 'get' the start key
+ * of the given region from the region server to detect when the region server becomes
+ * available. However, the first region has an empty array as the start key causing the
+ * Get c'tor to throw an exception as stated above. The end key cannot be used instead
+ * because it is an exclusive upper bound.
+ */
+ private static void blockUntilRegionIsOpened(Configuration conf, long timeout,
+ HRegionInfo hri) throws IOException, InterruptedException {
+ long start = System.currentTimeMillis();
+ HTable table = new HTable(conf, hri.getTableName());
+
+ try {
+ byte [] row = hri.getStartKey();
+ // Check for null/empty row. If we find one, use a key that is likely to
+ // be in first region. If key '0' happens not to be in the given region
+ // then an exception will be thrown.
+ if (row == null || row.length <= 0) row = new byte [] {'0'};
+ Get get = new Get(row);
+ while (System.currentTimeMillis() - start < timeout) {
+ try {
+ table.get(get);
+ break;
+ } catch(IOException ex) {
+ //wait some more
+ }
+ Threads.sleep(10);
+ }
+ } finally {
+ IOUtils.closeQuietly(table);
+ }
+ }
+
+ /**
+ * args contains a list of hbase table names. This program will split the hbase tables
+ * into regions and assign each region to a specific region server.
+ */
+ public static void main(String args[]) throws IOException, InterruptedException,
+ TableNotFoundException {
+ HBaseTestDataRegionAssigment assignment = new HBaseTestDataRegionAssigment();
+ for (String htable: args) {
+ assignment.performAssigment(htable);
+ }
+ assignment.close();
+ // Exit forcefully because of HDFS-6057. Otherwise, there the JVM won't exit due to a
+ // non-daemon thread still being up.
+ System.exit(0);
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/testdata/src/compat-minicluster-profile-3/java/org/apache/impala/datagenerator/HBaseTestDataRegionAssigment.java
----------------------------------------------------------------------
diff --git a/testdata/src/compat-minicluster-profile-3/java/org/apache/impala/datagenerator/HBaseTestDataRegionAssigment.java b/testdata/src/compat-minicluster-profile-3/java/org/apache/impala/datagenerator/HBaseTestDataRegionAssigment.java
new file mode 100644
index 0000000..fa24146
--- /dev/null
+++ b/testdata/src/compat-minicluster-profile-3/java/org/apache/impala/datagenerator/HBaseTestDataRegionAssigment.java
@@ -0,0 +1,331 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.impala.datagenerator;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.ClusterStatus;
+import org.apache.hadoop.hbase.HRegionLocation;
+import org.apache.hadoop.hbase.MetaTableAccessor;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.hadoop.hbase.util.PairOfSameType;
+import org.apache.hadoop.hbase.util.Threads;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Maps;
+
+/**
+ * Splits HBase tables into regions and deterministically assigns regions to region
+ * servers.
+ */
+class HBaseTestDataRegionAssigment {
+ public class TableNotFoundException extends Exception {
+ public TableNotFoundException(String s) {
+ super(s);
+ }
+ }
+
+ private final static Logger LOG = LoggerFactory.getLogger(
+ HBaseTestDataRegionAssigment.class);
+ private final Configuration conf;
+ private Connection connection = null;
+ private final Admin admin;
+ private final List<ServerName> sortedRS; // sorted list of region server name
+ private final String[] splitPoints = { "1", "3", "5", "7", "9"};
+
+ // Number of times to retry a series region-split/wait-for-split calls.
+ private final static int MAX_SPLIT_ATTEMPTS = 10;
+
+ // Maximum time in ms to wait for a region to be split.
+ private final static int WAIT_FOR_SPLIT_TIMEOUT = 10000;
+
+ public HBaseTestDataRegionAssigment() throws IOException {
+ conf = new Configuration();
+ connection = ConnectionFactory.createConnection(conf);
+ admin = connection.getAdmin();
+ ClusterStatus clusterStatus = admin.getClusterStatus();
+ List<ServerName> regionServerNames =
+ new ArrayList<ServerName>(clusterStatus.getServers());
+ ServerName master = clusterStatus.getMaster();
+ regionServerNames.remove(master);
+ sortedRS = new ArrayList<ServerName>(regionServerNames);
+ Collections.sort(sortedRS);
+ }
+
+ public void close() throws IOException {
+ admin.close();
+ }
+
+ /**
+ * Split the table regions according to splitPoints and pair up adjacent regions to the
+ * same server. Each region pair in ([unbound:1,1:3], [3:5,5:7], [7:9,9:unbound])
+ * will be on the same server.
+ * The table must have data loaded. We attempt to split even already partially split
+ * tables in order to facilitate recovery from partial transient failures.
+ */
+ public void performAssigment(String tableName) throws IOException,
+ InterruptedException, TableNotFoundException {
+ TableName table = TableName.valueOf(tableName);
+ if (!admin.tableExists(table)) {
+ throw new TableNotFoundException("Table " + tableName + " not found.");
+ }
+ if (admin.getRegions(table).size() <= splitPoints.length) {
+ // Split into regions
+ // The table has one region only to begin with. The logic of
+ // blockUntilRegionSplit requires that the input regionName has performed a split.
+ // If the table has already been split (i.e. regions count > 1), the same split
+ // call will be a no-op and this will cause blockUntilRegionSplit to break. In
+ // that case, swallow the resulting exception. Other exceptions will be re-thrown.
+ for (int i = 0; i < splitPoints.length; ++i) {
+ admin.majorCompact(table);
+ List<RegionInfo> regions = admin.getRegions(table);
+ RegionInfo splitRegion = regions.get(regions.size() - 1);
+ boolean done = false;
+ int attempt = 0;
+ for (; !done && attempt < MAX_SPLIT_ATTEMPTS; ++attempt) {
+ // HBase seems to not always properly receive/process this split RPC,
+ // so we need to retry the split/block several times.
+ try {
+ admin.split(splitRegion.getTable(), Bytes.toBytes(splitPoints[i]));
+ done = blockUntilRegionSplit(conf, WAIT_FOR_SPLIT_TIMEOUT,
+ splitRegion.getRegionName(), true);
+ } catch (IOException ex) {
+ if (!ex.getMessage().equals(
+ "should not give a splitkey which equals to startkey!")) {
+ throw ex;
+ }
+ done = true;
+ }
+ }
+ if (!done) {
+ throw new IllegalStateException(
+ String.format("Failed to split region '%s' after %s attempts (%d ms).",
+ splitRegion.getRegionNameAsString(), MAX_SPLIT_ATTEMPTS,
+ MAX_SPLIT_ATTEMPTS * WAIT_FOR_SPLIT_TIMEOUT));
+ }
+ LOG.info(String.format("Split region '%s' after %s attempts.",
+ splitRegion.getRegionNameAsString(), attempt));
+ }
+ }
+
+ // Sort the region by start key
+ List<RegionInfo> regions = admin.getRegions(table);
+ Preconditions.checkArgument(regions.size() == splitPoints.length + 1);
+ Collections.sort(regions, RegionInfo.COMPARATOR);
+ // Pair up two adjacent regions to the same region server. That is,
+ // region server 1 <- regions (unbound:1), (1:3)
+ // region server 2 <- regions (3:5), (5:7)
+ // region server 3 <- regions (7:9), (9:unbound)
+ HashMap<String, ServerName> expectedLocs = Maps.newHashMap();
+ for (int i = 0; i < regions.size(); ++i) {
+ RegionInfo regionInfo = regions.get(i);
+ int rsIdx = (i / 2) % sortedRS.size();
+ ServerName regionServerName = sortedRS.get(rsIdx);
+ LOG.info("Moving " + regionInfo.getRegionNameAsString() +
+ " to " + regionServerName.getAddress());
+ admin.move(regionInfo.getEncodedNameAsBytes(),
+ regionServerName.getServerName().getBytes());
+ expectedLocs.put(regionInfo.getRegionNameAsString(), regionServerName);
+ }
+
+ // admin.move() is an asynchronous operation. Wait for the move to complete.
+ // It should be done in 10sec.
+ int sleepCnt = 0;
+ while (true) {
+ int matched = 0;
+ List<Pair<RegionInfo, ServerName>> pairs =
+ MetaTableAccessor.getTableRegionsAndLocations(connection, table);
+ Preconditions.checkState(pairs.size() == regions.size());
+ for (Pair<RegionInfo, ServerName> pair: pairs) {
+ RegionInfo regionInfo = pair.getFirst();
+ String regionName = regionInfo.getRegionNameAsString();
+ ServerName serverName = pair.getSecond();
+ Preconditions.checkNotNull(expectedLocs.get(regionName));
+ LOG.info(regionName + " " + printKey(regionInfo.getStartKey()) + " -> " +
+ serverName.getAddress().toString() + ", expecting " +
+ expectedLocs.get(regionName));
+ if (expectedLocs.get(regionName).equals(serverName)) {
+ ++matched;
+ continue;
+ }
+ }
+ if (matched == regions.size()) break;
+ if (sleepCnt < 100) {
+ Thread.sleep(100);
+ ++sleepCnt;
+ continue;
+ }
+ throw new IllegalStateException(
+ String.format("Failed to assign regions to servers after 10 seconds."));
+ }
+
+ // Force a major compaction such that the HBase table is backed by deterministic
+ // physical artifacts (files, WAL, etc.). Our #rows estimate relies on the sizes of
+ // these physical artifacts.
+ LOG.info("Major compacting HBase table: " + tableName);
+ admin.majorCompact(table);
+ }
+
+ /**
+ * Returns non-printable characters in escaped octal, otherwise returns the characters.
+ */
+ public static String printKey(byte[] key) {
+ StringBuilder result = new StringBuilder();
+ for (int i = 0; i < key.length; ++i) {
+ if (!Character.isISOControl(key[i])) {
+ result.append((char) key[i]);
+ } else {
+ result.append("\\");
+ result.append(Integer.toOctalString(key[i]));
+ }
+ }
+ return result.toString();
+ }
+
+ /**
+ * The following static methods blockUntilRegionSplit, blockUntilRegionIsOpened
+ * and blockUntilRegionIsInMeta are copied from
+ * org.apache.hadoop.hbase.regionserver.TestEndToEndSplitTransaction
+ * to help block until a region split is completed.
+ *
+ * The original code was modified to return a true/false in case of success/failure.
+ *
+ * Blocks until the region split is complete in META and region server opens the
+ * daughters
+ */
+ private static boolean blockUntilRegionSplit(Configuration conf, long timeout,
+ final byte[] regionName, boolean waitForDaughters)
+ throws IOException, InterruptedException {
+ long start = System.currentTimeMillis();
+ RegionInfo daughterA = null, daughterB = null;
+ try (Connection conn = ConnectionFactory.createConnection(conf);
+ Table metaTable = conn.getTable(TableName.META_TABLE_NAME)) {
+ Result result = null;
+ while ((System.currentTimeMillis() - start) < timeout) {
+ result = metaTable.get(new Get(regionName));
+ if (result == null) {
+ break;
+ }
+ RegionInfo region = MetaTableAccessor.getRegionInfo(result);
+ if (region.isSplitParent()) {
+ PairOfSameType<RegionInfo> pair = MetaTableAccessor.getDaughterRegions(result);
+ daughterA = pair.getFirst();
+ daughterB = pair.getSecond();
+ break;
+ }
+ Threads.sleep(100);
+ }
+ if (daughterA == null || daughterB == null) return false;
+
+ //if we are here, this means the region split is complete or timed out
+ if (waitForDaughters) {
+ long rem = timeout - (System.currentTimeMillis() - start);
+ blockUntilRegionIsInMeta(conn, rem, daughterA);
+
+ rem = timeout - (System.currentTimeMillis() - start);
+ blockUntilRegionIsInMeta(conn, rem, daughterB);
+
+ rem = timeout - (System.currentTimeMillis() - start);
+ blockUntilRegionIsOpened(conf, rem, daughterA);
+
+ rem = timeout - (System.currentTimeMillis() - start);
+ blockUntilRegionIsOpened(conf, rem, daughterB);
+ }
+ }
+ return true;
+ }
+
+ private static void blockUntilRegionIsInMeta(Connection conn, long timeout,
+ RegionInfo hri) throws IOException, InterruptedException {
+ long start = System.currentTimeMillis();
+ while (System.currentTimeMillis() - start < timeout) {
+ HRegionLocation loc = MetaTableAccessor.getRegionLocation(conn, hri);
+ if (loc != null && !loc.getRegion().isOffline()) {
+ break;
+ }
+ Threads.sleep(10);
+ }
+ }
+
+ /**
+ * Starting with HBase 0.95.2 the Get class' c'tor no longer accepts
+ * empty key strings leading to the rather undesirable behavior that this method
+ * is not guaranteed to succeed. This method repeatedly attempts to 'get' the start key
+ * of the given region from the region server to detect when the region server becomes
+ * available. However, the first region has an empty array as the start key causing the
+ * Get c'tor to throw an exception as stated above. The end key cannot be used instead
+ * because it is an exclusive upper bound.
+ */
+ private static void blockUntilRegionIsOpened(Configuration conf, long timeout,
+ RegionInfo hri) throws IOException, InterruptedException {
+ long start = System.currentTimeMillis();
+ try (Connection conn = ConnectionFactory.createConnection(conf);
+ Table table = conn.getTable(hri.getTable())) {
+ byte [] row = hri.getStartKey();
+ // Check for null/empty row. If we find one, use a key that is likely to
+ // be in first region. If key '0' happens not to be in the given region
+ // then an exception will be thrown.
+ if (row == null || row.length <= 0) row = new byte [] {'0'};
+ Get get = new Get(row);
+ while (System.currentTimeMillis() - start < timeout) {
+ try {
+ table.get(get);
+ break;
+ } catch(IOException ex) {
+ //wait some more
+ }
+ Threads.sleep(10);
+ }
+ }
+ }
+
+ /**
+ * args contains a list of hbase table names. This program will split the hbase tables
+ * into regions and assign each region to a specific region server.
+ */
+ public static void main(String args[]) throws IOException, InterruptedException,
+ TableNotFoundException {
+ HBaseTestDataRegionAssigment assignment = new HBaseTestDataRegionAssigment();
+ for (String htable: args) {
+ assignment.performAssigment(htable);
+ }
+ assignment.close();
+ // Exit forcefully because of HDFS-6057. Otherwise, there the JVM won't exit due to a
+ // non-daemon thread still being up.
+ System.exit(0);
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/testdata/src/main/java/org/apache/impala/datagenerator/HBaseTestDataRegionAssigment.java
----------------------------------------------------------------------
diff --git a/testdata/src/main/java/org/apache/impala/datagenerator/HBaseTestDataRegionAssigment.java b/testdata/src/main/java/org/apache/impala/datagenerator/HBaseTestDataRegionAssigment.java
deleted file mode 100644
index 0e7f48e..0000000
--- a/testdata/src/main/java/org/apache/impala/datagenerator/HBaseTestDataRegionAssigment.java
+++ /dev/null
@@ -1,320 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.impala.datagenerator;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.NavigableMap;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.ClusterStatus;
-import org.apache.hadoop.hbase.HRegionInfo;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.client.HTable;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.util.PairOfSameType;
-import org.apache.hadoop.hbase.util.Threads;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Maps;
-
-/**
- * Splits HBase tables into regions and deterministically assigns regions to region
- * servers.
- */
-class HBaseTestDataRegionAssigment {
- public class TableNotFoundException extends Exception {
- public TableNotFoundException(String s) {
- super(s);
- }
- }
-
- private final static Logger LOG = LoggerFactory.getLogger(
- HBaseTestDataRegionAssigment.class);
- private final Configuration conf;
- private final HBaseAdmin hbaseAdmin;
- private final List<ServerName> sortedRS; // sorted list of region server name
- private final String[] splitPoints = { "1", "3", "5", "7", "9"};
-
- // Number of times to retry a series region-split/wait-for-split calls.
- private final static int MAX_SPLIT_ATTEMPTS = 10;
-
- // Maximum time in ms to wait for a region to be split.
- private final static int WAIT_FOR_SPLIT_TIMEOUT = 10000;
-
- public HBaseTestDataRegionAssigment() throws IOException {
- conf = new Configuration();
- hbaseAdmin = new HBaseAdmin(conf);
- ClusterStatus clusterStatus = hbaseAdmin.getClusterStatus();
- Collection<ServerName> regionServerNames = clusterStatus.getServers();
- sortedRS = new ArrayList<ServerName>(regionServerNames);
- Collections.sort(sortedRS);
- }
-
- public void close() throws IOException {
- hbaseAdmin.close();
- }
-
- /**
- * Split the table regions according to splitPoints and pair up adjacent regions to the
- * same server. Each region pair in ([unbound:1,1:3], [3:5,5:7], [7:9,9:unbound])
- * will be on the same server.
- * The table must have data loaded and only a single region.
- */
- public void performAssigment(String tableName) throws IOException,
- InterruptedException, TableNotFoundException {
- HTableDescriptor[] desc = hbaseAdmin.listTables(tableName);
- if (desc == null || desc.length == 0) {
- throw new TableNotFoundException("Table " + tableName + " not found.");
- }
-
- if (hbaseAdmin.getTableRegions(tableName.getBytes()).size() == 1) {
- // Split into regions
- // The table has one region only to begin with. The logic of
- // blockUntilRegionSplit requires that the input regionName has performed a split.
- // If the table has already been split (i.e. regions count > 1), the same split
- // call will be a no-op and this will cause blockUntilRegionSplit to break.
- for (int i = 0; i < splitPoints.length; ++i) {
- hbaseAdmin.majorCompact(tableName);
- List<HRegionInfo> regions = hbaseAdmin.getTableRegions(tableName.getBytes());
- HRegionInfo splitRegion = regions.get(regions.size() - 1);
- int attempt = 1;
- boolean done = false;
- while (!done && attempt < MAX_SPLIT_ATTEMPTS) {
- // HBase seems to not always properly receive/process this split RPC,
- // so we need to retry the split/block several times.
- hbaseAdmin.split(splitRegion.getRegionNameAsString(), splitPoints[i]);
- done = blockUntilRegionSplit(conf, WAIT_FOR_SPLIT_TIMEOUT,
- splitRegion.getRegionName(), true);
- Thread.sleep(100);
- ++attempt;
- }
- if (!done) {
- throw new IllegalStateException(
- String.format("Failed to split region '%s' after %s attempts.",
- splitRegion.getRegionNameAsString(), WAIT_FOR_SPLIT_TIMEOUT));
- }
- LOG.info(String.format("Split region '%s' after %s attempts.",
- splitRegion.getRegionNameAsString(), attempt));
- }
- }
-
- // Sort the region by start key
- List<HRegionInfo> regions = hbaseAdmin.getTableRegions(tableName.getBytes());
- Preconditions.checkArgument(regions.size() == splitPoints.length + 1);
- Collections.sort(regions);
-
- // Pair up two adjacent regions to the same region server. That is,
- // region server 1 <- regions (unbound:1), (1:3)
- // region server 2 <- regions (3:5), (5:7)
- // region server 3 <- regions (7:9), (9:unbound)
- NavigableMap<HRegionInfo, ServerName> expectedLocs = Maps.newTreeMap();
- for (int i = 0; i < regions.size(); ++i) {
- HRegionInfo regionInfo = regions.get(i);
- int rsIdx = (i / 2) % sortedRS.size();
- ServerName regionServerName = sortedRS.get(rsIdx);
- hbaseAdmin.move(regionInfo.getEncodedNameAsBytes(),
- regionServerName.getServerName().getBytes());
- expectedLocs.put(regionInfo, regionServerName);
- }
-
- // hbaseAdmin.move() is an asynchronous operation. HBase tests use sleep to wait for
- // the move to complete. It should be done in 10sec.
- int sleepCnt = 0;
- HTable hbaseTable = new HTable(conf, tableName);
- try {
- while(!expectedLocs.equals(hbaseTable.getRegionLocations()) &&
- sleepCnt < 100) {
- Thread.sleep(100);
- ++sleepCnt;
- }
- NavigableMap<HRegionInfo, ServerName> actualLocs = hbaseTable.getRegionLocations();
- Preconditions.checkArgument(expectedLocs.equals(actualLocs));
-
- // Log the actual region location map
- for (Map.Entry<HRegionInfo, ServerName> entry: actualLocs.entrySet()) {
- LOG.info(printKey(entry.getKey().getStartKey()) + " -> " +
- entry.getValue().getHostAndPort());
- }
-
- // Force a major compaction such that the HBase table is backed by deterministic
- // physical artifacts (files, WAL, etc.). Our #rows estimate relies on the sizes of
- // these physical artifacts.
- LOG.info("Major compacting HBase table: " + tableName);
- hbaseAdmin.majorCompact(tableName);
- } finally {
- IOUtils.closeQuietly(hbaseTable);
- }
- }
-
- /**
- * Returns non-printable characters in escaped octal, otherwise returns the characters.
- */
- public static String printKey(byte[] key) {
- StringBuilder result = new StringBuilder();
- for (int i = 0; i < key.length; ++i) {
- if (!Character.isISOControl(key[i])) {
- result.append((char) key[i]);
- } else {
- result.append("\\");
- result.append(Integer.toOctalString(key[i]));
- }
- }
- return result.toString();
- }
-
- /**
- * The following static methods blockUntilRegionSplit, getRegionRow,
- * blockUntilRegionIsOpened and blockUntilRegionIsInMeta are copied from
- * org.apache.hadoop.hbase.regionserver.TestEndToEndSplitTransaction
- * to help block until a region split is completed.
- *
- * The original code was modified to return a true/false in case of success/failure.
- *
- * Blocks until the region split is complete in META and region server opens the
- * daughters
- */
- private static boolean blockUntilRegionSplit(Configuration conf, long timeout,
- final byte[] regionName, boolean waitForDaughters)
- throws IOException, InterruptedException {
- long start = System.currentTimeMillis();
- HRegionInfo daughterA = null, daughterB = null;
- HTable metaTable = new HTable(conf, TableName.META_TABLE_NAME);
-
- try {
- while (System.currentTimeMillis() - start < timeout) {
- Result result = getRegionRow(metaTable, regionName);
- if (result == null) {
- break;
- }
-
- HRegionInfo region = HRegionInfo.getHRegionInfo(result);
- if(region.isSplitParent()) {
- PairOfSameType<HRegionInfo> pair = HRegionInfo.getDaughterRegions(result);
- daughterA = pair.getFirst();
- daughterB = pair.getSecond();
- break;
- }
- Threads.sleep(100);
- }
- if (daughterA == null || daughterB == null) return false;
-
- //if we are here, this means the region split is complete or timed out
- if (waitForDaughters) {
- long rem = timeout - (System.currentTimeMillis() - start);
- blockUntilRegionIsInMeta(metaTable, rem, daughterA);
-
- rem = timeout - (System.currentTimeMillis() - start);
- blockUntilRegionIsInMeta(metaTable, rem, daughterB);
-
- rem = timeout - (System.currentTimeMillis() - start);
- blockUntilRegionIsOpened(conf, rem, daughterA);
-
- rem = timeout - (System.currentTimeMillis() - start);
- blockUntilRegionIsOpened(conf, rem, daughterB);
- }
- } finally {
- IOUtils.closeQuietly(metaTable);
- }
- return true;
- }
-
- private static Result getRegionRow(HTable metaTable, byte[] regionName)
- throws IOException {
- Get get = new Get(regionName);
- return metaTable.get(get);
- }
-
- private static void blockUntilRegionIsInMeta(HTable metaTable, long timeout,
- HRegionInfo hri) throws IOException, InterruptedException {
- long start = System.currentTimeMillis();
- while (System.currentTimeMillis() - start < timeout) {
- Result result = getRegionRow(metaTable, hri.getRegionName());
- if (result != null) {
- HRegionInfo info = HRegionInfo.getHRegionInfo(result);
- if (info != null && !info.isOffline()) {
- break;
- }
- }
- Threads.sleep(10);
- }
- }
-
- /**
- * Starting with HBase 0.95.2 the Get class' c'tor no longer accepts
- * empty key strings leading to the rather undesirable behavior that this method
- * is not guaranteed to succeed. This method repeatedly attempts to 'get' the start key
- * of the given region from the region server to detect when the region server becomes
- * available. However, the first region has an empty array as the start key causing the
- * Get c'tor to throw an exception as stated above. The end key cannot be used instead
- * because it is an exclusive upper bound.
- */
- private static void blockUntilRegionIsOpened(Configuration conf, long timeout,
- HRegionInfo hri) throws IOException, InterruptedException {
- long start = System.currentTimeMillis();
- HTable table = new HTable(conf, hri.getTableName());
-
- try {
- byte [] row = hri.getStartKey();
- // Check for null/empty row. If we find one, use a key that is likely to
- // be in first region. If key '0' happens not to be in the given region
- // then an exception will be thrown.
- if (row == null || row.length <= 0) row = new byte [] {'0'};
- Get get = new Get(row);
- while (System.currentTimeMillis() - start < timeout) {
- try {
- table.get(get);
- break;
- } catch(IOException ex) {
- //wait some more
- }
- Threads.sleep(10);
- }
- } finally {
- IOUtils.closeQuietly(table);
- }
- }
-
- /**
- * args contains a list of hbase table names. This program will split the hbase tables
- * into regions and assign each region to a specific region server.
- */
- public static void main(String args[]) throws IOException, InterruptedException,
- TableNotFoundException {
- HBaseTestDataRegionAssigment assignment = new HBaseTestDataRegionAssigment();
- for (String htable: args) {
- assignment.performAssigment(htable);
- }
- assignment.close();
- // Exit forcefully because of HDFS-6057. Otherwise, there the JVM won't exit due to a
- // non-daemon thread still being up.
- System.exit(0);
- }
-}
-
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/testdata/workloads/functional-query/queries/QueryTest/parquet-filtering.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/parquet-filtering.test b/testdata/workloads/functional-query/queries/QueryTest/parquet-filtering.test
index 7a4876f..172f6b1 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/parquet-filtering.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/parquet-filtering.test
@@ -178,8 +178,7 @@ select count(*) from functional_parquet.alltypes where date_string_col = '01/01/
0
---- RUNTIME_PROFILE
aggregation(SUM, NumRowGroups): 24
-aggregation(SUM, NumDictFilteredRowGroups): 2
-aggregation(SUM, NumStatsFilteredRowGroups): 22
+aggregation(SUM, NumDictFilteredRowGroups): 24
====
---- QUERY
# string_col: All values pass
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/testdata/workloads/functional-query/queries/QueryTest/views-compatibility.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/views-compatibility.test b/testdata/workloads/functional-query/queries/QueryTest/views-compatibility.test
index d01f01c..d35b4bf 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/views-compatibility.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/views-compatibility.test
@@ -69,44 +69,44 @@ IMPALA=SUCCESS
HIVE=SUCCESS
====
---- CREATE_VIEW
-# Test that creating a view in Impala with limit and offset works in Impala and does not
-# work in Hive, which does not support offset.
+# Test that creating a view in Impala with limit and offset works in Impala;
+# it does not work in Hive 1, which does not support offset. Works in Hive 2.
create view test as
select id, int_col, string_col from functional.alltypesagg
order by int_col limit 10 offset 5
---- CREATE_VIEW_RESULTS
IMPALA=SUCCESS
-HIVE=FAILURE
+HIVE=SUCCESS_PROFILE_3_ONLY
---- QUERY_IMPALA_VIEW_RESULTS
IMPALA=SUCCESS
-HIVE=FAILURE
+HIVE=SUCCESS_PROFILE_3_ONLY
====
---- CREATE_VIEW
# Test that creating a view in Impala with "NULLS FIRST/LAST" works when the nulls
-# ordering is the default behavior. View creation in Hive will fail because the NULLS
-# FIRST/LAST is not supported.
+# ordering is the default behavior. View creation in Hive 1 will fail because the NULLS
+# FIRST/LAST is not supported; works in Hive 2.
create view test as
select id, int_col, string_col from functional.alltypesagg
order by int_col asc nulls last limit 10
---- CREATE_VIEW_RESULTS
IMPALA=SUCCESS
-HIVE=FAILURE
+HIVE=SUCCESS_PROFILE_3_ONLY
---- QUERY_IMPALA_VIEW_RESULTS
IMPALA=SUCCESS
HIVE=SUCCESS
====
---- CREATE_VIEW
-# A view created in Impala with "NULLS FIRST/LAST" will not work in Hive when the null
-# ordering is not the default.
+# A view created in Impala with "NULLS FIRST/LAST" will not work in Hive 1 when the null
+# ordering is not the default; works in Hive 2.
create view test as
select id, int_col, string_col from functional.alltypesagg
order by int_col desc nulls last limit 10
---- CREATE_VIEW_RESULTS
IMPALA=SUCCESS
-HIVE=FAILURE
+HIVE=SUCCESS_PROFILE_3_ONLY
---- QUERY_IMPALA_VIEW_RESULTS
IMPALA=SUCCESS
-HIVE=FAILURE
+HIVE=SUCCESS_PROFILE_3_ONLY
====
---- CREATE_VIEW
# Test that exotic column names are quoted in
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/tests/common/environ.py
----------------------------------------------------------------------
diff --git a/tests/common/environ.py b/tests/common/environ.py
index 347ea44..fc863dc 100644
--- a/tests/common/environ.py
+++ b/tests/common/environ.py
@@ -180,3 +180,7 @@ def specific_build_type_timeout(
else:
timeout_val = default_timeout
return timeout_val
+
+def is_hive_2():
+ """Returns True if IMPALA_MINICLUSTER_PROFILE in use provides Hive 2."""
+ return os.environ.get("IMPALA_MINICLUSTER_PROFILE", None) == "3"
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/tests/common/impala_test_suite.py
----------------------------------------------------------------------
diff --git a/tests/common/impala_test_suite.py b/tests/common/impala_test_suite.py
index 28d9c2e..d57b1cb 100644
--- a/tests/common/impala_test_suite.py
+++ b/tests/common/impala_test_suite.py
@@ -671,6 +671,14 @@ class ImpalaTestSuite(BaseTestSuite):
tmpdir = tempfile.mkdtemp(prefix="impala-tests-")
beeline_opts += ['--hiveconf', 'mapreduce.cluster.local.dir={0}'.format(tmpdir)]
try:
+ # Remove HADOOP_CLASSPATH from environment. Beeline doesn't need it,
+ # and doing so avoids Hadoop 3's classpath de-duplication code from
+ # placing $HADOOP_CONF_DIR too late in the classpath to get the right
+ # log4j configuration file picked up. Some log4j configuration files
+ # in Hadoop's jars send logging to stdout, confusing Impala's test
+ # framework.
+ env = os.environ.copy()
+ env.pop("HADOOP_CLASSPATH", None)
call = subprocess.Popen(
['beeline',
'--outputformat=csv2',
@@ -678,7 +686,13 @@ class ImpalaTestSuite(BaseTestSuite):
'-n', username,
'-e', stmt] + beeline_opts,
stdout=subprocess.PIPE,
- stderr=subprocess.PIPE)
+ stderr=subprocess.PIPE,
+ # Beeline in Hive 2.1 will read from stdin even when "-e"
+ # is specified; explicitly make sure there's nothing to
+ # read to avoid hanging, especially when running interactively
+ # with py.test.
+ stdin=file("/dev/null"),
+ env=env)
(stdout, stderr) = call.communicate()
call.wait()
if call.returncode != 0:
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/tests/metadata/test_views_compatibility.py
----------------------------------------------------------------------
diff --git a/tests/metadata/test_views_compatibility.py b/tests/metadata/test_views_compatibility.py
index 116eb10..11227dd 100644
--- a/tests/metadata/test_views_compatibility.py
+++ b/tests/metadata/test_views_compatibility.py
@@ -15,12 +15,14 @@
# specific language governing permissions and limitations
# under the License.
+import os
import pprint
import pytest
import shlex
from subprocess import call
from tests.beeswax.impala_beeswax import ImpalaBeeswaxException
+from tests.common.environ import is_hive_2
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.skip import SkipIfS3, SkipIfADLS, SkipIfIsilon, SkipIfLocal
from tests.common.test_dimensions import create_uncompressed_text_dimension
@@ -249,10 +251,15 @@ class ViewCompatTestCase(object):
exp_res = dict()
for line in lines:
components = line.partition("=")
- if (components[2].lower() == 'SUCCESS'.lower()):
+ component_value = components[2].upper()
+ if component_value == 'SUCCESS':
exp_res[components[0]] = True
- else:
+ elif component_value == 'SUCCESS_PROFILE_3_ONLY':
+ exp_res[components[0]] = is_hive_2()
+ elif component_value == 'FAILURE':
exp_res[components[0]] = False
+ else:
+ raise Exception("Unexpected result declared: " + line)
# check that the results section contains at least one entry
if not (lambda a, b: any(i in b for i in a)):
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/tests/query_test/test_mt_dop.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_mt_dop.py b/tests/query_test/test_mt_dop.py
index 38a8314..b6e08e5 100644
--- a/tests/query_test/test_mt_dop.py
+++ b/tests/query_test/test_mt_dop.py
@@ -100,6 +100,8 @@ class TestMtDopParquet(ImpalaTestSuite):
def test_parquet_filtering(self, vector):
"""IMPALA-4624: Test that dictionary filtering eliminates row groups correctly."""
vector.get_value('exec_option')['mt_dop'] = vector.get_value('mt_dop')
+ # Disable min-max stats to prevent interference with directionary filtering.
+ vector.get_value('exec_option')['parquet_read_statistics'] = '0'
self.run_test_case('QueryTest/parquet-filtering', vector)
class TestMtDopKudu(KuduTestSuite):
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/tests/query_test/test_partitioning.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_partitioning.py b/tests/query_test/test_partitioning.py
index 8fc3399..597e791 100644
--- a/tests/query_test/test_partitioning.py
+++ b/tests/query_test/test_partitioning.py
@@ -15,12 +15,14 @@
# specific language governing permissions and limitations
# under the License.
+import os
import pytest
from tests.beeswax.impala_beeswax import ImpalaBeeswaxException
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.skip import SkipIfS3, SkipIfADLS, SkipIfIsilon, SkipIfLocal
from tests.common.test_dimensions import create_single_exec_option_dimension
+from tests.common.environ import is_hive_2
# Tests to validate HDFS partitioning.
class TestPartitioning(ImpalaTestSuite):
@@ -59,9 +61,11 @@ class TestPartitioning(ImpalaTestSuite):
self.execute_query("create table %s (i int) partitioned by (b boolean)" % full_name)
- # Insert some data using Hive. Due to HIVE-6590, Hive may create multiple
+ # Insert some data using Hive. Due to HIVE-6590, Hive 1 may create multiple
# partitions, mapping to the same boolean literal value.
- # For example, Hive may create partitions: /b=FALSE and /b=false, etc
+ # For example, Hive may create partitions: /b=FALSE and /b=false, etc.
+ # This particular issue was fixed in Hive 2 though HIVE-6590 has not
+ # been resolved as of release 2.3.1, though it has been resolved in Hive master.
self.run_stmt_in_hive("INSERT INTO TABLE %s PARTITION(b=false) SELECT 1 from "\
"functional.alltypes limit 1" % full_name)
self.run_stmt_in_hive("INSERT INTO TABLE %s PARTITION(b=FALSE) SELECT 2 from "\
@@ -75,9 +79,13 @@ class TestPartitioning(ImpalaTestSuite):
# List the partitions. Show table stats returns 1 row for each partition + 1 summary
# row
result = self.execute_query("show table stats %s" % full_name)
- assert len(result.data) == 3 + 1
+ if is_hive_2():
+ assert len(result.data) == 2 + 1
+ else:
+ assert len(result.data) == 3 + 1
- # Verify Impala properly merges the results of the bad Hive metadata.
+ # Verify Impala properly merges the results of the Hive metadata,
+ # whether it be good (Hive 2) or bad (Hive 1).
assert '13' == self.execute_scalar("select sum(i) from %s" % full_name);
assert '10' == self.execute_scalar("select sum(i) from %s where b=true" % full_name)
assert '3' == self.execute_scalar("select sum(i) from %s where b=false" % full_name)
[3/4] impala git commit: IMPALA-4277: Support multiple versions of
Hadoop ecosystem
Posted by ta...@apache.org.
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/fe/src/compat-minicluster-profile-3/java/org/apache/impala/analysis/ParquetHelper.java
----------------------------------------------------------------------
diff --git a/fe/src/compat-minicluster-profile-3/java/org/apache/impala/analysis/ParquetHelper.java b/fe/src/compat-minicluster-profile-3/java/org/apache/impala/analysis/ParquetHelper.java
new file mode 100644
index 0000000..8c9bff8
--- /dev/null
+++ b/fe/src/compat-minicluster-profile-3/java/org/apache/impala/analysis/ParquetHelper.java
@@ -0,0 +1,341 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.impala.analysis;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Maps;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.parquet.hadoop.metadata.ParquetMetadata;
+import org.apache.parquet.hadoop.ParquetFileReader;
+import org.apache.parquet.schema.OriginalType;
+import org.apache.parquet.schema.PrimitiveType;
+
+import org.apache.impala.catalog.ArrayType;
+import org.apache.impala.catalog.MapType;
+import org.apache.impala.catalog.ScalarType;
+import org.apache.impala.catalog.StructField;
+import org.apache.impala.catalog.StructType;
+import org.apache.impala.catalog.Type;
+import org.apache.impala.common.AnalysisException;
+import org.apache.impala.common.FileSystemUtil;
+
+/**
+ * Provides extractParquetSchema() to extract a schema
+ * from a parquet file.
+ *
+ * Because Parquet's Java package changed between Parquet 1.5
+ * and 1.9, a second copy of this file, with "org.apache.parquet." replaced
+ * with "org.apache.org.apache.parquet." is generated by the build system.
+ */
+class ParquetHelper {
+ private final static String ERROR_MSG =
+ "Failed to convert Parquet type\n%s\nto an Impala %s type:\n%s\n";
+
+ /**
+ * Reads the first block from the given HDFS file and returns the Parquet schema.
+ * Throws Analysis exception for any failure, such as failing to read the file
+ * or failing to parse the contents.
+ */
+ private static org.apache.parquet.schema.MessageType loadParquetSchema(Path pathToFile)
+ throws AnalysisException {
+ try {
+ FileSystem fs = pathToFile.getFileSystem(FileSystemUtil.getConfiguration());
+ if (!fs.isFile(pathToFile)) {
+ throw new AnalysisException("Cannot infer schema, path is not a file: " +
+ pathToFile);
+ }
+ } catch (IOException e) {
+ throw new AnalysisException("Failed to connect to filesystem:" + e);
+ } catch (IllegalArgumentException e) {
+ throw new AnalysisException(e.getMessage());
+ }
+ ParquetMetadata readFooter = null;
+ try {
+ readFooter = ParquetFileReader.readFooter(FileSystemUtil.getConfiguration(),
+ pathToFile);
+ } catch (FileNotFoundException e) {
+ throw new AnalysisException("File not found: " + e);
+ } catch (IOException e) {
+ throw new AnalysisException("Failed to open file as a parquet file: " + e);
+ } catch (RuntimeException e) {
+ // Parquet throws a generic RuntimeException when reading a non-parquet file
+ if (e.toString().contains("is not a Parquet file")) {
+ throw new AnalysisException("File is not a parquet file: " + pathToFile);
+ }
+ // otherwise, who knows what we caught, throw it back up
+ throw e;
+ }
+ return readFooter.getFileMetaData().getSchema();
+ }
+
+ /**
+ * Converts a "primitive" Parquet type to an Impala type.
+ * A primitive type is a non-nested type with no annotations.
+ */
+ private static Type convertPrimitiveParquetType(org.apache.parquet.schema.Type parquetType)
+ throws AnalysisException {
+ Preconditions.checkState(parquetType.isPrimitive());
+ PrimitiveType prim = parquetType.asPrimitiveType();
+ switch (prim.getPrimitiveTypeName()) {
+ case BINARY: return Type.STRING;
+ case BOOLEAN: return Type.BOOLEAN;
+ case DOUBLE: return Type.DOUBLE;
+ case FIXED_LEN_BYTE_ARRAY:
+ throw new AnalysisException(
+ "Unsupported parquet type FIXED_LEN_BYTE_ARRAY for field " +
+ parquetType.getName());
+ case FLOAT: return Type.FLOAT;
+ case INT32: return Type.INT;
+ case INT64: return Type.BIGINT;
+ case INT96: return Type.TIMESTAMP;
+ default:
+ Preconditions.checkState(false, "Unexpected parquet primitive type: " +
+ prim.getPrimitiveTypeName());
+ return null;
+ }
+ }
+
+ /**
+ * Converts a Parquet group type to an Impala map Type. We support both standard
+ * Parquet map representations, as well as legacy. Legacy representations are handled
+ * according to this specification:
+ * https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#backward-compatibility-rules-1
+ *
+ * Standard representation of a map in Parquet:
+ * <optional | required> group <name> (MAP) { <-- outerGroup is pointing at this
+ * repeated group key_value {
+ * required <key-type> key;
+ * <optional | required> <value-type> value;
+ * }
+ * }
+ */
+ private static MapType convertMap(org.apache.parquet.schema.GroupType outerGroup)
+ throws AnalysisException {
+ if (outerGroup.getFieldCount() != 1){
+ throw new AnalysisException(String.format(ERROR_MSG, outerGroup.toString(),
+ "MAP", "The logical MAP type must have exactly 1 inner field."));
+ }
+
+ org.apache.parquet.schema.Type innerField = outerGroup.getType(0);
+ if (!innerField.isRepetition(org.apache.parquet.schema.Type.Repetition.REPEATED)){
+ throw new AnalysisException(String.format(ERROR_MSG, outerGroup.toString(),
+ "MAP", "The logical MAP type must have a repeated inner field."));
+ }
+ if (innerField.isPrimitive()) {
+ throw new AnalysisException(String.format(ERROR_MSG, outerGroup.toString(),
+ "MAP", "The inner field of the logical MAP type must be a group."));
+ }
+
+ org.apache.parquet.schema.GroupType innerGroup = innerField.asGroupType();
+ // It does not matter whether innerGroup has an annotation or not (for example it may
+ // be annotated with MAP_KEY_VALUE). We treat the case that innerGroup has an
+ // annotation and the case the innerGroup does not have an annotation the same.
+ if (innerGroup.getFieldCount() != 2) {
+ throw new AnalysisException(String.format(ERROR_MSG, outerGroup.toString(),
+ "MAP", "The inner field of the logical MAP type must have exactly 2 fields."));
+ }
+
+ org.apache.parquet.schema.Type key = innerGroup.getType(0);
+ if (!key.getName().equals("key")) {
+ throw new AnalysisException(String.format(ERROR_MSG, outerGroup.toString(),
+ "MAP", "The name of the first field of the inner field of the logical MAP " +
+ "type must be 'key'"));
+ }
+ if (!key.isPrimitive()) {
+ throw new AnalysisException(String.format(ERROR_MSG, outerGroup.toString(),
+ "MAP", "The key type of the logical MAP type must be primitive."));
+ }
+ org.apache.parquet.schema.Type value = innerGroup.getType(1);
+ if (!value.getName().equals("value")) {
+ throw new AnalysisException(String.format(ERROR_MSG, outerGroup.toString(),
+ "MAP", "The name of the second field of the inner field of the logical MAP " +
+ "type must be 'value'"));
+ }
+
+ return new MapType(convertParquetType(key), convertParquetType(value));
+ }
+
+ /**
+ * Converts a Parquet group type to an Impala struct Type.
+ */
+ private static StructType convertStruct(org.apache.parquet.schema.GroupType outerGroup)
+ throws AnalysisException {
+ ArrayList<StructField> structFields = new ArrayList<StructField>();
+ for (org.apache.parquet.schema.Type field: outerGroup.getFields()) {
+ StructField f = new StructField(field.getName(), convertParquetType(field));
+ structFields.add(f);
+ }
+ return new StructType(structFields);
+ }
+
+ /**
+ * Converts a Parquet group type to an Impala array Type. We can handle the standard
+ * representation, but also legacy representations for backwards compatibility.
+ * Legacy representations are handled according to this specification:
+ * https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#backward-compatibility-rules
+ *
+ * Standard representation of an array in Parquet:
+ * <optional | required> group <name> (LIST) { <-- outerGroup is pointing at this
+ * repeated group list {
+ * <optional | required> <element-type> element;
+ * }
+ * }
+ */
+ private static ArrayType convertArray(org.apache.parquet.schema.GroupType outerGroup)
+ throws AnalysisException {
+ if (outerGroup.getFieldCount() != 1) {
+ throw new AnalysisException(String.format(ERROR_MSG, outerGroup.toString(),
+ "LIST", "The logical LIST type must have exactly 1 inner field."));
+ }
+
+ org.apache.parquet.schema.Type innerField = outerGroup.getType(0);
+ if (!innerField.isRepetition(org.apache.parquet.schema.Type.Repetition.REPEATED)) {
+ throw new AnalysisException(String.format(ERROR_MSG, outerGroup.toString(),
+ "LIST", "The inner field of the logical LIST type must be repeated."));
+ }
+ if (innerField.isPrimitive() || innerField.getOriginalType() != null) {
+ // From the Parquet Spec:
+ // 1. If the repeated field is not a group then it's type is the element type.
+ //
+ // If innerField is a group, but originalType is not null, the element type is
+ // based on the logical type.
+ return new ArrayType(convertParquetType(innerField));
+ }
+
+ org.apache.parquet.schema.GroupType innerGroup = innerField.asGroupType();
+ if (innerGroup.getFieldCount() != 1) {
+ // From the Parquet Spec:
+ // 2. If the repeated field is a group with multiple fields, then it's type is a
+ // struct.
+ return new ArrayType(convertStruct(innerGroup));
+ }
+
+ return new ArrayType(convertParquetType(innerGroup.getType(0)));
+ }
+
+ /**
+ * Converts a "logical" Parquet type to an Impala column type.
+ * A Parquet type is considered logical when it has an annotation. The annotation is
+ * stored as a "OriginalType". The Parquet documentation refers to these as logical
+ * types, so we use that terminology here.
+ */
+ private static Type convertLogicalParquetType(org.apache.parquet.schema.Type parquetType)
+ throws AnalysisException {
+ OriginalType orig = parquetType.getOriginalType();
+ if (orig == OriginalType.LIST) {
+ return convertArray(parquetType.asGroupType());
+ }
+ if (orig == OriginalType.MAP || orig == OriginalType.MAP_KEY_VALUE) {
+ // MAP_KEY_VALUE annotation should not be used any more. However, according to the
+ // Parquet spec, some existing data incorrectly uses MAP_KEY_VALUE in place of MAP.
+ // For backward-compatibility, a group annotated with MAP_KEY_VALUE that is not
+ // contained by a MAP-annotated group should be handled as a MAP-annotated group.
+ return convertMap(parquetType.asGroupType());
+ }
+
+ PrimitiveType prim = parquetType.asPrimitiveType();
+ if (prim.getPrimitiveTypeName() == PrimitiveType.PrimitiveTypeName.BINARY &&
+ (orig == OriginalType.UTF8 || orig == OriginalType.ENUM)) {
+ // UTF8 is the type annotation Parquet uses for strings
+ // ENUM is the type annotation Parquet uses to indicate that
+ // the original data type, before conversion to parquet, had been enum.
+ // Applications which do not have enumerated types (e.g. Impala)
+ // should interpret it as a string.
+ // We check to make sure it applies to BINARY to avoid errors if there is a bad
+ // annotation.
+ return Type.STRING;
+ }
+
+ if (prim.getPrimitiveTypeName() == PrimitiveType.PrimitiveTypeName.INT32
+ || prim.getPrimitiveTypeName() == PrimitiveType.PrimitiveTypeName.INT64) {
+ // Map signed integer types to an supported Impala column type
+ switch (orig) {
+ case INT_8: return Type.TINYINT;
+ case INT_16: return Type.SMALLINT;
+ case INT_32: return Type.INT;
+ case INT_64: return Type.BIGINT;
+ }
+ }
+
+ if (orig == OriginalType.DECIMAL) {
+ return ScalarType.createDecimalType(prim.getDecimalMetadata().getPrecision(),
+ prim.getDecimalMetadata().getScale());
+ }
+
+ throw new AnalysisException(
+ "Unsupported logical parquet type " + orig + " (primitive type is " +
+ prim.getPrimitiveTypeName().name() + ") for field " +
+ parquetType.getName());
+ }
+
+ /**
+ * Converts a Parquet type into an Impala type.
+ */
+ private static Type convertParquetType(org.apache.parquet.schema.Type field)
+ throws AnalysisException {
+ Type type = null;
+ // TODO for 2.3: If a field is not annotated with LIST, it can still be sometimes
+ // interpreted as an array. The following 2 examples should be interpreted as an array
+ // of integers, but this is currently not done.
+ // 1. repeated int int_col;
+ // 2. required group int_arr {
+ // repeated group list {
+ // required int element;
+ // }
+ // }
+ if (field.getOriginalType() != null) {
+ type = convertLogicalParquetType(field);
+ } else if (field.isPrimitive()) {
+ type = convertPrimitiveParquetType(field);
+ } else {
+ // If field is not primitive, it must be a struct.
+ type = convertStruct(field.asGroupType());
+ }
+ return type;
+ }
+
+ /**
+ * Parses a Parquet file stored in HDFS and returns the corresponding Impala schema.
+ * This fails with an analysis exception if any errors occur reading the file,
+ * parsing the Parquet schema, or if the Parquet types cannot be represented in Impala.
+ */
+ static List<ColumnDef> extractParquetSchema(HdfsUri location)
+ throws AnalysisException {
+ org.apache.parquet.schema.MessageType parquetSchema = loadParquetSchema(location.getPath());
+ List<org.apache.parquet.schema.Type> fields = parquetSchema.getFields();
+ List<ColumnDef> schema = new ArrayList<ColumnDef>();
+
+ for (org.apache.parquet.schema.Type field: fields) {
+ Type type = convertParquetType(field);
+ Preconditions.checkNotNull(type);
+ String colName = field.getName();
+ Map<ColumnDef.Option, Object> option = Maps.newHashMap();
+ option.put(ColumnDef.Option.COMMENT, "Inferred from Parquet file.");
+ schema.add(new ColumnDef(colName, new TypeDef(type), option));
+ }
+ return schema;
+ }
+}
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/fe/src/compat-minicluster-profile-3/java/org/apache/impala/authorization/ImpalaActionFactory.java
----------------------------------------------------------------------
diff --git a/fe/src/compat-minicluster-profile-3/java/org/apache/impala/authorization/ImpalaActionFactory.java b/fe/src/compat-minicluster-profile-3/java/org/apache/impala/authorization/ImpalaActionFactory.java
new file mode 100644
index 0000000..da22386
--- /dev/null
+++ b/fe/src/compat-minicluster-profile-3/java/org/apache/impala/authorization/ImpalaActionFactory.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.impala.authorization;
+
+import org.apache.sentry.core.common.BitFieldAction;
+import org.apache.sentry.core.common.BitFieldActionFactory;
+import org.apache.sentry.core.model.db.AccessConstants;
+
+import java.util.List;
+
+/**
+ * Almost identical to HiveActionFactory, but extended to allow
+ * for "refresh".
+ */
+public class ImpalaActionFactory extends BitFieldActionFactory {
+
+ enum ActionType {
+ SELECT(AccessConstants.SELECT, 1),
+ INSERT(AccessConstants.INSERT, 2),
+ ALTER(AccessConstants.ALTER, 4),
+ CREATE(AccessConstants.CREATE, 8),
+ DROP(AccessConstants.DROP, 16),
+ INDEX(AccessConstants.INDEX, 32),
+ LOCK(AccessConstants.LOCK, 64),
+ // "refresh" isn't available in AccessConstants, so using an Impala constant.
+ REFRESH(Privilege.SentryAction.REFRESH.name(), 128),
+
+ // For the compatibility, ALL, ALL_STAR, SOME have the same binary value;
+ // They have the different names which are "ALL", "*", "+"
+ ALL(AccessConstants.ACTION_ALL, SELECT.getCode() | INSERT.getCode() | ALTER.getCode() | CREATE.getCode() |
+ DROP.getCode() | INDEX.getCode() | LOCK.getCode() | REFRESH.getCode()),
+ ALL_STAR(AccessConstants.ALL, ALL.getCode()),
+ SOME(AccessConstants.SOME, ALL.getCode());
+
+ final private String name;
+ final private int code;
+
+ ActionType(String name, int code) {
+ this.name = name;
+ this.code = code;
+ }
+
+ public int getCode() {
+ return code;
+ }
+
+ public String getName() {
+ return name;
+ }
+ }
+
+ public List<? extends BitFieldAction> getActionsByCode(int actionCode) {
+ return null;
+ }
+
+ public BitFieldAction getActionByName(String name) {
+ for (ActionType action : ActionType.values()) {
+ if (action.name.equalsIgnoreCase(name)) {
+ return new BitFieldAction(action.getName(), action.getCode());
+ }
+ }
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/fe/src/compat-minicluster-profile-3/java/org/apache/impala/authorization/ImpalaPrivilegeModel.java
----------------------------------------------------------------------
diff --git a/fe/src/compat-minicluster-profile-3/java/org/apache/impala/authorization/ImpalaPrivilegeModel.java b/fe/src/compat-minicluster-profile-3/java/org/apache/impala/authorization/ImpalaPrivilegeModel.java
new file mode 100644
index 0000000..0fd0a70
--- /dev/null
+++ b/fe/src/compat-minicluster-profile-3/java/org/apache/impala/authorization/ImpalaPrivilegeModel.java
@@ -0,0 +1,45 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.impala.authorization;
+
+import java.util.Map;
+
+import org.apache.sentry.core.common.BitFieldActionFactory;
+import org.apache.sentry.core.common.ImplyMethodType;
+import org.apache.sentry.core.model.db.HivePrivilegeModel;
+import org.apache.sentry.core.common.Model;
+
+/**
+ * Delegates to HivePrivilegeModel for getImplyMethodMap(), but
+ * adds "refresh" to the BitFieldActionFactory.
+ */
+public class ImpalaPrivilegeModel implements Model {
+ public static final ImpalaPrivilegeModel INSTANCE = new ImpalaPrivilegeModel();
+
+ private ImpalaActionFactory actionFactory = new ImpalaActionFactory();
+
+ @Override
+ public Map<String, ImplyMethodType> getImplyMethodMap() {
+ return HivePrivilegeModel.getInstance().getImplyMethodMap();
+ }
+
+ @Override
+ public BitFieldActionFactory getBitFieldActionFactory() {
+ return actionFactory;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/fe/src/compat-minicluster-profile-3/java/org/apache/impala/authorization/SentryAuthProvider.java
----------------------------------------------------------------------
diff --git a/fe/src/compat-minicluster-profile-3/java/org/apache/impala/authorization/SentryAuthProvider.java b/fe/src/compat-minicluster-profile-3/java/org/apache/impala/authorization/SentryAuthProvider.java
new file mode 100644
index 0000000..a4f0743
--- /dev/null
+++ b/fe/src/compat-minicluster-profile-3/java/org/apache/impala/authorization/SentryAuthProvider.java
@@ -0,0 +1,80 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.impala.authorization;
+
+import com.google.common.base.Preconditions;
+
+import org.apache.impala.catalog.AuthorizationPolicy;
+
+import org.apache.commons.lang.reflect.ConstructorUtils;
+import org.apache.sentry.core.common.Model;
+import org.apache.sentry.core.model.db.HivePrivilegeModel;
+import org.apache.sentry.policy.common.PolicyEngine;
+import org.apache.sentry.policy.engine.common.CommonPolicyEngine;
+import org.apache.sentry.provider.cache.SimpleCacheProviderBackend;
+import org.apache.sentry.provider.common.ProviderBackend;
+import org.apache.sentry.provider.common.ProviderBackendContext;
+import org.apache.sentry.provider.common.ResourceAuthorizationProvider;
+import org.apache.sentry.provider.file.SimpleFileProviderBackend;
+
+/**
+ * Wrapper to facilitate differences in Sentry APIs across Sentry versions.
+ */
+class SentryAuthProvider {
+ /*
+ * Creates a new ResourceAuthorizationProvider based on the given configuration.
+ */
+ static ResourceAuthorizationProvider createProvider(AuthorizationConfig config,
+ AuthorizationPolicy policy) {
+ try {
+ ProviderBackend providerBe;
+ // Create the appropriate backend provider.
+ if (config.isFileBasedPolicy()) {
+ providerBe = new SimpleFileProviderBackend(config.getSentryConfig().getConfig(),
+ config.getPolicyFile());
+ ProviderBackendContext context = new ProviderBackendContext();
+ providerBe.initialize(context);
+ } else {
+ // Note: The second parameter to the ProviderBackend is a "resourceFile" path
+ // which is not used by Impala. We cannot pass 'null' so instead pass an empty
+ // string.
+ providerBe = new SimpleCacheProviderBackend(config.getSentryConfig().getConfig(),
+ "");
+ Preconditions.checkNotNull(policy);
+ ProviderBackendContext context = new ProviderBackendContext();
+ context.setBindingHandle(policy);
+ providerBe.initialize(context);
+ }
+
+ CommonPolicyEngine engine =
+ new CommonPolicyEngine(providerBe);
+
+ // Try to create an instance of the specified policy provider class.
+ // Re-throw any exceptions that are encountered.
+ String policyFile = config.getPolicyFile() == null ? "" : config.getPolicyFile();
+
+ return (ResourceAuthorizationProvider) ConstructorUtils.invokeConstructor(
+ Class.forName(config.getPolicyProviderClassName()),
+ new Object[] {policyFile, engine, ImpalaPrivilegeModel.INSTANCE});
+ } catch (Exception e) {
+ // Re-throw as unchecked exception.
+ throw new IllegalStateException(
+ "Error creating ResourceAuthorizationProvider: ", e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/fe/src/compat-minicluster-profile-3/java/org/apache/impala/compat/MetastoreShim.java
----------------------------------------------------------------------
diff --git a/fe/src/compat-minicluster-profile-3/java/org/apache/impala/compat/MetastoreShim.java b/fe/src/compat-minicluster-profile-3/java/org/apache/impala/compat/MetastoreShim.java
new file mode 100644
index 0000000..3d69545
--- /dev/null
+++ b/fe/src/compat-minicluster-profile-3/java/org/apache/impala/compat/MetastoreShim.java
@@ -0,0 +1,127 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.impala.compat;
+
+import java.util.List;
+
+import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.IMetaStoreClient;
+import org.apache.hadoop.hive.metastore.MetaStoreUtils;
+import org.apache.hadoop.hive.metastore.Warehouse;
+import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hive.service.rpc.thrift.TGetColumnsReq;
+import org.apache.hive.service.rpc.thrift.TGetFunctionsReq;
+import org.apache.hive.service.rpc.thrift.TGetSchemasReq;
+import org.apache.hive.service.rpc.thrift.TGetTablesReq;
+import org.apache.impala.authorization.User;
+import org.apache.impala.common.ImpalaException;
+import org.apache.impala.common.Pair;
+import org.apache.impala.service.Frontend;
+import org.apache.impala.service.MetadataOp;
+import org.apache.impala.thrift.TMetadataOpRequest;
+import org.apache.impala.thrift.TResultSet;
+import org.apache.thrift.TException;
+
+/**
+ * A wrapper around some of Hive's Metastore API's to abstract away differences
+ * between major versions of Hive. This implements the shimmed methods for Hive 2.
+ */
+public class MetastoreShim {
+ /**
+ * Wrapper around MetaStoreUtils.validateName() to deal with added arguments.
+ */
+ public static boolean validateName(String name) {
+ return MetaStoreUtils.validateName(name, null);
+ }
+
+ /**
+ * Wrapper around IMetaStoreClient.alter_partition() to deal with added
+ * arguments.
+ */
+ public static void alterPartition(IMetaStoreClient client, Partition partition)
+ throws InvalidOperationException, MetaException, TException {
+ client.alter_partition(
+ partition.getDbName(), partition.getTableName(), partition, null);
+ }
+
+ /**
+ * Wrapper around IMetaStoreClient.alter_partitions() to deal with added
+ * arguments.
+ */
+ public static void alterPartitions(IMetaStoreClient client, String dbName,
+ String tableName, List<Partition> partitions)
+ throws InvalidOperationException, MetaException, TException {
+ client.alter_partitions(dbName, tableName, partitions, null);
+ }
+
+ /**
+ * Wrapper around MetaStoreUtils.updatePartitionStatsFast() to deal with added
+ * arguments.
+ */
+ public static void updatePartitionStatsFast(Partition partition, Warehouse warehouse)
+ throws MetaException {
+ MetaStoreUtils.updatePartitionStatsFast(partition, warehouse, null);
+ }
+
+ /**
+ * Return the maximum number of Metastore objects that should be retrieved in
+ * a batch.
+ */
+ public static String metastoreBatchRetrieveObjectsMaxConfigKey() {
+ return HiveConf.ConfVars.METASTORE_BATCH_RETRIEVE_OBJECTS_MAX.toString();
+ }
+
+ /**
+ * Return the key and value that should be set in the partition parameters to
+ * mark that the stats were generated automatically by a stats task.
+ */
+ public static Pair<String, String> statsGeneratedViaStatsTaskParam() {
+ return Pair.create(StatsSetupConst.STATS_GENERATED, StatsSetupConst.TASK);
+ }
+
+ public static TResultSet execGetFunctions(
+ Frontend frontend, TMetadataOpRequest request, User user) throws ImpalaException {
+ TGetFunctionsReq req = request.getGet_functions_req();
+ return MetadataOp.getFunctions(
+ frontend, req.getCatalogName(), req.getSchemaName(), req.getFunctionName(), user);
+ }
+
+ public static TResultSet execGetColumns(
+ Frontend frontend, TMetadataOpRequest request, User user) throws ImpalaException {
+ TGetColumnsReq req = request.getGet_columns_req();
+ return MetadataOp.getColumns(frontend, req.getCatalogName(), req.getSchemaName(),
+ req.getTableName(), req.getColumnName(), user);
+ }
+
+ public static TResultSet execGetTables(
+ Frontend frontend, TMetadataOpRequest request, User user) throws ImpalaException {
+ TGetTablesReq req = request.getGet_tables_req();
+ return MetadataOp.getTables(frontend, req.getCatalogName(), req.getSchemaName(),
+ req.getTableName(), req.getTableTypes(), user);
+ }
+
+ public static TResultSet execGetSchemas(
+ Frontend frontend, TMetadataOpRequest request, User user) throws ImpalaException {
+ TGetSchemasReq req = request.getGet_schemas_req();
+ return MetadataOp.getSchemas(
+ frontend, req.getCatalogName(), req.getSchemaName(), user);
+ }
+}
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/fe/src/compat-minicluster-profile-3/java/org/apache/impala/compat/MiniclusterProfile.java
----------------------------------------------------------------------
diff --git a/fe/src/compat-minicluster-profile-3/java/org/apache/impala/compat/MiniclusterProfile.java b/fe/src/compat-minicluster-profile-3/java/org/apache/impala/compat/MiniclusterProfile.java
new file mode 100644
index 0000000..9f9c36c
--- /dev/null
+++ b/fe/src/compat-minicluster-profile-3/java/org/apache/impala/compat/MiniclusterProfile.java
@@ -0,0 +1,25 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.impala.compat;
+
+/**
+ * Constant to tell us what Minicluster Profile we are built against.
+ */
+public class MiniclusterProfile {
+ public static final int MINICLUSTER_PROFILE = 3;
+}
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/fe/src/compat-minicluster-profile-3/java/org/apache/impala/util/SentryUtil.java
----------------------------------------------------------------------
diff --git a/fe/src/compat-minicluster-profile-3/java/org/apache/impala/util/SentryUtil.java b/fe/src/compat-minicluster-profile-3/java/org/apache/impala/util/SentryUtil.java
new file mode 100644
index 0000000..f85e890
--- /dev/null
+++ b/fe/src/compat-minicluster-profile-3/java/org/apache/impala/util/SentryUtil.java
@@ -0,0 +1,54 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.impala.util;
+
+import java.util.Set;
+
+import org.apache.sentry.core.common.exception.SentryAccessDeniedException;
+import org.apache.sentry.core.common.exception.SentryAlreadyExistsException;
+import org.apache.sentry.core.common.exception.SentryGroupNotFoundException;
+import org.apache.sentry.provider.db.service.thrift.SentryPolicyServiceClient;
+import org.apache.sentry.provider.db.service.thrift.TSentryRole;
+// See IMPALA-5540. Sentry over-shades itself (to avoid leaking Thrift),
+// causing this unusual package name. In the code below, we typically
+// check for either variant when it's available in the classpath.
+import sentry.org.apache.sentry.core.common.exception.SentryUserException;
+
+/**
+ * Wrapper to facilitate differences in Sentry APIs across Sentry versions.
+ */
+public class SentryUtil {
+ static boolean isSentryAlreadyExists(Exception e) {
+ return e instanceof SentryAlreadyExistsException || e instanceof
+ sentry.org.apache.sentry.core.common.exception.SentryAlreadyExistsException;
+ }
+
+ static boolean isSentryAccessDenied(Exception e) {
+ return e instanceof SentryAccessDeniedException || e instanceof
+ sentry.org.apache.sentry.core.common.exception.SentryAccessDeniedException;
+ }
+
+ public static boolean isSentryGroupNotFound(Exception e) {
+ return e instanceof SentryGroupNotFoundException;
+ }
+
+ static Set<TSentryRole> listRoles(SentryPolicyServiceClient client, String username)
+ throws SentryUserException {
+ return client.listAllRoles(username);
+ }
+}
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/fe/src/main/java/org/apache/impala/analysis/CreateTableLikeFileStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/CreateTableLikeFileStmt.java b/fe/src/main/java/org/apache/impala/analysis/CreateTableLikeFileStmt.java
index 5f43ec3..e92458e 100644
--- a/fe/src/main/java/org/apache/impala/analysis/CreateTableLikeFileStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/CreateTableLikeFileStmt.java
@@ -17,8 +17,6 @@
package org.apache.impala.analysis;
-import java.io.FileNotFoundException;
-import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -26,25 +24,12 @@ import java.util.Map;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsAction;
-import parquet.hadoop.ParquetFileReader;
-import parquet.hadoop.metadata.ParquetMetadata;
-import parquet.schema.OriginalType;
-import parquet.schema.PrimitiveType;
import org.apache.impala.authorization.Privilege;
-import org.apache.impala.catalog.ArrayType;
import org.apache.impala.catalog.HdfsCompression;
import org.apache.impala.catalog.HdfsFileFormat;
-import org.apache.impala.catalog.MapType;
-import org.apache.impala.catalog.ScalarType;
-import org.apache.impala.catalog.StructField;
-import org.apache.impala.catalog.StructType;
-import org.apache.impala.catalog.Type;
import org.apache.impala.common.AnalysisException;
-import org.apache.impala.common.FileSystemUtil;
import org.apache.impala.thrift.THdfsFileFormat;
@@ -56,8 +41,6 @@ import org.apache.impala.thrift.THdfsFileFormat;
public class CreateTableLikeFileStmt extends CreateTableStmt {
private final HdfsUri schemaLocation_;
private final THdfsFileFormat schemaFileFormat_;
- private final static String ERROR_MSG =
- "Failed to convert Parquet type\n%s\nto an Impala %s type:\n%s\n";
public CreateTableLikeFileStmt(CreateTableStmt createTableStmt,
THdfsFileFormat schemaFileFormat, HdfsUri schemaLocation) {
@@ -66,291 +49,6 @@ public class CreateTableLikeFileStmt extends CreateTableStmt {
schemaFileFormat_ = schemaFileFormat;
}
- /**
- * Reads the first block from the given HDFS file and returns the Parquet schema.
- * Throws Analysis exception for any failure, such as failing to read the file
- * or failing to parse the contents.
- */
- private static parquet.schema.MessageType loadParquetSchema(Path pathToFile)
- throws AnalysisException {
- try {
- FileSystem fs = pathToFile.getFileSystem(FileSystemUtil.getConfiguration());
- if (!fs.isFile(pathToFile)) {
- throw new AnalysisException("Cannot infer schema, path is not a file: " +
- pathToFile);
- }
- } catch (IOException e) {
- throw new AnalysisException("Failed to connect to filesystem:" + e);
- } catch (IllegalArgumentException e) {
- throw new AnalysisException(e.getMessage());
- }
- ParquetMetadata readFooter = null;
- try {
- readFooter = ParquetFileReader.readFooter(FileSystemUtil.getConfiguration(),
- pathToFile);
- } catch (FileNotFoundException e) {
- throw new AnalysisException("File not found: " + e);
- } catch (IOException e) {
- throw new AnalysisException("Failed to open file as a parquet file: " + e);
- } catch (RuntimeException e) {
- // Parquet throws a generic RuntimeException when reading a non-parquet file
- if (e.toString().contains("is not a Parquet file")) {
- throw new AnalysisException("File is not a parquet file: " + pathToFile);
- }
- // otherwise, who knows what we caught, throw it back up
- throw e;
- }
- return readFooter.getFileMetaData().getSchema();
- }
-
- /**
- * Converts a "primitive" Parquet type to an Impala type.
- * A primitive type is a non-nested type with no annotations.
- */
- private static Type convertPrimitiveParquetType(parquet.schema.Type parquetType)
- throws AnalysisException {
- Preconditions.checkState(parquetType.isPrimitive());
- PrimitiveType prim = parquetType.asPrimitiveType();
- switch (prim.getPrimitiveTypeName()) {
- case BINARY: return Type.STRING;
- case BOOLEAN: return Type.BOOLEAN;
- case DOUBLE: return Type.DOUBLE;
- case FIXED_LEN_BYTE_ARRAY:
- throw new AnalysisException(
- "Unsupported parquet type FIXED_LEN_BYTE_ARRAY for field " +
- parquetType.getName());
- case FLOAT: return Type.FLOAT;
- case INT32: return Type.INT;
- case INT64: return Type.BIGINT;
- case INT96: return Type.TIMESTAMP;
- default:
- Preconditions.checkState(false, "Unexpected parquet primitive type: " +
- prim.getPrimitiveTypeName());
- return null;
- }
- }
-
- /**
- * Converts a Parquet group type to an Impala map Type. We support both standard
- * Parquet map representations, as well as legacy. Legacy representations are handled
- * according to this specification:
- * https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#backward-compatibility-rules-1
- *
- * Standard representation of a map in Parquet:
- * <optional | required> group <name> (MAP) { <-- outerGroup is pointing at this
- * repeated group key_value {
- * required <key-type> key;
- * <optional | required> <value-type> value;
- * }
- * }
- */
- private static MapType convertMap(parquet.schema.GroupType outerGroup)
- throws AnalysisException {
- if (outerGroup.getFieldCount() != 1){
- throw new AnalysisException(String.format(ERROR_MSG, outerGroup.toString(),
- "MAP", "The logical MAP type must have exactly 1 inner field."));
- }
-
- parquet.schema.Type innerField = outerGroup.getType(0);
- if (!innerField.isRepetition(parquet.schema.Type.Repetition.REPEATED)){
- throw new AnalysisException(String.format(ERROR_MSG, outerGroup.toString(),
- "MAP", "The logical MAP type must have a repeated inner field."));
- }
- if (innerField.isPrimitive()) {
- throw new AnalysisException(String.format(ERROR_MSG, outerGroup.toString(),
- "MAP", "The inner field of the logical MAP type must be a group."));
- }
-
- parquet.schema.GroupType innerGroup = innerField.asGroupType();
- // It does not matter whether innerGroup has an annotation or not (for example it may
- // be annotated with MAP_KEY_VALUE). We treat the case that innerGroup has an
- // annotation and the case the innerGroup does not have an annotation the same.
- if (innerGroup.getFieldCount() != 2) {
- throw new AnalysisException(String.format(ERROR_MSG, outerGroup.toString(),
- "MAP", "The inner field of the logical MAP type must have exactly 2 fields."));
- }
-
- parquet.schema.Type key = innerGroup.getType(0);
- if (!key.getName().equals("key")) {
- throw new AnalysisException(String.format(ERROR_MSG, outerGroup.toString(),
- "MAP", "The name of the first field of the inner field of the logical MAP " +
- "type must be 'key'"));
- }
- if (!key.isPrimitive()) {
- throw new AnalysisException(String.format(ERROR_MSG, outerGroup.toString(),
- "MAP", "The key type of the logical MAP type must be primitive."));
- }
- parquet.schema.Type value = innerGroup.getType(1);
- if (!value.getName().equals("value")) {
- throw new AnalysisException(String.format(ERROR_MSG, outerGroup.toString(),
- "MAP", "The name of the second field of the inner field of the logical MAP " +
- "type must be 'value'"));
- }
-
- return new MapType(convertParquetType(key), convertParquetType(value));
- }
-
- /**
- * Converts a Parquet group type to an Impala struct Type.
- */
- private static StructType convertStruct(parquet.schema.GroupType outerGroup)
- throws AnalysisException {
- ArrayList<StructField> structFields = new ArrayList<StructField>();
- for (parquet.schema.Type field: outerGroup.getFields()) {
- StructField f = new StructField(field.getName(), convertParquetType(field));
- structFields.add(f);
- }
- return new StructType(structFields);
- }
-
- /**
- * Converts a Parquet group type to an Impala array Type. We can handle the standard
- * representation, but also legacy representations for backwards compatibility.
- * Legacy representations are handled according to this specification:
- * https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#backward-compatibility-rules
- *
- * Standard representation of an array in Parquet:
- * <optional | required> group <name> (LIST) { <-- outerGroup is pointing at this
- * repeated group list {
- * <optional | required> <element-type> element;
- * }
- * }
- */
- private static ArrayType convertArray(parquet.schema.GroupType outerGroup)
- throws AnalysisException {
- if (outerGroup.getFieldCount() != 1) {
- throw new AnalysisException(String.format(ERROR_MSG, outerGroup.toString(),
- "LIST", "The logical LIST type must have exactly 1 inner field."));
- }
-
- parquet.schema.Type innerField = outerGroup.getType(0);
- if (!innerField.isRepetition(parquet.schema.Type.Repetition.REPEATED)) {
- throw new AnalysisException(String.format(ERROR_MSG, outerGroup.toString(),
- "LIST", "The inner field of the logical LIST type must be repeated."));
- }
- if (innerField.isPrimitive() || innerField.getOriginalType() != null) {
- // From the Parquet Spec:
- // 1. If the repeated field is not a group then it's type is the element type.
- //
- // If innerField is a group, but originalType is not null, the element type is
- // based on the logical type.
- return new ArrayType(convertParquetType(innerField));
- }
-
- parquet.schema.GroupType innerGroup = innerField.asGroupType();
- if (innerGroup.getFieldCount() != 1) {
- // From the Parquet Spec:
- // 2. If the repeated field is a group with multiple fields, then it's type is a
- // struct.
- return new ArrayType(convertStruct(innerGroup));
- }
-
- return new ArrayType(convertParquetType(innerGroup.getType(0)));
- }
-
- /**
- * Converts a "logical" Parquet type to an Impala column type.
- * A Parquet type is considered logical when it has an annotation. The annotation is
- * stored as a "OriginalType". The Parquet documentation refers to these as logical
- * types, so we use that terminology here.
- */
- private static Type convertLogicalParquetType(parquet.schema.Type parquetType)
- throws AnalysisException {
- OriginalType orig = parquetType.getOriginalType();
- if (orig == OriginalType.LIST) {
- return convertArray(parquetType.asGroupType());
- }
- if (orig == OriginalType.MAP || orig == OriginalType.MAP_KEY_VALUE) {
- // MAP_KEY_VALUE annotation should not be used any more. However, according to the
- // Parquet spec, some existing data incorrectly uses MAP_KEY_VALUE in place of MAP.
- // For backward-compatibility, a group annotated with MAP_KEY_VALUE that is not
- // contained by a MAP-annotated group should be handled as a MAP-annotated group.
- return convertMap(parquetType.asGroupType());
- }
-
- PrimitiveType prim = parquetType.asPrimitiveType();
- if (prim.getPrimitiveTypeName() == PrimitiveType.PrimitiveTypeName.BINARY &&
- (orig == OriginalType.UTF8 || orig == OriginalType.ENUM)) {
- // UTF8 is the type annotation Parquet uses for strings
- // ENUM is the type annotation Parquet uses to indicate that
- // the original data type, before conversion to parquet, had been enum.
- // Applications which do not have enumerated types (e.g. Impala)
- // should interpret it as a string.
- // We check to make sure it applies to BINARY to avoid errors if there is a bad
- // annotation.
- return Type.STRING;
- }
-
- if (prim.getPrimitiveTypeName() == PrimitiveType.PrimitiveTypeName.INT32
- || prim.getPrimitiveTypeName() == PrimitiveType.PrimitiveTypeName.INT64) {
- // Map signed integer types to an supported Impala column type
- switch (orig) {
- case INT_8: return Type.TINYINT;
- case INT_16: return Type.SMALLINT;
- case INT_32: return Type.INT;
- case INT_64: return Type.BIGINT;
- }
- }
-
- if (orig == OriginalType.DECIMAL) {
- return ScalarType.createDecimalType(prim.getDecimalMetadata().getPrecision(),
- prim.getDecimalMetadata().getScale());
- }
-
- throw new AnalysisException(
- "Unsupported logical parquet type " + orig + " (primitive type is " +
- prim.getPrimitiveTypeName().name() + ") for field " +
- parquetType.getName());
- }
-
- /**
- * Converts a Parquet type into an Impala type.
- */
- private static Type convertParquetType(parquet.schema.Type field)
- throws AnalysisException {
- Type type = null;
- // TODO for 2.3: If a field is not annotated with LIST, it can still be sometimes
- // interpreted as an array. The following 2 examples should be interpreted as an array
- // of integers, but this is currently not done.
- // 1. repeated int int_col;
- // 2. required group int_arr {
- // repeated group list {
- // required int element;
- // }
- // }
- if (field.getOriginalType() != null) {
- type = convertLogicalParquetType(field);
- } else if (field.isPrimitive()) {
- type = convertPrimitiveParquetType(field);
- } else {
- // If field is not primitive, it must be a struct.
- type = convertStruct(field.asGroupType());
- }
- return type;
- }
-
- /**
- * Parses a Parquet file stored in HDFS and returns the corresponding Impala schema.
- * This fails with an analysis exception if any errors occur reading the file,
- * parsing the Parquet schema, or if the Parquet types cannot be represented in Impala.
- */
- private static List<ColumnDef> extractParquetSchema(HdfsUri location)
- throws AnalysisException {
- parquet.schema.MessageType parquetSchema = loadParquetSchema(location.getPath());
- List<parquet.schema.Type> fields = parquetSchema.getFields();
- List<ColumnDef> schema = new ArrayList<ColumnDef>();
-
- for (parquet.schema.Type field: fields) {
- Type type = convertParquetType(field);
- Preconditions.checkNotNull(type);
- String colName = field.getName();
- Map<ColumnDef.Option, Object> option = Maps.newHashMap();
- option.put(ColumnDef.Option.COMMENT, "Inferred from Parquet file.");
- schema.add(new ColumnDef(colName, new TypeDef(type), option));
- }
- return schema;
- }
-
@Override
public String toSql() {
ArrayList<String> colsSql = Lists.newArrayList();
@@ -376,7 +74,7 @@ public class CreateTableLikeFileStmt extends CreateTableStmt {
schemaLocation_.analyze(analyzer, Privilege.ALL, FsAction.READ);
switch (schemaFileFormat_) {
case PARQUET:
- getColumnDefs().addAll(extractParquetSchema(schemaLocation_));
+ getColumnDefs().addAll(ParquetHelper.extractParquetSchema(schemaLocation_));
break;
default:
throw new AnalysisException("Unsupported file type for schema inference: "
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/fe/src/main/java/org/apache/impala/analysis/InsertStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/InsertStmt.java b/fe/src/main/java/org/apache/impala/analysis/InsertStmt.java
index ad05751..43d582b 100644
--- a/fe/src/main/java/org/apache/impala/analysis/InsertStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/InsertStmt.java
@@ -494,10 +494,10 @@ public class InsertStmt extends StatementBase {
}
for (int colIdx = 0; colIdx < numClusteringCols; ++colIdx) {
Column col = hdfsTable.getColumns().get(colIdx);
- // Hive has a number of issues handling BOOLEAN partition columns (see HIVE-6590).
+ // Hive 1.x has a number of issues handling BOOLEAN partition columns (see HIVE-6590).
// Instead of working around the Hive bugs, INSERT is disabled for BOOLEAN
- // partitions in Impala. Once the Hive JIRA is resolved, we can remove this
- // analysis check.
+ // partitions in Impala when built against Hive 1. HIVE-6590 is currently resolved,
+ // but not in Hive 2.3.2, the latest release as of 3/17/2018.
if (col.getType() == Type.BOOLEAN) {
throw new AnalysisException(String.format("INSERT into table with BOOLEAN " +
"partition column (%s) is not supported: %s", col.getName(),
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/fe/src/main/java/org/apache/impala/authorization/AuthorizationChecker.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/authorization/AuthorizationChecker.java b/fe/src/main/java/org/apache/impala/authorization/AuthorizationChecker.java
index b6fc299..0331f7d 100644
--- a/fe/src/main/java/org/apache/impala/authorization/AuthorizationChecker.java
+++ b/fe/src/main/java/org/apache/impala/authorization/AuthorizationChecker.java
@@ -17,6 +17,7 @@
package org.apache.impala.authorization;
+import java.util.Collections;
import java.util.EnumSet;
import java.util.List;
import java.util.Set;
@@ -29,7 +30,6 @@ import org.apache.impala.common.InternalException;
import org.apache.sentry.core.common.ActiveRoleSet;
import org.apache.sentry.core.common.Subject;
import org.apache.sentry.core.model.db.DBModelAuthorizable;
-import org.apache.sentry.policy.db.SimpleDBPolicyEngine;
import org.apache.sentry.provider.cache.SimpleCacheProviderBackend;
import org.apache.sentry.provider.common.ProviderBackend;
import org.apache.sentry.provider.common.ProviderBackendContext;
@@ -39,6 +39,8 @@ import org.apache.sentry.provider.file.SimpleFileProviderBackend;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
+import org.apache.impala.util.SentryUtil;
+
/**
* Class used to check whether a user has access to a given resource.
*/
@@ -68,38 +70,7 @@ public class AuthorizationChecker {
*/
private static ResourceAuthorizationProvider createProvider(AuthorizationConfig config,
AuthorizationPolicy policy) {
- try {
- ProviderBackend providerBe;
- // Create the appropriate backend provider.
- if (config.isFileBasedPolicy()) {
- providerBe = new SimpleFileProviderBackend(config.getSentryConfig().getConfig(),
- config.getPolicyFile());
- } else {
- // Note: The second parameter to the ProviderBackend is a "resourceFile" path
- // which is not used by Impala. We cannot pass 'null' so instead pass an empty
- // string.
- providerBe = new SimpleCacheProviderBackend(config.getSentryConfig().getConfig(),
- "");
- Preconditions.checkNotNull(policy);
- ProviderBackendContext context = new ProviderBackendContext();
- context.setBindingHandle(policy);
- providerBe.initialize(context);
- }
-
- SimpleDBPolicyEngine engine =
- new SimpleDBPolicyEngine(config.getServerName(), providerBe);
-
- // Try to create an instance of the specified policy provider class.
- // Re-throw any exceptions that are encountered.
- String policyFile = config.getPolicyFile() == null ? "" : config.getPolicyFile();
- return (ResourceAuthorizationProvider) ConstructorUtils.invokeConstructor(
- Class.forName(config.getPolicyProviderClassName()),
- new Object[] {policyFile, engine});
- } catch (Exception e) {
- // Re-throw as unchecked exception.
- throw new IllegalStateException(
- "Error creating ResourceAuthorizationProvider: ", e);
- }
+ return SentryAuthProvider.createProvider(config, policy);
}
/*
@@ -113,7 +84,16 @@ public class AuthorizationChecker {
* local group mappings.
*/
public Set<String> getUserGroups(User user) throws InternalException {
- return provider_.getGroupMapping().getGroups(user.getShortName());
+ try {
+ return provider_.getGroupMapping().getGroups(user.getShortName());
+ } catch (Exception e) {
+ if (SentryUtil.isSentryGroupNotFound(e)) {
+ // Sentry 2.1+ throws exceptions when user does not exist; swallow the
+ // exception and just return an empty set for this case.
+ return Collections.emptySet();
+ }
+ throw new RuntimeException(e);
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/fe/src/main/java/org/apache/impala/catalog/AuthorizationPolicy.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/AuthorizationPolicy.java b/fe/src/main/java/org/apache/impala/catalog/AuthorizationPolicy.java
index bc83522..1c84973 100644
--- a/fe/src/main/java/org/apache/impala/catalog/AuthorizationPolicy.java
+++ b/fe/src/main/java/org/apache/impala/catalog/AuthorizationPolicy.java
@@ -294,6 +294,19 @@ public class AuthorizationPolicy implements PrivilegeCache {
return privileges;
}
+ /**
+ * Returns a set of privilege strings in Sentry format.
+ */
+ // This is an override for Sentry 2.1, but not for Sentry 1.x; we
+ // avoid annotation to support both.
+ // @Override
+ public Set<String> listPrivileges(Set<String> groups, Set<String> users,
+ ActiveRoleSet roleSet) {
+ /* User based roles and authorization hierarchy is not currently supported.
+ Fallback to listing privileges using groups. */
+ return listPrivileges(groups, roleSet);
+ }
+
@Override
public void close() {
// Nothing to do, but required by PrivilegeCache.
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/fe/src/main/java/org/apache/impala/util/SentryPolicyService.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/util/SentryPolicyService.java b/fe/src/main/java/org/apache/impala/util/SentryPolicyService.java
index f320cb9..f55ac30 100644
--- a/fe/src/main/java/org/apache/impala/util/SentryPolicyService.java
+++ b/fe/src/main/java/org/apache/impala/util/SentryPolicyService.java
@@ -19,8 +19,6 @@ package org.apache.impala.util;
import java.util.List;
-import org.apache.sentry.provider.db.SentryAccessDeniedException;
-import org.apache.sentry.provider.db.SentryAlreadyExistsException;
import org.apache.sentry.provider.db.service.thrift.SentryPolicyServiceClient;
import org.apache.sentry.provider.db.service.thrift.TSentryGrantOption;
import org.apache.sentry.provider.db.service.thrift.TSentryPrivilege;
@@ -124,10 +122,11 @@ public class SentryPolicyService {
} else {
client.get().dropRole(requestingUser.getShortName(), roleName);
}
- } catch (SentryAccessDeniedException e) {
- throw new AuthorizationException(String.format(ACCESS_DENIED_ERROR_MSG,
- requestingUser.getName(), "DROP_ROLE"));
} catch (Exception e) {
+ if (SentryUtil.isSentryAccessDenied(e)) {
+ throw new AuthorizationException(String.format(ACCESS_DENIED_ERROR_MSG,
+ requestingUser.getName(), "DROP_ROLE"));
+ }
throw new InternalException("Error dropping role: ", e);
} finally {
client.close();
@@ -151,14 +150,16 @@ public class SentryPolicyService {
SentryServiceClient client = new SentryServiceClient();
try {
client.get().createRole(requestingUser.getShortName(), roleName);
- } catch (SentryAccessDeniedException e) {
- throw new AuthorizationException(String.format(ACCESS_DENIED_ERROR_MSG,
- requestingUser.getName(), "CREATE_ROLE"));
- } catch (SentryAlreadyExistsException e) {
- if (ifNotExists) return;
- throw new InternalException("Error creating role: ", e);
} catch (Exception e) {
- throw new InternalException("Error creating role: ", e);
+ if (SentryUtil.isSentryAccessDenied(e)) {
+ throw new AuthorizationException(String.format(ACCESS_DENIED_ERROR_MSG,
+ requestingUser.getName(), "CREATE_ROLE"));
+ }
+ if (SentryUtil.isSentryAlreadyExists(e)) {
+ if (ifNotExists) return;
+ throw new InternalException("Error creating role: " + e.getMessage(), e);
+ }
+ throw new InternalException("Error creating role: " + e.getMessage(), e);
} finally {
client.close();
}
@@ -181,10 +182,11 @@ public class SentryPolicyService {
SentryServiceClient client = new SentryServiceClient();
try {
client.get().grantRoleToGroup(requestingUser.getShortName(), groupName, roleName);
- } catch (SentryAccessDeniedException e) {
- throw new AuthorizationException(String.format(ACCESS_DENIED_ERROR_MSG,
- requestingUser.getName(), "GRANT_ROLE"));
} catch (Exception e) {
+ if (SentryUtil.isSentryAccessDenied(e)) {
+ throw new AuthorizationException(String.format(ACCESS_DENIED_ERROR_MSG,
+ requestingUser.getName(), "GRANT_ROLE"));
+ }
throw new InternalException(
"Error making 'grantRoleToGroup' RPC to Sentry Service: ", e);
} finally {
@@ -210,10 +212,11 @@ public class SentryPolicyService {
try {
client.get().revokeRoleFromGroup(requestingUser.getShortName(),
groupName, roleName);
- } catch (SentryAccessDeniedException e) {
- throw new AuthorizationException(String.format(ACCESS_DENIED_ERROR_MSG,
- requestingUser.getName(), "REVOKE_ROLE"));
} catch (Exception e) {
+ if (SentryUtil.isSentryAccessDenied(e)) {
+ throw new AuthorizationException(String.format(ACCESS_DENIED_ERROR_MSG,
+ requestingUser.getName(), "REVOKE_ROLE"));
+ }
throw new InternalException(
"Error making 'revokeRoleFromGroup' RPC to Sentry Service: ", e);
} finally {
@@ -288,10 +291,11 @@ public class SentryPolicyService {
privilege.isHas_grant_opt());
break;
}
- } catch (SentryAccessDeniedException e) {
- throw new AuthorizationException(String.format(ACCESS_DENIED_ERROR_MSG,
- requestingUser.getName(), "GRANT_PRIVILEGE"));
} catch (Exception e) {
+ if (SentryUtil.isSentryAccessDenied(e)) {
+ throw new AuthorizationException(String.format(ACCESS_DENIED_ERROR_MSG,
+ requestingUser.getName(), "GRANT_PRIVILEGE"));
+ }
throw new InternalException(
"Error making 'grantPrivilege*' RPC to Sentry Service: ", e);
} finally {
@@ -355,10 +359,11 @@ public class SentryPolicyService {
null);
break;
}
- } catch (SentryAccessDeniedException e) {
- throw new AuthorizationException(String.format(ACCESS_DENIED_ERROR_MSG,
- requestingUser.getName(), "REVOKE_PRIVILEGE"));
} catch (Exception e) {
+ if (SentryUtil.isSentryAccessDenied(e)) {
+ throw new AuthorizationException(String.format(ACCESS_DENIED_ERROR_MSG,
+ requestingUser.getName(), "REVOKE_PRIVILEGE"));
+ }
throw new InternalException(
"Error making 'revokePrivilege*' RPC to Sentry Service: ", e);
} finally {
@@ -390,11 +395,13 @@ public class SentryPolicyService {
public List<TSentryRole> listAllRoles(User requestingUser) throws ImpalaException {
SentryServiceClient client = new SentryServiceClient();
try {
- return Lists.newArrayList(client.get().listRoles(requestingUser.getShortName()));
- } catch (SentryAccessDeniedException e) {
- throw new AuthorizationException(String.format(ACCESS_DENIED_ERROR_MSG,
- requestingUser.getName(), "LIST_ROLES"));
+ return Lists.newArrayList(SentryUtil.listRoles(client.get(),
+ requestingUser.getShortName()));
} catch (Exception e) {
+ if (SentryUtil.isSentryAccessDenied(e)) {
+ throw new AuthorizationException(String.format(ACCESS_DENIED_ERROR_MSG,
+ requestingUser.getName(), "LIST_ROLES"));
+ }
throw new InternalException("Error making 'listRoles' RPC to Sentry Service: ", e);
} finally {
client.close();
@@ -410,10 +417,11 @@ public class SentryPolicyService {
try {
return Lists.newArrayList(client.get().listAllPrivilegesByRoleName(
requestingUser.getShortName(), roleName));
- } catch (SentryAccessDeniedException e) {
- throw new AuthorizationException(String.format(ACCESS_DENIED_ERROR_MSG,
- requestingUser.getName(), "LIST_ROLE_PRIVILEGES"));
} catch (Exception e) {
+ if (SentryUtil.isSentryAccessDenied(e)) {
+ throw new AuthorizationException(String.format(ACCESS_DENIED_ERROR_MSG,
+ requestingUser.getName(), "LIST_ROLE_PRIVILEGES"));
+ }
throw new InternalException("Error making 'listAllPrivilegesByRoleName' RPC to " +
"Sentry Service: ", e);
} finally {
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/fe/src/test/java/org/apache/impala/analysis/AuthorizationTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/analysis/AuthorizationTest.java b/fe/src/test/java/org/apache/impala/analysis/AuthorizationTest.java
index e517172..39d207f 100644
--- a/fe/src/test/java/org/apache/impala/analysis/AuthorizationTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/AuthorizationTest.java
@@ -46,6 +46,7 @@ import org.apache.impala.common.FrontendTestBase;
import org.apache.impala.common.ImpalaException;
import org.apache.impala.common.InternalException;
import org.apache.impala.common.RuntimeEnv;
+import org.apache.impala.compat.MiniclusterProfile;
import org.apache.impala.service.Frontend;
import org.apache.impala.testutil.ImpaladTestCatalog;
import org.apache.impala.thrift.TColumnValue;
@@ -76,6 +77,7 @@ import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
@@ -1603,7 +1605,7 @@ public class AuthorizationTest extends FrontendTestBase {
// Expected output of DESCRIBE for a functional table.
// "*" is used when the output is variable such as time or user.
private static final List<String> EXPECTED_DESCRIBE_EXTENDED_ALLTYPESAGG =
- Lists.newArrayList(
+ ImmutableList.<String>builder().add(
"# col_name","data_type","comment",
"","NULL","NULL",
"id","int","NULL",
@@ -1629,15 +1631,22 @@ public class AuthorizationTest extends FrontendTestBase {
"Database:","functional","NULL",
"Owner:","*","NULL",
"CreateTime:","*","NULL",
- "LastAccessTime:","UNKNOWN","NULL",
- "Protect Mode:","None","NULL",
+ "LastAccessTime:","UNKNOWN","NULL")
+ // Account for minor differences between Hive 1 and Hive 2
+ .addAll(MiniclusterProfile.MINICLUSTER_PROFILE == 2
+ ? ImmutableList.<String>of("Protect Mode:", "None", "NULL")
+ : ImmutableList.<String>of())
+ .add(
"Retention:","0","NULL",
"Location:","hdfs://localhost:20500/test-warehouse/alltypesagg","NULL",
"Table Type:","EXTERNAL_TABLE","NULL",
"Table Parameters:","NULL","NULL",
"","DO_NOT_UPDATE_STATS","true",
- "","EXTERNAL","TRUE",
- "","STATS_GENERATED_VIA_STATS_TASK","true",
+ "","EXTERNAL","TRUE")
+ .addAll(MiniclusterProfile.MINICLUSTER_PROFILE == 2
+ ? ImmutableList.of("", "STATS_GENERATED_VIA_STATS_TASK", "true")
+ : ImmutableList.of("", "STATS_GENERATED", "TASK"))
+ .add(
"","numRows","11000",
"","totalSize","834279",
"","transient_lastDdlTime","*",
@@ -1654,12 +1663,12 @@ public class AuthorizationTest extends FrontendTestBase {
"","escape.delim","\\\\",
"","field.delim",",",
"","serialization.format",","
- );
+ ).build();
// Expected output of DESCRIBE for a functional table.
// "*" is used when the output is variable such as time or user.
private static final List<String> EXPECTED_DESCRIBE_EXTENDED_ALLTYPESSMALL =
- Lists.newArrayList(
+ ImmutableList.<String>builder().add(
"# col_name","data_type","comment",
"","NULL","NULL",
"id","int","NULL",
@@ -1674,14 +1683,20 @@ public class AuthorizationTest extends FrontendTestBase {
"Database:","functional","NULL",
"Owner:","*","NULL",
"CreateTime:","*","NULL",
- "LastAccessTime:","UNKNOWN","NULL",
- "Protect Mode:","None","NULL",
+ "LastAccessTime:","UNKNOWN","NULL")
+ .addAll(MiniclusterProfile.MINICLUSTER_PROFILE == 2
+ ? ImmutableList.<String>of("Protect Mode:", "None", "NULL")
+ : ImmutableList.<String>of())
+ .add(
"Retention:","0","NULL",
"Table Type:","EXTERNAL_TABLE","NULL",
"Table Parameters:","NULL","NULL",
"","DO_NOT_UPDATE_STATS","true",
- "","EXTERNAL","TRUE",
- "","STATS_GENERATED_VIA_STATS_TASK","true",
+ "","EXTERNAL","TRUE")
+ .addAll(MiniclusterProfile.MINICLUSTER_PROFILE == 2
+ ? ImmutableList.of("", "STATS_GENERATED_VIA_STATS_TASK", "true")
+ : ImmutableList.of("", "STATS_GENERATED", "TASK"))
+ .add(
"","numRows","100",
"","totalSize","6472",
"","transient_lastDdlTime","*",
@@ -1698,7 +1713,7 @@ public class AuthorizationTest extends FrontendTestBase {
"","escape.delim","\\\\",
"","field.delim",",",
"","serialization.format",","
- );
+ ).build();
@Test
public void TestDescribeTableResults() throws ImpalaException {
@@ -1735,7 +1750,8 @@ public class AuthorizationTest extends FrontendTestBase {
// Compares two arrays but skips an expected value that contains '*' since we need to
// compare output but some values change based on builds, environments, etc.
private void verifyOutputWithOptionalData(List<String> expected, List<String> actual) {
- Assert.assertEquals(expected.size(), actual.size());
+ Assert.assertEquals("Size difference. Expected: " + expected + " Actual: " + actual,
+ expected.size(), actual.size());
for (int idx = 0; idx < expected.size(); idx++) {
if (!expected.get(idx).equals("*")) {
Assert.assertEquals(expected.get(idx), actual.get(idx));
@@ -2135,7 +2151,13 @@ public class AuthorizationTest extends FrontendTestBase {
public void TestShortUsernameUsed() throws Exception {
// Different long variations of the same username.
List<User> users = Lists.newArrayList(
- new User(USER.getName() + "/abc.host.com@"),
+ // Hadoop 2 accepts kerberos names missing a realm, but insists
+ // on having a terminating '@' even when the default realm
+ // is intended. Hadoop 3 now has more normal name convetions,
+ // where to specify the default realm, everything after and
+ // including the '@' character is omitted.
+ new User(USER.getName() + "/abc.host.com" +
+ (MiniclusterProfile.MINICLUSTER_PROFILE == 3 ? "" : "@")),
new User(USER.getName() + "/abc.host.com@REAL.COM"),
new User(USER.getName() + "@REAL.COM"));
for (User user: users) {
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/fe/src/test/java/org/apache/impala/common/FrontendTestBase.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/common/FrontendTestBase.java b/fe/src/test/java/org/apache/impala/common/FrontendTestBase.java
index 28dc710..b3d16ba 100644
--- a/fe/src/test/java/org/apache/impala/common/FrontendTestBase.java
+++ b/fe/src/test/java/org/apache/impala/common/FrontendTestBase.java
@@ -54,6 +54,7 @@ import org.apache.impala.catalog.ScalarType;
import org.apache.impala.catalog.Table;
import org.apache.impala.catalog.Type;
import org.apache.impala.catalog.View;
+import org.apache.impala.compat.MiniclusterProfile;
import org.apache.impala.service.CatalogOpExecutor;
import org.apache.impala.service.Frontend;
import org.apache.impala.testutil.ImpaladTestCatalog;
@@ -407,9 +408,18 @@ public class FrontendTestBase {
} catch (Exception e) {
String errorString = e.getMessage();
Preconditions.checkNotNull(errorString, "Stack trace lost during exception.");
- Assert.assertTrue(
- "got error:\n" + errorString + "\nexpected:\n" + expectedErrorString,
- errorString.startsWith(expectedErrorString));
+ String msg = "got error:\n" + errorString + "\nexpected:\n" + expectedErrorString;
+ if (MiniclusterProfile.MINICLUSTER_PROFILE == 3) {
+ // Different versions of Hive have slightly different error messages;
+ // we normalize here as follows:
+ // 'No FileSystem for Scheme "x"' -> 'No FileSystem for scheme: x'
+ if (errorString.contains("No FileSystem for scheme ")) {
+ errorString = errorString.replace("\"", "");
+ errorString = errorString.replace("No FileSystem for scheme ",
+ "No FileSystem for scheme: ");
+ }
+ }
+ Assert.assertTrue(msg, errorString.startsWith(expectedErrorString));
return;
}
fail("Stmt didn't result in analysis error: " + stmt);
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/fe/src/test/resources/hive-log4j2.properties.template
----------------------------------------------------------------------
diff --git a/fe/src/test/resources/hive-log4j2.properties.template b/fe/src/test/resources/hive-log4j2.properties.template
new file mode 100644
index 0000000..2361f1d
--- /dev/null
+++ b/fe/src/test/resources/hive-log4j2.properties.template
@@ -0,0 +1,83 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+status = INFO
+name = HiveLog4j2
+packages = org.apache.hadoop.hive.ql.log
+
+# list of properties
+property.hive.log.level = INFO
+property.hive.root.logger = DRFA
+property.hive.log.dir = ${env:IMPALA_CLUSTER_LOGS_DIR}
+property.hive.log.file = hive.log
+property.hive.perflogger.log.level = INFO
+
+# list of all appenders
+appenders = console, DRFA
+
+# console appender
+appender.console.type = Console
+appender.console.name = console
+appender.console.target = SYSTEM_ERR
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = %d{ISO8601} %5p [%t] %c{2}: %m%n
+
+# daily rolling file appender
+appender.DRFA.type = RollingRandomAccessFile
+appender.DRFA.name = DRFA
+appender.DRFA.fileName = ${sys:hive.log.dir}/${sys:hive.log.file}
+# Use %pid in the filePattern to append <process-id>@<host-name> to the filename if you want separate log files for different CLI session
+appender.DRFA.filePattern = ${sys:hive.log.dir}/${sys:hive.log.file}.%d{yyyy-MM-dd}
+appender.DRFA.layout.type = PatternLayout
+appender.DRFA.layout.pattern = %d{ISO8601} %5p [%t] %c{2}: %m%n
+appender.DRFA.policies.type = Policies
+appender.DRFA.policies.time.type = TimeBasedTriggeringPolicy
+appender.DRFA.policies.time.interval = 1
+appender.DRFA.policies.time.modulate = true
+appender.DRFA.strategy.type = DefaultRolloverStrategy
+appender.DRFA.strategy.max = 30
+
+# list of all loggers
+loggers = NIOServerCnxn, ClientCnxnSocketNIO, DataNucleus, Datastore, JPOX, PerfLogger, AmazonAws, ApacheHttp
+
+logger.NIOServerCnxn.name = org.apache.zookeeper.server.NIOServerCnxn
+logger.NIOServerCnxn.level = WARN
+
+logger.ClientCnxnSocketNIO.name = org.apache.zookeeper.ClientCnxnSocketNIO
+logger.ClientCnxnSocketNIO.level = WARN
+
+logger.DataNucleus.name = DataNucleus
+logger.DataNucleus.level = ERROR
+
+logger.Datastore.name = Datastore
+logger.Datastore.level = ERROR
+
+logger.JPOX.name = JPOX
+logger.JPOX.level = ERROR
+
+logger.AmazonAws.name=com.amazonaws
+logger.AmazonAws.level = INFO
+
+logger.ApacheHttp.name=org.apache.http
+logger.ApacheHttp.level = INFO
+
+logger.PerfLogger.name = org.apache.hadoop.hive.ql.log.PerfLogger
+logger.PerfLogger.level = ${sys:hive.perflogger.log.level}
+
+# root logger
+rootLogger.level = ${sys:hive.log.level}
+rootLogger.appenderRefs = root
+rootLogger.appenderRef.root.ref = ${sys:hive.root.logger}
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/impala-parent/pom.xml
----------------------------------------------------------------------
diff --git a/impala-parent/pom.xml b/impala-parent/pom.xml
index 6dc2064..b806f2b 100644
--- a/impala-parent/pom.xml
+++ b/impala-parent/pom.xml
@@ -51,7 +51,7 @@ under the License.
<junit.version>4.12</junit.version>
<!-- Beware compatibility requirements with Thrift and
KMS; see IMPALA-4210. -->
- <httpcomponents.core.version>4.2.5</httpcomponents.core.version>
+ <httpcomponents.core.version>4.4.9</httpcomponents.core.version>
<yarn-extras.version>${project.version}</yarn-extras.version>
<eclipse.output.directory>eclipse-classes</eclipse.output.directory>
<guava.version>11.0.2</guava.version>
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/testdata/bin/run-hbase.sh
----------------------------------------------------------------------
diff --git a/testdata/bin/run-hbase.sh b/testdata/bin/run-hbase.sh
index f264b65..878fe4d 100755
--- a/testdata/bin/run-hbase.sh
+++ b/testdata/bin/run-hbase.sh
@@ -36,6 +36,9 @@ cat > ${HBASE_CONF_DIR}/hbase-env.sh <<EOF
export JAVA_HOME=${JAVA_HOME}
export HBASE_LOG_DIR=${HBASE_LOGDIR}
export HBASE_PID_DIR=${HBASE_LOGDIR}
+if [[ $IMPALA_MINICLUSTER_PROFILE == 3 ]]; then
+ export HBASE_CLASSPATH=${HADOOP_CLASSPATH}
+fi
EOF
# Put zookeeper things in the logs/cluster/zoo directory.
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/testdata/bin/run-hive-server.sh
----------------------------------------------------------------------
diff --git a/testdata/bin/run-hive-server.sh b/testdata/bin/run-hive-server.sh
index 42d95b5..49d1de2 100755
--- a/testdata/bin/run-hive-server.sh
+++ b/testdata/bin/run-hive-server.sh
@@ -63,8 +63,10 @@ done
${CLUSTER_BIN}/kill-hive-server.sh &> /dev/null
# Starts a Hive Metastore Server on the specified port.
-HADOOP_CLIENT_OPTS=-Xmx2024m hive --service metastore -p $HIVE_METASTORE_PORT \
- > ${LOGDIR}/hive-metastore.out 2>&1 &
+# To debug log4j2 loading issues, add to HADOOP_CLIENT_OPTS:
+# -Dorg.apache.logging.log4j.simplelog.StatusLogger.level=TRACE
+HADOOP_CLIENT_OPTS="-Xmx2024m -Dhive.log.file=hive-metastore.log" hive \
+ --service metastore -p $HIVE_METASTORE_PORT > ${LOGDIR}/hive-metastore.out 2>&1 &
# Wait for the Metastore to come up because HiveServer2 relies on it being live.
${CLUSTER_BIN}/wait-for-metastore.py --transport=${METASTORE_TRANSPORT}
@@ -72,7 +74,12 @@ ${CLUSTER_BIN}/wait-for-metastore.py --transport=${METASTORE_TRANSPORT}
if [ ${ONLY_METASTORE} -eq 0 ]; then
# Starts a HiveServer2 instance on the port specified by the HIVE_SERVER2_THRIFT_PORT
# environment variable.
- HADOOP_HEAPSIZE="512" hive --service hiveserver2 > ${LOGDIR}/hive-server2.out 2>&1 &
+ if [[ $IMPALA_MINICLUSTER_PROFILE == 2 ]]; then
+ HADOOP_HEAPSIZE="512" hive --service hiveserver2 > ${LOGDIR}/hive-server2.out 2>&1 &
+ elif [[ $IMPALA_MINICLUSTER_PROFILE == 3 ]]; then
+ HADOOP_CLIENT_OPTS="-Xmx2048m -Dhive.log.file=hive-server2.log" hive \
+ --service hiveserver2 > ${LOGDIR}/hive-server2.out 2>&1 &
+ fi
# Wait for the HiveServer2 service to come up because callers of this script
# may rely on it being available.
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/testdata/cluster/node_templates/cdh5/etc/hadoop/conf/kms-acls.xml.tmpl
----------------------------------------------------------------------
diff --git a/testdata/cluster/node_templates/cdh5/etc/hadoop/conf/kms-acls.xml.tmpl b/testdata/cluster/node_templates/cdh5/etc/hadoop/conf/kms-acls.xml.tmpl
deleted file mode 100644
index 0eefdb4..0000000
--- a/testdata/cluster/node_templates/cdh5/etc/hadoop/conf/kms-acls.xml.tmpl
+++ /dev/null
@@ -1,68 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<configuration>
- <property>
- <name>hadoop.kms.acl.CREATE</name>
- <value>*</value>
- </property>
-
- <property>
- <name>hadoop.kms.acl.DELETE</name>
- <value>*</value>
- </property>
-
- <property>
- <name>hadoop.kms.acl.ROLLOVER</name>
- <value>*</value>
- </property>
-
- <property>
- <name>hadoop.kms.acl.GET</name>
- <value>*</value>
- </property>
-
- <property>
- <name>hadoop.kms.acl.GET_KEYS</name>
- <value>*</value>
- </property>
-
- <property>
- <name>hadoop.kms.acl.GET_METADATA</name>
- <value>*</value>
- </property>
-
- <property>
- <name>hadoop.kms.acl.SET_KEY_MATERIAL</name>
- <value>*</value>
- </property>
-
- <property>
- <name>hadoop.kms.acl.GENERATE_EEK</name>
- <value>*</value>
- </property>
-
- <property>
- <name>hadoop.kms.acl.DECRYPT_EEK</name>
- <value>*</value>
- </property>
-
- <property>
- <name>default.key.acl.MANAGEMENT</name>
- <value>*</value>
- </property>
-
- <property>
- <name>default.key.acl.GENERATE_EEK</name>
- <value>*</value>
- </property>
-
- <property>
- <name>default.key.acl.DECRYPT_EEK</name>
- <value>*</value>
- </property>
-
- <property>
- <name>default.key.acl.READ</name>
- <value>*</value>
- </property>
-
-</configuration>
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/testdata/cluster/node_templates/cdh5/etc/hadoop/conf/kms-site.xml.tmpl
----------------------------------------------------------------------
diff --git a/testdata/cluster/node_templates/cdh5/etc/hadoop/conf/kms-site.xml.tmpl b/testdata/cluster/node_templates/cdh5/etc/hadoop/conf/kms-site.xml.tmpl
deleted file mode 100644
index 4027e32..0000000
--- a/testdata/cluster/node_templates/cdh5/etc/hadoop/conf/kms-site.xml.tmpl
+++ /dev/null
@@ -1,12 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<configuration>
- <property>
- <name>hadoop.kms.key.provider.uri</name>
- <value>jceks://file@/${NODE_DIR}/data/kms.keystore</value>
- </property>
-
- <property>
- <name>hadoop.kms.authentication.type</name>
- <value>simple</value>
- </property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/testdata/cluster/node_templates/cdh5/etc/hadoop/conf/yarn-site.xml.tmpl
----------------------------------------------------------------------
diff --git a/testdata/cluster/node_templates/cdh5/etc/hadoop/conf/yarn-site.xml.tmpl b/testdata/cluster/node_templates/cdh5/etc/hadoop/conf/yarn-site.xml.tmpl
deleted file mode 100644
index 0dd7bec..0000000
--- a/testdata/cluster/node_templates/cdh5/etc/hadoop/conf/yarn-site.xml.tmpl
+++ /dev/null
@@ -1,154 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-
-<!-- TODO: Remove any Llama-related configuration. Can this file be removed entirely? -->
-<configuration>
- <property>
- <name>yarn.resourcemanager.webapp.address</name>
- <value>0.0.0.0:${YARN_WEBUI_PORT}</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.address</name>
- <value>127.0.0.1:${NODEMANAGER_PORT}</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.delete.debug-delay-sec</name>
- <value>600</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.resource.memory-mb</name>
- <value>16384</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.resource.cpu-vcores</name>
- <value>16</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.nodemanagers.heartbeat-interval-ms</name>
- <value>100</value>
- </property>
-
- <property>
- <name>yarn.scheduler.fair.continuous-scheduling-enabled</name>
- <value>true</value>
- </property>
-
- <property>
- <name>yarn.scheduler.fair.assignmultiple</name>
- <value>true</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.scheduler.class</name>
- <value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.localizer.address</name>
- <value>127.0.0.1:${NODEMANAGER_LOCALIZER_PORT}</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.webapp.address</name>
- <value>127.0.0.1:${NODEMANAGER_WEBUI_PORT}</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.local-dirs</name>
- <value>${NODE_DIR}/data/yarn/local</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.log-dirs</name>
- <value>${NODE_DIR}/data/yarn/logs</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.aux-services</name>
- <value>mapreduce_shuffle</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.aux-services.mapreduce_shuffle.class</name>
- <value>org.apache.hadoop.mapred.ShuffleHandler</value>
- </property>
-
- <property>
- <name>yarn.log-aggregation-enable</name>
- <value>true</value>
- </property>
-
- <property>
- <description>List of directories to store localized files in.</description>
- <name>yarn.nodemanager.local-dirs</name>
- <value>${NODE_DIR}/var/lib/hadoop-yarn/cache/${user.name}/nm-local-dir</value>
- </property>
-
- <property>
- <description>Where to store container logs.</description>
- <name>yarn.nodemanager.log-dirs</name>
- <value>${NODE_DIR}/var/log/hadoop-yarn/containers</value>
- </property>
-
- <property>
- <description>Where to aggregate logs to.</description>
- <name>yarn.nodemanager.remote-app-log-dir</name>
- <value>${NODE_DIR}/var/log/hadoop-yarn/apps</value>
- </property>
-
- <property>
- <description>Classpath for typical applications.</description>
- <name>yarn.application.classpath</name>
- <value>
- ${HADOOP_CONF_DIR},
- ${HADOOP_HOME}/share/hadoop/tools/lib/*,
- ${HADOOP_HOME}/share/hadoop/common/*,
- ${HADOOP_HOME}/share/hadoop/common/lib/*,
- ${HADOOP_HOME}/share/hadoop/hdfs/*,
- ${HADOOP_HOME}/share/hdfs/common/lib/*,
- ${HADOOP_HOME}/share/hadoop/mapreduce/*,
- ${HADOOP_HOME}/share/hadoop/mapreduce/lib/*,
- ${HADOOP_HOME}/share/hadoop/yarn/*,
- ${HADOOP_HOME}/share/hadoop/yarn/lib/*,
- ${LZO_JAR_PATH}
- </value>
- </property>
-
- <!-- BEGIN Kerberos settings -->
-
- <!-- KERBEROS TODO: Add these to yarn.application.classpath.
- ${IMPALA_FE_DIR}/target/*,${HADOOP_LZO}/build/*,
- ${IMPALA_FE_DIR}/target/dependency/* -->
-
- <!-- ResourceManager security configs -->
- <property>
- <name>yarn.resourcemanager.keytab</name>
- <value>${KRB5_KTNAME}</value>
- </property>
-
- <property>
- <name>yarn.resourcemanager.principal</name>
- <value>${MINIKDC_PRINC_USER}</value>
- <!-- Sort of horrible: instead of the yarn principle, we'll use ${USER}
- so that we don't have a problem with file system permissions. -->
- </property>
-
- <!-- NodeManager security configs -->
- <property>
- <name>yarn.nodemanager.keytab</name>
- <value>${KRB5_KTNAME}</value>
- </property>
-
- <property>
- <name>yarn.nodemanager.principal</name>
- <value>${MINIKDC_PRINC_USER}</value>
- <!-- Also sort of horrible as per above -->
- </property>
- <!-- END Kerberos settings -->
-
-</configuration>
[4/4] impala git commit: IMPALA-4277: Support multiple versions of
Hadoop ecosystem
Posted by ta...@apache.org.
IMPALA-4277: Support multiple versions of Hadoop ecosystem
Adds support for building against two sets of Hadoop ecosystem
components. The control variable is IMPALA_MINICLUSTER_PROFILE_OVERRIDE,
which can either be set to 2 (for Hadoop 2, Hive 1, and so on) or 3 (for
Hadoop 3, Hive 2, and so on).
We intend (in a trivial follow-on change soon) to make 3 the new default
and to explicitly deprecate 2, but this change only does not switch the
default yet. We support both to facilitate a smoother transition, but
support will be removed soon in the Impala 3.x line.
The switch is done at build time, following the pattern from IMPALA-5184
(build fe against both Hive 1 & 2 APIs). Switching back and forth
requires running 'cmake' again. Doing this at build-time avoids
complicating the Java code with classloader configuration.
There are relatively few incompatible APIs. This implementation
encapsulates that by extracting some Java code into
fe/src/compat-minicluminicluster-profile-{2,3}. (This follows the
pattern established by IMPALA-5184, but, to avoid a proliferation
of directories, I've moved the Hive files into the same tree.)
pattern from IMPALA-5184 (build fe against both Hive 1 & 2 APIs). I
consolidated the Hive changes into the same directory structure.
For Maven, I introduced Maven "profiles" to handle the two cases where
the dependencies (and exclusions) differ. These are driven by the
$IMPALA_MINICLUSTER_PROFILE environment variable.
For Sentry, exception class names changed. We work around this by adding
"isSentry...(Exception)" methods with two different implementations.
Sentry is also doing some odd shading, whereby some exceptions are
"sentry.org.apache.sentry..."; we handle both. Similarly, the mechanism
to create a SentryAuthProvider is slightly different. The easiest way to
see the differences is to run:
diff -u fe/src/compat-minicluster-profile-{2,3}/java/org/apache/impala/util/SentryUtil.java
diff -u fe/src/compat-minicluster-profile-{2,3}/java/org/apache/impala/authorization/SentryAuthProvider.java
The Sentry work is based on a change by Zach Amsden.
In addition, we recently added an explicit "refresh" permission. In
Sentry 2, this required creating an ImpalaPrivilegeModel to capture
that. It's a slight customization of Hive's equivalent class.
For Parquet, the difference is even more mechanical. The package names
gone from "parquet" to "org.apache.parquet". The affected code
was extracted into ParquetHelper, but only one copy exists. The second
copy is generated at build-time using sed.
In the rare cases where we need to behave differently at runtime,
MiniclusterProfile.MINICLUSTER_PROFILE is a class which encapsulates
what version we were built aginst. One of the cases is the results
expected by various frontend tests. I avoided the issue by translating
one error string into another, which handled the diversion in one place,
rather than complicating the several locations which look for "No
FileSystem for scheme..." errors.
The HBase APIs we use for splitting regions at test time changed.
This patch includes a re-write of that code for the new APIs. This
piece was contributed by Zach Amsden.
To work with newer versions of dependencies, I updated the version of
httpcomponents.core we use to 4.4.9.
We (Thomas Tauber-Marshall and I) uploaded new Hadoop/Hive/Sentry/HBase
binaries to s3://native-toolchain, and amended the shell scripts to
launch the right things. There are minor mechanical differences. Some
of this was based on earlier work by Joe McDonnell and Zach Amsden.
Hive's logging is changed in Hive 2, necessitating creating a
log4j2.properties template and using it appropriately. Furthermore,
Hadoop3's new shell script re-writes do a certain amount of classpath
de-duplication, causing some issues with locating the relevant logging
configurations. Accomodations exist in the code to deal with that.
parquet-filtering.test was updated to turn off stats filtering. Older
Hive didn't write Parquet statistics, but newer Hive does. By turning
off stats filtering, we test what the test had intended to test.
For views-compatibility.test, it seems that Hive 2 has fixed certain
bugs that we were testing for in Hive. I've added a
HIVE=SUCCESS_PROFILE_3_ONLY mechanism to capture that.
For AuthorizationTest, different hive versions show slightly different
things for extended output.
To facilitate easier reviewing, the following files are 100% renames as identified by git; nothing
to see here.
rename fe/src/{compat-hive-1 => compat-minicluster-profile-2}/java/org/apache/hive/service/rpc/thrift/TGetCatalogsReq.java (100%)
rename fe/src/{compat-hive-1 => compat-minicluster-profile-2}/java/org/apache/hive/service/rpc/thrift/TGetColumnsReq.java (100%)
rename fe/src/{compat-hive-1 => compat-minicluster-profile-2}/java/org/apache/hive/service/rpc/thrift/TGetFunctionsReq.java (100%)
rename fe/src/{compat-hive-1 => compat-minicluster-profile-2}/java/org/apache/hive/service/rpc/thrift/TGetInfoReq.java (100%)
rename fe/src/{compat-hive-1 => compat-minicluster-profile-2}/java/org/apache/hive/service/rpc/thrift/TGetSchemasReq.java (100%)
rename fe/src/{compat-hive-1 => compat-minicluster-profile-2}/java/org/apache/hive/service/rpc/thrift/TGetTablesReq.java (100%)
rename fe/src/{compat-hive-1 => compat-minicluster-profile-2}/java/org/apache/impala/compat/MetastoreShim.java (100%)
rename fe/src/{compat-hive-2 => compat-minicluster-profile-3}/java/org/apache/impala/compat/MetastoreShim.java (100%)
rename testdata/cluster/node_templates/{cdh5 => common}/etc/hadoop/conf/kms-acls.xml.tmpl (100%)
rename testdata/cluster/node_templates/{cdh5 => common}/etc/hadoop/conf/kms-site.xml.tmpl (100%)
rename testdata/cluster/node_templates/{cdh5 => common}/etc/hadoop/conf/yarn-site.xml.tmpl (100%)
rename testdata/cluster/node_templates/{cdh5 => common}/etc/init.d/kudu-common (100%)
rename testdata/cluster/node_templates/{cdh5 => common}/etc/init.d/kudu-master (100%)
rename testdata/cluster/node_templates/{cdh5 => common}/etc/init.d/kudu-tserver (100%)
rename testdata/cluster/node_templates/{cdh5 => common}/etc/kudu/master.conf.tmpl (100%)
rename testdata/cluster/node_templates/{cdh5 => common}/etc/kudu/tserver.conf.tmpl (100%)
CreateTableLikeFileStmt had a chunk of code moved to ParquetHelper.java. This
was done manually, but without changing anything except what Java required in
terms of accessibility and boilerplate.
rewrite fe/src/main/java/org/apache/impala/analysis/CreateTableLikeFileStmt.java (80%)
copy fe/src/{main/java/org/apache/impala/analysis/CreateTableLikeFileStmt.java => compat-minicluster-profile-3/java/org/apache/impala/analysis/ParquetHelper.java} (77%)
Testing: Ran core & exhaustive tests with both profiles.
Cherry-picks: not for 2.x.
Change-Id: I7a2ab50331986c7394c2bbfd6c865232bca975f7
Reviewed-on: http://gerrit.cloudera.org:8080/9716
Reviewed-by: Alex Behm <al...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/783de170
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/783de170
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/783de170
Branch: refs/heads/master
Commit: 783de170c934be52109eccd88aac91e5a017e1ce
Parents: 466d126
Author: Philip Zeyliger <ph...@cloudera.com>
Authored: Fri Mar 2 16:16:06 2018 -0800
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Fri Mar 23 20:56:00 2018 +0000
----------------------------------------------------------------------
bin/create-test-configuration.sh | 14 +-
bin/impala-config.sh | 84 ++++-
bin/rat_exclude_files.txt | 2 +-
fe/pom.xml | 242 ++++++++++---
.../service/rpc/thrift/TGetCatalogsReq.java | 24 --
.../hive/service/rpc/thrift/TGetColumnsReq.java | 24 --
.../service/rpc/thrift/TGetFunctionsReq.java | 25 --
.../hive/service/rpc/thrift/TGetInfoReq.java | 24 --
.../hive/service/rpc/thrift/TGetSchemasReq.java | 24 --
.../hive/service/rpc/thrift/TGetTablesReq.java | 24 --
.../org/apache/impala/compat/MetastoreShim.java | 127 -------
.../org/apache/impala/compat/MetastoreShim.java | 127 -------
.../service/rpc/thrift/TGetCatalogsReq.java | 24 ++
.../hive/service/rpc/thrift/TGetColumnsReq.java | 24 ++
.../service/rpc/thrift/TGetFunctionsReq.java | 25 ++
.../hive/service/rpc/thrift/TGetInfoReq.java | 24 ++
.../hive/service/rpc/thrift/TGetSchemasReq.java | 24 ++
.../hive/service/rpc/thrift/TGetTablesReq.java | 24 ++
.../authorization/SentryAuthProvider.java | 74 ++++
.../org/apache/impala/compat/MetastoreShim.java | 127 +++++++
.../impala/compat/MiniclusterProfile.java | 25 ++
.../java/org/apache/impala/util/SentryUtil.java | 49 +++
.../apache/impala/analysis/ParquetHelper.java | 341 +++++++++++++++++++
.../authorization/ImpalaActionFactory.java | 79 +++++
.../authorization/ImpalaPrivilegeModel.java | 45 +++
.../authorization/SentryAuthProvider.java | 80 +++++
.../org/apache/impala/compat/MetastoreShim.java | 127 +++++++
.../impala/compat/MiniclusterProfile.java | 25 ++
.../java/org/apache/impala/util/SentryUtil.java | 54 +++
.../analysis/CreateTableLikeFileStmt.java | 304 +----------------
.../org/apache/impala/analysis/InsertStmt.java | 6 +-
.../authorization/AuthorizationChecker.java | 48 +--
.../impala/catalog/AuthorizationPolicy.java | 13 +
.../apache/impala/util/SentryPolicyService.java | 70 ++--
.../impala/analysis/AuthorizationTest.java | 50 ++-
.../apache/impala/common/FrontendTestBase.java | 16 +-
.../resources/hive-log4j2.properties.template | 83 +++++
impala-parent/pom.xml | 2 +-
testdata/bin/run-hbase.sh | 3 +
testdata/bin/run-hive-server.sh | 13 +-
.../cdh5/etc/hadoop/conf/kms-acls.xml.tmpl | 68 ----
.../cdh5/etc/hadoop/conf/kms-site.xml.tmpl | 12 -
.../cdh5/etc/hadoop/conf/yarn-site.xml.tmpl | 154 ---------
.../node_templates/cdh5/etc/init.d/kudu-common | 51 ---
.../node_templates/cdh5/etc/init.d/kudu-master | 33 --
.../node_templates/cdh5/etc/init.d/kudu-tserver | 33 --
.../cdh5/etc/kudu/master.conf.tmpl | 9 -
.../cdh5/etc/kudu/tserver.conf.tmpl | 11 -
.../cluster/node_templates/cdh6/etc/init.d/kms | 39 +++
.../common/etc/hadoop/conf/kms-acls.xml.tmpl | 68 ++++
.../common/etc/hadoop/conf/kms-site.xml.tmpl | 12 +
.../common/etc/hadoop/conf/yarn-site.xml.tmpl | 93 ++++-
.../common/etc/init.d/common.tmpl | 11 +-
.../common/etc/init.d/kudu-common | 51 +++
.../common/etc/init.d/kudu-master | 33 ++
.../common/etc/init.d/kudu-tserver | 33 ++
.../common/etc/init.d/yarn-common | 13 +-
.../common/etc/kudu/master.conf.tmpl | 9 +
.../common/etc/kudu/tserver.conf.tmpl | 11 +
testdata/pom.xml | 21 ++
.../HBaseTestDataRegionAssigment.java | 320 +++++++++++++++++
.../HBaseTestDataRegionAssigment.java | 331 ++++++++++++++++++
.../HBaseTestDataRegionAssigment.java | 320 -----------------
.../queries/QueryTest/parquet-filtering.test | 3 +-
.../queries/QueryTest/views-compatibility.test | 22 +-
tests/common/environ.py | 4 +
tests/common/impala_test_suite.py | 16 +-
tests/metadata/test_views_compatibility.py | 11 +-
tests/query_test/test_mt_dop.py | 2 +
tests/query_test/test_partitioning.py | 16 +-
70 files changed, 2759 insertions(+), 1571 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/bin/create-test-configuration.sh
----------------------------------------------------------------------
diff --git a/bin/create-test-configuration.sh b/bin/create-test-configuration.sh
index 97e05a3..b6781c1 100755
--- a/bin/create-test-configuration.sh
+++ b/bin/create-test-configuration.sh
@@ -89,13 +89,17 @@ rm -f authz-provider.ini
if [ $CREATE_METASTORE -eq 1 ]; then
echo "Creating postgresql database for Hive metastore"
- dropdb -U hiveuser ${METASTORE_DB} 2> /dev/null || true
+ dropdb -U hiveuser ${METASTORE_DB} || true
createdb -U hiveuser ${METASTORE_DB}
# Hive schema SQL scripts include other scripts using \i, which expects absolute paths.
# Switch to the scripts directory to make this work.
pushd ${HIVE_HOME}/scripts/metastore/upgrade/postgres
- psql -q -U hiveuser -d ${METASTORE_DB} -f hive-schema-1.1.0.postgres.sql
+ if [[ $IMPALA_MINICLUSTER_PROFILE == 2 ]]; then
+ psql -q -U hiveuser -d ${METASTORE_DB} -f hive-schema-1.1.0.postgres.sql
+ elif [[ $IMPALA_MINICLUSTER_PROFILE == 3 ]]; then
+ psql -q -U hiveuser -d ${METASTORE_DB} -f hive-schema-2.1.1.postgres.sql
+ fi
popd
# Increase the size limit of PARAM_VALUE from SERDE_PARAMS table to be able to create
# HBase tables with large number of columns.
@@ -160,7 +164,11 @@ fi
generate_config postgresql-hive-site.xml.template hive-site.xml
generate_config log4j.properties.template log4j.properties
-generate_config hive-log4j.properties.template hive-log4j.properties
+if [[ $IMPALA_MINICLUSTER_PROFILE == 3 ]]; then
+ generate_config hive-log4j2.properties.template hive-log4j2.properties
+else
+ generate_config hive-log4j.properties.template hive-log4j.properties
+fi
generate_config hbase-site.xml.template hbase-site.xml
generate_config authz-policy.ini.template authz-policy.ini
generate_config sentry-site.xml.template sentry-site.xml
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/bin/impala-config.sh
----------------------------------------------------------------------
diff --git a/bin/impala-config.sh b/bin/impala-config.sh
index e109573..e724604 100755
--- a/bin/impala-config.sh
+++ b/bin/impala-config.sh
@@ -166,19 +166,63 @@ export KUDU_JAVA_VERSION=1.7.0-cdh5.15.0-SNAPSHOT
# Versions of Hadoop ecosystem dependencies.
# ------------------------------------------
-export CDH_MAJOR_VERSION=5
-export IMPALA_HADOOP_VERSION=2.6.0-cdh5.15.0-SNAPSHOT
+# IMPALA_MINICLUSTER_PROFILE can have two values:
+# 2 represents:
+# Hadoop 2.6
+# HBase 1.2
+# Hive 1.1
+# Sentry 1.5
+# Parquet 1.5
+# Llama (used for Mini KDC) 1.0
+# 3 represents:
+# Hadoop 3.0
+# HBase 2.0
+# Hive 2.1
+# Sentry 2.0
+# Parquet 1.9
+#
+# Impala 3.x should default to profile 3 and mark profile 2 deprecated,
+# so that it may be removed in the 3.x line.
+# Currently, as the profile 3 is still being developed, we retain
+# profile 2 as the default.
+
+DEFAULT_MINICLUSTER_PROFILE=2
+: ${IMPALA_MINICLUSTER_PROFILE_OVERRIDE:=$DEFAULT_MINICLUSTER_PROFILE}
+
+if [[ $IMPALA_MINICLUSTER_PROFILE_OVERRIDE == 2 ]]; then
+ echo "IMPALA_MINICLUSTER_PROFILE=2 is deprecated and may be removed in Impala 3.x"
+
+ export IMPALA_MINICLUSTER_PROFILE=2
+ export CDH_MAJOR_VERSION=5
+ export IMPALA_HADOOP_VERSION=2.6.0-cdh5.15.0-SNAPSHOT
+ export IMPALA_HBASE_VERSION=1.2.0-cdh5.15.0-SNAPSHOT
+ export IMPALA_HIVE_VERSION=1.1.0-cdh5.15.0-SNAPSHOT
+ export IMPALA_SENTRY_VERSION=1.5.1-cdh5.15.0-SNAPSHOT
+ export IMPALA_PARQUET_VERSION=1.5.0-cdh5.15.0-SNAPSHOT
+ export IMPALA_LLAMA_MINIKDC_VERSION=1.0.0
+ export IMPALA_KITE_VERSION=1.0.0-cdh5.15.0-SNAPSHOT
+
+elif [[ $IMPALA_MINICLUSTER_PROFILE_OVERRIDE == 3 ]]; then
+ export IMPALA_MINICLUSTER_PROFILE=3
+ export CDH_MAJOR_VERSION=6
+ export IMPALA_HADOOP_VERSION=3.0.0-cdh6.x-SNAPSHOT
+ export IMPALA_HBASE_VERSION=2.0.0-cdh6.x-SNAPSHOT
+ export IMPALA_HIVE_VERSION=2.1.1-cdh6.x-SNAPSHOT
+ export IMPALA_SENTRY_VERSION=2.0.0-cdh6.x-SNAPSHOT
+ export IMPALA_PARQUET_VERSION=1.9.0-cdh6.x-SNAPSHOT
+ export IMPALA_AVRO_JAVA_VERSION=1.8.2-cdh6.x-SNAPSHOT
+ export IMPALA_LLAMA_MINIKDC_VERSION=1.0.0
+ export IMPALA_KITE_VERSION=1.0.0-cdh6.x-SNAPSHOT
+ # We continue using an older version of KUDU_JAVA_VERSION,
+ # as it includes support for DECIMAL.
+ # TODO: Jump to newer Kudu when it's published in Maven.
+fi
+
unset IMPALA_HADOOP_URL
-export IMPALA_HBASE_VERSION=1.2.0-cdh5.15.0-SNAPSHOT
unset IMPALA_HBASE_URL
-export IMPALA_HIVE_VERSION=1.1.0-cdh5.15.0-SNAPSHOT
unset IMPALA_HIVE_URL
-export IMPALA_SENTRY_VERSION=1.5.1-cdh5.15.0-SNAPSHOT
unset IMPALA_SENTRY_URL
-export IMPALA_PARQUET_VERSION=1.5.0-cdh5.15.0-SNAPSHOT
-export IMPALA_LLAMA_MINIKDC_VERSION=1.0.0
unset IMPALA_LLAMA_MINIKDC_URL
-export IMPALA_KITE_VERSION=1.0.0-cdh5.15.0-SNAPSHOT
# Source the branch and local config override files here to override any
# variables above or any variables below that allow overriding via environment
@@ -461,6 +505,12 @@ export HADOOP_CLASSPATH="${HADOOP_CLASSPATH-}:${HADOOP_HOME}/share/hadoop/tools/
export LZO_JAR_PATH="$HADOOP_LZO/build/hadoop-lzo-0.4.15.jar"
HADOOP_CLASSPATH+=":$LZO_JAR_PATH"
+if [[ $IMPALA_MINICLUSTER_PROFILE == 3 ]]; then
+ # Beware of adding entries from $HADOOP_HOME here, because they can change
+ # the order of the classpath, leading to configuration not showing up first.
+ HADOOP_CLASSPATH="$LZO_JAR_PATH"
+fi
+
export MINI_DFS_BASE_DATA_DIR="$IMPALA_HOME/cdh-${CDH_MAJOR_VERSION}-hdfs-data"
export PATH="$HADOOP_HOME/bin:$PATH"
@@ -475,7 +525,21 @@ export PATH="$HIVE_HOME/bin:$PATH"
# Allow overriding of Hive source location in case we want to build Impala without
# a complete Hive build.
export HIVE_SRC_DIR=${HIVE_SRC_DIR_OVERRIDE:-"${HIVE_HOME}/src"}
-export HIVE_CONF_DIR="$IMPALA_FE_DIR/src/test/resources"
+# To configure Hive logging, there's a hive-log4j2.properties[.template]
+# file in fe/src/test/resources. To get it into the classpath earlier
+# than the hive-log4j2.properties file included in some Hive jars,
+# we must set HIVE_CONF_DIR. Additionally, on Hadoop 3, because of
+# https://issues.apache.org/jira/browse/HADOOP-15019, when HIVE_CONF_DIR happens to equal
+# HADOOP_CONF_DIR, it gets de-duped out of its pole position in the CLASSPATH variable,
+# so we add an extra "./" into the path to avoid that. Use HADOOP_SHELL_SCRIPT_DEBUG=true
+# to debug issues like this. Hive may log something like:
+# Logging initialized using configuration in file:.../fe/src/test/resources/hive-log4j2.properties
+#
+# To debug log4j2 loading issues, add to HADOOP_CLIENT_OPTS:
+# -Dorg.apache.logging.log4j.simplelog.StatusLogger.level=TRACE
+#
+# We use a unique -Dhive.log.file to distinguish the HiveMetaStore and HiveServer2 logs.
+export HIVE_CONF_DIR="$IMPALA_FE_DIR/./src/test/resources"
# Hive looks for jar files in a single directory from HIVE_AUX_JARS_PATH plus
# any jars in AUX_CLASSPATH. (Or a list of jars in HIVE_AUX_JARS_PATH.)
@@ -498,7 +562,7 @@ export AUX_CLASSPATH="$AUX_CLASSPATH:$HBASE_HOME/lib/hbase-server-${IMPALA_HBASE
export AUX_CLASSPATH="$AUX_CLASSPATH:$HBASE_HOME/lib/hbase-protocol-${IMPALA_HBASE_VERSION}.jar"
export AUX_CLASSPATH="$AUX_CLASSPATH:$HBASE_HOME/lib/hbase-hadoop-compat-${IMPALA_HBASE_VERSION}.jar"
-export HBASE_CONF_DIR="$HIVE_CONF_DIR"
+export HBASE_CONF_DIR="$IMPALA_FE_DIR/src/test/resources"
# Set $THRIFT_HOME to the Thrift directory in toolchain.
export THRIFT_HOME="${IMPALA_TOOLCHAIN}/thrift-${IMPALA_THRIFT_VERSION}"
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/bin/rat_exclude_files.txt
----------------------------------------------------------------------
diff --git a/bin/rat_exclude_files.txt b/bin/rat_exclude_files.txt
index 5bb13f0..1819938 100644
--- a/bin/rat_exclude_files.txt
+++ b/bin/rat_exclude_files.txt
@@ -96,7 +96,7 @@ testdata/AllTypesErrorNoNulls/*.txt
*.parq
*.parquet
testdata/cluster/node_templates/cdh5/etc/hadoop/conf/*.xml.tmpl
-testdata/cluster/node_templates/cdh5/etc/kudu/*.conf.tmpl
+testdata/cluster/node_templates/common/etc/kudu/*.conf.tmpl
testdata/cluster/node_templates/common/etc/hadoop/conf/*.xml.tmpl
testdata/data/chars-formats.txt
testdata/data/chars-tiny.txt
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/fe/pom.xml
----------------------------------------------------------------------
diff --git a/fe/pom.xml b/fe/pom.xml
index 135ec77..a6d40cb 100644
--- a/fe/pom.xml
+++ b/fe/pom.xml
@@ -114,12 +114,6 @@ under the License.
<dependency>
<groupId>org.apache.sentry</groupId>
- <artifactId>sentry-policy-db</artifactId>
- <version>${sentry.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.sentry</groupId>
<artifactId>sentry-binding-hive</artifactId>
<version>${sentry.version}</version>
<exclusions>
@@ -138,12 +132,6 @@ under the License.
</dependency>
<dependency>
- <groupId>com.twitter</groupId>
- <artifactId>parquet-hadoop-bundle</artifactId>
- <version>${parquet.version}</version>
- </dependency>
-
- <dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
<version>${hbase.version}</version>
@@ -205,41 +193,14 @@ under the License.
<dependency>
<groupId>org.apache.hive</groupId>
- <artifactId>hive-hbase-handler</artifactId>
- <version>${hive.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.hive</groupId>
<artifactId>hive-service</artifactId>
<version>${hive.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
- <artifactId>hive-jdbc</artifactId>
- <version>${hive.version}</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-metastore</artifactId>
- <version>${hive.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-common</artifactId>
- <version>${hive.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.hive</groupId>
<artifactId>hive-serde</artifactId>
<version>${hive.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-exec</artifactId>
- <version>${hive.version}</version>
- </dependency>
<dependency>
<groupId>org.apache.kudu</groupId>
@@ -460,7 +421,7 @@ under the License.
-->
<source>${project.basedir}/generated-sources/gen-java</source>
<source>${project.build.directory}/generated-sources/cup</source>
- <source>${project.basedir}/src/compat-hive-${hive.major.version}/java</source>
+ <source>${project.basedir}/src/compat-minicluster-profile-${env.IMPALA_MINICLUSTER_PROFILE}/java</source>
</sources>
</configuration>
</execution>
@@ -652,5 +613,206 @@ under the License.
<maven.test.failure.ignore>true</maven.test.failure.ignore>
</properties>
</profile>
+
+ <profile>
+ <id>impala-minicluster-profile-2</id>
+ <activation>
+ <property>
+ <name>env.IMPALA_MINICLUSTER_PROFILE</name>
+ <value>2</value>
+ </property>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.sentry</groupId>
+ <artifactId>sentry-policy-db</artifactId>
+ <version>${sentry.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.twitter</groupId>
+ <artifactId>parquet-hadoop-bundle</artifactId>
+ <version>${parquet.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-exec</artifactId>
+ <version>${hive.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-common</artifactId>
+ <version>${hive.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-jdbc</artifactId>
+ <version>${hive.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-hbase-handler</artifactId>
+ <version>${hive.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-metastore</artifactId>
+ <version>${hive.version}</version>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <version>1.8</version>
+ <executions>
+ <execution>
+ <id>generate-minicluster-profile-2-compat-shims</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <configuration>
+ <!-- Generates ParquetHelper for minicluster profile 2. -->
+ <target>
+ <echo file="${project.build.directory}/gen-compat-shims.sh">
+ echo Generating shims for Minicluster Profile 2
+ mkdir -p generated-sources/gen-java/org/apache/impala/analysis
+ sed -e s,org.apache.parquet,parquet,g src/compat-minicluster-profile-3/java/org/apache/impala/analysis/ParquetHelper.java > generated-sources/gen-java/org/apache/impala/analysis/ParquetHelper.java
+ </echo>
+ <exec executable="bash" dir="${project.basedir}" failonerror="true">
+ <arg line="-ex ${project.build.directory}/gen-compat-shims.sh" />
+ </exec>
+ </target>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+ </profile>
+ <profile>
+ <id>impala-minicluster-profile-3</id>
+ <activation>
+ <property>
+ <name>env.IMPALA_MINICLUSTER_PROFILE</name>
+ <value>3</value>
+ </property>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.sentry</groupId>
+ <artifactId>sentry-policy-engine</artifactId>
+ <version>${sentry.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.parquet</groupId>
+ <artifactId>parquet-hadoop-bundle</artifactId>
+ <version>${parquet.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-exec</artifactId>
+ <version>${hive.version}</version>
+ <exclusions>
+ <!-- Impala uses log4j v1; avoid pulling in slf4j handling for log4j2 -->
+ <exclusion>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-slf4j-impl</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-common</artifactId>
+ <version>${hive.version}</version>
+ <exclusions>
+ <!-- Impala uses log4j v1; avoid pulling in slf4j handling for log4j2 -->
+ <exclusion>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-slf4j-impl</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-jdbc</artifactId>
+ <version>${hive.version}</version>
+ <scope>test</scope>
+ <exclusions>
+ <!-- Impala uses log4j v1; avoid pulling in slf4j handling for log4j2 -->
+ <exclusion>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-slf4j-impl</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-hbase-handler</artifactId>
+ <version>${hive.version}</version>
+ <exclusions>
+ <!-- Impala uses log4j v1; avoid pulling in slf4j handling for log4j2 -->
+ <exclusion>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-slf4j-impl</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-metastore</artifactId>
+ <version>${hive.version}</version>
+ <exclusions>
+ <!-- Impala uses log4j v1; avoid pulling in slf4j handling for log4j2 -->
+ <exclusion>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-slf4j-impl</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive.shims</groupId>
+ <artifactId>hive-shims-common</artifactId>
+ <version>${hive.version}</version>
+ <exclusions>
+ <!-- Impala uses log4j v1; avoid pulling in slf4j handling for log4j2 -->
+ <exclusion>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-slf4j-impl</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-enforcer-plugin</artifactId>
+ <version>3.0.0-M1</version>
+ <executions>
+ <execution>
+ <id>enforce-banned-dependencies</id>
+ <goals>
+ <goal>enforce</goal>
+ </goals>
+ <configuration>
+ <rules>
+ <bannedDependencies>
+ <excludes>
+ <exclude>org.apache.logging.log4j:log4j-slf4j-impl</exclude>
+ </excludes>
+ </bannedDependencies>
+ </rules>
+ <fail>true</fail>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
</profiles>
</project>
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/fe/src/compat-hive-1/java/org/apache/hive/service/rpc/thrift/TGetCatalogsReq.java
----------------------------------------------------------------------
diff --git a/fe/src/compat-hive-1/java/org/apache/hive/service/rpc/thrift/TGetCatalogsReq.java b/fe/src/compat-hive-1/java/org/apache/hive/service/rpc/thrift/TGetCatalogsReq.java
deleted file mode 100644
index 6a264bd..0000000
--- a/fe/src/compat-hive-1/java/org/apache/hive/service/rpc/thrift/TGetCatalogsReq.java
+++ /dev/null
@@ -1,24 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.hive.service.rpc.thrift;
-
-/**
- * Wrapper to allow the same code to instantiate the equivalent classes from Hive 1 and
- * Hive 2 APIs.
- */
-public class TGetCatalogsReq extends org.apache.hive.service.cli.thrift.TGetCatalogsReq {}
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/fe/src/compat-hive-1/java/org/apache/hive/service/rpc/thrift/TGetColumnsReq.java
----------------------------------------------------------------------
diff --git a/fe/src/compat-hive-1/java/org/apache/hive/service/rpc/thrift/TGetColumnsReq.java b/fe/src/compat-hive-1/java/org/apache/hive/service/rpc/thrift/TGetColumnsReq.java
deleted file mode 100644
index b35819a..0000000
--- a/fe/src/compat-hive-1/java/org/apache/hive/service/rpc/thrift/TGetColumnsReq.java
+++ /dev/null
@@ -1,24 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.hive.service.rpc.thrift;
-
-/**
- * Wrapper to allow the same code to instantiate the equivalent classes from Hive 1 and
- * Hive 2 APIs.
- */
-public class TGetColumnsReq extends org.apache.hive.service.cli.thrift.TGetColumnsReq {}
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/fe/src/compat-hive-1/java/org/apache/hive/service/rpc/thrift/TGetFunctionsReq.java
----------------------------------------------------------------------
diff --git a/fe/src/compat-hive-1/java/org/apache/hive/service/rpc/thrift/TGetFunctionsReq.java b/fe/src/compat-hive-1/java/org/apache/hive/service/rpc/thrift/TGetFunctionsReq.java
deleted file mode 100644
index 63424eb..0000000
--- a/fe/src/compat-hive-1/java/org/apache/hive/service/rpc/thrift/TGetFunctionsReq.java
+++ /dev/null
@@ -1,25 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.hive.service.rpc.thrift;
-
-/**
- * Wrapper to allow the same code to instantiate the equivalent classes from Hive 1 and
- * Hive 2 APIs.
- */
-public class TGetFunctionsReq
- extends org.apache.hive.service.cli.thrift.TGetFunctionsReq {}
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/fe/src/compat-hive-1/java/org/apache/hive/service/rpc/thrift/TGetInfoReq.java
----------------------------------------------------------------------
diff --git a/fe/src/compat-hive-1/java/org/apache/hive/service/rpc/thrift/TGetInfoReq.java b/fe/src/compat-hive-1/java/org/apache/hive/service/rpc/thrift/TGetInfoReq.java
deleted file mode 100644
index 708134d..0000000
--- a/fe/src/compat-hive-1/java/org/apache/hive/service/rpc/thrift/TGetInfoReq.java
+++ /dev/null
@@ -1,24 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.hive.service.rpc.thrift;
-
-/**
- * Wrapper to allow the same code to instantiate the equivalent classes from Hive 1 and
- * Hive 2 APIs.
- */
-public class TGetInfoReq extends org.apache.hive.service.cli.thrift.TGetInfoReq {}
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/fe/src/compat-hive-1/java/org/apache/hive/service/rpc/thrift/TGetSchemasReq.java
----------------------------------------------------------------------
diff --git a/fe/src/compat-hive-1/java/org/apache/hive/service/rpc/thrift/TGetSchemasReq.java b/fe/src/compat-hive-1/java/org/apache/hive/service/rpc/thrift/TGetSchemasReq.java
deleted file mode 100644
index 3b6ec26..0000000
--- a/fe/src/compat-hive-1/java/org/apache/hive/service/rpc/thrift/TGetSchemasReq.java
+++ /dev/null
@@ -1,24 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.hive.service.rpc.thrift;
-
-/**
- * Wrapper to allow the same code to instantiate the equivalent classes from Hive 1 and
- * Hive 2 APIs.
- */
-public class TGetSchemasReq extends org.apache.hive.service.cli.thrift.TGetSchemasReq {}
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/fe/src/compat-hive-1/java/org/apache/hive/service/rpc/thrift/TGetTablesReq.java
----------------------------------------------------------------------
diff --git a/fe/src/compat-hive-1/java/org/apache/hive/service/rpc/thrift/TGetTablesReq.java b/fe/src/compat-hive-1/java/org/apache/hive/service/rpc/thrift/TGetTablesReq.java
deleted file mode 100644
index fd309d4..0000000
--- a/fe/src/compat-hive-1/java/org/apache/hive/service/rpc/thrift/TGetTablesReq.java
+++ /dev/null
@@ -1,24 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.hive.service.rpc.thrift;
-
-/**
- * Wrapper to allow the same code to instantiate the equivalent classes from Hive 1 and
- * Hive 2 APIs.
- */
-public class TGetTablesReq extends org.apache.hive.service.cli.thrift.TGetTablesReq {}
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/fe/src/compat-hive-1/java/org/apache/impala/compat/MetastoreShim.java
----------------------------------------------------------------------
diff --git a/fe/src/compat-hive-1/java/org/apache/impala/compat/MetastoreShim.java b/fe/src/compat-hive-1/java/org/apache/impala/compat/MetastoreShim.java
deleted file mode 100644
index d0cd351..0000000
--- a/fe/src/compat-hive-1/java/org/apache/impala/compat/MetastoreShim.java
+++ /dev/null
@@ -1,127 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.impala.compat;
-
-import java.util.List;
-
-import org.apache.hadoop.hive.common.StatsSetupConst;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.IMetaStoreClient;
-import org.apache.hadoop.hive.metastore.MetaStoreUtils;
-import org.apache.hadoop.hive.metastore.Warehouse;
-import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hive.service.cli.thrift.TGetColumnsReq;
-import org.apache.hive.service.cli.thrift.TGetFunctionsReq;
-import org.apache.hive.service.cli.thrift.TGetSchemasReq;
-import org.apache.hive.service.cli.thrift.TGetTablesReq;
-import org.apache.impala.authorization.User;
-import org.apache.impala.common.Pair;
-import org.apache.impala.common.ImpalaException;
-import org.apache.impala.service.Frontend;
-import org.apache.impala.service.MetadataOp;
-import org.apache.impala.thrift.TMetadataOpRequest;
-import org.apache.impala.thrift.TResultSet;
-import org.apache.thrift.TException;
-
-/**
- * A wrapper around some of Hive's Metastore API's to abstract away differences
- * between major versions of Hive. This implements the shimmed methods for Hive 2.
- */
-public class MetastoreShim {
- /**
- * Wrapper around MetaStoreUtils.validateName() to deal with added arguments.
- */
- public static boolean validateName(String name) {
- return MetaStoreUtils.validateName(name);
- }
-
- /**
- * Wrapper around IMetaStoreClient.alter_partition() to deal with added
- * arguments.
- */
- public static void alterPartition(IMetaStoreClient client, Partition partition)
- throws InvalidOperationException, MetaException, TException {
- client.alter_partition(partition.getDbName(), partition.getTableName(), partition);
- }
-
- /**
- * Wrapper around IMetaStoreClient.alter_partitions() to deal with added
- * arguments.
- */
- public static void alterPartitions(IMetaStoreClient client, String dbName,
- String tableName, List<Partition> partitions)
- throws InvalidOperationException, MetaException, TException {
- client.alter_partitions(dbName, tableName, partitions);
- }
-
- /**
- * Wrapper around MetaStoreUtils.updatePartitionStatsFast() to deal with added
- * arguments.
- */
- public static void updatePartitionStatsFast(Partition partition, Warehouse warehouse)
- throws MetaException {
- MetaStoreUtils.updatePartitionStatsFast(partition, warehouse);
- }
-
- /**
- * Return the maximum number of Metastore objects that should be retrieved in
- * a batch.
- */
- public static String metastoreBatchRetrieveObjectsMaxConfigKey() {
- return HiveConf.ConfVars.METASTORE_BATCH_RETRIEVE_TABLE_PARTITION_MAX.toString();
- }
-
- /**
- * Return the key and value that should be set in the partition parameters to
- * mark that the stats were generated automatically by a stats task.
- */
- public static Pair<String, String> statsGeneratedViaStatsTaskParam() {
- return Pair.create(
- StatsSetupConst.STATS_GENERATED_VIA_STATS_TASK, StatsSetupConst.TRUE);
- }
-
- public static TResultSet execGetFunctions(
- Frontend frontend, TMetadataOpRequest request, User user) throws ImpalaException {
- TGetFunctionsReq req = request.getGet_functions_req();
- return MetadataOp.getFunctions(
- frontend, req.getCatalogName(), req.getSchemaName(), req.getFunctionName(), user);
- }
-
- public static TResultSet execGetColumns(
- Frontend frontend, TMetadataOpRequest request, User user) throws ImpalaException {
- TGetColumnsReq req = request.getGet_columns_req();
- return MetadataOp.getColumns(frontend, req.getCatalogName(), req.getSchemaName(),
- req.getTableName(), req.getColumnName(), user);
- }
-
- public static TResultSet execGetTables(
- Frontend frontend, TMetadataOpRequest request, User user) throws ImpalaException {
- TGetTablesReq req = request.getGet_tables_req();
- return MetadataOp.getTables(frontend, req.getCatalogName(), req.getSchemaName(),
- req.getTableName(), req.getTableTypes(), user);
- }
-
- public static TResultSet execGetSchemas(
- Frontend frontend, TMetadataOpRequest request, User user) throws ImpalaException {
- TGetSchemasReq req = request.getGet_schemas_req();
- return MetadataOp.getSchemas(
- frontend, req.getCatalogName(), req.getSchemaName(), user);
- }
-}
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/fe/src/compat-hive-2/java/org/apache/impala/compat/MetastoreShim.java
----------------------------------------------------------------------
diff --git a/fe/src/compat-hive-2/java/org/apache/impala/compat/MetastoreShim.java b/fe/src/compat-hive-2/java/org/apache/impala/compat/MetastoreShim.java
deleted file mode 100644
index 3d69545..0000000
--- a/fe/src/compat-hive-2/java/org/apache/impala/compat/MetastoreShim.java
+++ /dev/null
@@ -1,127 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.impala.compat;
-
-import java.util.List;
-
-import org.apache.hadoop.hive.common.StatsSetupConst;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.IMetaStoreClient;
-import org.apache.hadoop.hive.metastore.MetaStoreUtils;
-import org.apache.hadoop.hive.metastore.Warehouse;
-import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.Partition;
-import org.apache.hive.service.rpc.thrift.TGetColumnsReq;
-import org.apache.hive.service.rpc.thrift.TGetFunctionsReq;
-import org.apache.hive.service.rpc.thrift.TGetSchemasReq;
-import org.apache.hive.service.rpc.thrift.TGetTablesReq;
-import org.apache.impala.authorization.User;
-import org.apache.impala.common.ImpalaException;
-import org.apache.impala.common.Pair;
-import org.apache.impala.service.Frontend;
-import org.apache.impala.service.MetadataOp;
-import org.apache.impala.thrift.TMetadataOpRequest;
-import org.apache.impala.thrift.TResultSet;
-import org.apache.thrift.TException;
-
-/**
- * A wrapper around some of Hive's Metastore API's to abstract away differences
- * between major versions of Hive. This implements the shimmed methods for Hive 2.
- */
-public class MetastoreShim {
- /**
- * Wrapper around MetaStoreUtils.validateName() to deal with added arguments.
- */
- public static boolean validateName(String name) {
- return MetaStoreUtils.validateName(name, null);
- }
-
- /**
- * Wrapper around IMetaStoreClient.alter_partition() to deal with added
- * arguments.
- */
- public static void alterPartition(IMetaStoreClient client, Partition partition)
- throws InvalidOperationException, MetaException, TException {
- client.alter_partition(
- partition.getDbName(), partition.getTableName(), partition, null);
- }
-
- /**
- * Wrapper around IMetaStoreClient.alter_partitions() to deal with added
- * arguments.
- */
- public static void alterPartitions(IMetaStoreClient client, String dbName,
- String tableName, List<Partition> partitions)
- throws InvalidOperationException, MetaException, TException {
- client.alter_partitions(dbName, tableName, partitions, null);
- }
-
- /**
- * Wrapper around MetaStoreUtils.updatePartitionStatsFast() to deal with added
- * arguments.
- */
- public static void updatePartitionStatsFast(Partition partition, Warehouse warehouse)
- throws MetaException {
- MetaStoreUtils.updatePartitionStatsFast(partition, warehouse, null);
- }
-
- /**
- * Return the maximum number of Metastore objects that should be retrieved in
- * a batch.
- */
- public static String metastoreBatchRetrieveObjectsMaxConfigKey() {
- return HiveConf.ConfVars.METASTORE_BATCH_RETRIEVE_OBJECTS_MAX.toString();
- }
-
- /**
- * Return the key and value that should be set in the partition parameters to
- * mark that the stats were generated automatically by a stats task.
- */
- public static Pair<String, String> statsGeneratedViaStatsTaskParam() {
- return Pair.create(StatsSetupConst.STATS_GENERATED, StatsSetupConst.TASK);
- }
-
- public static TResultSet execGetFunctions(
- Frontend frontend, TMetadataOpRequest request, User user) throws ImpalaException {
- TGetFunctionsReq req = request.getGet_functions_req();
- return MetadataOp.getFunctions(
- frontend, req.getCatalogName(), req.getSchemaName(), req.getFunctionName(), user);
- }
-
- public static TResultSet execGetColumns(
- Frontend frontend, TMetadataOpRequest request, User user) throws ImpalaException {
- TGetColumnsReq req = request.getGet_columns_req();
- return MetadataOp.getColumns(frontend, req.getCatalogName(), req.getSchemaName(),
- req.getTableName(), req.getColumnName(), user);
- }
-
- public static TResultSet execGetTables(
- Frontend frontend, TMetadataOpRequest request, User user) throws ImpalaException {
- TGetTablesReq req = request.getGet_tables_req();
- return MetadataOp.getTables(frontend, req.getCatalogName(), req.getSchemaName(),
- req.getTableName(), req.getTableTypes(), user);
- }
-
- public static TResultSet execGetSchemas(
- Frontend frontend, TMetadataOpRequest request, User user) throws ImpalaException {
- TGetSchemasReq req = request.getGet_schemas_req();
- return MetadataOp.getSchemas(
- frontend, req.getCatalogName(), req.getSchemaName(), user);
- }
-}
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/fe/src/compat-minicluster-profile-2/java/org/apache/hive/service/rpc/thrift/TGetCatalogsReq.java
----------------------------------------------------------------------
diff --git a/fe/src/compat-minicluster-profile-2/java/org/apache/hive/service/rpc/thrift/TGetCatalogsReq.java b/fe/src/compat-minicluster-profile-2/java/org/apache/hive/service/rpc/thrift/TGetCatalogsReq.java
new file mode 100644
index 0000000..6a264bd
--- /dev/null
+++ b/fe/src/compat-minicluster-profile-2/java/org/apache/hive/service/rpc/thrift/TGetCatalogsReq.java
@@ -0,0 +1,24 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.hive.service.rpc.thrift;
+
+/**
+ * Wrapper to allow the same code to instantiate the equivalent classes from Hive 1 and
+ * Hive 2 APIs.
+ */
+public class TGetCatalogsReq extends org.apache.hive.service.cli.thrift.TGetCatalogsReq {}
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/fe/src/compat-minicluster-profile-2/java/org/apache/hive/service/rpc/thrift/TGetColumnsReq.java
----------------------------------------------------------------------
diff --git a/fe/src/compat-minicluster-profile-2/java/org/apache/hive/service/rpc/thrift/TGetColumnsReq.java b/fe/src/compat-minicluster-profile-2/java/org/apache/hive/service/rpc/thrift/TGetColumnsReq.java
new file mode 100644
index 0000000..b35819a
--- /dev/null
+++ b/fe/src/compat-minicluster-profile-2/java/org/apache/hive/service/rpc/thrift/TGetColumnsReq.java
@@ -0,0 +1,24 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.hive.service.rpc.thrift;
+
+/**
+ * Wrapper to allow the same code to instantiate the equivalent classes from Hive 1 and
+ * Hive 2 APIs.
+ */
+public class TGetColumnsReq extends org.apache.hive.service.cli.thrift.TGetColumnsReq {}
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/fe/src/compat-minicluster-profile-2/java/org/apache/hive/service/rpc/thrift/TGetFunctionsReq.java
----------------------------------------------------------------------
diff --git a/fe/src/compat-minicluster-profile-2/java/org/apache/hive/service/rpc/thrift/TGetFunctionsReq.java b/fe/src/compat-minicluster-profile-2/java/org/apache/hive/service/rpc/thrift/TGetFunctionsReq.java
new file mode 100644
index 0000000..63424eb
--- /dev/null
+++ b/fe/src/compat-minicluster-profile-2/java/org/apache/hive/service/rpc/thrift/TGetFunctionsReq.java
@@ -0,0 +1,25 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.hive.service.rpc.thrift;
+
+/**
+ * Wrapper to allow the same code to instantiate the equivalent classes from Hive 1 and
+ * Hive 2 APIs.
+ */
+public class TGetFunctionsReq
+ extends org.apache.hive.service.cli.thrift.TGetFunctionsReq {}
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/fe/src/compat-minicluster-profile-2/java/org/apache/hive/service/rpc/thrift/TGetInfoReq.java
----------------------------------------------------------------------
diff --git a/fe/src/compat-minicluster-profile-2/java/org/apache/hive/service/rpc/thrift/TGetInfoReq.java b/fe/src/compat-minicluster-profile-2/java/org/apache/hive/service/rpc/thrift/TGetInfoReq.java
new file mode 100644
index 0000000..708134d
--- /dev/null
+++ b/fe/src/compat-minicluster-profile-2/java/org/apache/hive/service/rpc/thrift/TGetInfoReq.java
@@ -0,0 +1,24 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.hive.service.rpc.thrift;
+
+/**
+ * Wrapper to allow the same code to instantiate the equivalent classes from Hive 1 and
+ * Hive 2 APIs.
+ */
+public class TGetInfoReq extends org.apache.hive.service.cli.thrift.TGetInfoReq {}
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/fe/src/compat-minicluster-profile-2/java/org/apache/hive/service/rpc/thrift/TGetSchemasReq.java
----------------------------------------------------------------------
diff --git a/fe/src/compat-minicluster-profile-2/java/org/apache/hive/service/rpc/thrift/TGetSchemasReq.java b/fe/src/compat-minicluster-profile-2/java/org/apache/hive/service/rpc/thrift/TGetSchemasReq.java
new file mode 100644
index 0000000..3b6ec26
--- /dev/null
+++ b/fe/src/compat-minicluster-profile-2/java/org/apache/hive/service/rpc/thrift/TGetSchemasReq.java
@@ -0,0 +1,24 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.hive.service.rpc.thrift;
+
+/**
+ * Wrapper to allow the same code to instantiate the equivalent classes from Hive 1 and
+ * Hive 2 APIs.
+ */
+public class TGetSchemasReq extends org.apache.hive.service.cli.thrift.TGetSchemasReq {}
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/fe/src/compat-minicluster-profile-2/java/org/apache/hive/service/rpc/thrift/TGetTablesReq.java
----------------------------------------------------------------------
diff --git a/fe/src/compat-minicluster-profile-2/java/org/apache/hive/service/rpc/thrift/TGetTablesReq.java b/fe/src/compat-minicluster-profile-2/java/org/apache/hive/service/rpc/thrift/TGetTablesReq.java
new file mode 100644
index 0000000..fd309d4
--- /dev/null
+++ b/fe/src/compat-minicluster-profile-2/java/org/apache/hive/service/rpc/thrift/TGetTablesReq.java
@@ -0,0 +1,24 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.hive.service.rpc.thrift;
+
+/**
+ * Wrapper to allow the same code to instantiate the equivalent classes from Hive 1 and
+ * Hive 2 APIs.
+ */
+public class TGetTablesReq extends org.apache.hive.service.cli.thrift.TGetTablesReq {}
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/fe/src/compat-minicluster-profile-2/java/org/apache/impala/authorization/SentryAuthProvider.java
----------------------------------------------------------------------
diff --git a/fe/src/compat-minicluster-profile-2/java/org/apache/impala/authorization/SentryAuthProvider.java b/fe/src/compat-minicluster-profile-2/java/org/apache/impala/authorization/SentryAuthProvider.java
new file mode 100644
index 0000000..4793516
--- /dev/null
+++ b/fe/src/compat-minicluster-profile-2/java/org/apache/impala/authorization/SentryAuthProvider.java
@@ -0,0 +1,74 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.impala.authorization;
+
+import com.google.common.base.Preconditions;
+import org.apache.sentry.policy.db.SimpleDBPolicyEngine;
+import org.apache.commons.lang.reflect.ConstructorUtils;
+import org.apache.impala.catalog.AuthorizationPolicy;
+import org.apache.sentry.provider.file.SimpleFileProviderBackend;
+import org.apache.sentry.policy.common.PolicyEngine;
+import org.apache.sentry.provider.cache.SimpleCacheProviderBackend;
+import org.apache.sentry.provider.common.ProviderBackend;
+import org.apache.sentry.provider.common.ProviderBackendContext;
+import org.apache.sentry.provider.common.ResourceAuthorizationProvider;
+
+/**
+ * Wrapper to facilitate differences in Sentry APIs across
+ * Sentry versions.
+ */
+class SentryAuthProvider {
+ /*
+ * Creates a new ResourceAuthorizationProvider based on the given configuration.
+ */
+ static ResourceAuthorizationProvider createProvider(AuthorizationConfig config,
+ AuthorizationPolicy policy) {
+ try {
+ ProviderBackend providerBe;
+ // Create the appropriate backend provider.
+ if (config.isFileBasedPolicy()) {
+ providerBe = new SimpleFileProviderBackend(config.getSentryConfig().getConfig(),
+ config.getPolicyFile());
+ } else {
+ // Note: The second parameter to the ProviderBackend is a "resourceFile" path
+ // which is not used by Impala. We cannot pass 'null' so instead pass an empty
+ // string.
+ providerBe = new SimpleCacheProviderBackend(config.getSentryConfig().getConfig(),
+ "");
+ Preconditions.checkNotNull(policy);
+ ProviderBackendContext context = new ProviderBackendContext();
+ context.setBindingHandle(policy);
+ providerBe.initialize(context);
+ }
+
+ SimpleDBPolicyEngine engine =
+ new SimpleDBPolicyEngine(config.getServerName(), providerBe);
+
+ // Try to create an instance of the specified policy provider class.
+ // Re-throw any exceptions that are encountered.
+ String policyFile = config.getPolicyFile() == null ? "" : config.getPolicyFile();
+ return (ResourceAuthorizationProvider) ConstructorUtils.invokeConstructor(
+ Class.forName(config.getPolicyProviderClassName()),
+ new Object[] {policyFile, engine});
+ } catch (Exception e) {
+ // Re-throw as unchecked exception.
+ throw new IllegalStateException(
+ "Error creating ResourceAuthorizationProvider: ", e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/fe/src/compat-minicluster-profile-2/java/org/apache/impala/compat/MetastoreShim.java
----------------------------------------------------------------------
diff --git a/fe/src/compat-minicluster-profile-2/java/org/apache/impala/compat/MetastoreShim.java b/fe/src/compat-minicluster-profile-2/java/org/apache/impala/compat/MetastoreShim.java
new file mode 100644
index 0000000..d0cd351
--- /dev/null
+++ b/fe/src/compat-minicluster-profile-2/java/org/apache/impala/compat/MetastoreShim.java
@@ -0,0 +1,127 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.impala.compat;
+
+import java.util.List;
+
+import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.IMetaStoreClient;
+import org.apache.hadoop.hive.metastore.MetaStoreUtils;
+import org.apache.hadoop.hive.metastore.Warehouse;
+import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hive.service.cli.thrift.TGetColumnsReq;
+import org.apache.hive.service.cli.thrift.TGetFunctionsReq;
+import org.apache.hive.service.cli.thrift.TGetSchemasReq;
+import org.apache.hive.service.cli.thrift.TGetTablesReq;
+import org.apache.impala.authorization.User;
+import org.apache.impala.common.Pair;
+import org.apache.impala.common.ImpalaException;
+import org.apache.impala.service.Frontend;
+import org.apache.impala.service.MetadataOp;
+import org.apache.impala.thrift.TMetadataOpRequest;
+import org.apache.impala.thrift.TResultSet;
+import org.apache.thrift.TException;
+
+/**
+ * A wrapper around some of Hive's Metastore API's to abstract away differences
+ * between major versions of Hive. This implements the shimmed methods for Hive 2.
+ */
+public class MetastoreShim {
+ /**
+ * Wrapper around MetaStoreUtils.validateName() to deal with added arguments.
+ */
+ public static boolean validateName(String name) {
+ return MetaStoreUtils.validateName(name);
+ }
+
+ /**
+ * Wrapper around IMetaStoreClient.alter_partition() to deal with added
+ * arguments.
+ */
+ public static void alterPartition(IMetaStoreClient client, Partition partition)
+ throws InvalidOperationException, MetaException, TException {
+ client.alter_partition(partition.getDbName(), partition.getTableName(), partition);
+ }
+
+ /**
+ * Wrapper around IMetaStoreClient.alter_partitions() to deal with added
+ * arguments.
+ */
+ public static void alterPartitions(IMetaStoreClient client, String dbName,
+ String tableName, List<Partition> partitions)
+ throws InvalidOperationException, MetaException, TException {
+ client.alter_partitions(dbName, tableName, partitions);
+ }
+
+ /**
+ * Wrapper around MetaStoreUtils.updatePartitionStatsFast() to deal with added
+ * arguments.
+ */
+ public static void updatePartitionStatsFast(Partition partition, Warehouse warehouse)
+ throws MetaException {
+ MetaStoreUtils.updatePartitionStatsFast(partition, warehouse);
+ }
+
+ /**
+ * Return the maximum number of Metastore objects that should be retrieved in
+ * a batch.
+ */
+ public static String metastoreBatchRetrieveObjectsMaxConfigKey() {
+ return HiveConf.ConfVars.METASTORE_BATCH_RETRIEVE_TABLE_PARTITION_MAX.toString();
+ }
+
+ /**
+ * Return the key and value that should be set in the partition parameters to
+ * mark that the stats were generated automatically by a stats task.
+ */
+ public static Pair<String, String> statsGeneratedViaStatsTaskParam() {
+ return Pair.create(
+ StatsSetupConst.STATS_GENERATED_VIA_STATS_TASK, StatsSetupConst.TRUE);
+ }
+
+ public static TResultSet execGetFunctions(
+ Frontend frontend, TMetadataOpRequest request, User user) throws ImpalaException {
+ TGetFunctionsReq req = request.getGet_functions_req();
+ return MetadataOp.getFunctions(
+ frontend, req.getCatalogName(), req.getSchemaName(), req.getFunctionName(), user);
+ }
+
+ public static TResultSet execGetColumns(
+ Frontend frontend, TMetadataOpRequest request, User user) throws ImpalaException {
+ TGetColumnsReq req = request.getGet_columns_req();
+ return MetadataOp.getColumns(frontend, req.getCatalogName(), req.getSchemaName(),
+ req.getTableName(), req.getColumnName(), user);
+ }
+
+ public static TResultSet execGetTables(
+ Frontend frontend, TMetadataOpRequest request, User user) throws ImpalaException {
+ TGetTablesReq req = request.getGet_tables_req();
+ return MetadataOp.getTables(frontend, req.getCatalogName(), req.getSchemaName(),
+ req.getTableName(), req.getTableTypes(), user);
+ }
+
+ public static TResultSet execGetSchemas(
+ Frontend frontend, TMetadataOpRequest request, User user) throws ImpalaException {
+ TGetSchemasReq req = request.getGet_schemas_req();
+ return MetadataOp.getSchemas(
+ frontend, req.getCatalogName(), req.getSchemaName(), user);
+ }
+}
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/fe/src/compat-minicluster-profile-2/java/org/apache/impala/compat/MiniclusterProfile.java
----------------------------------------------------------------------
diff --git a/fe/src/compat-minicluster-profile-2/java/org/apache/impala/compat/MiniclusterProfile.java b/fe/src/compat-minicluster-profile-2/java/org/apache/impala/compat/MiniclusterProfile.java
new file mode 100644
index 0000000..330035e
--- /dev/null
+++ b/fe/src/compat-minicluster-profile-2/java/org/apache/impala/compat/MiniclusterProfile.java
@@ -0,0 +1,25 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.impala.compat;
+
+/**
+ * Constant to tell us what Minicluster Profile we are built against.
+ */
+public class MiniclusterProfile {
+ public static final int MINICLUSTER_PROFILE = 2;
+}
http://git-wip-us.apache.org/repos/asf/impala/blob/783de170/fe/src/compat-minicluster-profile-2/java/org/apache/impala/util/SentryUtil.java
----------------------------------------------------------------------
diff --git a/fe/src/compat-minicluster-profile-2/java/org/apache/impala/util/SentryUtil.java b/fe/src/compat-minicluster-profile-2/java/org/apache/impala/util/SentryUtil.java
new file mode 100644
index 0000000..5d2eadc
--- /dev/null
+++ b/fe/src/compat-minicluster-profile-2/java/org/apache/impala/util/SentryUtil.java
@@ -0,0 +1,49 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+package org.apache.impala.util;
+
+import java.util.Set;
+
+import org.apache.sentry.provider.db.SentryAccessDeniedException;
+import org.apache.sentry.provider.db.SentryAlreadyExistsException;
+import org.apache.sentry.provider.db.service.thrift.SentryPolicyServiceClient;
+import org.apache.sentry.provider.db.service.thrift.TSentryRole;
+import org.apache.sentry.SentryUserException;
+
+/**
+ * Wrapper to facilitate differences in Sentry APIs across Sentry versions.
+ */
+public class SentryUtil {
+ static boolean isSentryAlreadyExists(Exception e) {
+ return e instanceof SentryAlreadyExistsException;
+ }
+
+ static boolean isSentryAccessDenied(Exception e) {
+ return e instanceof SentryAccessDeniedException;
+ }
+
+ public static boolean isSentryGroupNotFound(Exception e) {
+ // Sentry 1.5 does not have this exception
+ return false;
+ }
+
+ static Set<TSentryRole> listRoles(SentryPolicyServiceClient client, String username)
+ throws SentryUserException {
+ return client.listRoles(username);
+ }
+}