You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by mi...@apache.org on 2026/01/06 18:07:01 UTC
(impala) 05/05: IMPALA-12349: Support Apache Hive 2.x in Impala
This is an automated email from the ASF dual-hosted git repository.
michaelsmith pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit ee67ede314b25f789c9e14ce7fc7b00e6641fc0f
Author: ttttttz <24...@qq.com>
AuthorDate: Sun Sep 1 17:10:32 2024 +0800
IMPALA-12349: Support Apache Hive 2.x in Impala
Like IMPALA-10871, this patch adds MetastoreShim to support Apache
Hive 2.x. At the build time, based on the environment variable
IMPALA_HIVE_DIST_TYPE one of the three shims is added to as source
using the fe/pom.xml build plugin. And select the dependencies
related to Hive in the fe/pom.xml based on the environment variable
IMPALA_HIVE_MAJOR_VERSION. There are some duplicate classes under
compat-apache-hive2 directory, e.g. fe/src/compat-apache-hive-2/java/
org/apache/impala/catalog/events/MetastoreEvents.java duplicates
fe/src/main/java/org/apache/impala/catalog/events/MetastoreEvents.java
The class in compat-apache-hive2 is a simplified version that works
with Apache Hive 2.x. So we don't need to extract lots of Hive-dependent
codes in MetastoreEvents.java into the metastore shim. Due to this,
the build process simply remove the original source code when building
on Apache Hive 2. Additionally, it should be noted that all
the code in the fe/src/compat-apache-hive-2/java/org/apache/hadoop/hive
directory comes from Apache Hive 3.x, original
source: https://github.com/apache/hive/blob/branch-3.1
In order to reduce the unnecessary intrusion into the code, skip all
tests when building with Apache Hive 2.x. If wanting to build Impala
adapted to Apache Hive 2.x, please set the following environment variables
before `source bin/impala-config.sh`:
export USE_APACHE_COMPONENTS=true
export USE_APACHE_HIVE_2=true
TODO:
- IMPALA-14581: Support testing related to Apache Hive 2 in the minicluster.
Testing:
- Compile using the -package option to obtain the package. After
deployment, perform all types of query tests, including SELECT, INSERT,
CREATE TABLE, ALTER TABLE, COMPUTE STATUS, etc. In addition, comprehensive
testing has been conducted on the metadata auto-synchronization functionality.
The tests confirm that all event types are supported except for AlterDatabaseEvent,
AllocWriteIdEvent, AbortTxnEvent, PseudoAbortTxnEvent, and CommitCompactionEvent.
It is worth noting that these unsupported events are not generated in Apache Hive 2,
so their lack of processing support does not impact the functionality.
Change-Id: Ib5f104dc8d131835b8118b9d54077471db65681c
Reviewed-on: http://gerrit.cloudera.org:8080/21760
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
bin/bootstrap_toolchain.py | 3 +-
bin/create-test-configuration.sh | 4 +-
bin/impala-config.sh | 41 +-
bin/jenkins/build-all-flag-combinations.sh | 23 +-
bin/rat_exclude_files.txt | 1 +
buildall.sh | 3 +
common/function-registry/CMakeLists.txt | 2 +-
fe/pom.xml | 531 ++-
.../hadoop/hive/common/ValidReaderWriteIdList.java | 270 ++
.../hadoop/hive/common/ValidTxnWriteIdList.java | 110 +
.../hadoop/hive/common/ValidWriteIdList.java | 133 +
.../hive/metastore/api/TableValidWriteIds.java | 965 ++++++
.../impala/catalog/Hive2MetastoreShimBase.java} | 179 +-
.../impala/catalog/events/MetastoreEvents.java | 3587 ++++++++++++++++++++
.../impala/compat/HiveEsriGeospatialBuiltins.java | 29 +
.../org/apache/impala/compat/MetastoreShim.java | 965 ++++++
.../java/org/apache/thrift/TConfiguration.java | 91 +
.../thrift/transport/TEndpointTransport.java | 97 +
.../impala/analysis/ConvertTableToIcebergStmt.java | 6 +-
.../org/apache/impala/catalog/ColumnStats.java | 5 +-
.../org/apache/impala/catalog/FeCatalogUtils.java | 16 +
.../java/org/apache/impala/catalog/HdfsTable.java | 2 +-
.../impala/catalog/Hive3MetastoreShimBase.java | 37 +
.../main/java/org/apache/impala/catalog/Table.java | 3 +-
.../catalog/events/MetastoreEventsProcessor.java | 2 +-
.../apache/impala/catalog/local/LocalTable.java | 3 +-
.../apache/impala/catalog/paimon/PaimonUtil.java | 11 +-
.../apache/impala/service/CatalogOpExecutor.java | 22 +-
.../impala/util/HiveMetadataFormatUtils.java | 3 +-
java/calcite-planner/pom.xml | 33 +
java/impala-package/pom.xml | 37 +
java/pom.xml | 16 +
java/shaded-deps/hive-exec/pom.xml | 59 +-
testdata/bin/patch_hive2.sh | 71 +
testdata/cluster/hive2/patch0-HIVE-18755.diff | 306 ++
tests/common/environ.py | 3 +-
36 files changed, 7413 insertions(+), 256 deletions(-)
diff --git a/bin/bootstrap_toolchain.py b/bin/bootstrap_toolchain.py
index c58a17b6c..f3f0c8381 100755
--- a/bin/bootstrap_toolchain.py
+++ b/bin/bootstrap_toolchain.py
@@ -506,7 +506,8 @@ def get_hadoop_downloads():
cluster_components = []
use_apache_hadoop = os.environ["USE_APACHE_HADOOP"] == "true"
use_apache_hbase = os.environ["USE_APACHE_HBASE"] == "true"
- use_apache_hive = os.environ["USE_APACHE_HIVE_3"] == "true"
+ use_apache_hive = (os.environ["USE_APACHE_HIVE_3"] == "true"
+ or os.environ["USE_APACHE_HIVE_2"] == "true")
use_apache_tez = os.environ["USE_APACHE_TEZ"] == "true"
use_apache_ranger = os.environ["USE_APACHE_RANGER"] == "true"
use_apache_ozone = os.environ["USE_APACHE_OZONE"] == "true"
diff --git a/bin/create-test-configuration.sh b/bin/create-test-configuration.sh
index 6eb51fde7..ad6675aed 100755
--- a/bin/create-test-configuration.sh
+++ b/bin/create-test-configuration.sh
@@ -201,7 +201,7 @@ if [ $CREATE_METASTORE -eq 1 ]; then
# Use schematool to initialize the metastore db schema. It detects the Hive
# version and invokes the appropriate scripts
- CLASSPATH={$CLASSPATH}:${CONFIG_DIR} ${HIVE_HOME}/bin/schematool -initSchema -dbType \
+ CLASSPATH=${CLASSPATH}:${CONFIG_DIR} ${HIVE_HOME}/bin/schematool -initSchema -dbType \
postgres 1>${IMPALA_CLUSTER_LOGS_DIR}/schematool.log 2>&1
# TODO: We probably don't need to do this anymore
# Increase the size limit of PARAM_VALUE from SERDE_PARAMS table to be able to create
@@ -213,7 +213,7 @@ fi
if [ $UPGRADE_METASTORE_DB -eq 1 ]; then
echo "Upgrading the schema of metastore db ${METASTORE_DB}. Check \
${IMPALA_CLUSTER_LOGS_DIR}/schematool.log for details."
- CLASSPATH={$CLASSPATH}:${CONFIG_DIR} ${HIVE_HOME}/bin/schematool -upgradeSchema \
+ CLASSPATH=${CLASSPATH}:${CONFIG_DIR} ${HIVE_HOME}/bin/schematool -upgradeSchema \
-dbType postgres 1>${IMPALA_CLUSTER_LOGS_DIR}/schematool.log 2>&1
fi
diff --git a/bin/impala-config.sh b/bin/impala-config.sh
index beda12811..5c0b5e09a 100755
--- a/bin/impala-config.sh
+++ b/bin/impala-config.sh
@@ -272,6 +272,8 @@ export APACHE_RANGER_VERSION=2.4.0
export APACHE_TEZ_VERSION=0.10.2
export APACHE_HIVE_3_VERSION=3.1.3
export APACHE_HIVE_3_STORAGE_API_VERSION=2.7.0
+export APACHE_HIVE_2_VERSION=2.3.9
+export APACHE_HIVE_2_STORAGE_API_VERSION=2.4.0
export APACHE_OZONE_VERSION=1.4.0
# Java dependencies that are not also runtime components. Declaring versions here allows
@@ -400,7 +402,18 @@ if ${USE_APACHE_COMPONENTS:=false}; then
export IMPALA_TEZ_VERSION=${APACHE_TEZ_VERSION}
export USE_APACHE_HADOOP=true
export USE_APACHE_HBASE=true
- export USE_APACHE_HIVE_3=true
+ export USE_APACHE_HIVE_3=${USE_APACHE_HIVE_3:-false}
+ export USE_APACHE_HIVE_2=${USE_APACHE_HIVE_2:-false}
+ if [[ "${USE_APACHE_HIVE_3,,}" != "true" && "${USE_APACHE_HIVE_2,,}" != "true" ]]; then
+ echo "ERROR: The major version of Apache Hive must be specified:"
+ echo " Apache Hive 2.x: export USE_APACHE_HIVE_2=true"
+ echo " Apache Hive 3.x: export USE_APACHE_HIVE_3=true"
+ return 1
+ fi
+ if [[ "${USE_APACHE_HIVE_3,,}" == "true" && "${USE_APACHE_HIVE_2,,}" == "true" ]]; then
+ echo "ERROR: The major version of Apache Hive is ambiguous."
+ return 1
+ fi
export USE_APACHE_TEZ=true
export USE_APACHE_RANGER=true
export USE_APACHE_OZONE=true
@@ -423,6 +436,7 @@ else
export USE_APACHE_HADOOP=${USE_APACHE_HADOOP:=false}
export USE_APACHE_HBASE=${USE_APACHE_HBASE:=false}
export USE_APACHE_HIVE_3=${USE_APACHE_HIVE_3:=false}
+ export USE_APACHE_HIVE_2=${USE_APACHE_HIVE_2:=false}
export USE_APACHE_TEZ=${USE_APACHE_TEZ:=false}
export USE_APACHE_RANGER=${USE_APACHE_RANGER:=false}
export USE_APACHE_OZONE=${USE_APACHE_OZONE:=false}
@@ -444,6 +458,14 @@ if $USE_APACHE_HIVE_3; then
export IMPALA_HIVE_URL=${APACHE_HIVE_3_URL-}
export IMPALA_HIVE_SOURCE_URL=${APACHE_HIVE_3_SOURCE_URL-}
export IMPALA_HIVE_STORAGE_API_VERSION=${APACHE_HIVE_3_STORAGE_API_VERSION}
+elif $USE_APACHE_HIVE_2; then
+ # When USE_APACHE_HIVE_2 is set we use the apache hive version to build as well
+ # as deploy in the minicluster
+ export IMPALA_HIVE_DIST_TYPE="apache-hive-2"
+ export IMPALA_HIVE_VERSION=${APACHE_HIVE_2_VERSION}
+ export IMPALA_HIVE_URL=${APACHE_HIVE_2_URL-}
+ export IMPALA_HIVE_SOURCE_URL=${APACHE_HIVE_2_SOURCE_URL-}
+ export IMPALA_HIVE_STORAGE_API_VERSION=${APACHE_HIVE_2_STORAGE_API_VERSION}
else
# CDP hive version is used to build and deploy in minicluster when USE_APACHE_HIVE_* is
# false
@@ -465,7 +487,7 @@ fi
# infra/python/deps/requirements.txt.
export IMPALA_THRIFT_CPP_VERSION=0.16.0-p7
unset IMPALA_THRIFT_CPP_URL
-if $USE_APACHE_HIVE_3; then
+if $USE_APACHE_HIVE_3 || $USE_APACHE_HIVE_2; then
# Apache Hive 3 clients can't run on thrift versions >= 0.14 (IMPALA-11801)
export IMPALA_THRIFT_POM_VERSION=0.11.0
export IMPALA_THRIFT_JAVA_VERSION=${IMPALA_THRIFT_POM_VERSION}-p5
@@ -483,10 +505,9 @@ unset IMPALA_THRIFT_PY_URL
# disable tests and functionality.
export IMPALA_HIVE_MAJOR_VERSION=$(echo "$IMPALA_HIVE_VERSION" | cut -d . -f 1)
-# Hive 1 and 2 are no longer supported.
-if [[ "${IMPALA_HIVE_MAJOR_VERSION}" == "1" ||
- "${IMPALA_HIVE_MAJOR_VERSION}" == "2" ]]; then
- echo "Hive 1 and 2 are no longer supported"
+# Hive 1 is no longer supported.
+if [[ "${IMPALA_HIVE_MAJOR_VERSION}" == "1" ]]; then
+ echo "Hive 1 is no longer supported"
return 1
fi
@@ -718,7 +739,7 @@ DEFAULT_NODES_DIR="$IMPALA_HOME/testdata/cluster/cdh$CDH_MAJOR_VERSION$UNIQUE_FS
export IMPALA_CLUSTER_NODES_DIR="${IMPALA_CLUSTER_NODES_DIR-$DEFAULT_NODES_DIR}"
ESCAPED_DB_UID=$(sed "s/[^0-9a-zA-Z]/_/g" <<< "$UNIQUE_FS_LABEL$IMPALA_HOME")
-if $USE_APACHE_HIVE_3; then
+if $USE_APACHE_HIVE_3 || $USE_APACHE_HIVE_2; then
export HIVE_HOME="$APACHE_COMPONENTS_HOME/apache-hive-${IMPALA_HIVE_VERSION}-bin"
export HIVE_SRC_DIR="$APACHE_COMPONENTS_HOME/apache-hive-${IMPALA_HIVE_VERSION}-src"
# if apache hive is being used change the metastore db name, so we don't have to
@@ -734,8 +755,12 @@ else
export METASTORE_DB=${METASTORE_DB-"$(cut -c-59 <<< HMS$ESCAPED_DB_UID)_cdp"}
fi
# Set the path to the hive_metastore.thrift which is used to build thrift code
-export HIVE_METASTORE_THRIFT_DIR=${HIVE_METASTORE_THRIFT_DIR_OVERRIDE:-\
+if $USE_APACHE_HIVE_2; then
+ export HIVE_METASTORE_THRIFT_DIR=$HIVE_SRC_DIR/metastore/if
+else
+ export HIVE_METASTORE_THRIFT_DIR=${HIVE_METASTORE_THRIFT_DIR_OVERRIDE:-\
"$HIVE_SRC_DIR/standalone-metastore/src/main/thrift"}
+fi
if $USE_APACHE_TEZ; then
export TEZ_HOME="$APACHE_COMPONENTS_HOME/apache-tez-${IMPALA_TEZ_VERSION}-bin"
else
diff --git a/bin/jenkins/build-all-flag-combinations.sh b/bin/jenkins/build-all-flag-combinations.sh
index 8f7361342..842492efd 100755
--- a/bin/jenkins/build-all-flag-combinations.sh
+++ b/bin/jenkins/build-all-flag-combinations.sh
@@ -49,8 +49,11 @@ CONFIGS=(
"-skiptests -noclean -tsan"
"-skiptests -noclean -ubsan -so -ninja"
# USE_APACHE_HIVE_3=true build:
- "-skiptests -noclean -use_apache_components"
- "-notests -noclean -use_apache_components -package"
+ "-skiptests -noclean -use_apache_components -use_apache_hive_3"
+ "-notests -noclean -use_apache_components -use_apache_hive_3 -package"
+ # USE_APACHE_HIVE_2=true build:
+ "-skiptests -noclean -use_apache_components -use_apache_hive_2"
+ "-notests -noclean -use_apache_components -use_apache_hive_2 -package"
)
FAILED=""
@@ -79,6 +82,22 @@ for CONFIG in "${CONFIGS[@]}"; do
fi
DESCRIPTION="Options $CONFIG USE_APACHE_COMPONENTS=$USE_APACHE_COMPONENTS"
+ CONFIG2=${CONFIG/-use_apache_hive_3/}
+ if [[ "$CONFIG" != "$CONFIG2" ]]; then
+ CONFIG=$CONFIG2
+ export USE_APACHE_HIVE_3=true
+ export USE_APACHE_HIVE_2=false
+ fi
+ DESCRIPTION="Options $CONFIG USE_APACHE_HIVE_3=$USE_APACHE_HIVE_3"
+
+ CONFIG2=${CONFIG/-use_apache_hive_2/}
+ if [[ "$CONFIG" != "$CONFIG2" ]]; then
+ CONFIG=$CONFIG2
+ export USE_APACHE_HIVE_2=true
+ export USE_APACHE_HIVE_3=false
+ fi
+ DESCRIPTION="Options $CONFIG USE_APACHE_HIVE_2=$USE_APACHE_HIVE_2"
+
if [[ $# == 1 && $1 == "--dryrun" ]]; then
echo $DESCRIPTION
continue
diff --git a/bin/rat_exclude_files.txt b/bin/rat_exclude_files.txt
index eb4dfd1b0..e84296dbe 100644
--- a/bin/rat_exclude_files.txt
+++ b/bin/rat_exclude_files.txt
@@ -223,6 +223,7 @@ tests/shell/test_close_queries.sql
tests/shell/test_file_comments.sql
tests/shell/test_file_no_comments.sql
tests/shell/test_var_substitution.sql
+testdata/cluster/hive2/*.diff
# symlink to testdata/workloads/tpcds/queries
testdata/workloads/tpcds_partitioned/queries
diff --git a/buildall.sh b/buildall.sh
index 70f905c51..756ad6fc8 100755
--- a/buildall.sh
+++ b/buildall.sh
@@ -463,6 +463,9 @@ bootstrap_dependencies() {
if [[ "${USE_APACHE_HIVE_3}" = true ]]; then
"$IMPALA_HOME/testdata/bin/patch_hive.sh"
fi
+ if [[ "${USE_APACHE_HIVE_2}" = true ]]; then
+ "$IMPALA_HOME/testdata/bin/patch_hive2.sh"
+ fi
}
# Build the Impala frontend and its dependencies.
diff --git a/common/function-registry/CMakeLists.txt b/common/function-registry/CMakeLists.txt
index 55faeb59d..5a54c6e64 100644
--- a/common/function-registry/CMakeLists.txt
+++ b/common/function-registry/CMakeLists.txt
@@ -61,7 +61,7 @@ add_custom_command(
add_custom_target(function-registry ALL DEPENDS ${CODE_GEN_OUTPUT})
-if($ENV{USE_APACHE_HIVE_3} STREQUAL "false")
+if($ENV{USE_APACHE_HIVE_3} STREQUAL "false" AND $ENV{USE_APACHE_HIVE_2} STREQUAL "false")
add_custom_target(geospatial-udf-wrappers ALL DEPENDS ${GEOSPATIAL_CODE_GEN_OUTPUT})
add_dependencies(geospatial-udf-wrappers function-registry)
else()
diff --git a/fe/pom.xml b/fe/pom.xml
index ee7fe44ae..043c6fa3c 100644
--- a/fe/pom.xml
+++ b/fe/pom.xml
@@ -362,102 +362,6 @@ under the License.
<artifactId>datasketches-java</artifactId>
</dependency>
- <!-- Needed for tests like JdbcTest which instantiates HiveDriver -->
- <dependency>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-jdbc</artifactId>
- <version>${hive.version}</version>
- <scope>test</scope>
- <exclusions>
- <!-- Exclusions needed for Hive 3.1.3 -->
- <exclusion>
- <groupId>org.apache.hbase</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-slf4j-impl</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-1.2-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- <!-- We should exclude hive-serde since it brings along a
- different version of flatbuffers causing problems for loading tables -->
- <exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-serde</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-metastore</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hive.shims</groupId>
- <artifactId>hive-shims-0.20</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.ant</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- <exclusion>
- <!-- IMPALA-9108: Avoid pulling in leveldbjni, which is unneeded. -->
- <groupId>org.fusesource.leveldbjni</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.cloudera</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-classification</artifactId>
- <version>${hive.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-standalone-metastore</artifactId>
- <version>${hive.version}</version>
- <exclusions>
- <!-- Impala uses log4j v1; avoid pulling in slf4j handling for log4j2 -->
- <exclusion>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-slf4j-impl</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-1.2-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-serde</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-shims</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.cloudera</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
<!-- needed for JobConf, which HiveConf inherits from -->
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -1105,6 +1009,367 @@ under the License.
</build>
<profiles>
+ <!--
+ Select the dependencies related to Hive based on the
+ environment variable IMPALA_HIVE_MAJOR_VERSION.
+ -->
+ <profile>
+ <id>hive-major-version-3</id>
+ <activation>
+ <property>
+ <name>env.IMPALA_HIVE_MAJOR_VERSION</name>
+ <value>3</value>
+ </property>
+ </activation>
+ <dependencies>
+ <!-- Needed for tests like JdbcTest which instantiates HiveDriver -->
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-jdbc</artifactId>
+ <version>${hive.version}</version>
+ <scope>test</scope>
+ <exclusions>
+ <!-- Exclusions needed for Hive 3.1.3 -->
+ <exclusion>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-slf4j-impl</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-1.2-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <!-- We should exclude hive-serde since it brings along a
+ different version of flatbuffers causing problems for loading tables -->
+ <exclusion>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-serde</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-metastore</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hive.shims</groupId>
+ <artifactId>hive-shims-0.20</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.ant</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <!-- IMPALA-9108: Avoid pulling in leveldbjni, which is unneeded. -->
+ <groupId>org.fusesource.leveldbjni</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.cloudera</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-classification</artifactId>
+ <version>${hive.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-standalone-metastore</artifactId>
+ <version>${hive.version}</version>
+ <exclusions>
+ <!-- Impala uses log4j v1; avoid pulling in slf4j handling for log4j2 -->
+ <exclusion>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-slf4j-impl</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-1.2-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-serde</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-shims</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.cloudera</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+ </profile>
+
+ <profile>
+ <id>hive-major-version-2</id>
+ <activation>
+ <property>
+ <name>env.IMPALA_HIVE_MAJOR_VERSION</name>
+ <value>2</value>
+ </property>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-service</artifactId>
+ <version>${hive.version}</version>
+ <!--
+ hive-service pulls in an incoherent (not ${hadoop.version}) version of
+ hadoop-client via this dependency, which we don't need.
+ -->
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-llap-server</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-exec</artifactId>
+ </exclusion>
+ <!-- https://issues.apache.org/jira/browse/HADOOP-14903 -->
+ <exclusion>
+ <groupId>net.minidev</groupId>
+ <artifactId>json-smart</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.calcite.avatica</groupId>
+ <artifactId>avatica</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.calcite</groupId>
+ <artifactId>calcite-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.calcite</groupId>
+ <artifactId>calcite-avatica</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.calcite</groupId>
+ <artifactId>calcite-linq4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>ant</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <!-- IMPALA-9468: Avoid pulling in netty for security reasons -->
+ <groupId>io.netty</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-serde</artifactId>
+ <version>${hive.version}</version>
+ <exclusions>
+ <!-- https://issues.apache.org/jira/browse/HADOOP-14903 -->
+ <exclusion>
+ <groupId>net.minidev</groupId>
+ <artifactId>json-smart</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <!-- IMPALA-9108: Avoid pulling in leveldbjni, which is unneeded. -->
+ <groupId>org.fusesource.leveldbjni</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <!-- IMPALA-9468: Avoid pulling in netty for security reasons -->
+ <groupId>io.netty</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-common</artifactId>
+ <version>${hive.version}</version>
+ <exclusions>
+ <!-- Impala uses reload4j v1; avoid pulling in log4j2 -->
+ <exclusion>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <!-- https://issues.apache.org/jira/browse/HADOOP-14903 -->
+ <exclusion>
+ <groupId>net.minidev</groupId>
+ <artifactId>json-smart</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.ant</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <!-- IMPALA-9468: Avoid pulling in netty for security reasons -->
+ <groupId>io.netty</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-jdbc</artifactId>
+ <version>${hive.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <!-- Impala uses reload4j v1; avoid pulling in log4j2 -->
+ <exclusion>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>net.minidev</groupId>
+ <artifactId>json-smart</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.ant</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <!-- IMPALA-9468: Avoid pulling in netty for security reasons -->
+ <groupId>io.netty</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <!-- IMPALA-9108: Avoid pulling in leveldbjni, which is unneeded. -->
+ <groupId>org.fusesource.leveldbjni</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-metastore</artifactId>
+ <version>${hive.version}</version>
+ <scope>system</scope>
+ <systemPath>${hive.home}/lib/hive-metastore-${hive.version}.jar</systemPath>
+ <exclusions>
+ <!-- Impala uses reload4j v1; avoid pulling in log4j2 -->
+ <exclusion>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <!-- https://issues.apache.org/jira/browse/HADOOP-14903 -->
+ <exclusion>
+ <groupId>net.minidev</groupId>
+ <artifactId>json-smart</artifactId>
+ </exclusion>
+ <exclusion>
+ <!-- IMPALA-9468: Avoid pulling in netty for security reasons -->
+ <groupId>io.netty</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hive.shims</groupId>
+ <artifactId>hive-shims-common</artifactId>
+ <version>${hive.version}</version>
+ <exclusions>
+ <!-- Impala uses reload4j v1; avoid pulling in log4j2 -->
+ <exclusion>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <!-- IMPALA-9108: Avoid pulling in leveldbjni, which is unneeded. -->
+ <groupId>org.fusesource.leveldbjni</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <!-- IMPALA-9468: Avoid pulling in netty for security reasons -->
+ <groupId>io.netty</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+ </profile>
+
<profile>
<id>thrift-home-defined</id>
<activation>
@@ -1200,6 +1465,80 @@ under the License.
</build>
</profile>
+ <profile>
+ <id>apache-hive-2</id>
+ <activation>
+ <property>
+ <name>env.IMPALA_HIVE_DIST_TYPE</name>
+ <value>apache-hive-2</value>
+ </property>
+ </activation>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <version>1.8</version>
+ <executions>
+ <!-- Deleting the implementation of MetastoreEvents.java in hive3 -->
+ <execution>
+ <id>delete-hive3-metastore-events</id>
+ <phase>process-sources</phase>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <configuration>
+ <target>
+ <echo message="Deleting the MetastoreEvents.java used for adapting to hive-3.x"/>
+ <delete file="${project.basedir}/src/main/java/org/apache/impala/catalog/events/MetastoreEvents.java"/>
+ </target>
+ </configuration>
+ </execution>
+ <execution>
+ <id>copy-hive-metastore-2-dep</id>
+ <phase>package</phase>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <configuration>
+ <target>
+ <echo message="Copying hive-metastore-2.x.jar"/>
+ <copy file="${hive.home}/lib/hive-metastore-${hive.version}.jar"
+ tofile="${project.build.directory}/dependency/hive-metastore-${hive.version}.jar"/>
+ </target>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <excludes>
+ <exclude>**/org/apache/impala/catalog/metastore/*.java</exclude>
+ <exclude>**/org/apache/impala/catalog/CatalogHmsAPIHelper.java</exclude>
+ <exclude>**/org/apache/impala/catalog/CompactionInfoLoader.java</exclude>
+ <exclude>**/org/apache/impala/catalog/Hive3MetastoreShimBase.java</exclude>
+ </excludes>
+ </configuration>
+ <executions>
+ <execution>
+ <id>default-testCompile</id>
+ <phase>test-compile</phase>
+ <goals>
+ <goal>testCompile</goal>
+ </goals>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+
<!-- Profile which is automatically activated when building from
within Eclipse based on the presence of the m2e.version
property -->
diff --git a/fe/src/compat-apache-hive-2/java/org/apache/hadoop/hive/common/ValidReaderWriteIdList.java b/fe/src/compat-apache-hive-2/java/org/apache/hadoop/hive/common/ValidReaderWriteIdList.java
new file mode 100644
index 000000000..cbe53c6c8
--- /dev/null
+++ b/fe/src/compat-apache-hive-2/java/org/apache/hadoop/hive/common/ValidReaderWriteIdList.java
@@ -0,0 +1,270 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ // This code is copied from apache/hive and only some modifications were
+ // made to the code format to pass the code check.
+ // Original Source:
+ // https://github.com/apache/hive/blob/branch-3.1/storage-api/src/java/
+ // org/apache/hadoop/hive/common/ValidReaderWriteIdList.java
+
+package org.apache.hadoop.hive.common;
+
+import java.util.Arrays;
+import java.util.BitSet;
+
+/**
+ * An implementation of {@link ValidWriteIdList} for use by readers.
+ * This class will view a write id as valid only if it maps to committed transaction.
+ * Write ids of both open and aborted transactions will be seen as invalid.
+ */
+public class ValidReaderWriteIdList implements ValidWriteIdList {
+
+ private String tableName; // Full table name of format <db_name>.<table_name>
+ protected long[] exceptions;
+ protected BitSet abortedBits; // BitSet for flagging aborted write ids. Bit is true if
+ // aborted, false if open
+ //default value means there are no open write ids in the snapshot
+ private long minOpenWriteId = Long.MAX_VALUE;
+ protected long highWatermark;
+
+ public ValidReaderWriteIdList() {
+ this(null, new long[0], new BitSet(),
+ Long.MAX_VALUE, Long.MAX_VALUE);
+ }
+
+ /**
+ * Used if there are no open write ids in the snapshot.
+ */
+ public ValidReaderWriteIdList(String tableName, long[] exceptions, BitSet abortedBits,
+ long highWatermark) {
+ this(tableName, exceptions, abortedBits, highWatermark, Long.MAX_VALUE);
+ }
+ public ValidReaderWriteIdList(String tableName,
+ long[] exceptions, BitSet abortedBits, long highWatermark,
+ long minOpenWriteId) {
+ this.tableName = tableName;
+ if (exceptions.length > 0) {
+ this.minOpenWriteId = minOpenWriteId;
+ }
+ this.exceptions = exceptions;
+ this.abortedBits = abortedBits;
+ this.highWatermark = highWatermark;
+ }
+
+ public ValidReaderWriteIdList(String value) {
+ readFromString(value);
+ }
+
+ @Override
+ public boolean isWriteIdValid(long writeId) {
+ if (writeId > highWatermark) {
+ return false;
+ }
+ return Arrays.binarySearch(exceptions, writeId) < 0;
+ }
+
+ /**
+ * We cannot use a base file if its range contains an open write id.
+ * @param writeId from base_xxxx
+ */
+ @Override
+ public boolean isValidBase(long writeId) {
+ return (writeId < minOpenWriteId)
+ && (writeId <= highWatermark);
+ }
+ @Override
+ public RangeResponse isWriteIdRangeValid(long minWriteId, long maxWriteId) {
+ // check the easy cases first
+ if (minWriteId > highWatermark) {
+ return RangeResponse.NONE;
+ } else if (exceptions.length > 0 && exceptions[0] > maxWriteId) {
+ return RangeResponse.ALL;
+ }
+
+ // since the exceptions and the range in question overlap, count the
+ // exceptions in the range
+ long count = Math.max(0, maxWriteId - highWatermark);
+ for(long txn: exceptions) {
+ if (minWriteId <= txn && txn <= maxWriteId) {
+ count += 1;
+ }
+ }
+
+ if (count == 0) {
+ return RangeResponse.ALL;
+ } else if (count == (maxWriteId - minWriteId + 1)) {
+ return RangeResponse.NONE;
+ } else {
+ return RangeResponse.SOME;
+ }
+ }
+
+ @Override
+ public String toString() {
+ return writeToString();
+ }
+
+ // Format is <table_name>:<hwm>:<minOpenWriteId>:<open_writeids>:<abort_writeids>
+ @Override
+ public String writeToString() {
+ StringBuilder buf = new StringBuilder();
+ if (tableName == null) {
+ buf.append("null");
+ } else {
+ buf.append(tableName);
+ }
+ buf.append(':');
+ buf.append(highWatermark);
+ buf.append(':');
+ buf.append(minOpenWriteId);
+ if (exceptions.length == 0) {
+ buf.append(':'); // separator for open write ids
+ buf.append(':'); // separator for aborted write ids
+ } else {
+ StringBuilder open = new StringBuilder();
+ StringBuilder abort = new StringBuilder();
+ for (int i = 0; i < exceptions.length; i++) {
+ if (abortedBits.get(i)) {
+ if (abort.length() > 0) {
+ abort.append(',');
+ }
+ abort.append(exceptions[i]);
+ } else {
+ if (open.length() > 0) {
+ open.append(',');
+ }
+ open.append(exceptions[i]);
+ }
+ }
+ buf.append(':');
+ buf.append(open);
+ buf.append(':');
+ buf.append(abort);
+ }
+ return buf.toString();
+ }
+
+ @Override
+ public void readFromString(String src) {
+ if (src == null || src.length() == 0) {
+ highWatermark = Long.MAX_VALUE;
+ exceptions = new long[0];
+ abortedBits = new BitSet();
+ } else {
+ String[] values = src.split(":");
+ tableName = values[0];
+ if (tableName.equalsIgnoreCase("null")) {
+ tableName = null;
+ }
+ highWatermark = Long.parseLong(values[1]);
+ minOpenWriteId = Long.parseLong(values[2]);
+ String[] openWriteIds = new String[0];
+ String[] abortedWriteIds = new String[0];
+ if (values.length < 4) {
+ openWriteIds = new String[0];
+ abortedWriteIds = new String[0];
+ } else if (values.length == 4) {
+ if (!values[3].isEmpty()) {
+ openWriteIds = values[3].split(",");
+ }
+ } else {
+ if (!values[3].isEmpty()) {
+ openWriteIds = values[3].split(",");
+ }
+ if (!values[4].isEmpty()) {
+ abortedWriteIds = values[4].split(",");
+ }
+ }
+ exceptions = new long[openWriteIds.length + abortedWriteIds.length];
+ int i = 0;
+ for (String open : openWriteIds) {
+ exceptions[i++] = Long.parseLong(open);
+ }
+ for (String abort : abortedWriteIds) {
+ exceptions[i++] = Long.parseLong(abort);
+ }
+ Arrays.sort(exceptions);
+ abortedBits = new BitSet(exceptions.length);
+ for (String abort : abortedWriteIds) {
+ int index = Arrays.binarySearch(exceptions, Long.parseLong(abort));
+ abortedBits.set(index);
+ }
+ }
+ }
+
+ @Override
+ public String getTableName() {
+ return tableName;
+ }
+
+ @Override
+ public long getHighWatermark() {
+ return highWatermark;
+ }
+
+ @Override
+ public long[] getInvalidWriteIds() {
+ return exceptions;
+ }
+
+ @Override
+ public Long getMinOpenWriteId() {
+ return minOpenWriteId == Long.MAX_VALUE ?
+ null : minOpenWriteId;
+ }
+
+ @Override
+ public boolean isWriteIdAborted(long writeId) {
+ int index = Arrays.binarySearch(exceptions, writeId);
+ return index >= 0 && abortedBits.get(index);
+ }
+
+ @Override
+ public RangeResponse isWriteIdRangeAborted(long minWriteId, long maxWriteId) {
+ // check the easy cases first
+ if (highWatermark < minWriteId) {
+ return RangeResponse.NONE;
+ }
+
+ int count = 0; // number of aborted txns found in exceptions
+
+ // traverse the aborted txns list, starting at first aborted txn index
+ for (int i = abortedBits.nextSetBit(0); i >= 0; i = abortedBits.nextSetBit(i + 1)) {
+ long abortedTxnId = exceptions[i];
+ if (abortedTxnId > maxWriteId) { // we've already gone beyond the specified range
+ break;
+ }
+ if (abortedTxnId >= minWriteId && abortedTxnId <= maxWriteId) {
+ count++;
+ }
+ }
+
+ if (count == 0) {
+ return RangeResponse.NONE;
+ } else if (count == (maxWriteId - minWriteId + 1)) {
+ return RangeResponse.ALL;
+ } else {
+ return RangeResponse.SOME;
+ }
+ }
+
+ public ValidReaderWriteIdList updateHighWatermark(long value) {
+ return new ValidReaderWriteIdList(tableName, exceptions, abortedBits, value,
+ minOpenWriteId);
+ }
+}
diff --git a/fe/src/compat-apache-hive-2/java/org/apache/hadoop/hive/common/ValidTxnWriteIdList.java b/fe/src/compat-apache-hive-2/java/org/apache/hadoop/hive/common/ValidTxnWriteIdList.java
new file mode 100644
index 000000000..6dc6c59a9
--- /dev/null
+++ b/fe/src/compat-apache-hive-2/java/org/apache/hadoop/hive/common/ValidTxnWriteIdList.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// This code is copied from apache/hive and only some modifications were
+// made to the code format to pass the code check.
+// Original Source:
+// https://github.com/apache/hive/tree/branch-3.1/storage-api/src/java/
+// org/apache/hadoop/hive/common/ValidTxnWriteIdList.java
+
+package org.apache.hadoop.hive.common;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * An implementation to store and manage list of ValidWriteIds for each tables read by
+ * current transaction.
+ */
+public class ValidTxnWriteIdList {
+ /**
+ * Key used to store valid write id list for all the operated tables in a
+ * {@link org.apache.hadoop.conf.Configuration} object.
+ */
+ public static final String VALID_TABLES_WRITEIDS_KEY = "hive.txn.tables.valid.writeids";
+
+ // Transaction for which the list of tables valid write Ids are populated
+ private Long txnId;
+
+ // Map of valid write ids list for all the tables read by the current txn
+ // Key is full table name string of format <db_name>.<table_name>
+ private Map<String, ValidWriteIdList> tablesValidWriteIdList = new HashMap<>();
+ public ValidTxnWriteIdList(Long txnId) {
+ this.txnId = txnId;
+ }
+
+ public ValidTxnWriteIdList(String value) {
+ readFromString(value);
+ }
+
+ @Override
+ public String toString() {
+ return writeToString();
+ }
+
+ public void addTableValidWriteIdList(ValidWriteIdList validWriteIds) {
+ tablesValidWriteIdList.put(validWriteIds.getTableName(), validWriteIds);
+ }
+
+ // Input fullTableName is of format <db_name>.<table_name>
+ public ValidWriteIdList getTableValidWriteIdList(String fullTableName) {
+ if (tablesValidWriteIdList.containsKey(fullTableName)) {
+ return tablesValidWriteIdList.get(fullTableName);
+ }
+ return null;
+ }
+
+ // Each ValidWriteIdList is separated with "$" and each one maps to one table
+ // Format <txnId>$<table_name>:<hwm>:<minOpenWriteId>:<open_writeids>
+ // :<abort_writeids>$<table_name>...
+ private void readFromString(String src) {
+ if ((src == null) || (src.length() == 0)) {
+ return;
+ }
+ String[] tblWriteIdStrList = src.split("\\$");
+ assert(tblWriteIdStrList.length >= 1);
+
+ // First $ separated substring would be txnId and the rest are ValidReaderWriteIdList
+ this.txnId = Long.parseLong(tblWriteIdStrList[0]);
+ for (int index = 1; index < tblWriteIdStrList.length; index++) {
+ String tableStr = tblWriteIdStrList[index];
+ ValidWriteIdList validWriteIdList = new ValidReaderWriteIdList(tableStr);
+ addTableValidWriteIdList(validWriteIdList);
+ }
+ }
+
+ // Each ValidWriteIdList is separated with "$" and each one maps to one table
+ // Format <txnId>$<table_name>:<hwm>:<minOpenWriteId>:<open_writeids>
+ // :<abort_writeids>$<table_name>...
+ private String writeToString() {
+ // First $ separated substring will be txnId and the rest are ValidReaderWriteIdList
+ StringBuilder buf = new StringBuilder(txnId.toString());
+ int index = 0;
+ for (HashMap.Entry<String, ValidWriteIdList> entry
+ : tablesValidWriteIdList.entrySet()) {
+ if (index < tablesValidWriteIdList.size()) {
+ buf.append('$');
+ }
+ buf.append(entry.getValue().writeToString());
+
+ // Separator for multiple tables' ValidWriteIdList. Also, skip it for last entry.
+ index++;
+ }
+ return buf.toString();
+ }
+}
diff --git a/fe/src/compat-apache-hive-2/java/org/apache/hadoop/hive/common/ValidWriteIdList.java b/fe/src/compat-apache-hive-2/java/org/apache/hadoop/hive/common/ValidWriteIdList.java
new file mode 100644
index 000000000..673a5231e
--- /dev/null
+++ b/fe/src/compat-apache-hive-2/java/org/apache/hadoop/hive/common/ValidWriteIdList.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// This code is copied from apache/hive and only some modifications were
+// made to the code format to pass the code check.
+// Original Source:
+// https://github.com/apache/hive/tree/branch-3.1/storage-api/src/java/
+// org/apache/hadoop/hive/common/ValidWriteIdList.java
+
+package org.apache.hadoop.hive.common;
+
+/**
+ * Models the list of transactions that should be included in a snapshot.
+ * It is modelled as a high water mark, which is the largest write id that
+ * has been committed and a list of write ids that are not included.
+ */
+public interface ValidWriteIdList {
+
+ /**
+ * Key used to store valid write id list in a
+ * {@link org.apache.hadoop.conf.Configuration} object.
+ */
+ String VALID_WRITEIDS_KEY = "hive.txn.valid.writeids";
+
+ /**
+ * The response to a range query. NONE means no values in this range match,
+ * SOME mean that some do, and ALL means that every value does.
+ */
+ enum RangeResponse {NONE, SOME, ALL};
+
+ /**
+ * Indicates whether a given write ID is valid. Note that valid may have different
+ * meanings for different implementations, as some will only want to see
+ * committed transactions and some
+ * both committed and aborted.
+ * @param writeId write ID of the table
+ * @return true if valid, false otherwise
+ */
+ boolean isWriteIdValid(long writeId);
+
+ /**
+ * Returns {@code true} if such base file can be used to materialize the snapshot
+ * represented by
+ * this {@code ValidWriteIdList}.
+ * @param writeId highest write ID in a given base_xxxx file
+ * @return true if the base file can be used
+ */
+ boolean isValidBase(long writeId);
+
+ /**
+ * Find out if a range of write ids are valid. Note that valid may have different
+ * meanings for different implementations, as some will only want to see committed
+ * transactions and some
+ * both committed and aborted.
+ * @param minWriteId minimum write ID to look for, inclusive
+ * @param maxWriteId maximum write ID to look for, inclusive
+ * @return Indicate whether none, some, or all of these transactions are valid.
+ */
+ RangeResponse isWriteIdRangeValid(long minWriteId, long maxWriteId);
+
+ /**
+ * Write this ValidWriteIdList into a string. This should produce a string that
+ * can be used by {@link #readFromString(String)} to populate a ValidWriteIdList.
+ * @return the list as a string
+ */
+ String writeToString();
+
+ /**
+ * Populate this ValidWriteIdList from the string. It is assumed that the string
+ * was created via {@link #writeToString()} and the exceptions list is sorted.
+ * @param src source string.
+ */
+ void readFromString(String src);
+
+ /**
+ * Get the table for which the ValidWriteIdList is formed
+ * @return table name (<db_name>.<table_name>) associated with
+ * ValidWriteIdList.
+ */
+ String getTableName();
+
+ /**
+ * Get the largest write id used.
+ * @return largest write id used
+ */
+ long getHighWatermark();
+
+ /**
+ * Get the list of write ids under the high water mark that are not valid.
+ * Note that invalid
+ * may have different meanings for different implementations, as some will
+ * only want to see open
+ * transactions and some both open and aborted.
+ * @return a list of invalid write ids
+ */
+ long[] getInvalidWriteIds();
+
+ /**
+ * Indicates whether a given write maps to aborted transaction.
+ * @param writeId write id to be validated
+ * @return true if aborted, false otherwise
+ */
+ boolean isWriteIdAborted(long writeId);
+
+ /**
+ * Find out if a range of write ids are aborted.
+ * @param minWriteId minimum write Id to look for, inclusive
+ * @param maxWriteId maximum write Id to look for, inclusive
+ * @return Indicate whether none, some, or all of these write ids are aborted.
+ */
+ RangeResponse isWriteIdRangeAborted(long minWriteId, long maxWriteId);
+
+ /**
+ * The smallest open write id.
+ * @return smallest Open write Id in this set, {@code null} if there is none.
+ */
+ Long getMinOpenWriteId();
+}
diff --git a/fe/src/compat-apache-hive-2/java/org/apache/hadoop/hive/metastore/api/TableValidWriteIds.java b/fe/src/compat-apache-hive-2/java/org/apache/hadoop/hive/metastore/api/TableValidWriteIds.java
new file mode 100644
index 000000000..8f8c84993
--- /dev/null
+++ b/fe/src/compat-apache-hive-2/java/org/apache/hadoop/hive/metastore/api/TableValidWriteIds.java
@@ -0,0 +1,965 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Autogenerated by Thrift Compiler (0.9.3)
+ *
+ * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+ * @generated
+ */
+
+// This code is copied from apache/hive and only some modifications were
+// made to the code format to pass the code check.
+// Original Source:
+// https://github.com/apache/hive/blob/branch-3.1/standalone-metastore/src/gen/thrift/
+// gen-javabean/org/apache/hadoop/hive/metastore/api/TableValidWriteIds.java
+
+package org.apache.hadoop.hive.metastore.api;
+
+import org.apache.thrift.scheme.IScheme;
+import org.apache.thrift.scheme.SchemeFactory;
+import org.apache.thrift.scheme.StandardScheme;
+
+import org.apache.thrift.scheme.TupleScheme;
+import org.apache.thrift.protocol.TTupleProtocol;
+import org.apache.thrift.protocol.TProtocolException;
+import org.apache.thrift.EncodingUtils;
+import org.apache.thrift.TException;
+import org.apache.thrift.async.AsyncMethodCallback;
+import org.apache.thrift.server.AbstractNonblockingServer.*;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.HashMap;
+import java.util.EnumMap;
+import java.util.Set;
+import java.util.HashSet;
+import java.util.EnumSet;
+import java.util.Collections;
+import java.util.BitSet;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import javax.annotation.Generated;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
+@Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
+@org.apache.hadoop.classification.InterfaceAudience.Public
+@org.apache.hadoop.classification.InterfaceStability.Stable
+public class TableValidWriteIds
+ implements org.apache.thrift.TBase<TableValidWriteIds, TableValidWriteIds._Fields>,
+ java.io.Serializable, Cloneable, Comparable<TableValidWriteIds> {
+ private static final org.apache.thrift.protocol.TStruct STRUCT_DESC =
+ new org.apache.thrift.protocol.TStruct("TableValidWriteIds");
+
+ private static final org.apache.thrift.protocol.TField FULL_TABLE_NAME_FIELD_DESC =
+ new org.apache.thrift.protocol.TField("fullTableName",
+ org.apache.thrift.protocol.TType.STRING, (short)1);
+ private static final org.apache.thrift.protocol
+ .TField WRITE_ID_HIGH_WATER_MARK_FIELD_DESC =
+ new org.apache.thrift.protocol.TField("writeIdHighWaterMark",
+ org.apache.thrift.protocol.TType.I64, (short)2);
+ private static final org.apache.thrift.protocol.TField INVALID_WRITE_IDS_FIELD_DESC =
+ new org.apache.thrift.protocol.TField("invalidWriteIds",
+ org.apache.thrift.protocol.TType.LIST, (short)3);
+ private static final org.apache.thrift.protocol.TField MIN_OPEN_WRITE_ID_FIELD_DESC =
+ new org.apache.thrift.protocol.TField("minOpenWriteId",
+ org.apache.thrift.protocol.TType.I64, (short)4);
+ private static final org.apache.thrift.protocol.TField ABORTED_BITS_FIELD_DESC =
+ new org.apache.thrift.protocol.TField("abortedBits",
+ org.apache.thrift.protocol.TType.STRING, (short)5);
+
+ private static final Map<Class<? extends IScheme>, SchemeFactory> schemes =
+ new HashMap<Class<? extends IScheme>, SchemeFactory>();
+ static {
+ schemes.put(StandardScheme.class, new TableValidWriteIdsStandardSchemeFactory());
+ schemes.put(TupleScheme.class, new TableValidWriteIdsTupleSchemeFactory());
+ }
+
+ private String fullTableName; // required
+ private long writeIdHighWaterMark; // required
+ private List<Long> invalidWriteIds; // required
+ private long minOpenWriteId; // optional
+ private ByteBuffer abortedBits; // required
+
+ /** The set of fields this struct contains, along with convenience methods for
+ * finding and manipulating them. */
+ public enum _Fields implements org.apache.thrift.TFieldIdEnum {
+ FULL_TABLE_NAME((short)1, "fullTableName"),
+ WRITE_ID_HIGH_WATER_MARK((short)2, "writeIdHighWaterMark"),
+ INVALID_WRITE_IDS((short)3, "invalidWriteIds"),
+ MIN_OPEN_WRITE_ID((short)4, "minOpenWriteId"),
+ ABORTED_BITS((short)5, "abortedBits");
+
+ private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
+
+ static {
+ for (_Fields field : EnumSet.allOf(_Fields.class)) {
+ byName.put(field.getFieldName(), field);
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, or null if its not found.
+ */
+ public static _Fields findByThriftId(int fieldId) {
+ switch(fieldId) {
+ case 1: // FULL_TABLE_NAME
+ return FULL_TABLE_NAME;
+ case 2: // WRITE_ID_HIGH_WATER_MARK
+ return WRITE_ID_HIGH_WATER_MARK;
+ case 3: // INVALID_WRITE_IDS
+ return INVALID_WRITE_IDS;
+ case 4: // MIN_OPEN_WRITE_ID
+ return MIN_OPEN_WRITE_ID;
+ case 5: // ABORTED_BITS
+ return ABORTED_BITS;
+ default:
+ return null;
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, throwing an exception
+ * if it is not found.
+ */
+ public static _Fields findByThriftIdOrThrow(int fieldId) {
+ _Fields fields = findByThriftId(fieldId);
+ if (fields == null) throw new IllegalArgumentException("Field "
+ + fieldId + " doesn't exist!");
+ return fields;
+ }
+
+ /**
+ * Find the _Fields constant that matches name, or null if its not found.
+ */
+ public static _Fields findByName(String name) {
+ return byName.get(name);
+ }
+
+ private final short _thriftId;
+ private final String _fieldName;
+
+ _Fields(short thriftId, String fieldName) {
+ _thriftId = thriftId;
+ _fieldName = fieldName;
+ }
+
+ public short getThriftFieldId() {
+ return _thriftId;
+ }
+
+ public String getFieldName() {
+ return _fieldName;
+ }
+ }
+
+ // isset id assignments
+ private static final int __WRITEIDHIGHWATERMARK_ISSET_ID = 0;
+ private static final int __MINOPENWRITEID_ISSET_ID = 1;
+ private byte __isset_bitfield = 0;
+ private static final _Fields optionals[] = {_Fields.MIN_OPEN_WRITE_ID};
+ public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData>
+ metaDataMap;
+ static {
+ Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap =
+ new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
+ tmpMap.put(_Fields.FULL_TABLE_NAME,
+ new org.apache.thrift.meta_data.FieldMetaData("fullTableName",
+ org.apache.thrift.TFieldRequirementType.REQUIRED,
+ new org.apache.thrift.meta_data.FieldValueMetaData(
+ org.apache.thrift.protocol.TType.STRING)));
+ tmpMap.put(_Fields.WRITE_ID_HIGH_WATER_MARK,
+ new org.apache.thrift.meta_data.FieldMetaData("writeIdHighWaterMark",
+ org.apache.thrift.TFieldRequirementType.REQUIRED,
+ new org.apache.thrift.meta_data.FieldValueMetaData(
+ org.apache.thrift.protocol.TType.I64)));
+ tmpMap.put(_Fields.INVALID_WRITE_IDS,
+ new org.apache.thrift.meta_data.FieldMetaData("invalidWriteIds",
+ org.apache.thrift.TFieldRequirementType.REQUIRED,
+ new org.apache.thrift.meta_data.ListMetaData(
+ org.apache.thrift.protocol.TType.LIST,
+ new org.apache.thrift.meta_data.FieldValueMetaData(
+ org.apache.thrift.protocol.TType.I64))));
+ tmpMap.put(_Fields.MIN_OPEN_WRITE_ID,
+ new org.apache.thrift.meta_data.FieldMetaData("minOpenWriteId",
+ org.apache.thrift.TFieldRequirementType.OPTIONAL,
+ new org.apache.thrift.meta_data.FieldValueMetaData(
+ org.apache.thrift.protocol.TType.I64)));
+ tmpMap.put(_Fields.ABORTED_BITS, new org.apache.thrift.meta_data.FieldMetaData(
+ "abortedBits", org.apache.thrift.TFieldRequirementType.REQUIRED,
+ new org.apache.thrift.meta_data.FieldValueMetaData(
+ org.apache.thrift.protocol.TType.STRING , true)));
+ metaDataMap = Collections.unmodifiableMap(tmpMap);
+ org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(
+ TableValidWriteIds.class, metaDataMap);
+ }
+
+ public TableValidWriteIds() {
+ }
+
+ public TableValidWriteIds(
+ String fullTableName,
+ long writeIdHighWaterMark,
+ List<Long> invalidWriteIds,
+ ByteBuffer abortedBits)
+ {
+ this();
+ this.fullTableName = fullTableName;
+ this.writeIdHighWaterMark = writeIdHighWaterMark;
+ setWriteIdHighWaterMarkIsSet(true);
+ this.invalidWriteIds = invalidWriteIds;
+ this.abortedBits = org.apache.thrift.TBaseHelper.copyBinary(abortedBits);
+ }
+
+ /**
+ * Performs a deep copy on <i>other</i>.
+ */
+ public TableValidWriteIds(TableValidWriteIds other) {
+ __isset_bitfield = other.__isset_bitfield;
+ if (other.isSetFullTableName()) {
+ this.fullTableName = other.fullTableName;
+ }
+ this.writeIdHighWaterMark = other.writeIdHighWaterMark;
+ if (other.isSetInvalidWriteIds()) {
+ List<Long> __this__invalidWriteIds = new ArrayList<Long>(other.invalidWriteIds);
+ this.invalidWriteIds = __this__invalidWriteIds;
+ }
+ this.minOpenWriteId = other.minOpenWriteId;
+ if (other.isSetAbortedBits()) {
+ this.abortedBits = org.apache.thrift.TBaseHelper.copyBinary(other.abortedBits);
+ }
+ }
+
+ public TableValidWriteIds deepCopy() {
+ return new TableValidWriteIds(this);
+ }
+
+ @Override
+ public void clear() {
+ this.fullTableName = null;
+ setWriteIdHighWaterMarkIsSet(false);
+ this.writeIdHighWaterMark = 0;
+ this.invalidWriteIds = null;
+ setMinOpenWriteIdIsSet(false);
+ this.minOpenWriteId = 0;
+ this.abortedBits = null;
+ }
+
+ public String getFullTableName() {
+ return this.fullTableName;
+ }
+
+ public void setFullTableName(String fullTableName) {
+ this.fullTableName = fullTableName;
+ }
+
+ public void unsetFullTableName() {
+ this.fullTableName = null;
+ }
+
+ /** Returns true if field fullTableName is set (has been assigned a value)
+ * and false otherwise */
+ public boolean isSetFullTableName() {
+ return this.fullTableName != null;
+ }
+
+ public void setFullTableNameIsSet(boolean value) {
+ if (!value) {
+ this.fullTableName = null;
+ }
+ }
+
+ public long getWriteIdHighWaterMark() {
+ return this.writeIdHighWaterMark;
+ }
+
+ public void setWriteIdHighWaterMark(long writeIdHighWaterMark) {
+ this.writeIdHighWaterMark = writeIdHighWaterMark;
+ setWriteIdHighWaterMarkIsSet(true);
+ }
+
+ public void unsetWriteIdHighWaterMark() {
+ __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield,
+ __WRITEIDHIGHWATERMARK_ISSET_ID);
+ }
+
+ /** Returns true if field writeIdHighWaterMark is set (has been assigned a value)
+ * and false otherwise */
+ public boolean isSetWriteIdHighWaterMark() {
+ return EncodingUtils.testBit(__isset_bitfield, __WRITEIDHIGHWATERMARK_ISSET_ID);
+ }
+
+ public void setWriteIdHighWaterMarkIsSet(boolean value) {
+ __isset_bitfield = EncodingUtils.setBit(__isset_bitfield,
+ __WRITEIDHIGHWATERMARK_ISSET_ID,
+ value);
+ }
+
+ public int getInvalidWriteIdsSize() {
+ return (this.invalidWriteIds == null) ?
+ 0 : this.invalidWriteIds.size();
+ }
+
+ public java.util.Iterator<Long> getInvalidWriteIdsIterator() {
+ return (this.invalidWriteIds == null) ? null : this.invalidWriteIds.iterator();
+ }
+
+ public void addToInvalidWriteIds(long elem) {
+ if (this.invalidWriteIds == null) {
+ this.invalidWriteIds = new ArrayList<Long>();
+ }
+ this.invalidWriteIds.add(elem);
+ }
+
+ public List<Long> getInvalidWriteIds() {
+ return this.invalidWriteIds;
+ }
+
+ public void setInvalidWriteIds(List<Long> invalidWriteIds) {
+ this.invalidWriteIds =
+ invalidWriteIds;
+ }
+
+ public void unsetInvalidWriteIds() {
+ this.invalidWriteIds = null;
+ }
+
+ /** Returns true if field invalidWriteIds is set (has been assigned a value) and
+ * false otherwise */
+ public boolean isSetInvalidWriteIds() {
+ return this.invalidWriteIds != null;
+ }
+
+ public void setInvalidWriteIdsIsSet(boolean value) {
+ if (!value) {
+ this.invalidWriteIds = null;
+ }
+ }
+
+ public long getMinOpenWriteId() {
+ return this.minOpenWriteId;
+ }
+
+ public void setMinOpenWriteId(long minOpenWriteId) {
+ this.minOpenWriteId = minOpenWriteId;
+ setMinOpenWriteIdIsSet(true);
+ }
+
+ public void unsetMinOpenWriteId() {
+ __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield,
+ __MINOPENWRITEID_ISSET_ID);
+ }
+
+ /** Returns true if field minOpenWriteId is set (has been assigned a value) and false
+ * otherwise */
+ public boolean isSetMinOpenWriteId() {
+ return EncodingUtils.testBit(__isset_bitfield,
+ __MINOPENWRITEID_ISSET_ID);
+ }
+
+ public void setMinOpenWriteIdIsSet(boolean value) {
+ __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __MINOPENWRITEID_ISSET_ID,
+ value);
+ }
+
+ public byte[] getAbortedBits() {
+ setAbortedBits(org.apache.thrift.TBaseHelper.rightSize(abortedBits));
+ return abortedBits == null ? null : abortedBits.array();
+ }
+
+ public ByteBuffer bufferForAbortedBits() {
+ return org.apache.thrift.TBaseHelper
+ .copyBinary(abortedBits);
+ }
+
+ public void setAbortedBits(byte[] abortedBits) {
+ this.abortedBits = abortedBits == null ? (ByteBuffer)null : ByteBuffer.wrap(Arrays
+ .copyOf(abortedBits, abortedBits.length));
+ }
+
+ public void setAbortedBits(ByteBuffer abortedBits) {
+ this.abortedBits = org.apache.thrift.TBaseHelper.copyBinary(abortedBits);
+ }
+
+ public void unsetAbortedBits() {
+ this.abortedBits = null;
+ }
+
+ /** Returns true if field abortedBits is set (has been assigned a value) and false
+ * otherwise */
+ public boolean isSetAbortedBits() {
+ return this.abortedBits != null;
+ }
+
+ public void setAbortedBitsIsSet(boolean value) {
+ if (!value) {
+ this.abortedBits = null;
+ }
+ }
+
+ public void setFieldValue(_Fields field, Object value) {
+ switch (field) {
+ case FULL_TABLE_NAME:
+ if (value == null) {
+ unsetFullTableName();
+ } else {
+ setFullTableName((String)value);
+ }
+ break;
+
+ case WRITE_ID_HIGH_WATER_MARK:
+ if (value == null) {
+ unsetWriteIdHighWaterMark();
+ } else {
+ setWriteIdHighWaterMark((Long)value);
+ }
+ break;
+
+ case INVALID_WRITE_IDS:
+ if (value == null) {
+ unsetInvalidWriteIds();
+ } else {
+ setInvalidWriteIds((List<Long>)value);
+ }
+ break;
+
+ case MIN_OPEN_WRITE_ID:
+ if (value == null) {
+ unsetMinOpenWriteId();
+ } else {
+ setMinOpenWriteId((Long)value);
+ }
+ break;
+
+ case ABORTED_BITS:
+ if (value == null) {
+ unsetAbortedBits();
+ } else {
+ setAbortedBits((ByteBuffer)value);
+ }
+ break;
+
+ }
+ }
+
+ public Object getFieldValue(_Fields field) {
+ switch (field) {
+ case FULL_TABLE_NAME:
+ return getFullTableName();
+
+ case WRITE_ID_HIGH_WATER_MARK:
+ return getWriteIdHighWaterMark();
+
+ case INVALID_WRITE_IDS:
+ return getInvalidWriteIds();
+
+ case MIN_OPEN_WRITE_ID:
+ return getMinOpenWriteId();
+
+ case ABORTED_BITS:
+ return getAbortedBits();
+
+ }
+ throw new IllegalStateException();
+ }
+
+ /** Returns true if field corresponding to fieldID is set (has been assigned a
+ * value) and false otherwise */
+ public boolean isSet(_Fields field) {
+ if (field == null) {
+ throw new IllegalArgumentException();
+ }
+
+ switch (field) {
+ case FULL_TABLE_NAME:
+ return isSetFullTableName();
+ case WRITE_ID_HIGH_WATER_MARK:
+ return isSetWriteIdHighWaterMark();
+ case INVALID_WRITE_IDS:
+ return isSetInvalidWriteIds();
+ case MIN_OPEN_WRITE_ID:
+ return isSetMinOpenWriteId();
+ case ABORTED_BITS:
+ return isSetAbortedBits();
+ }
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public boolean equals(Object that) {
+ if (that == null)
+ return false;
+ if (that instanceof TableValidWriteIds)
+ return this.equals((TableValidWriteIds)that);
+ return false;
+ }
+
+ public boolean equals(TableValidWriteIds that) {
+ if (that == null)
+ return false;
+
+ boolean this_present_fullTableName = true && this.isSetFullTableName();
+ boolean that_present_fullTableName = true && that.isSetFullTableName();
+ if (this_present_fullTableName || that_present_fullTableName) {
+ if (!(this_present_fullTableName && that_present_fullTableName))
+ return false;
+ if (!this.fullTableName.equals(that.fullTableName))
+ return false;
+ }
+
+ boolean this_present_writeIdHighWaterMark = true;
+ boolean that_present_writeIdHighWaterMark = true;
+ if (this_present_writeIdHighWaterMark || that_present_writeIdHighWaterMark) {
+ if (!(this_present_writeIdHighWaterMark && that_present_writeIdHighWaterMark))
+ return false;
+ if (this.writeIdHighWaterMark != that.writeIdHighWaterMark)
+ return false;
+ }
+
+ boolean this_present_invalidWriteIds = true && this.isSetInvalidWriteIds();
+ boolean that_present_invalidWriteIds = true && that.isSetInvalidWriteIds();
+ if (this_present_invalidWriteIds || that_present_invalidWriteIds) {
+ if (!(this_present_invalidWriteIds && that_present_invalidWriteIds))
+ return false;
+ if (!this.invalidWriteIds.equals(that.invalidWriteIds))
+ return false;
+ }
+
+ boolean this_present_minOpenWriteId = true && this.isSetMinOpenWriteId();
+ boolean that_present_minOpenWriteId = true && that.isSetMinOpenWriteId();
+ if (this_present_minOpenWriteId || that_present_minOpenWriteId) {
+ if (!(this_present_minOpenWriteId && that_present_minOpenWriteId))
+ return false;
+ if (this.minOpenWriteId != that.minOpenWriteId)
+ return false;
+ }
+
+ boolean this_present_abortedBits = true && this.isSetAbortedBits();
+ boolean that_present_abortedBits = true && that.isSetAbortedBits();
+ if (this_present_abortedBits || that_present_abortedBits) {
+ if (!(this_present_abortedBits && that_present_abortedBits))
+ return false;
+ if (!this.abortedBits.equals(that.abortedBits))
+ return false;
+ }
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ List<Object> list = new ArrayList<Object>();
+
+ boolean present_fullTableName = true && (isSetFullTableName());
+ list.add(present_fullTableName);
+ if (present_fullTableName)
+ list.add(fullTableName);
+
+ boolean present_writeIdHighWaterMark = true;
+ list.add(present_writeIdHighWaterMark);
+ if (present_writeIdHighWaterMark)
+ list.add(writeIdHighWaterMark);
+
+ boolean present_invalidWriteIds = true && (isSetInvalidWriteIds());
+ list.add(present_invalidWriteIds);
+ if (present_invalidWriteIds)
+ list.add(invalidWriteIds);
+
+ boolean present_minOpenWriteId = true && (isSetMinOpenWriteId());
+ list.add(present_minOpenWriteId);
+ if (present_minOpenWriteId)
+ list.add(minOpenWriteId);
+
+ boolean present_abortedBits = true && (isSetAbortedBits());
+ list.add(present_abortedBits);
+ if (present_abortedBits)
+ list.add(abortedBits);
+
+ return list.hashCode();
+ }
+
+ @Override
+ public int compareTo(TableValidWriteIds other) {
+ if (!getClass().equals(other.getClass())) {
+ return getClass().getName().compareTo(other.getClass().getName());
+ }
+
+ int lastComparison = 0;
+
+ lastComparison = Boolean.valueOf(isSetFullTableName()).compareTo(
+ other.isSetFullTableName());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetFullTableName()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.fullTableName,
+ other.fullTableName);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ lastComparison = Boolean.valueOf(isSetWriteIdHighWaterMark()).compareTo(
+ other.isSetWriteIdHighWaterMark());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetWriteIdHighWaterMark()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(
+ this.writeIdHighWaterMark, other.writeIdHighWaterMark);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ lastComparison = Boolean.valueOf(isSetInvalidWriteIds()).compareTo(
+ other.isSetInvalidWriteIds());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetInvalidWriteIds()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.invalidWriteIds,
+ other.invalidWriteIds);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ lastComparison = Boolean.valueOf(isSetMinOpenWriteId()).compareTo(
+ other.isSetMinOpenWriteId());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetMinOpenWriteId()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.minOpenWriteId,
+ other.minOpenWriteId);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ lastComparison = Boolean.valueOf(isSetAbortedBits()).compareTo(
+ other.isSetAbortedBits());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetAbortedBits()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.abortedBits,
+ other.abortedBits);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ return 0;
+ }
+
+ public _Fields fieldForId(int fieldId) {
+ return _Fields.findByThriftId(fieldId);
+ }
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot)
+ throws org.apache.thrift.TException {
+ schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot)
+ throws org.apache.thrift.TException {
+ schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder("TableValidWriteIds(");
+ boolean first = true;
+
+ sb.append("fullTableName:");
+ if (this.fullTableName == null) {
+ sb.append("null");
+ } else {
+ sb.append(this.fullTableName);
+ }
+ first = false;
+ if (!first) sb.append(", ");
+ sb.append("writeIdHighWaterMark:");
+ sb.append(this.writeIdHighWaterMark);
+ first = false;
+ if (!first) sb.append(", ");
+ sb.append("invalidWriteIds:");
+ if (this.invalidWriteIds == null) {
+ sb.append("null");
+ } else {
+ sb.append(this.invalidWriteIds);
+ }
+ first = false;
+ if (isSetMinOpenWriteId()) {
+ if (!first) sb.append(", ");
+ sb.append("minOpenWriteId:");
+ sb.append(this.minOpenWriteId);
+ first = false;
+ }
+ if (!first) sb.append(", ");
+ sb.append("abortedBits:");
+ if (this.abortedBits == null) {
+ sb.append("null");
+ } else {
+ org.apache.thrift.TBaseHelper.toString(this.abortedBits, sb);
+ }
+ first = false;
+ sb.append(")");
+ return sb.toString();
+ }
+
+ public void validate() throws org.apache.thrift.TException {
+ // check for required fields
+ if (!isSetFullTableName()) {
+ throw new org.apache.thrift.protocol.TProtocolException(
+ "Required field 'fullTableName' is unset! Struct:" + toString());
+ }
+
+ if (!isSetWriteIdHighWaterMark()) {
+ throw new org.apache.thrift.protocol.TProtocolException(
+ "Required field 'writeIdHighWaterMark' is unset! Struct:" + toString());
+ }
+
+ if (!isSetInvalidWriteIds()) {
+ throw new org.apache.thrift.protocol.TProtocolException(
+ "Required field 'invalidWriteIds' is unset! Struct:" + toString());
+ }
+
+ if (!isSetAbortedBits()) {
+ throw new org.apache.thrift.protocol.TProtocolException(
+ "Required field 'abortedBits' is unset! Struct:" + toString());
+ }
+
+ // check for sub-struct validity
+ }
+
+ private void writeObject(java.io.ObjectOutputStream out)
+ throws java.io.IOException {
+ try {
+ write(new org.apache.thrift.protocol.TCompactProtocol(
+ new org.apache.thrift.transport.TIOStreamTransport(out)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private void readObject(java.io.ObjectInputStream in)
+ throws java.io.IOException, ClassNotFoundException {
+ try {
+ // it doesn't seem like you should have to do this, but java serialization is wacky,
+ // and doesn't call the default constructor.
+ __isset_bitfield = 0;
+ read(new org.apache.thrift.protocol.TCompactProtocol(
+ new org.apache.thrift.transport.TIOStreamTransport(in)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private static class TableValidWriteIdsStandardSchemeFactory implements
+ SchemeFactory {
+ public TableValidWriteIdsStandardScheme getScheme() {
+ return new TableValidWriteIdsStandardScheme();
+ }
+ }
+
+ private static class TableValidWriteIdsStandardScheme
+ extends StandardScheme<TableValidWriteIds> {
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot,
+ TableValidWriteIds struct)
+ throws org.apache.thrift.TException {
+ org.apache.thrift.protocol.TField schemeField;
+ iprot.readStructBegin();
+ while (true)
+ {
+ schemeField = iprot.readFieldBegin();
+ if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
+ break;
+ }
+ switch (schemeField.id) {
+ case 1: // FULL_TABLE_NAME
+ if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
+ struct.fullTableName = iprot.readString();
+ struct.setFullTableNameIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
+ case 2: // WRITE_ID_HIGH_WATER_MARK
+ if (schemeField.type == org.apache.thrift.protocol.TType.I64) {
+ struct.writeIdHighWaterMark = iprot.readI64();
+ struct.setWriteIdHighWaterMarkIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
+ case 3: // INVALID_WRITE_IDS
+ if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
+ {
+ org.apache.thrift.protocol.TList _list610 = iprot.readListBegin();
+ struct.invalidWriteIds = new ArrayList<Long>(_list610.size);
+ long _elem611;
+ for (int _i612 = 0; _i612 < _list610.size; ++_i612)
+ {
+ _elem611 = iprot.readI64();
+ struct.invalidWriteIds.add(_elem611);
+ }
+ iprot.readListEnd();
+ }
+ struct.setInvalidWriteIdsIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
+ case 4: // MIN_OPEN_WRITE_ID
+ if (schemeField.type == org.apache.thrift.protocol.TType.I64) {
+ struct.minOpenWriteId = iprot.readI64();
+ struct.setMinOpenWriteIdIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
+ case 5: // ABORTED_BITS
+ if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
+ struct.abortedBits = iprot.readBinary();
+ struct.setAbortedBitsIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
+ default:
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ iprot.readFieldEnd();
+ }
+ iprot.readStructEnd();
+ struct.validate();
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot,
+ TableValidWriteIds struct)
+ throws org.apache.thrift.TException {
+ struct.validate();
+
+ oprot.writeStructBegin(STRUCT_DESC);
+ if (struct.fullTableName != null) {
+ oprot.writeFieldBegin(FULL_TABLE_NAME_FIELD_DESC);
+ oprot.writeString(struct.fullTableName);
+ oprot.writeFieldEnd();
+ }
+ oprot.writeFieldBegin(WRITE_ID_HIGH_WATER_MARK_FIELD_DESC);
+ oprot.writeI64(struct.writeIdHighWaterMark);
+ oprot.writeFieldEnd();
+ if (struct.invalidWriteIds != null) {
+ oprot.writeFieldBegin(INVALID_WRITE_IDS_FIELD_DESC);
+ {
+ oprot.writeListBegin(new org.apache.thrift.protocol.TList(
+ org.apache.thrift.protocol.TType.I64,
+ struct.invalidWriteIds.size()));
+ for (long _iter613 : struct.invalidWriteIds)
+ {
+ oprot.writeI64(_iter613);
+ }
+ oprot.writeListEnd();
+ }
+ oprot.writeFieldEnd();
+ }
+ if (struct.isSetMinOpenWriteId()) {
+ oprot.writeFieldBegin(MIN_OPEN_WRITE_ID_FIELD_DESC);
+ oprot.writeI64(struct.minOpenWriteId);
+ oprot.writeFieldEnd();
+ }
+ if (struct.abortedBits != null) {
+ oprot.writeFieldBegin(ABORTED_BITS_FIELD_DESC);
+ oprot.writeBinary(struct.abortedBits);
+ oprot.writeFieldEnd();
+ }
+ oprot.writeFieldStop();
+ oprot.writeStructEnd();
+ }
+
+ }
+
+ private static class TableValidWriteIdsTupleSchemeFactory implements SchemeFactory {
+ public TableValidWriteIdsTupleScheme getScheme() {
+ return new TableValidWriteIdsTupleScheme();
+ }
+ }
+
+ private static class TableValidWriteIdsTupleScheme
+ extends TupleScheme<TableValidWriteIds> {
+
+ @Override
+ public void write(org.apache.thrift.protocol.TProtocol prot,
+ TableValidWriteIds struct)
+ throws org.apache.thrift.TException {
+ TTupleProtocol oprot = (TTupleProtocol) prot;
+ oprot.writeString(struct.fullTableName);
+ oprot.writeI64(struct.writeIdHighWaterMark);
+ {
+ oprot.writeI32(struct.invalidWriteIds.size());
+ for (long _iter614 : struct.invalidWriteIds)
+ {
+ oprot.writeI64(_iter614);
+ }
+ }
+ oprot.writeBinary(struct.abortedBits);
+ BitSet optionals = new BitSet();
+ if (struct.isSetMinOpenWriteId()) {
+ optionals.set(0);
+ }
+ oprot.writeBitSet(optionals, 1);
+ if (struct.isSetMinOpenWriteId()) {
+ oprot.writeI64(struct.minOpenWriteId);
+ }
+ }
+
+ @Override
+ public void read(org.apache.thrift.protocol.TProtocol prot,
+ TableValidWriteIds struct)
+ throws org.apache.thrift.TException {
+ TTupleProtocol iprot = (TTupleProtocol) prot;
+ struct.fullTableName = iprot.readString();
+ struct.setFullTableNameIsSet(true);
+ struct.writeIdHighWaterMark = iprot.readI64();
+ struct.setWriteIdHighWaterMarkIsSet(true);
+ {
+ org.apache.thrift.protocol.TList _list615 =
+ new org.apache.thrift.protocol.TList(
+ org.apache.thrift.protocol.TType.I64, iprot.readI32());
+ struct.invalidWriteIds = new ArrayList<Long>(_list615.size);
+ long _elem616;
+ for (int _i617 = 0; _i617 < _list615.size; ++_i617)
+ {
+ _elem616 = iprot.readI64();
+ struct.invalidWriteIds.add(_elem616);
+ }
+ }
+ struct.setInvalidWriteIdsIsSet(true);
+ struct.abortedBits = iprot.readBinary();
+ struct.setAbortedBitsIsSet(true);
+ BitSet incoming = iprot.readBitSet(1);
+ if (incoming.get(0)) {
+ struct.minOpenWriteId = iprot.readI64();
+ struct.setMinOpenWriteIdIsSet(true);
+ }
+ }
+ }
+
+}
+
diff --git a/fe/src/main/java/org/apache/impala/catalog/Hive3MetastoreShimBase.java b/fe/src/compat-apache-hive-2/java/org/apache/impala/catalog/Hive2MetastoreShimBase.java
similarity index 84%
copy from fe/src/main/java/org/apache/impala/catalog/Hive3MetastoreShimBase.java
copy to fe/src/compat-apache-hive-2/java/org/apache/impala/catalog/Hive2MetastoreShimBase.java
index 447636ea1..751e54935 100644
--- a/fe/src/main/java/org/apache/impala/catalog/Hive3MetastoreShimBase.java
+++ b/fe/src/compat-apache-hive-2/java/org/apache/impala/catalog/Hive2MetastoreShimBase.java
@@ -21,6 +21,7 @@ import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
+import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.BitSet;
@@ -29,6 +30,9 @@ import java.util.EnumSet;
import java.util.List;
import java.util.stream.Collectors;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.common.FileUtils;
import org.apache.hadoop.hive.common.StatsSetupConst;
import org.apache.hadoop.hive.common.ValidReaderWriteIdList;
import org.apache.hadoop.hive.common.ValidTxnList;
@@ -39,8 +43,10 @@ import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.hive.metastore.LockRequestBuilder;
import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.Warehouse;
+import org.apache.hadoop.hive.metastore.api.Decimal;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.InsertEventRequestData;
import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
import org.apache.hadoop.hive.metastore.api.LockComponent;
import org.apache.hadoop.hive.metastore.api.LockRequest;
@@ -50,14 +56,13 @@ import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.NoSuchLockException;
import org.apache.hadoop.hive.metastore.api.NoSuchTxnException;
import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.PrincipalType;
import org.apache.hadoop.hive.metastore.api.SQLForeignKey;
import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.api.TableValidWriteIds;
import org.apache.hadoop.hive.metastore.api.TxnAbortedException;
-import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
-import org.apache.hadoop.hive.metastore.utils.FileUtils;
-import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
+import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.ql.metadata.ForeignKeyInfo;
import org.apache.hadoop.hive.ql.metadata.PrimaryKeyInfo;
import org.apache.hive.service.rpc.thrift.TGetColumnsReq;
@@ -65,6 +70,7 @@ import org.apache.hive.service.rpc.thrift.TGetFunctionsReq;
import org.apache.hive.service.rpc.thrift.TGetSchemasReq;
import org.apache.hive.service.rpc.thrift.TGetTablesReq;
import org.apache.impala.authorization.User;
+import org.apache.impala.catalog.events.MetastoreEventProcessorConfig;
import org.apache.impala.common.ImpalaException;
import org.apache.impala.common.Pair;
import org.apache.impala.common.TransactionException;
@@ -77,16 +83,16 @@ import org.apache.impala.thrift.TValidWriteIdList;
import org.apache.impala.util.AcidUtils;
import org.apache.impala.util.AcidUtils.TblTransaction;
import org.apache.impala.util.HiveMetadataFormatUtils;
+import org.apache.impala.util.MetaStoreUtil;
import org.apache.thrift.TException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
/**
- * Base class for Hive 3 MetastoreShim.
+ * Base class for Hive 2 MetastoreShim.
*/
-public class Hive3MetastoreShimBase {
- private static final Logger LOG = LoggerFactory.getLogger(Hive3MetastoreShimBase.class);
+public class Hive2MetastoreShimBase {
+ private static final Logger LOG = LoggerFactory.getLogger(Hive2MetastoreShimBase.class);
protected static final String EXTWRITE = "EXTWRITE";
protected static final String EXTREAD = "EXTREAD";
@@ -100,7 +106,7 @@ public class Hive3MetastoreShimBase {
protected static final String HIVEMQT = "HIVEMQT";
// Virtual View
protected static final String HIVESQL = "HIVESQL";
- protected static final long MAJOR_VERSION = 3;
+ protected static final long MAJOR_VERSION = 2;
protected static boolean capabilitiestSet_ = false;
// Max sleep interval during acquiring an ACID lock.
@@ -113,6 +119,8 @@ public class Hive3MetastoreShimBase {
// Not final, as it makes sense to set it based on role + instance, see IMPALA-8853.
public static String TRANSACTION_USER_ID = "Impala";
+ public static String HIVE_METASTORE_WAREHOUSE_EXTERNAL = "";
+
/**
* Initializes and returns a TblTransaction object for table 'tbl'. Opens a new
* transaction if txnId is not valid.
@@ -120,39 +128,17 @@ public class Hive3MetastoreShimBase {
public static TblTransaction createTblTransaction(
IMetaStoreClient client, Table tbl, long txnId)
throws TransactionException {
- TblTransaction tblTxn = new TblTransaction();
- try {
- if (txnId <= 0) {
- txnId = openTransaction(client);
- tblTxn.ownsTxn = true;
- }
- tblTxn.txnId = txnId;
- tblTxn.writeId =
- allocateTableWriteId(client, txnId, tbl.getDbName(), tbl.getTableName());
- tblTxn.validWriteIds =
- getValidWriteIdListInTxn(client, tbl.getDbName(), tbl.getTableName(), txnId);
- return tblTxn;
- } catch (TException e) {
- if (tblTxn.ownsTxn) {
- abortTransactionNoThrow(client, tblTxn.txnId);
- }
- throw new TransactionException(
- String.format(HMS_RPC_ERROR_FORMAT_STR, "createTblTransaction"), e);
- }
+ throw new UnsupportedOperationException("createTblTransaction");
}
static public void commitTblTransactionIfNeeded(IMetaStoreClient client,
TblTransaction tblTxn) throws TransactionException {
- if (tblTxn.ownsTxn) {
- commitTransaction(client, tblTxn.txnId);
- }
+ throw new UnsupportedOperationException("commitTblTransactionIfNeeded");
}
static public void abortTblTransactionIfNeeded(IMetaStoreClient client,
TblTransaction tblTxn) {
- if (tblTxn.ownsTxn) {
- abortTransactionNoThrow(client, tblTxn.txnId);
- }
+ throw new UnsupportedOperationException("abortTblTransactionIfNeeded");
}
/**
@@ -194,8 +180,7 @@ public class Hive3MetastoreShimBase {
public static void createTableWithConstraints(IMetaStoreClient client,
Table newTbl, List<SQLPrimaryKey> primaryKeys, List<SQLForeignKey> foreignKeys)
throws InvalidOperationException, MetaException, TException {
- client.createTableWithConstraints(newTbl, primaryKeys, foreignKeys, null, null,
- null, null);
+ client.createTableWithConstraints(newTbl, primaryKeys, foreignKeys);
}
/**
@@ -204,16 +189,14 @@ public class Hive3MetastoreShimBase {
*/
public static void updatePartitionStatsFast(Partition partition, Table tbl,
Warehouse warehouse) throws MetaException {
- MetaStoreUtils.updatePartitionStatsFast(partition, tbl, warehouse, /*madeDir*/false,
- /*forceRecompute*/false,
- /*environmentContext*/null, /*isCreate*/false);
+ MetaStoreUtils.updatePartitionStatsFast(partition, warehouse, null);
}
/**
* Return the maximum number of Metastore objects that should be retrieved in a batch.
*/
public static String metastoreBatchRetrieveObjectsMaxConfigKey() {
- return MetastoreConf.ConfVars.BATCH_RETRIEVE_OBJECTS_MAX.toString();
+ return "metastore.batch.retrieve.table.partition.max";
}
/**
@@ -300,10 +283,7 @@ public class Hive3MetastoreShimBase {
}
}
- // hive-3 introduces a catalog object in hive
- // Impala only supports the default catalog of hive
- private static final String DEFAULT_CATALOG_NAME = MetaStoreUtils
- .getDefaultCatalog(MetastoreConf.newMetastoreConf());
+ private static final String DEFAULT_CATALOG_NAME = "hive";
/**
* Gets the name of the default catalog from metastore configuration.
@@ -563,18 +543,8 @@ public class Hive3MetastoreShimBase {
private static String getValidWriteIdListInTxn(IMetaStoreClient client, String dbName,
String tblName, long txnId)
throws TException {
- ValidTxnList txns = client.getValidTxns(txnId);
- String tableFullName = dbName + "." + tblName;
- List<TableValidWriteIds> writeIdsObj = client.getValidWriteIds(
- Lists.newArrayList(tableFullName), txns.toString());
- ValidTxnWriteIdList validTxnWriteIdList = new ValidTxnWriteIdList(txnId);
- for (TableValidWriteIds tableWriteIds : writeIdsObj) {
- validTxnWriteIdList.addTableValidWriteIdList(
- createValidReaderWriteIdList(tableWriteIds));
- }
- String validWriteIds =
- validTxnWriteIdList.getTableValidWriteIdList(tableFullName).writeToString();
- return validWriteIds;
+ throw new UnsupportedOperationException(
+ "getValidWriteIdListInTxn is not supported.");
}
/**
@@ -770,11 +740,7 @@ public class Hive3MetastoreShimBase {
*/
public static long allocateTableWriteId(IMetaStoreClient client, long txnId,
String dbName, String tableName) throws TransactionException {
- try {
- return client.allocateTableWriteId(txnId, dbName, tableName);
- } catch (Exception e) {
- throw new TransactionException(e.getMessage());
- }
+ throw new UnsupportedOperationException("allocateTableWriteId is not supported.");
}
/**
@@ -793,22 +759,8 @@ public class Hive3MetastoreShimBase {
*/
private static ValidReaderWriteIdList createValidReaderWriteIdList(
TableValidWriteIds tableWriteIds) {
- String fullTableName = tableWriteIds.getFullTableName();
- long highWater = tableWriteIds.getWriteIdHighWaterMark();
- List<Long> invalids = tableWriteIds.getInvalidWriteIds();
- BitSet abortedBits = BitSet.valueOf(tableWriteIds.getAbortedBits());
- long[] exceptions = new long[invalids.size()];
- int i = 0;
- for (long writeId : invalids) {
- exceptions[i++] = writeId;
- }
- if (tableWriteIds.isSetMinOpenWriteId()) {
- return new ValidReaderWriteIdList(fullTableName, exceptions, abortedBits,
- highWater, tableWriteIds.getMinOpenWriteId());
- } else {
- return new ValidReaderWriteIdList(fullTableName, exceptions, abortedBits,
- highWater);
- }
+ throw new UnsupportedOperationException(
+ "createValidReaderWriteIdList is not supported.");
}
/**
@@ -822,18 +774,7 @@ public class Hive3MetastoreShimBase {
*/
public static String getPathForNewTable(Database db, Table tbl)
throws MetaException {
- Warehouse wh = new Warehouse(new HiveConf());
- // Non transactional tables are all translated to external tables by HMS's default
- // transformer (HIVE-22158). Note that external tables can't be transactional.
- // So the request and result of the default transformer is:
- // non transactional managed table => external table
- // non transactional external table => external table
- // transactional managed table => managed table
- // transactional external table (not allowed)
- boolean isExternal = !AcidUtils.isTransactionalTable(tbl.getParameters());
- // TODO(IMPALA-9088): deal with customized transformer in HMS.
- return wh.getDefaultTablePath(db, tbl.getTableName().toLowerCase(), isExternal)
- .toString();
+ return new Path(db.getLocationUri(), tbl.getTableName().toLowerCase()).toString();
}
/**
@@ -844,41 +785,43 @@ public class Hive3MetastoreShimBase {
}
/**
- * Constructs a new AlterPartitionsInfo object.
+ * Wrapper around MetaStoreUtils.isExternalTable()
*/
- public static class AlterPartitionsInfo {
- private final org.apache.hadoop.hive.metastore.api.Table msTable;
- private final List<org.apache.hadoop.hive.metastore.api.Partition> partitions;
- private final boolean isTruncate;
+ public static boolean isExternalTable(Table table) {
+ return MetaStoreUtils.isExternalTable(table);
+ }
- public AlterPartitionsInfo(org.apache.hadoop.hive.metastore.api.Table msTable,
- List<org.apache.hadoop.hive.metastore.api.Partition> partitions,
- boolean isTruncate) {
- this.msTable = msTable;
- this.partitions = partitions;
- this.isTruncate = isTruncate;
- }
+ public static Decimal newDecimal(short scale, ByteBuffer unscaled) {
+ return new Decimal(unscaled, scale);
+ }
- /**
- * Returns the Thrift representation of the table.
- */
- public org.apache.hadoop.hive.metastore.api.Table getMsTable() {
- return msTable;
- }
+ /**
+ * Used for hive-3 only
+ */
+ public static PrincipalType getTableOwnerType(Table table) {
+ return null;
+ }
- /**
- * Returns the list of Thrift partition objects affected by the reload.
- * Can be null or empty if not applicable.
- */
- public List<org.apache.hadoop.hive.metastore.api.Partition> getPartitions() {
- return partitions;
- }
+ /**
+ * Used for hive-3 only
+ */
+ public static void setInsertEventRequestDataReplace(
+ InsertEventRequestData insertEventRequestData, boolean isInsertOverwrite) {
+ }
+
+ /**
+ * Used for hive-3 only
+ */
+ public static void setTableOwnerType(Table table, PrincipalType principalType) {
+ }
- /**
- * Returns true if the reload operation was due to a TRUNCATE TABLE.
- */
- public boolean isTruncate() {
- return isTruncate;
+ public static String getMetastoreConfigValue(
+ IMetaStoreClient client, String config, String defaultVal) throws TException {
+ String metastoreDefaultCatalogNameKey = MetastoreEventProcessorConfig
+ .METASTORE_DEFAULT_CATALOG_NAME.getValidator().getConfigKey();
+ if (StringUtils.equals(config, metastoreDefaultCatalogNameKey)) {
+ return DEFAULT_CATALOG_NAME;
}
+ return MetaStoreUtil.getMetastoreConfigValue(client, config, defaultVal);
}
}
diff --git a/fe/src/compat-apache-hive-2/java/org/apache/impala/catalog/events/MetastoreEvents.java b/fe/src/compat-apache-hive-2/java/org/apache/impala/catalog/events/MetastoreEvents.java
new file mode 100644
index 000000000..605acbb12
--- /dev/null
+++ b/fe/src/compat-apache-hive-2/java/org/apache/impala/catalog/events/MetastoreEvents.java
@@ -0,0 +1,3587 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.impala.catalog.events;
+
+import com.codahale.metrics.Timer;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Objects;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.NotificationEvent;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.messaging.AddPartitionMessage;
+import org.apache.hadoop.hive.metastore.messaging.AlterPartitionMessage;
+import org.apache.hadoop.hive.metastore.messaging.CreateTableMessage;
+import org.apache.hadoop.hive.metastore.messaging.DropPartitionMessage;
+import org.apache.hadoop.hive.metastore.messaging.InsertMessage;
+import org.apache.hadoop.hive.metastore.messaging.json.JSONAlterTableMessage;
+import org.apache.hadoop.hive.metastore.messaging.json.JSONCreateDatabaseMessage;
+import org.apache.hadoop.hive.metastore.messaging.json.JSONDropDatabaseMessage;
+import org.apache.hadoop.hive.metastore.messaging.json.JSONDropTableMessage;
+import org.apache.hadoop.hive.metastore.MetaStoreUtils;
+import org.apache.impala.analysis.TableName;
+import org.apache.impala.catalog.CatalogException;
+import org.apache.impala.catalog.CatalogServiceCatalog;
+import org.apache.impala.catalog.DatabaseNotFoundException;
+import org.apache.impala.catalog.Db;
+import org.apache.impala.catalog.FeCatalogUtils;
+import org.apache.impala.catalog.FileMetadataLoadOpts;
+import org.apache.impala.catalog.HdfsTable;
+import org.apache.impala.catalog.IncompleteTable;
+import org.apache.impala.catalog.MetastoreClientInstantiationException;
+import org.apache.impala.catalog.MetaStoreClientPool;
+import org.apache.impala.catalog.TableLoadingException;
+import org.apache.impala.catalog.TableNotFoundException;
+import org.apache.impala.catalog.TableNotLoadedException;
+import org.apache.impala.catalog.TableWriteId;
+import org.apache.impala.common.Metrics;
+import org.apache.impala.common.PrintUtils;
+import org.apache.impala.common.Reference;
+import org.apache.impala.compat.MetastoreShim;
+import org.apache.impala.hive.common.MutableValidWriteIdList;
+
+import static org.apache.impala.catalog.Table.TBL_EVENTS_PROCESS_DURATION;
+
+import org.apache.impala.service.BackendConfig;
+import org.apache.impala.service.CatalogOpExecutor;
+import org.apache.impala.thrift.TPartitionKeyValue;
+import org.apache.impala.thrift.TTableName;
+import org.apache.impala.util.AcidUtils;
+import org.apache.impala.util.DebugUtils;
+import org.apache.impala.util.MetaStoreUtil;
+import org.apache.impala.util.NoOpEventSequence;
+import org.slf4j.LoggerFactory;
+import org.slf4j.Logger;
+
+/**
+ * Main class which provides Metastore event objects for various event types. Also
+ * provides a MetastoreEventFactory to get or create the event instances for a given event
+ * type
+ */
+public class MetastoreEvents {
+
+ /**
+ * This enum contains keys for parameters added in Metastore entities, relevant for
+ * event processing. When eventProcessor is instantiated, we make sure during config
+ * validation that these parameters are not filtered out through the Metastore config
+ * EVENT_NOTIFICATION_PARAMETERS_EXCLUDE_PATTERNS.
+ */
+ public enum MetastoreEventPropertyKey {
+ // key to be used for catalog version in table properties for detecting self-events
+ CATALOG_VERSION("impala.events.catalogVersion"),
+ // key to be used for catalog service id for detecting self-events
+ CATALOG_SERVICE_ID("impala.events.catalogServiceId"),
+ // flag to be set in the table/database parameters to disable event based metadata
+ // sync. Note the this is a user-facing property. Any changes to this key name
+ // will break backwards compatibility
+ DISABLE_EVENT_HMS_SYNC("impala.disableHmsSync");
+
+ private String key_;
+
+ MetastoreEventPropertyKey(String key) { this.key_ = key; }
+
+ public String getKey() { return key_; }
+ }
+
+ public enum MetastoreEventType {
+ // It's intended that some eventTypes have a suffix of "_EVENT". This is consistent
+ // with Hive so the type matching can use string equal checks.
+ CREATE_TABLE("CREATE_TABLE"),
+ DROP_TABLE("DROP_TABLE"),
+ ALTER_TABLE("ALTER_TABLE"),
+ CREATE_DATABASE("CREATE_DATABASE"),
+ DROP_DATABASE("DROP_DATABASE"),
+ ALTER_DATABASE("ALTER_DATABASE"),
+ ADD_PARTITION("ADD_PARTITION"),
+ ALTER_PARTITION("ALTER_PARTITION"),
+ ALTER_PARTITIONS("ALTER_PARTITIONS"),
+ DROP_PARTITION("DROP_PARTITION"),
+ INSERT("INSERT"),
+ INSERT_PARTITIONS("INSERT_PARTITIONS"),
+ RELOAD("RELOAD"),
+ ALLOC_WRITE_ID_EVENT("ALLOC_WRITE_ID_EVENT"),
+ COMMIT_TXN("COMMIT_TXN"),
+ ABORT_TXN("ABORT_TXN"),
+ COMMIT_COMPACTION_EVENT("COMMIT_COMPACTION_EVENT"),
+ OTHER("OTHER");
+
+ private final String eventType_;
+
+ MetastoreEventType(String msEventType) {
+ this.eventType_ = msEventType;
+ }
+
+ @Override
+ public String toString() {
+ return eventType_;
+ }
+
+ /**
+ * Returns the MetastoreEventType from a given string value of event from Metastore's
+ * NotificationEvent.eventType. If none of the supported MetastoreEventTypes match,
+ * return OTHER
+ *
+ * @param eventType EventType value from the <code>NotificationEvent</code>
+ */
+ public static MetastoreEventType from(String eventType) {
+ for (MetastoreEventType metastoreEventType : values()) {
+ if (metastoreEventType.eventType_.equalsIgnoreCase(eventType)) {
+ return metastoreEventType;
+ }
+ }
+ return OTHER;
+ }
+ }
+
+ /**
+ * Factory class to create various MetastoreEvents.
+ */
+ public static class MetastoreEventFactory implements EventFactory {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(MetastoreEventFactory.class);
+
+ // catalog service instance to be used for creating eventHandlers
+ protected final CatalogServiceCatalog catalog_;
+ // metrics registry to be made available for each events to publish metrics
+ //protected final Metrics metrics_;
+ // catalogOpExecutor needed for the create/drop events for table and database.
+ protected final CatalogOpExecutor catalogOpExecutor_;
+ private static MetastoreEventFactory INSTANCE = null;
+
+ public MetastoreEventFactory(CatalogOpExecutor catalogOpExecutor) {
+ this.catalogOpExecutor_ = Preconditions.checkNotNull(catalogOpExecutor);
+ this.catalog_ = Preconditions.checkNotNull(catalogOpExecutor.getCatalog());
+ }
+
+ /**
+ * creates instance of <code>MetastoreEvent</code> used to process a given event type.
+ * If the event type is unknown, returns a IgnoredEvent
+ */
+ public MetastoreEvent get(NotificationEvent event, Metrics metrics)
+ throws MetastoreNotificationException {
+ Preconditions.checkNotNull(event.getEventType());
+ MetastoreEventType metastoreEventType =
+ MetastoreEventType.from(event.getEventType());
+ if (BackendConfig.INSTANCE.getHMSEventIncrementalRefreshTransactionalTable()) {
+ switch (metastoreEventType) {
+ case ALLOC_WRITE_ID_EVENT:
+ return new AllocWriteIdEvent(catalogOpExecutor_, metrics, event);
+ case COMMIT_TXN:
+ return new MetastoreShim.CommitTxnEvent(catalogOpExecutor_, metrics, event);
+ case ABORT_TXN:
+ return new AbortTxnEvent(catalogOpExecutor_, metrics, event);
+ }
+ }
+ switch (metastoreEventType) {
+ case CREATE_TABLE:
+ return new CreateTableEvent(catalogOpExecutor_, metrics, event);
+ case DROP_TABLE:
+ return new DropTableEvent(catalogOpExecutor_, metrics, event);
+ case ALTER_TABLE:
+ return new AlterTableEvent(catalogOpExecutor_, metrics, event);
+ case CREATE_DATABASE:
+ return new CreateDatabaseEvent(catalogOpExecutor_, metrics, event);
+ case DROP_DATABASE:
+ return new DropDatabaseEvent(catalogOpExecutor_, metrics, event);
+ case ALTER_DATABASE:
+ return new AlterDatabaseEvent(catalogOpExecutor_, metrics, event);
+ case ADD_PARTITION:
+ return new AddPartitionEvent(catalogOpExecutor_, metrics, event);
+ case DROP_PARTITION:
+ return new DropPartitionEvent(catalogOpExecutor_, metrics, event);
+ case ALTER_PARTITION:
+ return new AlterPartitionEvent(catalogOpExecutor_, metrics, event);
+ case RELOAD:
+ return new ReloadEvent(catalogOpExecutor_, metrics, event);
+ case INSERT:
+ return new InsertEvent(catalogOpExecutor_, metrics, event);
+ case COMMIT_COMPACTION_EVENT:
+ return new CommitCompactionEvent(catalogOpExecutor_, metrics, event);
+ default:
+ // ignore all the unknown events by creating a IgnoredEvent
+ return new IgnoredEvent(catalogOpExecutor_, metrics, event);
+ }
+ }
+
+ /**
+ * Given a list of notification events, returns a list of <code>MetastoreEvent</code>
+ * In case there are create events which are followed by drop events for the same
+ * object, the create events are filtered out. The drop events do not need to be
+ * filtered out
+ *
+ * This is needed to avoid the replay problem. For example, if catalog created and
+ * removed a table, the create event received will try to add the object again. This
+ * table will be visible until the drop table event is processed. This can be avoided
+ * by "looking ahead" in the event stream to see if the table with the same name was
+ * dropped. In such a case, the create event can be ignored
+ *
+ * @param events NotificationEvents fetched from metastore
+ * @param metrics Metrics to update while filtering events
+ * @return A list of MetastoreEvents corresponding to the given the NotificationEvents
+ * @throws MetastoreNotificationException if a NotificationEvent could not be
+ * parsed into MetastoreEvent
+ */
+ List<MetastoreEvent> getFilteredEvents(List<NotificationEvent> events,
+ Metrics metrics) throws MetastoreNotificationException {
+ Preconditions.checkNotNull(events);
+ if (events.isEmpty()) return Collections.emptyList();
+
+ if (StringUtils.isNotEmpty(BackendConfig.INSTANCE.debugActions())) {
+ DebugUtils.executeDebugAction(
+ BackendConfig.INSTANCE.debugActions(), DebugUtils.GET_FILTERED_EVENTS_DELAY);
+ }
+
+ List<MetastoreEvent> metastoreEvents = new ArrayList<>(events.size());
+ for (NotificationEvent event : events) {
+ metastoreEvents.add(get(event, metrics));
+ }
+ // filter out the create events which has a corresponding drop event later
+ int sizeBefore = metastoreEvents.size();
+ int numFilteredEvents = 0;
+ int i = 0;
+ while (i < metastoreEvents.size()) {
+ MetastoreEvent currentEvent = metastoreEvents.get(i);
+ String catalogName = currentEvent.getCatalogName();
+ String eventDb = currentEvent.getDbName();
+ String eventTbl = currentEvent.getTableName();
+ if (catalogName != null && !MetastoreShim.isDefaultCatalog(catalogName)) {
+ // currently Impala doesn't support custom hive catalogs and hence we should
+ // ignore all the events which are on non-default catalog namespaces.
+ LOG.debug(currentEvent.debugString(
+ "Filtering out this event since it is on a non-default hive catalog %s",
+ catalogName));
+ metastoreEvents.remove(i);
+ numFilteredEvents++;
+ } else if ((eventDb != null && catalog_.isBlacklistedDb(eventDb)) || (
+ eventTbl != null && catalog_.isBlacklistedTable(eventDb, eventTbl))) {
+ // if the event is on blacklisted db or table we should filter it out
+ String blacklistedObject = eventTbl != null ? new TableName(eventDb,
+ eventTbl).toString() : eventDb;
+ LOG.info(currentEvent.debugString("Filtering out this event since it is on a "
+ + "blacklisted database or table %s", blacklistedObject));
+ metastoreEvents.remove(i);
+ numFilteredEvents++;
+ } else {
+ i++;
+ }
+ }
+ LOG.info(String.format("Total number of events received: %d Total number of events "
+ + "filtered out: %d", sizeBefore, numFilteredEvents));
+ if (numFilteredEvents > 0) {
+ metrics.getCounter(MetastoreEventsProcessor.EVENTS_SKIPPED_METRIC)
+ .inc(numFilteredEvents);
+ LOG.debug("Incremented skipped metric to "
+ + metrics.getCounter(MetastoreEventsProcessor.EVENTS_SKIPPED_METRIC)
+ .getCount());
+ }
+ return createBatchEvents(metastoreEvents, metrics);
+ }
+
+ /**
+ * This method flushes all in-progress batches for tables from the specified
+ * database from the pendingTableEventsMap to the sortedFinalBatches.
+ */
+ void flushBatchesForDb(
+ Map<String, Map<String, MetastoreEvent>> pendingTableEventsMap,
+ TreeMap<Long, MetastoreEvent> sortedFinalBatches, String dbName) {
+ String lowerDbName = dbName.toLowerCase();
+ Map<String, MetastoreEvent> dbMap = pendingTableEventsMap.get(lowerDbName);
+ if (dbMap != null) {
+ // Flush out any pending events in the database map and delete it
+ for (MetastoreEvent event : dbMap.values()) {
+ sortedFinalBatches.put(event.getEventId(), event);
+ }
+ pendingTableEventsMap.remove(lowerDbName);
+ }
+ }
+
+ /**
+ * This method flushes any in-progress batch for the specified table
+ * from the pendingTableEventsMap to the sortedFinalBatches.
+ */
+ void flushBatchForTable(
+ Map<String, Map<String, MetastoreEvent>> pendingTableEventsMap,
+ TreeMap<Long, MetastoreEvent> sortedFinalBatches, Table table) {
+ // Produce the lower-cased fully qualified table name
+ String dbName = table.getDbName().toLowerCase();
+ String tableName = table.getTableName().toLowerCase();
+ Map<String, MetastoreEvent> dbMap = pendingTableEventsMap.get(dbName);
+ if (dbMap != null) {
+ MetastoreEvent tableEvent = dbMap.get(tableName);
+ if (tableEvent != null) {
+ sortedFinalBatches.put(tableEvent.getEventId(), tableEvent);
+ dbMap.remove(tableName);
+ // If this was the last table, delete the DB map
+ if (dbMap.isEmpty()) {
+ pendingTableEventsMap.remove(dbName);
+ }
+ }
+ }
+ }
+
+ /**
+ * Event batching is done on a per-table basis to allow more batching in
+ * interleaved circumstances. Single-table events still follow the same rules
+ * for batching, but certain events cross table boundaries and should cut
+ * batches across multiple tables. This method detects cross-table events and
+ * cuts the appropriate batches. Currently, it handles drop database, alter
+ * database, and alter table rename. It is a no-op for events that are not
+ * cross-table.
+ */
+ void cutBatchesForCrossTableEvents(MetastoreEvent event,
+ Map<String, Map<String, MetastoreEvent>> pendingTableEventsMap,
+ TreeMap<Long, MetastoreEvent> sortedFinalBatches) {
+ // drop database - cuts any existing batches for tables in that database
+ // alter database - cuts any existing batches for tables in the database
+ // alter table rename - cuts any existing batches for the source or destination
+ // table
+ if (event instanceof DropDatabaseEvent || event instanceof AlterDatabaseEvent) {
+ // Any batched events for tables from this database need to be flushed
+ // before emitting the AlterDatabaseEvent or DropDatabaseEvent.
+ flushBatchesForDb(pendingTableEventsMap, sortedFinalBatches, event.getDbName());
+ } else if (event instanceof AlterTableEvent) {
+ AlterTableEvent alterTable = (AlterTableEvent) event;
+ if (alterTable.isRename()) {
+ // Flush any batched events for the source table.
+ Table beforeTable = alterTable.getBeforeTable();
+ flushBatchForTable(pendingTableEventsMap, sortedFinalBatches, beforeTable);
+
+ // Flush any batched events for the destination table. Given that the
+ // destination table must not exist when doing this rename, valid sequences
+ // are already handled implicitly by the existing batch-breaking logic
+ // (combined with the sorting of the final batches). This does the flushing
+ // explicitly in case there are any edge cases not handled by the existing
+ // mechanisms.
+ Table afterTable = alterTable.getAfterTable();
+ flushBatchForTable(pendingTableEventsMap, sortedFinalBatches, afterTable);
+ }
+ }
+ }
+
+ /**
+ * This method batches together any eligible events from the given list of
+ * {@code MetastoreEvent}. The returned list may or may not contain batch
+ * events depending on whether it finds any events which could be batched together.
+ * Events on a table do not need to be contiguous to be batched, but there must
+ * not be an intervening event that cuts the batch.
+ */
+ @VisibleForTesting
+ List<MetastoreEvent> createBatchEvents(List<MetastoreEvent> events, Metrics metrics) {
+ if (events.size() < 2) return events;
+ // We can batch certain events on the same table as long as there is no
+ // intervening event that cuts the batch. To allow this non-contiguous batching,
+ // we keep state for each table separately. This is a two-level structure with
+ // the first layer keyed on the database name and the second layer keyed on the
+ // table name. This makes it possible to flush all entries for a database
+ // efficiently. Both database name and table name are lowercased to make this case
+ // insensitive.
+ //
+ // Entries in this hash map are still pending and can accept more entries into
+ // a batch when eligible. Each time an event is added a batch, it changes the
+ // ending Event ID, so this holds the pending batches until the batch is finalized.
+ // When the batch is finalized (either by an event that cuts the batch or by
+ // running out of events), it is moved to the sortedFinalBatches.
+ Map<String, Map<String, MetastoreEvent>> pendingTableEventsMap = new HashMap<>();
+
+ // The output events need to be monotonically increasing in their Event IDs,
+ // so we insert the resulting batches into a TreeMap and use that to produce
+ // the output list. Examples:
+ // 1. Basic ordering
+ // Suppose there are inserts on 4 different tables (Event ID in parens):
+ // A(1), B(2), C(3), D(4)
+ // The sorting will emit those in the same order they arrived. This also applies
+ // to any contiguous batching.
+ // 2. Interleaved events
+ // Suppose there are interleaved events that can be batched for different tables:
+ // A(1), B(2), A(3), B(4)
+ // The sorting will emit the batches in order of ascending ending Event ID, i.e.
+ // A(1-3), B(2-4)
+ // Since the ending Event ID of a batch changes each time an extra event is added,
+ // this structure should only contain finalized batches that can't change.
+ TreeMap<Long, MetastoreEvent> sortedFinalBatches = new TreeMap<>();
+
+ for (MetastoreEvent next : events) {
+ // Events that impact multiple tables need special handling to cut event batches
+ // for all impacted tables. This logic is in addition to the regular branch
+ // cutting logic that happens on a single-table basis.
+ cutBatchesForCrossTableEvents(next, pendingTableEventsMap, sortedFinalBatches);
+
+ if (!(next instanceof MetastoreTableEvent)) {
+ // No batching for non-table events
+ sortedFinalBatches.put(next.getEventId(), next);
+ continue;
+ }
+ String dbName = next.getDbName().toLowerCase();
+ String tableName = next.getTableName().toLowerCase();
+ // First, lookup the dbMap or create it if it doesn't exist
+ Map<String, MetastoreEvent> dbMap =
+ pendingTableEventsMap.computeIfAbsent(dbName, k -> new HashMap<>());
+ // Second, find the table entry in the dbMap
+ MetastoreEvent current = dbMap.get(tableName);
+ if (current != null) {
+ // Check if the next metastore event for the table can be batched into the
+ // current event for the table.
+ if (!current.canBeBatched(next)) {
+ // Events cannot be batched. Flush the current event in the table map to the
+ // output and put the next element into the table map.
+ sortedFinalBatches.put(current.getEventId(), current);
+ dbMap.put(tableName, next);
+ } else {
+ // next can be batched into current event
+ dbMap.put(tableName,
+ Preconditions.checkNotNull(current.addToBatchEvents(next)));
+ }
+ } else {
+ // New entry for this table
+ dbMap.put(tableName, next);
+ }
+ }
+ // Flush out any pending events
+ for (Map<String, MetastoreEvent> dbMap : pendingTableEventsMap.values()) {
+ for (MetastoreEvent event : dbMap.values()) {
+ sortedFinalBatches.put(event.getEventId(), event);
+ }
+ }
+
+ // We defer logging about the batches created until the end so that we can output
+ // them in the sorted order used for the actual output list.
+ List<MetastoreEvent> batchedEventList =
+ new ArrayList<>(sortedFinalBatches.values());
+ for (MetastoreEvent event : batchedEventList) {
+ if (event.getNumberOfEvents() > 1) {
+ Preconditions.checkState(event instanceof BatchPartitionEvent);
+ BatchPartitionEvent batchEvent = (BatchPartitionEvent) event;
+ batchEvent.infoLog("Created a batch event for {} events between {} and {}",
+ batchEvent.getNumberOfEvents(), batchEvent.getFirstEventId(),
+ batchEvent.getLastEventId());
+ metrics.getCounter(MetastoreEventsProcessor.NUMBER_OF_BATCH_EVENTS).inc();
+ }
+ }
+ return batchedEventList;
+ }
+ }
+
+ // A factory class for creating metastore events for syncing to latest event id
+ // We can't reuse existing event factory because of the following scenario:
+ // A ddl is executed from Impala shell. As a result of it, a self event generated
+ // for it should be ignored by event factory in MetastoreEventProcessor. If
+ // MetastoreEventProcessor also uses EventFactoryForSyncToLatestEvent then it would
+ // skip self event check and end up re processing the event which defeats the purpose
+ // of self event code. The reason for this behavior is - ddl ops from Impala shell
+ // i.e catalogOpExecutor don't sync db/table to latest event id. After
+ // IMPALA-10976 is merged, we can use one event factory and disable self event
+ // check in that.
+
+ public static class EventFactoryForSyncToLatestEvent extends
+ MetastoreEvents.MetastoreEventFactory {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(MetastoreEventFactory.class);
+
+ public EventFactoryForSyncToLatestEvent(CatalogOpExecutor catalogOpExecutor) {
+ super(catalogOpExecutor);
+ }
+
+ public MetastoreEvent get(NotificationEvent event, Metrics metrics)
+ throws MetastoreNotificationException {
+ Preconditions.checkNotNull(event.getEventType());
+ MetastoreEventType metastoreEventType =
+ MetastoreEventType.from(event.getEventType());
+ switch (metastoreEventType) {
+ case ALTER_DATABASE:
+ return new AlterDatabaseEvent(catalogOpExecutor_, metrics, event) {
+ @Override
+ protected boolean isSelfEvent() {
+ return false;
+ }
+ };
+ case ALTER_TABLE:
+ return new AlterTableEvent(catalogOpExecutor_, metrics, event) {
+ @Override
+ protected boolean isSelfEvent() {
+ return false;
+ }
+ };
+ case ADD_PARTITION:
+ return new AddPartitionEvent(catalogOpExecutor_, metrics, event) {
+ @Override
+ public boolean isSelfEvent() {
+ return false;
+ }
+ };
+ case ALTER_PARTITION:
+ return new AlterPartitionEvent(catalogOpExecutor_, metrics, event) {
+ @Override
+ public boolean isSelfEvent() {
+ return false;
+ }
+ };
+ case DROP_PARTITION:
+ return new DropPartitionEvent(catalogOpExecutor_, metrics, event) {
+ @Override
+ public boolean isSelfEvent() {
+ return false;
+ }
+ };
+ case INSERT:
+ return new InsertEvent(catalogOpExecutor_, metrics, event) {
+ @Override
+ public boolean isSelfEvent() {
+ return false;
+ }
+ };
+ default:
+ return super.get(event, metrics);
+ }
+ }
+ }
+
+
+ /**
+ * Abstract base class for all MetastoreEvents. A MetastoreEvent is a object used to
+ * process a NotificationEvent received from metastore. It is self-contained with all
+ * the information needed to take action on catalog based on a the given
+ * NotificationEvent
+ */
+ public static abstract class MetastoreEvent {
+
+ // String.format compatible string to prepend event id and type
+ private static final String STR_FORMAT_EVENT_ID_TYPE =
+ "EventId: %d EventType: %s Target: %s. ";
+
+ // logger format compatible string to prepend to a log formatted message
+ private static final String LOG_FORMAT_EVENT_ID_TYPE =
+ "EventId: {} EventType: {} Target: {}. ";
+
+ protected static final String CLUSTER_WIDE_TARGET = "CLUSTER_WIDE";
+
+ // CatalogServiceCatalog instance on which the event needs to be acted upon
+ protected final CatalogServiceCatalog catalog_;
+
+ protected final CatalogOpExecutor catalogOpExecutor_;
+
+ // the notification received from metastore which is processed by this
+ protected final NotificationEvent event_;
+
+ // Logger available for all the sub-classes
+ protected final Logger LOG = LoggerFactory.getLogger(this.getClass());
+
+ // catalog name from the event
+ protected final String catalogName_;
+
+ // dbName from the event
+ protected String dbName_;
+
+ // tblName from the event
+ protected String tblName_;
+
+ // eventId of the event. Used instead of calling getter on event_ everytime
+ private long eventId_;
+
+ // eventType from the NotificationEvent or in case of batch events set using
+ // the setter for this field
+ private MetastoreEventType eventType_;
+
+ // Actual notificationEvent object received from Metastore
+ protected final NotificationEvent metastoreNotificationEvent_;
+
+ // metrics registry so that events can add metrics
+ protected final Metrics metrics_;
+
+ // Event creation time. It is initialized in constructor. It is not used for
+ // DerivedMetastoreEvent.
+ protected final long creationTime_;
+
+ // Event dispatch time. It is initialized after dispatching actual event. It is not
+ // set for DerivedMetastoreEvent.
+ protected long dispatchTime_;
+
+ // Whether it is delimiter event. It is set only when hierarchical event processing is
+ // enabled. Delimiter is a kind of metastore event that do not require event
+ // processing. Delimeter event can be:
+ // 1. A CommitTxnEvent that do not have any write event info for a given transaction.
+ // 2. An AbortTxnEvent that do not have write ids for a given transaction.
+ // 3. An IgnoredEvent.
+ // An event is determined and marked as delimiter in EventExecutorService#dispatch()
+ protected boolean isDelimiter_;
+
+ protected MetastoreEvent(CatalogOpExecutor catalogOpExecutor, Metrics metrics,
+ NotificationEvent event) {
+ this.catalogOpExecutor_ = catalogOpExecutor;
+ this.catalog_ = catalogOpExecutor_.getCatalog();
+ this.event_ = event;
+ this.eventId_ = event_.getEventId();
+ this.eventType_ = MetastoreEventType.from(event.getEventType());
+ // certain event types in Hive-3 like COMMIT_TXN may not have dbName set
+ this.catalogName_ = event.getCatName();
+ this.dbName_ = event.getDbName();
+ this.tblName_ = event.getTableName();
+ this.metastoreNotificationEvent_ = event;
+ this.metrics_ = metrics;
+ this.creationTime_ = System.currentTimeMillis();
+ }
+
+ /**
+ * Gets metastore event creation time.
+ * @return Creation time
+ */
+ long getCreationTime() {
+ return creationTime_;
+ }
+
+ /**
+ * Gets metastore event dispatch time.
+ * @return Dispatch time
+ */
+ long getDispatchTime() {
+ if (this instanceof DerivedMetastoreEvent) {
+ return ((DerivedMetastoreEvent) this).getActualEvent().dispatchTime_;
+ }
+ return dispatchTime_;
+ }
+
+ /**
+ * Sets the metastore event dispatch time. It can be initialized only once after
+ * dispatching the event.
+ * @param dispatchTime Time when event is dispatched
+ */
+ void setDispatchTime(long dispatchTime) {
+ Preconditions.checkState(dispatchTime_ == 0, "Dispatch time can be set only once");
+ Preconditions.checkState(!(this instanceof DerivedMetastoreEvent),
+ "Not allowed to set on derived metastore event currently");
+ dispatchTime_ = dispatchTime;
+ }
+
+ /**
+ * To determine whether it is a delimiter event.
+ * @return True if delimiter event. False otherwise
+ */
+ public boolean isDelimiter() {
+ return isDelimiter_;
+ }
+
+ /**
+ * Set the event as delimiter.
+ * @param delimiter
+ */
+ public void setDelimiter(boolean delimiter) {
+ isDelimiter_ = delimiter;
+ }
+
+ public long getEventId() { return eventId_; }
+
+ public long getEventTime() { return event_.getEventTime(); }
+
+ public MetastoreEventType getEventType() { return eventType_; }
+
+ /**
+ * Certain events like {@link BatchPartitionEvent} batch a group of events
+ * to create a batch event type. This method is used to override the event type
+ * in such cases since the event type is not really derived from NotificationEvent.
+ */
+ public void setEventType(MetastoreEventType type) {
+ this.eventType_ = type;
+ }
+
+ public String getCatalogName() { return catalogName_; }
+
+ public String getDbName() { return dbName_; }
+
+ public String getTableName() { return tblName_; }
+
+ public String getTargetName() {
+ if (dbName_ == null && tblName_ == null) return CLUSTER_WIDE_TARGET;
+ if (tblName_ == null) return dbName_;
+ return dbName_ + "." + tblName_;
+ }
+
+ public String getEventDesc() {
+ return eventType_ + " event " + eventId_;
+ }
+
+ public CatalogOpExecutor getCatalogOpExecutor() {
+ return catalogOpExecutor_;
+ }
+
+ public NotificationEvent getEvent() {
+ return event_;
+ }
+
+ public Metrics getMetrics() {
+ return metrics_;
+ }
+
+ public boolean isDatabaseEvent() {
+ return eventType_ == MetastoreEventType.CREATE_DATABASE ||
+ eventType_ == MetastoreEventType.DROP_DATABASE ||
+ eventType_ == MetastoreEventType.ALTER_DATABASE;
+ }
+
+ /**
+ * Method to inject error randomly for certain events during the processing.
+ * It is used for testing purpose.
+ */
+ void injectErrorIfNeeded() {
+ if (catalog_.getFailureEventsForTesting().contains(eventType_.toString())) {
+ double random = Math.random();
+ if (random < BackendConfig.INSTANCE.getProcessEventFailureRatio()) {
+ throw new RuntimeException("Event processing failed due to error injection");
+ }
+ }
+ }
+
+ /**
+ * Process this event if it is enabled based on the flags on this object
+ *
+ * @throws CatalogException If Catalog operations fail
+ * @throws MetastoreNotificationException If NotificationEvent parsing fails
+ */
+ public void processIfEnabled()
+ throws CatalogException, MetastoreNotificationException {
+ boolean isNotDerivedEvent = !(this instanceof DerivedMetastoreEvent);
+ if (isEventProcessingDisabled()) {
+ infoLog("Skipping this event because of flag evaluation");
+ if (isNotDerivedEvent) {
+ metrics_.getCounter(MetastoreEventsProcessor.EVENTS_SKIPPED_METRIC).inc();
+ debugLog("Incremented skipped metric to " + metrics_.getCounter(
+ MetastoreEventsProcessor.EVENTS_SKIPPED_METRIC).getCount());
+ }
+ return;
+ }
+ if (BackendConfig.INSTANCE.enableSyncToLatestEventOnDdls()) {
+ if (shouldSkipWhenSyncingToLatestEventId()) {
+ if (isNotDerivedEvent) {
+ metrics_.getCounter(MetastoreEventsProcessor.EVENTS_SKIPPED_METRIC).inc();
+ }
+ return;
+ }
+ }
+ DebugUtils.executeDebugAction(
+ BackendConfig.INSTANCE.debugActions(), DebugUtils.EVENT_PROCESSING_DELAY);
+ process();
+ injectErrorIfNeeded();
+ }
+
+ /**
+ * Checks if the given event can be batched into this event. Default behavior is
+ * to return false which can be overridden by a sub-class.
+ *
+ * @param event The event under consideration to be batched into this event.
+ * @return false if event cannot be batched into this event; otherwise true.
+ */
+ protected boolean canBeBatched(MetastoreEvent event) { return false; }
+
+ /**
+ * Adds the given event into the batch of events represented by this event. Default
+ * implementation is to return null. Sub-classes must override this method to
+ * implement batching.
+ *
+ * @param event The event which needs to be added to the batch.
+ * @return The batch event which represents all the events batched into this event
+ * until now including the given event.
+ */
+ protected MetastoreEvent addToBatchEvents(MetastoreEvent event) { return null; }
+
+ /**
+ * Returns the number of events represented by this event. For most events this is
+ * 1. In case of batch events this could be more than 1.
+ */
+ protected int getNumberOfEvents() { return 1; }
+
+ /**
+ * Certain events like ALTER_TABLE or ALTER_PARTITION implement logic to ignore
+ * some events because they are not interesting from catalogd's perspective.
+ * @return true if this event can be skipped.
+ */
+ protected boolean canBeSkipped() { return false; }
+
+ /**
+ * In case of batch events, this method can be used override the {@code eventType_}
+ * field which is used for logging purposes.
+ */
+ protected MetastoreEventType getBatchEventType() { return null; }
+
+ /**
+ * Evaluates whether processing of this event should be skipped
+ * if sync to latest event id is enabled. The event should
+ * be skipped if the db/table is already synced atleast till
+ * this event.
+ * @return true if the event should be skipped. False
+ * otherwise
+ * @throws CatalogException
+ */
+ protected abstract boolean shouldSkipWhenSyncingToLatestEventId()
+ throws CatalogException;
+
+ /**
+ * Process the information available in the NotificationEvent to take appropriate
+ * action on Catalog
+ *
+ * @throws MetastoreNotificationException in case of event parsing errors out
+ * @throws CatalogException in case catalog operations could not be performed
+ */
+ protected abstract void process()
+ throws MetastoreNotificationException, CatalogException;
+
+ /**
+ * Process event failure handles the exception occurred during processing.
+ * @param e Exception occurred during process
+ * @return Returns true when failure is handled. Otherwise, false
+ */
+ protected abstract boolean onFailure(Exception e);
+
+ protected boolean canInvalidateTable(Exception e) {
+ if (e instanceof MetastoreClientInstantiationException) {
+ // All runtime exceptions except this exception are considered for invalidation.
+ return false;
+ }
+ // This RuntimeException covers all the exceptions from
+ // com.google.common.base.Preconditions methods. IllegalStateException is one such
+ // exception seen in IMPALA-12827.
+ return (e instanceof RuntimeException) || (e instanceof CatalogException)
+ || (e instanceof MetastoreNotificationNeedsInvalidateException);
+ }
+
+ /**
+ * Helper method to get debug string with helpful event information prepended to the
+ * message. This can be used to generate helpful exception messages
+ *
+ * @param msgFormatString String value to be used in String.format() for the given
+ * message
+ * @param args args to the <code>String.format()</code> for the given
+ * msgFormatString
+ */
+ protected String debugString(String msgFormatString, Object... args) {
+ String formatString =
+ new StringBuilder(STR_FORMAT_EVENT_ID_TYPE).append(msgFormatString).toString();
+ Object[] formatArgs = getLogFormatArgs(args);
+ return String.format(formatString, formatArgs);
+ }
+
+ /**
+ * Helper method to generate the format args after prepending the event id and type
+ */
+ private Object[] getLogFormatArgs(Object[] args) {
+ Object[] formatArgs = new Object[args.length + 3];
+ formatArgs[0] = getEventId();
+ formatArgs[1] = getEventType();
+ String target = getTargetName();
+ // AlterPartitionEvent is on a single partition so we can also log the partition
+ // name. Add/DropPartitionEvent might have multiple partitions so ignore them here.
+ if (this instanceof AlterPartitionEvent) {
+ target += ":" + ((AlterPartitionEvent) this).getPartName();
+ }
+ formatArgs[2] = target;
+ int i = 3;
+ for (Object arg : args) {
+ formatArgs[i] = arg;
+ i++;
+ }
+ return formatArgs;
+ }
+
+ /**
+ * Logs at info level the given log formatted string and its args. The log formatted
+ * string should have {} pair at the appropriate location in the string for each arg
+ * value provided. This method prepends the event id and event type before logging the
+ * message. No-op if the log level is not at INFO
+ */
+ protected void infoLog(String logFormattedStr, Object... args) {
+ if (!LOG.isInfoEnabled()) return;
+ String formatString =
+ new StringBuilder(LOG_FORMAT_EVENT_ID_TYPE).append(logFormattedStr).toString();
+ Object[] formatArgs = getLogFormatArgs(args);
+ LOG.info(formatString, formatArgs);
+ }
+
+ /**
+ * Similar to infoLog excepts logs at debug level
+ */
+ protected void debugLog(String logFormattedStr, Object... args) {
+ if (!LOG.isDebugEnabled()) return;
+ String formatString =
+ new StringBuilder(LOG_FORMAT_EVENT_ID_TYPE).append(logFormattedStr).toString();
+ Object[] formatArgs = getLogFormatArgs(args);
+ LOG.debug(formatString, formatArgs);
+ }
+
+ /**
+ * Similar to infoLog excepts logs at trace level
+ */
+ protected void traceLog(String logFormattedStr, Object... args) {
+ if (!LOG.isTraceEnabled()) return;
+ String formatString =
+ new StringBuilder(LOG_FORMAT_EVENT_ID_TYPE).append(logFormattedStr).toString();
+ Object[] formatArgs = getLogFormatArgs(args);
+ LOG.trace(formatString, formatArgs);
+ }
+
+ /**
+ * Similar to infoLog excepts logs at warn level
+ */
+ protected void warnLog(String logFormattedStr, Object... args) {
+ if (!LOG.isWarnEnabled()) return;
+ String formatString =
+ new StringBuilder(LOG_FORMAT_EVENT_ID_TYPE).append(logFormattedStr).toString();
+ Object[] formatArgs = getLogFormatArgs(args);
+ LOG.warn(formatString, formatArgs);
+ }
+
+ /**
+ * Method to log error for an event
+ */
+ protected void errorLog(String logFormattedStr, Object... args) {
+ if (!LOG.isErrorEnabled()) return;
+ String formatString =
+ new StringBuilder(LOG_FORMAT_EVENT_ID_TYPE).append(logFormattedStr).toString();
+ Object[] formatArgs = getLogFormatArgs(args);
+ LOG.error(formatString, formatArgs);
+ }
+
+ /**
+ * Search for a inverse event (for example drop_table is a inverse event for
+ * create_table) for this event from a given list of notificationEvents starting for
+ * the startIndex. This is useful for skipping certain events from processing
+ *
+ * @param events List of NotificationEvents to be searched
+ * @return true if the object is removed after this event, else false
+ */
+ protected boolean isRemovedAfter(List<MetastoreEvent> events) {
+ return false;
+ }
+
+ /**
+ * Returns true if event based sync is disabled for this table/database associated
+ * with this event
+ */
+ protected abstract boolean isEventProcessingDisabled();
+
+ protected abstract SelfEventContext getSelfEventContext();
+
+ /**
+ * This method detects if this event is self-generated or not (see class
+ * documentation of <code>MetastoreEventProcessor</code> to understand what a
+ * self-event is).
+ *
+ * In order to determine this, it compares the value of catalogVersion from the
+ * event with the list of pending version numbers stored in the catalog
+ * database/table. The event could be generated by another instance of CatalogService
+ * which can potentially have the same versionNumber. In order to resolve such
+ * conflict, it compares the CatalogService's serviceId before comparing the version
+ * number. If it is determined that this is indeed a self-event, this method also
+ * clears the version number from the catalog database/table's list of pending
+ * versions for in-flight events. This is needed so that a subsequent event with the
+ * same service id or version number is not incorrectly determined as a self-event. A
+ * subsequent event with the same serviceId and versionNumber is most likely generated
+ * by a non-Impala system because it cached the table object having those values of
+ * serviceId and version. More details on complete flow of self-event handling
+ * logic can be read in <code>MetastoreEventsProcessor</code> documentation.
+ *
+ * @return True if this event is a self-generated event. If the returned value is
+ * true, this method also clears the version number from the catalog database/table.
+ * Returns false if the version numbers or service id don't match
+ */
+ protected boolean isSelfEvent() {
+ try {
+ if (catalog_.evaluateSelfEvent(getSelfEventContext())) {
+ metrics_.getCounter(MetastoreEventsProcessor.EVENTS_SKIPPED_METRIC)
+ .inc(getNumberOfEvents());
+ infoLog("Incremented events skipped counter to {}",
+ metrics_.getCounter(MetastoreEventsProcessor.EVENTS_SKIPPED_METRIC)
+ .getCount());
+ return true;
+ }
+ } catch (CatalogException e) {
+ debugLog("Received exception {}. Ignoring self-event evaluation",
+ e.getMessage());
+ }
+ return false;
+ }
+
+ public final boolean isDropEvent() {
+ return (this instanceof DropTableEvent ||
+ this instanceof DropDatabaseEvent ||
+ this instanceof DropPartitionEvent);
+ }
+
+ @Override
+ public String toString() {
+ return String.format(STR_FORMAT_EVENT_ID_TYPE, eventId_, eventType_);
+ }
+
+ protected boolean isOlderThanLastSyncEventId(MetastoreEvent event) {
+ org.apache.impala.catalog.Table tbl = catalog_.getTableNoThrow(dbName_, tblName_);
+ if (tbl != null && tbl.getLastSyncedEventId() >= event.getEventId()) {
+ return true;
+ }
+ return false;
+ }
+ }
+
+ public static String getStringProperty(
+ Map<String, String> params, String key, String defaultVal) {
+ if (params == null) return defaultVal;
+ return params.getOrDefault(key, defaultVal);
+ }
+
+ /**
+ * Interface for derived metastore event
+ */
+ public interface DerivedMetastoreEvent {
+ // Gets the actual metastore event.
+ MetastoreEvent getActualEvent();
+
+ // Marks the derived event as processed.
+ void markProcessed();
+
+ // Determines whether all the derived event associated with actual event are
+ // processed.
+ boolean isAllDerivedEventsProcessed();
+ }
+
+ /**
+ *
+ */
+ public static class DerivedMetastoreEventContext {
+ // Total count of derived table events associated to actual table event. When a
+ // derived event is processed, this count is decremented by 1. Once the count becomes
+ // 0, actual table event is treated as processed.
+ private final AtomicInteger derivedEventsCount_;
+
+ // Actual metastore event the event is derived from.
+ private final MetastoreEvent actualEvent_;
+
+ public DerivedMetastoreEventContext(MetastoreEvent actualEvent,
+ int derivedEventsCount) {
+ Preconditions.checkArgument(actualEvent != null);
+ derivedEventsCount_ = new AtomicInteger(derivedEventsCount);
+ actualEvent_ = actualEvent;
+ }
+
+ public DerivedMetastoreEventContext(MetastoreEvent actualEvent) {
+ Preconditions.checkArgument(actualEvent != null);
+ derivedEventsCount_ = new AtomicInteger(0);
+ actualEvent_ = actualEvent;
+ }
+
+ /**
+ * Sets the derived metastore event count. Need to be initialized only once.
+ * @param count Derived metastore event count
+ */
+ public void setDerivedEventsCount(int count) {
+ Preconditions.checkState(derivedEventsCount_.get() == 0,
+ "Derived events count can be set only once");
+ derivedEventsCount_.set(count);
+ }
+
+ public MetastoreEvent getActualEvent() {
+ return actualEvent_;
+ }
+ }
+
+ /**
+ * Abstract class for derived metastore table event. A single actual MetastoreEvent can
+ * have multiple DerivedMetastoreTableEvent associated with it.
+ */
+ public static abstract class DerivedMetastoreTableEvent extends MetastoreTableEvent
+ implements DerivedMetastoreEvent {
+
+ private final DerivedMetastoreEventContext context_;
+
+ // Whether the current derived table event is processed.
+ private boolean isProcessed_ = false;
+
+ DerivedMetastoreTableEvent(CatalogOpExecutor catalogOpExecutor, Metrics metrics,
+ NotificationEvent event, DerivedMetastoreEventContext context) {
+ super(catalogOpExecutor, metrics, event);
+ Preconditions.checkArgument(context.derivedEventsCount_ != null);
+ context_ = context;
+ }
+
+ DerivedMetastoreTableEvent(DerivedMetastoreEventContext context,
+ NotificationEvent event) {
+ this(context.getActualEvent().getCatalogOpExecutor(),
+ context.getActualEvent().getMetrics(), event, context);
+ }
+
+ protected DerivedMetastoreTableEvent(DerivedMetastoreEventContext context) {
+ this(context.getActualEvent().getCatalogOpExecutor(),
+ context.getActualEvent().getMetrics(), context.getActualEvent().getEvent(),
+ context);
+ }
+
+ /**
+ * Gets the actual metastore event.
+ * @return Actual metastore event this event is associated to
+ */
+ @Override
+ public MetastoreEvent getActualEvent() {
+ return context_.getActualEvent();
+ }
+
+ /**
+ * Marks the current derived event as processed.
+ * <p>
+ * This method must be called only once per derived event instance. It decrements the
+ * count of remaining derived events associated with the actual event.
+ */
+ @Override
+ public void markProcessed() {
+ Preconditions.checkState(!isProcessed_, "Event already processed");
+ isProcessed_ = true;
+ Preconditions.checkState(context_.derivedEventsCount_.decrementAndGet() >= 0);
+ }
+
+ /**
+ * Determines whether the all the derived event associated with actual event are
+ * processed. The actual event is treated as processed only when all its associated
+ * derived events are processed.
+ *
+ * @return True if all derived events are processed. False otherwise.
+ */
+ @Override
+ public boolean isAllDerivedEventsProcessed() {
+ return context_.derivedEventsCount_.get() == 0;
+ }
+ }
+
+ /**
+ * Base class for all the table events
+ */
+ public static abstract class MetastoreTableEvent extends MetastoreEvent {
+ // tbl object from the Notification event, corresponds to the before tableObj in
+ // case of alter events
+ protected org.apache.hadoop.hive.metastore.api.Table msTbl_;
+
+ // A boolean flag used in alter table event to record if the file metadata reload
+ // can be skipped for certain type of alter table statements
+ protected boolean skipFileMetadataReload_ = false;
+
+ // in case of partition batch events, this method can be overridden to return
+ // the partition object from the events which are batched together.
+ protected Partition getPartitionForBatching() { return null; }
+
+ protected MetastoreTableEvent(CatalogOpExecutor catalogOpExecutor, Metrics metrics,
+ NotificationEvent event) {
+ super(catalogOpExecutor, metrics, event);
+ if (!(this instanceof DerivedMetastoreEvent)) {
+ Preconditions.checkNotNull(dbName_, "Database name cannot be null");
+ }
+ }
+
+ public Table getTable() {
+ return msTbl_;
+ }
+
+ /**
+ * Util method to return the fully qualified table name which is of the format
+ * dbName.tblName for this event
+ */
+ protected String getFullyQualifiedTblName() {
+ return new TableName(dbName_, tblName_).toString();
+ }
+
+ /**
+ * Checks if the table level property is set in the parameters of the table from the
+ * event. If it is available, it takes precedence over database level flag for this
+ * table. If the table level property is not set, returns the value from the database
+ * level property.f
+ *
+ * @return Boolean value of the table property with the key
+ * <code>MetastoreEventPropertyKey.DISABLE_EVENT_HMS_SYNC</code>. Else,
+ * returns the database property which is associated with this table. Returns
+ * false if neither of the properties are set.
+ */
+ @Override
+ protected boolean isEventProcessingDisabled() {
+ Preconditions.checkNotNull(msTbl_);
+ Boolean tblProperty = getHmsSyncProperty(msTbl_);
+ if (tblProperty != null) {
+ infoLog("Found table level flag {} is set to {} for table {}",
+ MetastoreEventPropertyKey.DISABLE_EVENT_HMS_SYNC.getKey(),
+ tblProperty.toString(),
+ getFullyQualifiedTblName());
+ return tblProperty;
+ }
+ // if the tbl property is not set check at db level
+ String dbFlagVal = catalog_.getDbProperty(dbName_,
+ MetastoreEventPropertyKey.DISABLE_EVENT_HMS_SYNC.getKey());
+ if (dbFlagVal != null) {
+ // no need to spew unnecessary logs. Most tables/databases are expected to not
+ // have this flag set when event based HMS polling is enabled
+ debugLog("Table level flag is not set. Db level flag {} is {} for "
+ + "database {}",
+ MetastoreEventPropertyKey.DISABLE_EVENT_HMS_SYNC.getKey(),
+ dbFlagVal, dbName_);
+ }
+ // flag value of null also returns false
+ return Boolean.valueOf(dbFlagVal);
+ }
+
+ /**
+ * Gets the value of the parameter with the key
+ * <code>MetastoreEventPropertyKey.DISABLE_EVENT_HMS_SYNC</code> from the given
+ * table
+ *
+ * @return the Boolean value of the property with the key
+ * <code>MetastoreEventPropertyKey.DISABLE_EVENT_HMS_SYNC</code> if it is
+ * available else returns null
+ */
+ public static Boolean getHmsSyncProperty(
+ org.apache.hadoop.hive.metastore.api.Table tbl) {
+ if (!tbl.isSetParameters()) return null;
+ String val =
+ tbl.getParameters()
+ .get(MetastoreEventPropertyKey.DISABLE_EVENT_HMS_SYNC.getKey());
+ if (val == null || val.isEmpty()) return null;
+ return Boolean.valueOf(val);
+ }
+
+ /**
+ * Util method to create partition key-value map from HMS Partition objects.
+ */
+ protected static List<TPartitionKeyValue> getTPartitionSpecFromHmsPartition(
+ org.apache.hadoop.hive.metastore.api.Table msTbl, Partition partition) {
+ List<TPartitionKeyValue> tPartSpec = new ArrayList<>();
+ List<org.apache.hadoop.hive.metastore.api.FieldSchema> fsList =
+ msTbl.getPartitionKeys();
+ List<String> partVals = partition.getValues();
+ Preconditions.checkNotNull(partVals);
+ Preconditions.checkState(fsList.size() == partVals.size());
+ for (int i = 0; i < fsList.size(); i++) {
+ tPartSpec.add(new TPartitionKeyValue(fsList.get(i).getName(), partVals.get(i)));
+ }
+ return tPartSpec;
+ }
+
+ /*
+ * Helper function to initiate a table reload on Catalog. Re-throws the exception if
+ * the catalog operation throws.
+ */
+ protected boolean reloadTableFromCatalog(boolean isTransactional)
+ throws CatalogException {
+ try {
+ if (!catalog_.reloadTableIfExists(dbName_, tblName_, getEventDesc(), getEventId(),
+ skipFileMetadataReload_, null)) {
+ debugLog("Automatic refresh on table {} failed as the table "
+ + "either does not exist anymore or is not in loaded state.",
+ getFullyQualifiedTblName());
+ return false;
+ }
+ } catch (TableLoadingException | DatabaseNotFoundException e) {
+ // there could be many reasons for receiving a tableLoading exception,
+ // eg. table doesn't exist in HMS anymore or table schema is not supported
+ // or Kudu threw an exception due to some internal error. There is not much
+ // we can do here other than log it appropriately.
+ debugLog("Table {} was not refreshed due to error {}",
+ getFullyQualifiedTblName(), e.getMessage());
+ return false;
+ }
+ String tblStr = isTransactional ? "transactional table" : "table";
+ infoLog("Refreshed {} {}", tblStr, getFullyQualifiedTblName());
+ metrics_.getCounter(MetastoreEventsProcessor.NUMBER_OF_TABLE_REFRESHES).inc();
+ return true;
+ }
+
+ /**
+ * Reloads the partitions provided, only if the table is loaded and if the partitions
+ * exist in catalogd.
+ * @param partitions the list of Partition objects which need to be reloaded.
+ * @param fileMetadataLoadOpts: describes how to reload file metadata for partitions
+ * @param reason The reason for reload operation which is used for logging by
+ * catalogd.
+ * @param batch flag to show if the function is called by the batch event process
+ */
+ protected void reloadPartitions(List<Partition> partitions,
+ FileMetadataLoadOpts fileMetadataLoadOpts, String reason, boolean batch)
+ throws CatalogException {
+ int skippedEvent = batch ? partitions.size() : 1;
+ try {
+ int numPartsRefreshed = catalogOpExecutor_.reloadPartitionsIfExist(getEventId(),
+ getEventType().toString(), dbName_, tblName_, partitions, reason,
+ fileMetadataLoadOpts);
+ if (numPartsRefreshed > 0) {
+ metrics_.getCounter(MetastoreEventsProcessor.NUMBER_OF_PARTITION_REFRESHES)
+ .inc(numPartsRefreshed);
+ } else if (numPartsRefreshed == -1) {
+ debugLog("Ignoring the event since table {} is not loadded or " +
+ "table was removed latter in catalog or table is synced."
+ , getFullyQualifiedTblName());
+ metrics_.getCounter(MetastoreEventsProcessor.EVENTS_SKIPPED_METRIC)
+ .inc(skippedEvent);
+ }
+ } catch (TableNotLoadedException e) {
+ debugLog("Ignoring the event since table {} is not loaded",
+ getFullyQualifiedTblName());
+ metrics_.getCounter(MetastoreEventsProcessor.EVENTS_SKIPPED_METRIC)
+ .inc(skippedEvent);
+ } catch (DatabaseNotFoundException | TableNotFoundException e) {
+ debugLog("Ignoring the event since table {} is not found",
+ getFullyQualifiedTblName());
+ metrics_.getCounter(MetastoreEventsProcessor.EVENTS_SKIPPED_METRIC)
+ .inc(skippedEvent);
+ }
+ }
+
+ /**
+ * Reloads partitions from the event if they exist. Does not fetch those partitions
+ * from HMS. Does NOT reload file metadata
+ * @param partitions: Partitions to be reloaded
+ * @param reason: Reason for reloading. Mainly used for logging in catalogD
+ * @throws CatalogException
+ */
+ protected void reloadPartitionsFromEvent(List<Partition> partitions, String reason)
+ throws CatalogException {
+ try {
+ int numPartsRefreshed = catalogOpExecutor_.reloadPartitionsFromEvent(getEventId(),
+ dbName_, tblName_, partitions, reason);
+ if (numPartsRefreshed > 0) {
+ metrics_.getCounter(MetastoreEventsProcessor.NUMBER_OF_PARTITION_REFRESHES)
+ .inc(numPartsRefreshed);
+ } else if (numPartsRefreshed == -1) {
+ metrics_.getCounter(MetastoreEventsProcessor.EVENTS_SKIPPED_METRIC).inc();
+ }
+ } catch (TableNotLoadedException e) {
+ debugLog("Ignoring the event since table {} is not loaded",
+ getFullyQualifiedTblName());
+ metrics_.getCounter(MetastoreEventsProcessor.EVENTS_SKIPPED_METRIC).inc();
+ } catch (DatabaseNotFoundException | TableNotFoundException e) {
+ debugLog("Ignoring the event since table {} is not found",
+ getFullyQualifiedTblName());
+ metrics_.getCounter(MetastoreEventsProcessor.EVENTS_SKIPPED_METRIC).inc();
+ }
+ }
+
+ protected void reloadPartitionsFromNames(List<String> partitionNames, String reason,
+ FileMetadataLoadOpts fileMetadataLoadOpts) throws CatalogException {
+ try {
+ int numPartsRefreshed = catalogOpExecutor_.reloadPartitionsFromNamesIfExists(
+ getEventId(), getEventType().toString(), dbName_, tblName_, partitionNames,
+ reason, fileMetadataLoadOpts);
+ if (numPartsRefreshed > 0) {
+ metrics_.getCounter(MetastoreEventsProcessor.NUMBER_OF_PARTITION_REFRESHES)
+ .inc(numPartsRefreshed);
+ } else if (numPartsRefreshed == -1) {
+ metrics_.getCounter(MetastoreEventsProcessor.EVENTS_SKIPPED_METRIC).inc();
+ }
+ } catch (TableNotLoadedException e) {
+ debugLog("Ignoring the event since table {} is not loaded",
+ getFullyQualifiedTblName());
+ metrics_.getCounter(MetastoreEventsProcessor.EVENTS_SKIPPED_METRIC).inc();
+ } catch (DatabaseNotFoundException | TableNotFoundException e) {
+ debugLog("Ignoring the event since table {} is not found",
+ getFullyQualifiedTblName());
+ metrics_.getCounter(MetastoreEventsProcessor.EVENTS_SKIPPED_METRIC).inc();
+ }
+ }
+
+ /**
+ * To decide whether to skip processing this event, fetch table from cache
+ * and compare the last synced event id of cache table with this event id.
+ * Skip this event if the table is already synced till this event id. Otherwise,
+ * process this event.
+ * @return true if processing of this event should be skipped. False otherwise
+ * @throws CatalogException
+ */
+ protected boolean shouldSkipWhenSyncingToLatestEventId() throws CatalogException {
+ Preconditions.checkState(
+ BackendConfig.INSTANCE.enableSyncToLatestEventOnDdls(),
+ "sync to latest event flag is not set to true");
+ long eventId = this.getEventId();
+ Preconditions.checkState(eventId > 0,
+ "Invalid event id %s. Should be greater than "
+ + "0", eventId);
+ org.apache.impala.catalog.Table tbl = null;
+ try {
+ tbl = catalog_.getTable(dbName_, tblName_);
+ if (tbl == null) {
+ infoLog("Skipping on table {}.{} since it does not exist in cache", dbName_,
+ tblName_);
+ return true;
+ }
+ // During alter table rename, the renamed table is created as Incomplete table
+ // with create event id set to alter_table event id (i.e not -1) and therefore
+ // should *NOT* be skipped in event processing
+ if (tbl instanceof IncompleteTable && tbl.getLastSyncedEventId() == -1) {
+ infoLog("Skipping on an incomplete table {} since last synced event id is "
+ + "set to {}", tbl.getFullName(), tbl.getLastSyncedEventId());
+ return true;
+ }
+ } catch (DatabaseNotFoundException e) {
+ infoLog("Skipping on table {} because db {} not found in cache", tblName_,
+ dbName_);
+ return true;
+ }
+ boolean shouldSkip = false;
+ // do not acquire read lock on tbl because lastSyncedEventId_ is volatile.
+ // It is fine if this method returns false because at the time of actual
+ // processing of the event, we would again check lastSyncedEventId_ after
+ // acquiring write lock on table and if the table was already synced till
+ // this event id, the event processing would be skipped.
+ if (tbl.getLastSyncedEventId() >= eventId) {
+ infoLog("Skipping on table {} since it is already synced till event id {}",
+ tbl.getFullName(), tbl.getLastSyncedEventId());
+ shouldSkip = true;
+ }
+ return shouldSkip;
+ }
+
+ /**
+ * Overrides parent's isSelfEvent method. If the event turns out to be a self event
+ * then this implementation checks and sets table's lastSyncedEvent if it is less
+ * than this event id. It is done so that when syncing table to latest event id on
+ * subsequent ddl operations, the self event is not processed again.
+ * @return
+ */
+ @Override
+ protected boolean isSelfEvent() {
+ boolean isSelfEvent = super.isSelfEvent();
+ if (!isSelfEvent || !BackendConfig.INSTANCE.enableSyncToLatestEventOnDdls()) {
+ return isSelfEvent;
+ }
+ org.apache.impala.catalog.Table tbl = null;
+ try {
+ tbl = catalog_.getTable(getDbName(), getTableName());
+
+ if (tbl != null && catalog_.tryWriteLock(tbl)) {
+ catalog_.getLock().writeLock().unlock();
+ if (tbl.getLastSyncedEventId() < getEventId()) {
+ infoLog("is a self event. last synced event id for "
+ + "table {} is {}. Setting it to {}", tbl.getFullName(),
+ tbl.getLastSyncedEventId(), getEventId());
+ tbl.setLastSyncedEventId(getEventId());
+ }
+ }
+ } catch (CatalogException e) {
+ debugLog("ignoring exception when trying to set latest event for a self event "
+ + "on table {}.{}", getDbName(), getTableName(), e);
+ } finally {
+ catalogOpExecutor_.UnlockWriteLockIfErronouslyLocked();
+ if (tbl != null && tbl.isWriteLockedByCurrentThread()) {
+ tbl.releaseWriteLock();
+ }
+ }
+ return true;
+ }
+
+ protected boolean isOlderEvent(Partition partitionEventObj) {
+ if (!BackendConfig.INSTANCE.enableSkippingOlderEvents()) {
+ return false;
+ }
+ org.apache.impala.catalog.Table tbl = null;
+ try {
+ tbl = catalog_.getTable(dbName_, tblName_);
+ if (tbl == null || tbl instanceof IncompleteTable) {
+ return false;
+ }
+ if (getEventId() > 0 && getEventId() <= tbl.getCreateEventId()) {
+ // Older event, so this event will be skipped.
+ metrics_.getCounter(MetastoreEventsProcessor.EVENTS_SKIPPED_METRIC).inc();
+ infoLog("Table: {} createEventId: {} is >= to the current " +
+ "eventId: {}. Incremented skipped metric to {}", tbl.getFullName(),
+ tbl.getCreateEventId(), getEventId(),
+ metrics_.getCounter(MetastoreEventsProcessor.EVENTS_SKIPPED_METRIC)
+ .getCount());
+ return true;
+ }
+ // Always check the lastRefreshEventId on the table first for table level refresh
+ boolean canSkip = tbl.getLastRefreshEventId() >= getEventId();
+ try {
+ if (!canSkip && partitionEventObj != null) {
+ tbl.takeReadLock();
+ canSkip = catalog_.isPartitionLoadedAfterEvent(dbName_, tblName_,
+ partitionEventObj, getEventId());
+ }
+ if (canSkip) {
+ metrics_.getCounter(MetastoreEventsProcessor.EVENTS_SKIPPED_METRIC)
+ .inc(getNumberOfEvents());
+ String messageStr = partitionEventObj == null ? "Skipping the event since " +
+ "the table " + dbName_ + "." + tblName_ + " has last refresh id as " +
+ tbl.getLastRefreshEventId() + ". Comparing it with current event " +
+ getEventId() + ". " : "";
+ infoLog("{}Incremented events skipped counter to {}", messageStr,
+ metrics_.getCounter(MetastoreEventsProcessor.EVENTS_SKIPPED_METRIC)
+ .getCount());
+ return true;
+ }
+ } finally {
+ if (tbl.isReadLockedByCurrentThread()) {
+ tbl.releaseReadLock();
+ }
+ }
+ } catch (CatalogException e) {
+ debugLog("ignoring exception while checking if it is an older event "
+ + "on table {}.{}", dbName_, tblName_, e);
+ }
+ return false;
+ }
+
+ @Override
+ protected void process() throws MetastoreNotificationException, CatalogException {
+ Timer.Context context = null;
+ org.apache.impala.catalog.Table tbl = catalog_.getTableNoThrow(dbName_, tblName_);
+ if (tbl != null && !(tbl instanceof IncompleteTable)) {
+ context = tbl.getMetrics().getTimer(TBL_EVENTS_PROCESS_DURATION).time();
+ }
+ try {
+ processTableEvent();
+ } finally {
+ if (context != null) {
+ context.stop();
+ }
+ }
+ }
+
+ @Override
+ protected boolean onFailure(Exception e) {
+ if (!BackendConfig.INSTANCE.isInvalidateMetadataOnEventProcessFailureEnabled()) {
+ return false;
+ }
+ boolean isInvalidate = canInvalidateTable(e);
+ if (isInvalidate) {
+ errorLog("Invalidate table {}.{} due to exception during event processing",
+ dbName_, tblName_, e);
+ catalog_.invalidateTableIfExists(dbName_, tblName_);
+ }
+ return isInvalidate;
+ }
+
+ protected abstract void processTableEvent() throws MetastoreNotificationException,
+ CatalogException;
+ }
+
+ /**
+ * Base class for all the database events
+ */
+ public static abstract class MetastoreDatabaseEvent extends MetastoreEvent {
+ MetastoreDatabaseEvent(CatalogOpExecutor catalogOpExecutor, Metrics metrics,
+ NotificationEvent event) {
+ super(catalogOpExecutor, metrics, event);
+ Preconditions.checkNotNull(dbName_, debugString("Database name cannot be null"));
+ debugLog("Creating event {} of type {} on database {}", getEventId(),
+ getEventType(), dbName_);
+ }
+
+ /**
+ * Even though there is a database level property
+ * <code>MetastoreEventPropertyKey.DISABLE_EVENT_HMS_SYNC</code> it is only used
+ * for tables within that
+ * database. As such this property does not control if the database level DDLs are
+ * skipped or not.
+ *
+ * @return false
+ */
+ @Override
+ protected boolean isEventProcessingDisabled() {
+ return false;
+ }
+
+ protected boolean shouldSkipWhenSyncingToLatestEventId() throws CatalogException {
+ return false;
+ }
+
+ /**
+ * Overrides parent's isSelfEvent method. If the event turns out to be a self event
+ * then this implementation checks and sets db's lastSyncedEvent if it is less
+ * than this event id. It is done so that when syncing db to latest event id on
+ * subsequent ddl operations, the self event is not processed again.
+ * @return
+ */
+ @Override
+ protected boolean isSelfEvent() {
+ boolean isSelfEvent = super.isSelfEvent();
+ if (!isSelfEvent || !BackendConfig.INSTANCE.enableSyncToLatestEventOnDdls()) {
+ return isSelfEvent;
+ }
+ Db db = null;
+ try {
+ db = catalog_.getDb(getDbName());
+ if (db != null && catalog_.tryLockDb(db)) {
+ catalog_.getLock().writeLock().unlock();
+ if (db.getLastSyncedEventId() < getEventId()) {
+ infoLog("is a self event. last synced event id for "
+ + "db {} is {}. Setting it to {}", getDbName(),
+ db.getLastSyncedEventId(), getEventId());
+ db.setLastSyncedEventId(getEventId());
+ }
+ }
+ } finally {
+ catalogOpExecutor_.UnlockWriteLockIfErronouslyLocked();
+ if (db != null && db.isLockHeldByCurrentThread()) {
+ db.getLock().unlock();
+ }
+ }
+ return true;
+ }
+
+ @Override
+ protected boolean onFailure(Exception e) {
+ // TODO: Need to check db event failure in future
+ return false;
+ }
+ }
+
+ /**
+ * MetastoreEvent for CREATE_TABLE event type
+ */
+ public static class CreateTableEvent extends MetastoreTableEvent {
+
+ public static final String EVENT_TYPE = "CREATE_TABLE";
+ /**
+ * Prevent instantiation from outside should use MetastoreEventFactory instead
+ */
+ private CreateTableEvent(CatalogOpExecutor catalogOpExecutor, Metrics metrics,
+ NotificationEvent event) throws MetastoreNotificationException {
+ super(catalogOpExecutor, metrics, event);
+ Preconditions.checkArgument(MetastoreEventType.CREATE_TABLE.equals(getEventType()));
+ Preconditions
+ .checkNotNull(event.getMessage(), debugString("Event message is null"));
+ CreateTableMessage createTableMessage =
+ MetastoreEventsProcessor.getMessageDeserializer()
+ .getCreateTableMessage(event.getMessage());
+ try {
+ msTbl_ = createTableMessage.getTableObj();
+ } catch (Exception e) {
+ throw new MetastoreNotificationException(
+ debugString("Unable to deserialize the event message"), e);
+ }
+ }
+
+ protected CreateTableEvent(CatalogOpExecutor catalogOpExecutor, Metrics metrics,
+ NotificationEvent event, org.apache.hadoop.hive.metastore.api.Table table) {
+ super(catalogOpExecutor, metrics, event);
+ Preconditions.checkArgument(getEventType() == MetastoreEventType.CREATE_TABLE);
+ msTbl_ = table;
+ }
+
+ @Override
+ public SelfEventContext getSelfEventContext() {
+ throw new UnsupportedOperationException("Self-event evaluation is unnecessary for"
+ + " this event type");
+ }
+
+ /**
+ * If the table provided in the catalog does not exist in the catalog, this method
+ * will create it. If the table in the catalog already exists, it relies of the
+ * creationTime of the Metastore Table to resolve the conflict. If the catalog table's
+ * creation time is less than creationTime of the table from the event, it will be
+ * overridden. Else, it will ignore the event
+ */
+ @Override
+ public void processTableEvent() throws MetastoreNotificationException {
+ // check if the table exists already. This could happen in corner cases of the
+ // table being dropped and recreated with the same name or in case this event is
+ // a self-event (see description of self-event in the class documentation of
+ // MetastoreEventsProcessor)
+ try {
+ if (catalogOpExecutor_.addTableIfNotRemovedLater(getEventId(), msTbl_)) {
+ infoLog("Successfully added table {}", getFullyQualifiedTblName());
+ metrics_.getCounter(MetastoreEventsProcessor.NUMBER_OF_TABLES_ADDED).inc();
+ } else {
+ metrics_.getCounter(MetastoreEventsProcessor.EVENTS_SKIPPED_METRIC).inc();
+ debugLog("Incremented skipped metric to " + metrics_
+ .getCounter(MetastoreEventsProcessor.EVENTS_SKIPPED_METRIC).getCount());
+ }
+ } catch (DatabaseNotFoundException ex) {
+ // This is an incorrect setup where DB is not found in cache for a table event.
+ // Don't put the event processor into error state, instead ignore this event.
+ errorLog("Create table {}.{} failed because the database does not exist cache." +
+ " Ignoring the CREATE_TABLE event", dbName_, tblName_, ex);
+ } catch (CatalogException e) {
+ // if a DatabaseNotFoundException is caught here it means either we incorrectly
+ // determined that the event needs to be processed instead of skipped, or we
+ // somehow missed the previous create database event.
+ throw new MetastoreNotificationException(
+ debugString("Unable to process event"), e);
+ }
+ }
+
+ @Override
+ protected boolean onFailure(Exception e) {
+ return false;
+ }
+
+ @Override
+ public boolean isRemovedAfter(List<MetastoreEvent> events) {
+ Preconditions.checkNotNull(events);
+ for (MetastoreEvent event : events) {
+ if (event.eventType_.equals(MetastoreEventType.DROP_TABLE)) {
+ DropTableEvent dropTableEvent = (DropTableEvent) event;
+ if (dbName_.equalsIgnoreCase(dropTableEvent.dbName_) && tblName_
+ .equalsIgnoreCase(dropTableEvent.tblName_)) {
+ infoLog("Found table {} is removed later in event {} type {}", tblName_,
+ dropTableEvent.getEventId(), dropTableEvent.getEventType());
+ return true;
+ }
+ } else if (event.eventType_.equals(MetastoreEventType.ALTER_TABLE)) {
+ // renames are implemented as a atomic (drop+create) so rename events can
+ // also be treated as a inverse event of the create_table event. Consider a
+ // DDL op sequence like create table, alter table rename from impala. Since
+ // the rename operation is internally implemented as drop+add, processing a
+ // create table event on this cluster will show up the table for small window
+ // of time, until the actual rename event is processed. If however, we ignore
+ // the create table event, the alter rename event just becomes a addIfNotExists
+ // event which is valid for both a self-event and external event cases
+ AlterTableEvent alterTableEvent = (AlterTableEvent) event;
+ if (alterTableEvent.isRename_
+ && dbName_.equalsIgnoreCase(alterTableEvent.msTbl_.getDbName())
+ && tblName_.equalsIgnoreCase(alterTableEvent.msTbl_.getTableName())) {
+ infoLog("Found table {} is renamed later in event {} type {}", tblName_,
+ alterTableEvent.getEventId(), alterTableEvent.getEventType());
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ protected boolean shouldSkipWhenSyncingToLatestEventId() {
+ return false;
+ }
+ }
+
+ /**
+ * Metastore event handler for INSERT events. Handles insert events at both table
+ * and partition scopes.
+ */
+ public static class InsertEvent extends MetastoreTableEvent {
+
+ // Represents the partition for this insert. Null if the table is unpartitioned.
+ private Partition insertPartition_;
+
+ /**
+ * Prevent instantiation from outside should use MetastoreEventFactory instead
+ */
+ @VisibleForTesting
+ InsertEvent(CatalogOpExecutor catalogOpExecutor, Metrics metrics,
+ NotificationEvent event) throws MetastoreNotificationException {
+ super(catalogOpExecutor, metrics, event);
+ Preconditions.checkArgument(MetastoreEventType.INSERT.equals(getEventType()));
+ InsertMessage insertMessage =
+ MetastoreEventsProcessor.getMessageDeserializer()
+ .getInsertMessage(event.getMessage());
+ try (MetaStoreClientPool.MetaStoreClient client = catalog_.getMetaStoreClient()) {
+ String dbName = insertMessage.getDB();
+ String tableName = insertMessage.getTable();
+ msTbl_ = client.getHiveClient().getTable(dbName, tableName);
+ if (!insertMessage.getPartitionKeyValues().isEmpty()) {
+ List<String> partitionKeys = msTbl_.getPartitionKeys().stream()
+ .map(FieldSchema::getName).collect(Collectors.toList());
+ String partitionName = FeCatalogUtils.getPartitionName(partitionKeys,
+ insertMessage.getPartitionKeyValues());
+ insertPartition_ = client.getHiveClient().getPartition(dbName,
+ tableName, partitionName);
+ }
+ } catch (Exception e) {
+ throw new MetastoreNotificationException(debugString("Unable to "
+ + "parse insert message"), e);
+ }
+ }
+
+ @Override
+ protected MetastoreEventType getBatchEventType() {
+ return MetastoreEventType.INSERT_PARTITIONS;
+ }
+
+ @Override
+ protected Partition getPartitionForBatching() { return insertPartition_; }
+
+ @Override
+ public boolean canBeBatched(MetastoreEvent event) {
+ if (!(event instanceof InsertEvent)) return false;
+ if (isOlderThanLastSyncEventId(event)) return false;
+ InsertEvent insertEvent = (InsertEvent) event;
+ // make sure that the event is on the same table
+ if (!getFullyQualifiedTblName().equalsIgnoreCase(
+ insertEvent.getFullyQualifiedTblName())) {
+ return false;
+ }
+ // we currently only batch partition level insert events
+ if (this.insertPartition_ == null || insertEvent.insertPartition_ == null) {
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public MetastoreEvent addToBatchEvents(MetastoreEvent event) {
+ if (!(event instanceof InsertEvent)) return null;
+ BatchPartitionEvent<InsertEvent> batchEvent = new BatchPartitionEvent<>(
+ this);
+ Preconditions.checkState(batchEvent.canBeBatched(event));
+ batchEvent.addToBatchEvents(event);
+ return batchEvent;
+ }
+
+ @Override
+ public SelfEventContext getSelfEventContext() {
+ if (insertPartition_ != null) {
+ // create selfEventContext for insert partition event
+ List<TPartitionKeyValue> tPartSpec =
+ getTPartitionSpecFromHmsPartition(msTbl_, insertPartition_);
+ return new SelfEventContext(dbName_, tblName_, Arrays.asList(tPartSpec),
+ insertPartition_.getParameters(), Arrays.asList(getEventId()));
+ } else {
+ // create selfEventContext for insert table event
+ return new SelfEventContext(
+ dbName_, tblName_, null, msTbl_.getParameters(),
+ Arrays.asList(getEventId()));
+ }
+ }
+
+ @Override
+ public void processTableEvent() throws MetastoreNotificationException {
+ if (isSelfEvent()) {
+ infoLog("Not processing the insert event as it is a self-event");
+ return;
+ }
+
+ if (isOlderEvent(insertPartition_)) {
+ infoLog("Not processing the insert event {} as it is an older event",
+ getEventId());
+ return;
+ }
+ // Reload the whole table if it's a transactional table or materialized view.
+ // Materialized views are treated as a special case because it causes problems
+ // on the reloading partition logic which expects it to be a HdfsTable.
+ if (AcidUtils.isTransactionalTable(msTbl_.getParameters())
+ || MetaStoreUtils.isMaterializedViewTable(msTbl_)) {
+ insertPartition_ = null;
+ }
+
+ if (insertPartition_ != null) {
+ processPartitionInserts();
+ } else {
+ processTableInserts();
+ }
+ }
+
+ /**
+ * Process partition inserts
+ */
+ private void processPartitionInserts() throws MetastoreNotificationException {
+ // For partitioned table, refresh the partition only.
+ Preconditions.checkNotNull(insertPartition_);
+ try {
+ // Ignore event if table or database is not in catalog. Throw exception if
+ // refresh fails. If the partition does not exist in metastore the reload
+ // method below removes it from the catalog
+ // forcing file metadata reload so that new files (due to insert) are reflected
+ // HdfsPartition
+ reloadPartitions(Arrays.asList(insertPartition_),
+ FileMetadataLoadOpts.FORCE_LOAD, getEventDesc(), false);
+ } catch (CatalogException e) {
+ throw new MetastoreNotificationNeedsInvalidateException(debugString("Refresh "
+ + "partition on table {} partition {} failed. Event processing cannot "
+ + "continue. Issue an invalidate metadata command to reset the event "
+ + "processor state.", getFullyQualifiedTblName(),
+ Joiner.on(',').join(insertPartition_.getValues())), e);
+ }
+ }
+
+ /**
+ * Process unpartitioned table inserts
+ */
+ private void processTableInserts() throws MetastoreNotificationException {
+ // For non-partitioned tables, refresh the whole table.
+ Preconditions.checkState(insertPartition_ == null);
+ try {
+ boolean notSkipped = reloadTableFromCatalog(false);
+ if (!notSkipped) {
+ metrics_.getCounter(MetastoreEventsProcessor.EVENTS_SKIPPED_METRIC).inc();
+ }
+ } catch (CatalogException e) {
+ throw new MetastoreNotificationNeedsInvalidateException(
+ debugString("Refresh table {} failed. Event processing "
+ + "cannot continue. Issue an invalidate metadata command to reset "
+ + "the event processor state.", getFullyQualifiedTblName()), e);
+ }
+ }
+ }
+
+ /**
+ * MetastoreEvent for ALTER_TABLE event type
+ */
+ public static class AlterTableEvent extends MetastoreTableEvent {
+ public static final String EVENT_TYPE = "ALTER_TABLE";
+ protected org.apache.hadoop.hive.metastore.api.Table tableBefore_;
+ // the table object after alter operation, as parsed from the NotificationEvent
+ protected org.apache.hadoop.hive.metastore.api.Table tableAfter_;
+ // true if this alter event was due to a rename operation
+ protected final boolean isRename_;
+ // value of event sync flag for this table before the alter operation
+ private final Boolean eventSyncBeforeFlag_;
+ // value of the event sync flag if available at this table after the alter operation
+ private final Boolean eventSyncAfterFlag_;
+ // value of the db flag at the time of event creation
+ private final boolean dbFlagVal;
+ // true if this alter event was due to a truncate operation in metastore
+ private final boolean isTruncateOp_;
+
+ /**
+ * Prevent instantiation from outside should use MetastoreEventFactory instead
+ */
+ @VisibleForTesting
+ AlterTableEvent(CatalogOpExecutor catalogOpExecutor, Metrics metrics,
+ NotificationEvent event) throws MetastoreNotificationException {
+ super(catalogOpExecutor, metrics, event);
+ Preconditions.checkArgument(MetastoreEventType.ALTER_TABLE.equals(getEventType()));
+ JSONAlterTableMessage alterTableMessage =
+ (JSONAlterTableMessage) MetastoreEventsProcessor.getMessageDeserializer()
+ .getAlterTableMessage(event.getMessage());
+ try {
+ msTbl_ = Preconditions.checkNotNull(alterTableMessage.getTableObjBefore());
+ tableAfter_ = Preconditions.checkNotNull(alterTableMessage.getTableObjAfter());
+ tableBefore_ = Preconditions.checkNotNull(alterTableMessage.getTableObjBefore());
+ isTruncateOp_ = false;
+ } catch (Exception e) {
+ throw new MetastoreNotificationException(
+ debugString("Unable to parse the alter table message"), e);
+ }
+ // this is a rename event if either dbName or tblName of before and after object
+ // changed
+ isRename_ = !msTbl_.getDbName().equalsIgnoreCase(tableAfter_.getDbName())
+ || !msTbl_.getTableName().equalsIgnoreCase(tableAfter_.getTableName());
+ eventSyncBeforeFlag_ = getHmsSyncProperty(msTbl_);
+ eventSyncAfterFlag_ = getHmsSyncProperty(tableAfter_);
+ dbFlagVal =
+ Boolean.valueOf(catalog_.getDbProperty(dbName_,
+ MetastoreEventPropertyKey.DISABLE_EVENT_HMS_SYNC.getKey()));
+ }
+
+ public boolean isRename() { return isRename_; }
+
+ public Table getBeforeTable() { return tableBefore_; }
+
+ public Table getAfterTable() { return tableAfter_; }
+
+ @Override
+ protected SelfEventContext getSelfEventContext() {
+ return new SelfEventContext(tableAfter_.getDbName(), tableAfter_.getTableName(),
+ tableAfter_.getParameters());
+ }
+
+ private void processRename() throws CatalogException {
+ if (!isRename_) return;
+ Reference<Boolean> oldTblRemoved = new Reference<>();
+ Reference<Boolean> newTblAdded = new Reference<>();
+ catalogOpExecutor_
+ .renameTableFromEvent(getEventId(), tableBefore_, tableAfter_, oldTblRemoved,
+ newTblAdded);
+
+ // Only bump the skipped metric if the old table is not removed and the new table
+ // is not added. Not doing this in other cases since we need to either remove the
+ // old table or add the new table, which is processing the event.
+ if (!oldTblRemoved.getRef() && !newTblAdded.getRef()) {
+ metrics_.getCounter(MetastoreEventsProcessor.EVENTS_SKIPPED_METRIC).inc();
+ debugLog("Incremented skipped metric to " + metrics_
+ .getCounter(MetastoreEventsProcessor.EVENTS_SKIPPED_METRIC).getCount());
+ } else {
+ if (oldTblRemoved.getRef()) {
+ metrics_.getCounter(MetastoreEventsProcessor.NUMBER_OF_TABLES_REMOVED).inc();
+ }
+ if (newTblAdded.getRef()) {
+ metrics_.getCounter(MetastoreEventsProcessor.NUMBER_OF_TABLES_ADDED).inc();
+ }
+ }
+ }
+
+ /**
+ * If the alter table event is generated because of table rename then event
+ * should *NOT* be skipped if old table is not synced this till event AND
+ * new table doesn't exist in cache. Skip otherwise
+ * @return true if event should be skipped. false otherwise
+ * @throws CatalogException
+ */
+ protected boolean shouldSkipWhenSyncingToLatestEventId() throws CatalogException {
+ // always process rename since renameTableFromEvent will make sure that
+ // the old table was removed and new was added
+ if (isRename_) {
+ return false;
+ }
+ return super.shouldSkipWhenSyncingToLatestEventId();
+ }
+
+ /**
+ * If the ALTER_TABLE event is due a table rename, this method removes the old table
+ * and creates a new table with the new name. Else, this just issues a refresh
+ * table on the tblName from the event
+ */
+ @Override
+ public void processTableEvent() throws MetastoreNotificationException,
+ CatalogException {
+ if (isRename_) {
+ processRename();
+ return;
+ }
+
+ // Determine whether this is an event which we have already seen or if it is a new
+ // event
+ if (isSelfEvent()) {
+ infoLog("Not processing the event as it is a self-event");
+ return;
+ }
+ // Ignore the event if this is a trivial event. See javadoc for
+ // canBeSkipped() for examples.
+ if (canBeSkipped()) {
+ metrics_.getCounter(MetastoreEventsProcessor.EVENTS_SKIPPED_METRIC).inc();
+ infoLog("Not processing this event as it only modifies some table parameters "
+ + "which can be ignored.");
+ return;
+ }
+
+ if (isOlderEvent(null)) {
+ infoLog("Not processing the alter table event {} as it is an older event",
+ getEventId());
+ return;
+ }
+ skipFileMetadataReload_ = !isTruncateOp_ && canSkipFileMetadataReload(tableBefore_,
+ tableAfter_);
+ long startNs = System.nanoTime();
+ if (wasEventSyncTurnedOn()) {
+ handleEventSyncTurnedOn();
+ } else {
+ // in case of table level alters from external systems it is better to do a full
+ // refresh, eg. this could be due to as simple as adding a new parameter or a
+ // full blown adding or changing column type
+ // rename is already handled above
+ boolean notSkipped = reloadTableFromCatalog(false);
+ if (!notSkipped) {
+ metrics_.getCounter(MetastoreEventsProcessor.EVENTS_SKIPPED_METRIC).inc();
+ }
+ }
+ long durationNs = System.nanoTime() - startNs;
+ // Log event details for those triggered slow reload.
+ if (durationNs > HdfsTable.LOADING_WARNING_TIME_NS) {
+ warnLog("Slow event processing. Duration: {}. TableBefore: {}. " +
+ "TableAfter: {}", PrintUtils.printTimeNs(durationNs),
+ tableBefore_.toString(), tableAfter_.toString());
+ }
+ }
+
+ @Override
+ protected boolean onFailure(Exception e) {
+ if (!BackendConfig.INSTANCE.isInvalidateMetadataOnEventProcessFailureEnabled()
+ || !canInvalidateTable(e)) {
+ return false;
+ }
+ if (isRename()) {
+ /* In case of rename table, not invalidating tables due of following reasons:
+ 1. When failure happened before old table is removed from catalog,
+ invalidateTableIfExists may trigger table load for a non-existing table later.
+ 2. When failure happened before adding new table to catalog,
+ invalidateTableIfExists does not invalidate table
+ */
+ errorLog(
+ "Rename table {}.{} to {}.{} failed due to exception during event processing",
+ tableBefore_.getDbName(), tableBefore_.getTableName(), dbName_, tblName_, e);
+ return false;
+ }
+ return super.onFailure(e);
+ }
+
+ private void handleEventSyncTurnedOn() throws DatabaseNotFoundException,
+ MetastoreNotificationNeedsInvalidateException {
+ // check if the table exists or not. 1) if the table doesn't exist create an
+ // incomplete instance of the table. 2) If the table exists, there can be two
+ // scenarios a) current table eventId is greater than table's createEventId,
+ // then we should mark the table as stale. b) current table eventId <= table's
+ // createEventId, then we should ignore the event as it is an older event.
+ org.apache.impala.catalog.Table tbl = catalog_.getTableNoThrow(dbName_, tblName_);
+ if (tbl == null) { // table doesn't exist. Go with option (1)
+ if (catalogOpExecutor_.addTableIfNotRemovedLater(getEventId(), msTbl_)) {
+ infoLog("Successfully added table {}", getFullyQualifiedTblName());
+ metrics_.getCounter(MetastoreEventsProcessor.NUMBER_OF_TABLES_ADDED).inc();
+ } else {
+ metrics_.getCounter(MetastoreEventsProcessor.EVENTS_SKIPPED_METRIC).inc();
+ debugLog("Incremented skipped metric to " +
+ metrics_.getCounter(MetastoreEventsProcessor.EVENTS_SKIPPED_METRIC)
+ .getCount());
+ }
+ } else if (tbl instanceof IncompleteTable) {
+ // No-Op
+ } else if (getEventId() > tbl.getCreateEventId()) {
+ catalog_.invalidateTable(tbl.getTableName().toThrift(), new Reference<>(),
+ new Reference<>(), NoOpEventSequence.INSTANCE, getEventId());
+ LOG.info("Table " + tbl.getFullName() + " is invalidated from catalog cache" +
+ " since eventSync is turned on for this table.");
+ } else {
+ // Unknown state of metadata object, make event processor go into error state
+ throw new MetastoreNotificationNeedsInvalidateException(debugString(
+ "Detected that event sync was turned on for the table %s "
+ + "with createEventId %s. This event should have been skipped as stale "
+ + "event. Event processing cannot be continued further. Issue a "
+ + "invalidate metadata command to reset the event processing state",
+ getFullyQualifiedTblName(), tbl.getCreateEventId()));
+ }
+ }
+
+ /**
+ * This method checks if the reloading of file metadata can be skipped for an alter
+ * statement. This method accepts two arguments, 1) pre-modified HMS table instance
+ * 2) post-modified HMS table instance and compare what really changed in the alter
+ * event.
+ */
+ private boolean canSkipFileMetadataReload(
+ org.apache.hadoop.hive.metastore.api.Table beforeTable,
+ org.apache.hadoop.hive.metastore.api.Table afterTable) {
+ Set<String> whitelistedTblProperties = catalog_.getWhitelistedTblProperties();
+ // If the whitelisted table properties are empty, then we skip this optimization
+ if (whitelistedTblProperties.isEmpty()) {
+ return false;
+ }
+
+ boolean incrementalAcidRefresh =
+ BackendConfig.INSTANCE.getHMSEventIncrementalRefreshTransactionalTable();
+ boolean unpartitioned = afterTable.getPartitionKeysSize() == 0;
+ if (!incrementalAcidRefresh && unpartitioned
+ && AcidUtils.isTransactionalTable(afterTable.getParameters())) {
+ // In case of ACID tables no INSERT event is generated. If flag
+ // hms_event_incremental_refresh_transactional_table is false, then transaction
+ // related events are ignored (including COMMIT_TXN), so Impala has to rely on
+ // ALTER_TABLE events to detect INSERTs to unpartitioned tables (IMPALA-12835).
+ return false;
+ }
+
+ // There are lot of other alter statements which doesn't require file metadata
+ // reload but these are the most common types for alter statements.
+ if (!Objects.equals(beforeTable.getSd(), afterTable.getSd())) {
+ if (!isTrivialSdPropsChanged(beforeTable.getSd(), afterTable.getSd())) {
+ return false;
+ }
+ }
+ if (isCustomTblPropsChanged(whitelistedTblProperties, beforeTable, afterTable)) {
+ return false;
+ }
+ infoLog("Skipping reloading of file metadata for table {}.{} since SD and " +
+ "whitelistedTblProperties has not changed.", dbName_, tblName_);
+ return true;
+ }
+
+ private boolean isFieldSchemaChanged(
+ org.apache.hadoop.hive.metastore.api.Table beforeTable,
+ org.apache.hadoop.hive.metastore.api.Table afterTable) {
+ List<FieldSchema> beforeCols = beforeTable.getSd().getCols();
+ List<FieldSchema> afterCols = afterTable.getSd().getCols();
+ // check if columns are added or removed
+ if (beforeCols.size() != afterCols.size()) {
+ infoLog("Change in number of columns detected for table {}.{} from {} to {}. " +
+ "So file metadata reload can be skipped.", dbName_, tblName_,
+ beforeCols.size(), afterCols.size());
+ return true;
+ }
+ // check if columns are replaced or column definition is changed
+ // Field schema's comment is rarely used, so it'll be ignored
+ for (int i = 0; i < beforeCols.size(); i++) {
+ if (!beforeCols.get(i).getName().equals(afterCols.get(i).getName()) ||
+ !beforeCols.get(i).getType().equals(afterCols.get(i).getType())) {
+ infoLog("Change in table schema detected for table {}.{} from {} ({}) " +
+ "to {} ({}). So file metadata reload can be skipped.", dbName_, tblName_,
+ beforeCols.get(i).getName(), beforeCols.get(i).getType(),
+ afterCols.get(i).getName(), afterCols.get(i).getType());
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private boolean isTableOwnerChanged(String ownerBefore, String ownerAfter) {
+ if (!Objects.equals(ownerBefore, ownerAfter)) {
+ infoLog("Change in Ownership detected for table {}.{}, oldOwner: {}, " +
+ "newOwner: {}. So file metadata reload can be skipped.",
+ dbName_, tblName_, ownerBefore, ownerAfter);
+ return true;
+ }
+ return false;
+ }
+
+ // Check if the whitelisted properties are changed during the alter statement
+ private boolean isCustomTblPropsChanged(Set<String> whitelistedTblProperties,
+ org.apache.hadoop.hive.metastore.api.Table beforeTable,
+ org.apache.hadoop.hive.metastore.api.Table afterTable) {
+ for (String whitelistConfig : whitelistedTblProperties) {
+ String configBefore = beforeTable.getParameters().get(whitelistConfig);
+ String configAfter = afterTable.getParameters().get(whitelistConfig);
+ if (!Objects.equals(configBefore, configAfter)) {
+ infoLog("Change in whitelisted table properties detected for table {}.{} " +
+ "whitelisted config: {}, value before: {}, value after: {}. So file " +
+ "metadata should be reloaded.", dbName_, tblName_, whitelistConfig,
+ configBefore, configAfter);
+ return true;
+ }
+ }
+ return false;
+ }
+
+ // Check if the trivial SD properties are changed during the alter statement.
+ // Also, the caller should make sure that 'beforeSd' and 'afterSd' are not equal.
+ private boolean isTrivialSdPropsChanged(StorageDescriptor beforeSd,
+ StorageDescriptor afterSd) {
+ Preconditions.checkNotNull(beforeSd, "beforeSd is null");
+ Preconditions.checkNotNull(afterSd, "afterSd is null");
+ StorageDescriptor previousSD = normalizeStorageDescriptor(beforeSd.deepCopy());
+ StorageDescriptor currentSD = normalizeStorageDescriptor(afterSd.deepCopy());
+ if (!Objects.equals(previousSD, currentSD)) {
+ infoLog("Non-trivial change in table storage descriptor (SD) detected for " +
+ "table {}.{}. So file metadata should be reloaded. SD before: {}, SD " +
+ "after: {}", dbName_, tblName_, beforeSd.toString(), afterSd.toString());
+ return false;
+ }
+ infoLog("Trivial changes in table storage descriptor (SD) detected for table " +
+ "{}.{}. So file metadata reload can be skipped.", dbName_, tblName_);
+ return true;
+ }
+
+ /**
+ * Normalize the storage descriptor by unsetting the trivial fields in SD like
+ * columns, compressed, numBuckets, bucketCols,SkewedInfo, and
+ * setStoredAsSubDirectories.
+ */
+ private StorageDescriptor normalizeStorageDescriptor(StorageDescriptor sd) {
+ sd.unsetCols();
+ sd.unsetCompressed();
+ sd.unsetNumBuckets();
+ sd.unsetBucketCols();
+ sd.unsetSortCols();
+ sd.unsetSkewedInfo();
+ // setStoredAsSubDirectories = null or 'false' are trivial changes, so we normalize
+ // this value to false if it is null.
+ if (!sd.isSetStoredAsSubDirectories()) {
+ sd.setStoredAsSubDirectories(false);
+ }
+ return sd;
+ }
+
+ /**
+ * Detects a event sync flag was turned on in this event
+ */
+ private boolean wasEventSyncTurnedOn() {
+ // the eventsync flag was not changed
+ if (Objects.equals(eventSyncBeforeFlag_, eventSyncAfterFlag_)) return false;
+ // eventSync after flag is null or if it is explicitly set to false
+ if ((eventSyncAfterFlag_ == null && !dbFlagVal) || !eventSyncAfterFlag_) {
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ protected boolean canBeSkipped() {
+ // Certain alter events just modify some parameters such as
+ // "transient_lastDdlTime" in Hive. For eg: the alter table event generated
+ // along with insert events. Check if the alter table event is such a trivial
+ // event by setting those parameters equal before and after the event and
+ // comparing the objects.
+
+ // alter table event from truncate ops always can't be skipped.
+ if (isTruncateOp_) {
+ return false;
+ }
+
+ // Avoid modifying the object from event.
+ org.apache.hadoop.hive.metastore.api.Table tblAfter = tableAfter_.deepCopy();
+ setTrivialParameters(tableBefore_.getParameters(), tblAfter.getParameters());
+ return tblAfter.equals(tableBefore_);
+ }
+
+ private String qualify(TTableName tTableName) {
+ return new TableName(tTableName.db_name, tTableName.table_name).toString();
+ }
+
+ /**
+ * In case of alter table events, it is possible that the alter event is generated
+ * because user changed the value of the parameter
+ * <code>MetastoreEventPropertyKey.DISABLE_EVENT_HMS_SYNC</code>. If the
+ * parameter is unchanged, it doesn't
+ * matter if you use the before or after table object here since the eventual action
+ * is going be refresh or rename. If however, the parameter is changed, couple of
+ * things could happen. The flag changes from unset/false to true or it changes from
+ * true to false/unset. In the first case, we want to process the event (and ignore
+ * subsequent events on this table). In the second case, we should process the event
+ * (as well as all the subsequent events on the table). So we always process this
+ * event when the value of the flag is changed.
+ *
+ * @return true, if this event needs to be skipped. false if this event needs to be
+ * processed.
+ */
+ @Override
+ protected boolean isEventProcessingDisabled() {
+ // if the event sync flag was changed then we always process this event
+ if (!Objects.equals(eventSyncBeforeFlag_, eventSyncAfterFlag_)) {
+ infoLog("Before flag value {} after flag value {} changed for table {}",
+ eventSyncBeforeFlag_, eventSyncAfterFlag_, getFullyQualifiedTblName());
+ return false;
+ }
+ // flag is unchanged, use the default impl from base class
+ return super.isEventProcessingDisabled();
+ }
+ }
+
+ /**
+ * MetastoreEvent for the DROP_TABLE event type
+ */
+ public static class DropTableEvent extends MetastoreTableEvent {
+
+ public static final String EVENT_TYPE = "DROP_TABLE";
+
+ /**
+ * Prevent instantiation from outside should use MetastoreEventFactory instead
+ */
+ private DropTableEvent(CatalogOpExecutor catalogOpExecutor, Metrics metrics,
+ NotificationEvent event) throws MetastoreNotificationException {
+ super(catalogOpExecutor, metrics, event);
+ Preconditions.checkArgument(MetastoreEventType.DROP_TABLE.equals(getEventType()));
+ JSONDropTableMessage dropTableMessage =
+ (JSONDropTableMessage) MetastoreEventsProcessor.getMessageDeserializer()
+ .getDropTableMessage(event.getMessage());
+ try {
+ String dbName = dropTableMessage.getDB();
+ String tableName = dropTableMessage.getTable();
+ msTbl_ = new Table();
+ msTbl_.setDbName(dbName);
+ msTbl_.setTableName(tableName);
+ } catch (Exception e) {
+ throw new MetastoreNotificationException(debugString(
+ "Could not parse event message. "
+ + "Check if %s is set to true in metastore configuration",
+ MetastoreEventsProcessor.HMS_ADD_THRIFT_OBJECTS_IN_EVENTS_CONFIG_KEY), e);
+ }
+ }
+
+ protected DropTableEvent(CatalogOpExecutor catalogOpExecutor, Metrics metrics,
+ NotificationEvent event, org.apache.hadoop.hive.metastore.api.Table table) {
+ super(catalogOpExecutor, metrics, event);
+ Preconditions.checkArgument(getEventType() == MetastoreEventType.DROP_TABLE);
+ msTbl_ = table;
+ }
+
+ @Override
+ public SelfEventContext getSelfEventContext() {
+ throw new UnsupportedOperationException("self-event evaluation is not needed for "
+ + "this event type");
+ }
+
+ /**
+ * Process the drop table event type. If the table from the event doesn't exist in the
+ * catalog, ignore the event. If the table exists in the catalog, compares the
+ * createTime of the table in catalog with the createTime of the table from the event
+ * and remove the catalog table if there is a match. If the catalog table is a
+ * incomplete table it is removed as well. The creation_time from HMS is unfortunately
+ * in seconds granularity, which means there is a limitation that we cannot
+ * distinguish between tables which are created with the same name within a second.
+ * So a sequence of create_table, drop_table, create_table happening within the
+ * same second might cause false positives on drop_table event processing. This is
+ * not a huge problem since the tables will eventually be created when the
+ * create events are processed but there will be a non-zero amount of time when the
+ * table will not be existing in catalog.
+ * TODO: IMPALA-12646, to track average process time for drop operations.
+ */
+ @Override
+ public void processTableEvent() throws MetastoreNotificationException {
+ Reference<Boolean> tblRemovedLater = new Reference<>();
+ boolean removedTable;
+ removedTable = catalogOpExecutor_
+ .removeTableIfNotAddedLater(getEventId(), msTbl_.getDbName(),
+ msTbl_.getTableName(), tblRemovedLater);
+ if (removedTable) {
+ infoLog("Successfully removed table {}", getFullyQualifiedTblName());
+ metrics_.getCounter(MetastoreEventsProcessor.NUMBER_OF_TABLES_REMOVED).inc();
+ } else {
+ metrics_.getCounter(MetastoreEventsProcessor.EVENTS_SKIPPED_METRIC).inc();
+ debugLog("Incremented skipped metric to " + metrics_
+ .getCounter(MetastoreEventsProcessor.EVENTS_SKIPPED_METRIC).getCount());
+ }
+ }
+
+ @Override
+ protected boolean onFailure(Exception e) {
+ return false;
+ }
+ }
+
+ /**
+ * MetastoreEvent for CREATE_DATABASE event type
+ */
+ public static class CreateDatabaseEvent extends MetastoreDatabaseEvent {
+
+ public static final String EVENT_TYPE = "CREATE_DATABASE";
+ // metastore database object as parsed from NotificationEvent message
+ private final Database createdDatabase_;
+
+ /**
+ * Prevent instantiation from outside should use MetastoreEventFactory instead
+ */
+ private CreateDatabaseEvent(CatalogOpExecutor catalogOpExecutor, Metrics metrics,
+ NotificationEvent event) throws MetastoreNotificationException {
+ super(catalogOpExecutor, metrics, event);
+ Preconditions.checkArgument(
+ MetastoreEventType.CREATE_DATABASE.equals(getEventType()));
+ JSONCreateDatabaseMessage createDatabaseMessage =
+ (JSONCreateDatabaseMessage) MetastoreEventsProcessor.getMessageDeserializer()
+ .getCreateDatabaseMessage(event.getMessage());
+ try (MetaStoreClientPool.MetaStoreClient client = catalog_.getMetaStoreClient()) {
+ String dbName = createDatabaseMessage.getDB();
+ createdDatabase_ = Preconditions.checkNotNull(client.getHiveClient()
+ .getDatabase(dbName));
+ } catch (Exception e) {
+ throw new MetastoreNotificationException(debugString(
+ "Database object is null in the event. "
+ + "This could be a metastore configuration problem. "
+ + "Check if %s is set to true in metastore configuration",
+ MetastoreEventsProcessor.HMS_ADD_THRIFT_OBJECTS_IN_EVENTS_CONFIG_KEY), e);
+ }
+ }
+
+ public Database getDatabase() { return createdDatabase_; }
+
+ @Override
+ public SelfEventContext getSelfEventContext() {
+ throw new UnsupportedOperationException("Self-event evaluation is unnecessary for"
+ + " this event type");
+ }
+
+ /**
+ * Processes the create database event by adding the Db object from the event if it
+ * does not exist in the catalog already.
+ */
+ @Override
+ public void process() {
+ boolean dbAdded = catalogOpExecutor_
+ .addDbIfNotRemovedLater(getEventId(), createdDatabase_);
+ if (!dbAdded) {
+ debugLog(
+ "Database {} was not added since it either exists or was "
+ + "removed since the event was generated", dbName_);
+ metrics_.getCounter(MetastoreEventsProcessor.EVENTS_SKIPPED_METRIC).inc();
+ debugLog("Incremented skipped metric to " + metrics_
+ .getCounter(MetastoreEventsProcessor.EVENTS_SKIPPED_METRIC).getCount());
+ } else {
+ infoLog("Successfully added database {}", dbName_);
+ metrics_.getCounter(MetastoreEventsProcessor.NUMBER_OF_DATABASES_ADDED).inc();
+ }
+ }
+ }
+
+ /**
+ * MetastoreEvent for ALTER_DATABASE event type
+ */
+ public static class AlterDatabaseEvent extends MetastoreDatabaseEvent {
+ public static final String EVENT_TYPE = "ALTER_DATABASE";
+ // metastore database object as parsed from NotificationEvent message
+ private final Database alteredDatabase_;
+
+ /**
+ * Prevent instantiation from outside should use MetastoreEventFactory instead
+ */
+ private AlterDatabaseEvent(CatalogOpExecutor catalogOpExecutor, Metrics metrics,
+ NotificationEvent event) throws MetastoreNotificationException {
+ super(catalogOpExecutor, metrics, event);
+ throw new UnsupportedOperationException("AlterDatabaseEvent is not supported.");
+ }
+
+ /**
+ * Processes the alter database event by replacing the catalog cached Db object with
+ * the Db object from the event
+ */
+ @Override
+ public void process() throws CatalogException {
+ throw new UnsupportedOperationException("AlterDatabaseEvent is not supported.");
+ }
+
+ @Override
+ protected SelfEventContext getSelfEventContext() {
+ throw new UnsupportedOperationException("AlterDatabaseEvent is not supported.");
+ }
+
+ /**
+ * Skip processing this event if either db does not exist in cache or is already
+ * synced atleast to this event id.
+ * @return
+ * @throws CatalogException
+ */
+ @Override
+ protected boolean shouldSkipWhenSyncingToLatestEventId() throws CatalogException {
+ Preconditions.checkState(
+ BackendConfig.INSTANCE.enableSyncToLatestEventOnDdls(),
+ "sync to latest event id flag should be set");
+ long eventId = this.getEventId();
+ Db db = catalog_.getDb(getDbName());
+ if (db == null) {
+ infoLog("Skipping since db {} does not exist in cache", getDbName());
+ return true;
+ }
+ if (!catalog_.tryLockDb(db)) {
+ throw new CatalogException(String.format("Couldn't acquire lock on db %s",
+ db.getName()));
+ }
+ catalog_.getLock().writeLock().unlock();
+ boolean shouldSkip = false;
+ if (db.getLastSyncedEventId() >= eventId) {
+ infoLog("Skipping on db {} since db is already synced till event id {}",
+ getDbName(), db.getLastSyncedEventId());
+ shouldSkip = true;
+ }
+ db.getLock().unlock();
+ return shouldSkip;
+ }
+ }
+
+ /**
+ * MetastoreEvent for the DROP_DATABASE event
+ */
+ public static class DropDatabaseEvent extends MetastoreDatabaseEvent {
+
+ public static final String EVENT_TYPE = "DROP_DATABASE";
+ // Metastore database object as parsed from NotificationEvent message
+ private final Database droppedDatabase_;
+ /**
+ * Prevent instantiation from outside should use MetastoreEventFactory instead
+ */
+ private DropDatabaseEvent(
+ CatalogOpExecutor catalogOpExecutor, Metrics metrics, NotificationEvent event)
+ throws MetastoreNotificationException {
+ super(catalogOpExecutor, metrics, event);
+ Preconditions.checkArgument(
+ MetastoreEventType.DROP_DATABASE.equals(getEventType()));
+ JSONDropDatabaseMessage dropDatabaseMessage =
+ (JSONDropDatabaseMessage) MetastoreEventsProcessor.getMessageDeserializer()
+ .getDropDatabaseMessage(event.getMessage());
+ try {
+ droppedDatabase_ =
+ Preconditions
+ .checkNotNull(MetastoreShim.getDatabaseObject(dropDatabaseMessage));
+ } catch (Exception e) {
+ throw new MetastoreNotificationException(debugString(
+ "Database object is null in the event. "
+ + "This could be a metastore configuration problem. "
+ + "Check if %s is set to true in metastore configuration",
+ MetastoreEventsProcessor.HMS_ADD_THRIFT_OBJECTS_IN_EVENTS_CONFIG_KEY), e);
+ }
+ }
+
+ @Override
+ public SelfEventContext getSelfEventContext() {
+ throw new UnsupportedOperationException("Self-event evaluation is not needed for "
+ + "this event");
+ }
+
+ @Override
+ protected boolean shouldSkipWhenSyncingToLatestEventId() {
+ return false;
+ }
+
+ /**
+ * Process the drop database event. This handler removes the db object from catalog
+ * only if the CREATION_TIME of the catalog's database object is lesser than or equal
+ * to that of the database object present in the notification event. If the
+ * CREATION_TIME of the catalog's DB object is greater than that of the notification
+ * event's DB object, it means that the Database object present in the catalog is a
+ * later version and we can skip the event. (For instance, when user does a create db,
+ * drop db and create db again with the same dbName.).
+ * The creation_time from HMS is unfortunately in seconds granularity, which means
+ * there is a limitation that we cannot distinguish between databases which are
+ * created with the same name within a second. So a sequence of create_database,
+ * drop_database, create_database happening within the same second might cause
+ * false positives on drop_database event processing. This is not a huge problem
+ * since the databases will eventually be created when the create events are
+ * processed but there will be a non-zero amount of time when the database will not
+ * be existing in catalog.
+ */
+ @Override
+ public void process() {
+ boolean dbRemoved = catalogOpExecutor_
+ .removeDbIfNotAddedLater(getEventId(), droppedDatabase_.getName());
+ if (dbRemoved) {
+ infoLog("Removed Database {} ", dbName_);
+ metrics_.getCounter(MetastoreEventsProcessor.NUMBER_OF_DATABASES_REMOVED).inc();
+ } else {
+ metrics_.getCounter(MetastoreEventsProcessor.EVENTS_SKIPPED_METRIC).inc();
+ debugLog("Incremented skipped metric to " + metrics_
+ .getCounter(MetastoreEventsProcessor.EVENTS_SKIPPED_METRIC).getCount());
+ }
+ }
+ }
+
+ /**
+ * Returns a list of parameters that are set by Hive for tables/partitions that can be
+ * ignored to determine if the alter table/partition event is a trivial one.
+ */
+ @VisibleForTesting
+ static final List<String> parametersToIgnore =
+ new ImmutableList.Builder<String>()
+ .add("transient_lastDdlTime")
+ .add("totalSize")
+ .add("numFilesErasureCoded")
+ .add("numFiles")
+ .build();
+
+ /**
+ * Util method that sets the parameters that can be ignored equal before and after
+ * event.
+ */
+ private static void setTrivialParameters(Map<String, String> parametersBefore,
+ Map<String, String> parametersAfter) {
+ for (String parameter: parametersToIgnore) {
+ String val = parametersBefore.get(parameter);
+ if (val == null) {
+ parametersAfter.remove(parameter);
+ } else {
+ parametersAfter.put(parameter, val);
+ }
+ }
+ }
+ public static class AddPartitionEvent extends MetastoreTableEvent {
+
+ public static final String EVENT_TYPE = "ADD_PARTITION";
+ private final List<Partition> addedPartitions_;
+ private final List<List<TPartitionKeyValue>> partitionKeyVals_;
+
+ /**
+ * Prevent instantiation from outside should use MetastoreEventFactory instead
+ */
+ private AddPartitionEvent(CatalogOpExecutor catalogOpExecutor, Metrics metrics,
+ NotificationEvent event) throws MetastoreNotificationException {
+ super(catalogOpExecutor, metrics, event);
+ Preconditions.checkState(getEventType().equals(MetastoreEventType.ADD_PARTITION));
+ if (event.getMessage() == null) {
+ throw new IllegalStateException(debugString("Event message is null"));
+ }
+ try {
+ AddPartitionMessage addPartitionMessage_ =
+ MetastoreEventsProcessor.getMessageDeserializer()
+ .getAddPartitionMessage(event.getMessage());
+ addedPartitions_ =
+ Lists.newArrayList(addPartitionMessage_.getPartitionObjs());
+ // it is possible that the added partitions is empty in certain cases. See
+ // IMPALA-8847 for example
+ msTbl_ = addPartitionMessage_.getTableObj();
+ MetaStoreUtil.replaceSchemaFromTable(addedPartitions_, msTbl_);
+ partitionKeyVals_ = new ArrayList<>(addedPartitions_.size());
+ for (Partition part : addedPartitions_) {
+ partitionKeyVals_.add(getTPartitionSpecFromHmsPartition(msTbl_, part));
+ }
+ } catch (Exception ex) {
+ throw new MetastoreNotificationException(ex);
+ }
+ }
+
+ @Override
+ public SelfEventContext getSelfEventContext() {
+ // self event evaluation is only done for transactional tables currently.
+ // for non-transactional tables we use the partition level createEventId
+ if (AcidUtils.isTransactionalTable(msTbl_.getParameters())) {
+ Map<String, String> params = new HashMap<>();
+ // all the partitions are added as one transaction and hence we expect all the
+ // added partitions to have the same catalog service identifiers. Using the first
+ // one for the params is enough for the purpose of self-event evaluation
+ if (!addedPartitions_.isEmpty()) {
+ params.putAll(addedPartitions_.get(0).getParameters());
+ }
+ return new SelfEventContext(dbName_, tblName_, partitionKeyVals_,
+ params);
+ }
+ throw new UnsupportedOperationException("Self-event evaluation is unnecessary for"
+ + " this event type");
+ }
+
+ @Override
+ public void processTableEvent() throws MetastoreNotificationException,
+ CatalogException {
+ // bail out early if there are not partitions to process
+ if (addedPartitions_.isEmpty()) {
+ metrics_.getCounter(MetastoreEventsProcessor.EVENTS_SKIPPED_METRIC).inc();
+ infoLog("Partition list is empty. Ignoring this event.");
+ return;
+ }
+ try {
+ // Reload the whole table if it's a transactional table and incremental
+ // refresh is not enabled. Materialized views are treated as a special case
+ // because it's possible to receive partition event on MVs, but they are
+ // regular views in Impala. That cause problems on the reloading partition
+ // logic which expects it to be a HdfsTable.
+ boolean incrementalRefresh =
+ BackendConfig.INSTANCE.getHMSEventIncrementalRefreshTransactionalTable();
+ if ((AcidUtils.isTransactionalTable(msTbl_.getParameters()) && !isSelfEvent() &&
+ !incrementalRefresh) || MetaStoreUtils.isMaterializedViewTable(msTbl_)) {
+ boolean notSkipped = reloadTableFromCatalog(true);
+ if (!notSkipped) {
+ metrics_.getCounter(MetastoreEventsProcessor.EVENTS_SKIPPED_METRIC).inc();
+ }
+ } else {
+ // HMS adds partitions in a transactional way. This means there may be multiple
+ // HMS partition objects in an add_partition event. We try to do the same here
+ // by refreshing all those partitions in a loop. If any partition refresh fails,
+ // we throw MetastoreNotificationNeedsInvalidateException exception. We skip
+ // refresh of the partitions if the table is not present in the catalog.
+ int numPartsAdded = catalogOpExecutor_
+ .addPartitionsIfNotRemovedLater(getEventId(), dbName_, tblName_,
+ addedPartitions_, "ADD_PARTITION");
+ if (numPartsAdded != 0) {
+ infoLog("Successfully added {} partitions to table {}",
+ numPartsAdded, getFullyQualifiedTblName());
+ metrics_.getCounter(MetastoreEventsProcessor.NUMBER_OF_PARTITIONS_ADDED)
+ .inc(numPartsAdded);
+ } else {
+ metrics_.getCounter(MetastoreEventsProcessor.EVENTS_SKIPPED_METRIC).inc();
+ debugLog("Incremented skipped metric to {} since no partitions were added.",
+ metrics_.getCounter(MetastoreEventsProcessor.EVENTS_SKIPPED_METRIC)
+ .getCount());
+ }
+ }
+ } catch (CatalogException e) {
+ throw new MetastoreNotificationNeedsInvalidateException(debugString("Failed to "
+ + "refresh newly added partitions of table '%s'. Event processing cannot "
+ + "continue. Issue an invalidate metadata command to reset event "
+ + "processor.", getFullyQualifiedTblName()), e);
+ }
+ }
+
+ public List<Partition> getPartitions() {
+ return addedPartitions_;
+ }
+ }
+
+ public static class AlterPartitionEvent extends MetastoreTableEvent {
+ public static final String EVENT_TYPE = "ALTER_PARTITION";
+ // the Partition object before alter operation, as parsed from the NotificationEvent
+ private final org.apache.hadoop.hive.metastore.api.Partition partitionBefore_;
+ // the Partition object after alter operation, as parsed from the NotificationEvent
+ private final org.apache.hadoop.hive.metastore.api.Partition partitionAfter_;
+ // the version number from the partition parameters of the event.
+ private final long versionNumberFromEvent_;
+ // the service id from the partition parameters of the event.
+ private final String serviceIdFromEvent_;
+ // true if this alter event was due to a truncate operation in metastore
+ private final boolean isTruncateOp_;
+ private final String partName_;
+
+ /**
+ * Prevent instantiation from outside should use MetastoreEventFactory instead
+ */
+ private AlterPartitionEvent(CatalogOpExecutor catalogOpExecutor, Metrics metrics,
+ NotificationEvent event) throws MetastoreNotificationException {
+ super(catalogOpExecutor, metrics, event);
+ Preconditions.checkState(getEventType().equals(MetastoreEventType.ALTER_PARTITION));
+ Preconditions.checkNotNull(event.getMessage());
+ AlterPartitionMessage alterPartitionMessage =
+ MetastoreEventsProcessor.getMessageDeserializer()
+ .getAlterPartitionMessage(event.getMessage());
+
+ try (MetaStoreClientPool.MetaStoreClient client = catalog_.getMetaStoreClient()) {
+ partitionBefore_ =
+ Preconditions.checkNotNull(alterPartitionMessage.getPtnObjBefore());
+ partitionAfter_ =
+ Preconditions.checkNotNull(alterPartitionMessage.getPtnObjAfter());
+ isTruncateOp_ = false;
+ String dbName = alterPartitionMessage.getDB();
+ String tableName = alterPartitionMessage.getTable();
+ msTbl_ = client.getHiveClient().getTable(dbName, tableName);
+ Map<String, String> parameters = partitionAfter_.getParameters();
+ versionNumberFromEvent_ = Long.parseLong(
+ MetastoreEvents.getStringProperty(parameters,
+ MetastoreEventPropertyKey.CATALOG_VERSION.getKey(), "-1"));
+ serviceIdFromEvent_ = MetastoreEvents.getStringProperty(
+ parameters, MetastoreEventPropertyKey.CATALOG_SERVICE_ID.getKey(), "");
+ partName_ = HdfsTable.constructPartitionName(getTPartitionSpecFromHmsPartition(
+ msTbl_, partitionAfter_));
+ } catch (Exception e) {
+ throw new MetastoreNotificationException(
+ debugString("Unable to parse the alter partition message"), e);
+ }
+ }
+
+ public String getPartName() { return partName_; }
+
+ @Override
+ protected MetastoreEventType getBatchEventType() {
+ return MetastoreEventType.ALTER_PARTITIONS;
+ }
+
+ @Override
+ protected Partition getPartitionForBatching() { return partitionAfter_; }
+
+ @Override
+ public boolean canBeBatched(MetastoreEvent event) {
+ if (!(event instanceof AlterPartitionEvent)) return false;
+ if (isOlderThanLastSyncEventId(event)) return false;
+ AlterPartitionEvent alterPartitionEvent = (AlterPartitionEvent) event;
+ // make sure that the event is on the same table
+ if (!getFullyQualifiedTblName().equalsIgnoreCase(
+ alterPartitionEvent.getFullyQualifiedTblName())) {
+ return false;
+ }
+
+ // in case of ALTER_PARTITION we only batch together the events
+ // which have same versionNumber and serviceId from the event. This simplifies
+ // the self-event evaluation for the batch since either the whole batch is
+ // self-events or not.
+ Map<String, String> parametersFromEvent =
+ alterPartitionEvent.partitionAfter_.getParameters();
+ long versionNumberOfEvent = Long.parseLong(
+ MetastoreEvents.getStringProperty(parametersFromEvent,
+ MetastoreEventPropertyKey.CATALOG_VERSION.getKey(), "-1"));
+ String serviceIdOfEvent = MetastoreEvents.getStringProperty(parametersFromEvent,
+ MetastoreEventPropertyKey.CATALOG_SERVICE_ID.getKey(), "");
+ return versionNumberFromEvent_ == versionNumberOfEvent
+ && serviceIdFromEvent_.equals(serviceIdOfEvent);
+ }
+
+ @Override
+ public MetastoreEvent addToBatchEvents(MetastoreEvent event) {
+ if (!(event instanceof AlterPartitionEvent)) return null;
+ BatchPartitionEvent<AlterPartitionEvent> batchEvent = new BatchPartitionEvent<>(
+ this);
+ Preconditions.checkState(batchEvent.canBeBatched(event));
+ batchEvent.addToBatchEvents(event);
+ return batchEvent;
+ }
+
+ @Override
+ public void processTableEvent() throws MetastoreNotificationException,
+ CatalogException {
+ if (isSelfEvent()) {
+ infoLog("Not processing the event as it is a self-event");
+ return;
+ }
+
+ // Ignore the event if this is a trivial event. See javadoc for
+ // isTrivialAlterPartitionEvent() for examples.
+ if (canBeSkipped()) {
+ metrics_.getCounter(MetastoreEventsProcessor.EVENTS_SKIPPED_METRIC).inc();
+ infoLog("Not processing this event as it only modifies some partition "
+ + "parameters which can be ignored.");
+ return;
+ }
+
+ if (isOlderEvent(partitionBefore_)) {
+ infoLog("Not processing the alter partition event {} as it is an older event",
+ getEventId());
+ return;
+ }
+ // Reload the whole table if it's a transactional table or materialized view.
+ // Materialized views are treated as a special case because it's possible to
+ // receive partition event on MVs, but they are regular views in Impala. That
+ // cause problems on the reloading partition logic which expects it to be a
+ // HdfsTable.
+ if (AcidUtils.isTransactionalTable(msTbl_.getParameters())
+ || MetaStoreUtils.isMaterializedViewTable(msTbl_)) {
+ reloadTransactionalTable();
+ } else {
+ // Refresh the partition that was altered.
+ Preconditions.checkNotNull(partitionAfter_);
+ try {
+ // load file metadata only if storage descriptor of partitionAfter_ differs
+ // from sd of HdfsPartition. If the alter_partition event type is of truncate
+ // then force load the file metadata.
+ FileMetadataLoadOpts fileMetadataLoadOpts =
+ isTruncateOp_ ? FileMetadataLoadOpts.FORCE_LOAD :
+ FileMetadataLoadOpts.LOAD_IF_SD_CHANGED;
+ reloadPartitions(Arrays.asList(partitionAfter_), fileMetadataLoadOpts,
+ getEventDesc(), false);
+ } catch (CatalogException e) {
+ throw new MetastoreNotificationNeedsInvalidateException(
+ debugString("Refresh partition on table {} partition {} failed.",
+ getFullyQualifiedTblName(), partName_), e);
+ }
+ }
+ }
+
+ @Override
+ protected boolean canBeSkipped() {
+ // Certain alter events just modify some parameters such as
+ // "transient_lastDdlTime" in Hive. For eg: the alter table event generated
+ // along with insert events. Check if the alter table event is such a trivial
+ // event by setting those parameters equal before and after the event and
+ // comparing the objects.
+
+ // alter partition event from truncate ops always can't be skipped.
+ if (isTruncateOp_) {
+ return false;
+ }
+
+ // Avoid modifying the object from event.
+ Partition afterPartition = partitionAfter_.deepCopy();
+ setTrivialParameters(partitionBefore_.getParameters(),
+ afterPartition.getParameters());
+ return afterPartition.equals(partitionBefore_);
+ }
+
+ @Override
+ public SelfEventContext getSelfEventContext() {
+ return new SelfEventContext(dbName_, tblName_,
+ Arrays.asList(getTPartitionSpecFromHmsPartition(msTbl_, partitionAfter_)),
+ partitionAfter_.getParameters());
+ }
+
+ private void reloadTransactionalTable() throws CatalogException {
+ boolean incrementalRefresh =
+ BackendConfig.INSTANCE.getHMSEventIncrementalRefreshTransactionalTable();
+ if (incrementalRefresh) {
+ reloadPartitionsFromEvent(Collections.singletonList(partitionAfter_),
+ getEventDesc() + " FOR TRANSACTIONAL TABLE");
+ } else {
+ boolean notSkipped = reloadTableFromCatalog(true);
+ if (!notSkipped) {
+ metrics_.getCounter(MetastoreEventsProcessor.EVENTS_SKIPPED_METRIC).inc();
+ }
+ }
+ }
+ }
+
+ /**
+ * This event represents a batch of events of type T. The batch of events is
+ * initialized from a single initial event called baseEvent. More events can be added
+ * to the batch using {@code addToBatchEvents} method. Currently we only support
+ * ALTER_PARTITION and INSERT partition events for batching.
+ * @param <T> The type of event which is batched by this event.
+ */
+ public static class BatchPartitionEvent<T extends MetastoreTableEvent> extends
+ MetastoreTableEvent {
+ private final T baseEvent_;
+ private final List<T> batchedEvents_ = new ArrayList<>();
+
+ private BatchPartitionEvent(T baseEvent) {
+ super(baseEvent.catalogOpExecutor_, baseEvent.metrics_, baseEvent.event_);
+ this.msTbl_ = baseEvent.msTbl_;
+ this.baseEvent_ = baseEvent;
+ batchedEvents_.add(baseEvent);
+ // override the eventType_ to represent that this is a batch of events.
+ setEventType(baseEvent.getBatchEventType());
+ }
+
+ @Override
+ public MetastoreEvent addToBatchEvents(MetastoreEvent event) {
+ Preconditions.checkState(canBeBatched(event));
+ batchedEvents_.add((T) event);
+ return this;
+ }
+
+ @Override
+ public int getNumberOfEvents() { return batchedEvents_.size(); }
+
+ /**
+ * Return the event id of this batch event. We return the last eventId
+ * from this batch which is important since it is used to determined the event
+ * id for fetching next set of events from metastore.
+ */
+ @Override
+ public long getEventId() {
+ Preconditions.checkState(!batchedEvents_.isEmpty());
+ return batchedEvents_.get(batchedEvents_.size()-1).getEventId();
+ }
+
+ /**
+ * Same as the above but returns the event time.
+ */
+ @Override
+ public long getEventTime() {
+ Preconditions.checkState(!batchedEvents_.isEmpty());
+ return batchedEvents_.get(batchedEvents_.size() - 1).getEventTime();
+ }
+
+ @Override
+ public String getEventDesc() {
+ if (batchedEvents_.isEmpty()) return "empty event batch";
+ int batch = batchedEvents_.size();
+ // E.g. ALTER_PARTITIONS 4 events from 76697 to 76703
+ return String.format("%s %d events from %d to %d", getEventType(), batch,
+ batchedEvents_.get(0).getEventId(), batchedEvents_.get(batch - 1).getEventId());
+ }
+
+ /**
+ *
+ * @param event The event under consideration to be batched into this event. It can
+ * be added to the batch if it can be batched into the last event of the
+ * current batch.
+ * @return true if we can add the event to the current batch; else false.
+ */
+ @Override
+ public boolean canBeBatched(MetastoreEvent event) {
+ Preconditions.checkState(!batchedEvents_.isEmpty());
+ return batchedEvents_.get(batchedEvents_.size()-1).canBeBatched(event);
+ }
+
+ @VisibleForTesting
+ List<T> getBatchEvents() { return batchedEvents_; }
+
+ @Override
+ protected void processTableEvent() throws MetastoreNotificationException,
+ CatalogException {
+ if (isSelfEvent()) {
+ infoLog("Not processing the event as it is a self-event");
+ return;
+ }
+ // Ignore the event if this is a trivial event. See javadoc for
+ // isTrivialAlterPartitionEvent() for examples.
+ List<T> eventsToProcess = new ArrayList<>();
+ List<Partition> partitionEventsToForceReload = new ArrayList<>();
+ for (T event : batchedEvents_) {
+ if (isOlderEvent(event.getPartitionForBatching())) {
+ infoLog("Not processing the current event id {} as it is an older event",
+ event.getEventId());
+ continue;
+ }
+ boolean isTruncateOp = (event instanceof AlterPartitionEvent &&
+ ((AlterPartitionEvent)event).isTruncateOp_);
+ if (isTruncateOp) {
+ partitionEventsToForceReload.add(event.getPartitionForBatching());
+ } else if (!event.canBeSkipped()){
+ eventsToProcess.add(event);
+ }
+ }
+ int notSkippedNum = eventsToProcess.size() + partitionEventsToForceReload.size();
+ int skippedNum = batchedEvents_.size() - notSkippedNum;
+ if (skippedNum > 0) {
+ metrics_.getCounter(MetastoreEventsProcessor.EVENTS_SKIPPED_METRIC)
+ .inc(skippedNum);
+ }
+ if (eventsToProcess.isEmpty() && partitionEventsToForceReload.isEmpty()) {
+ LOG.info(
+ "Ignoring {} events between event id {} and {} since they modify parameters"
+ + " which can be ignored", getNumberOfEvents(), getFirstEventId(),
+ getLastEventId());
+ return;
+ }
+
+ // Reload the whole table if it's a transactional table.
+ if (AcidUtils.isTransactionalTable(msTbl_.getParameters())) {
+ boolean notSkipped = reloadTableFromCatalog(true);
+ if (!notSkipped) {
+ metrics_.getCounter(MetastoreEventsProcessor.EVENTS_SKIPPED_METRIC)
+ .inc(eventsToProcess.size() + partitionEventsToForceReload.size());
+ }
+ } else {
+ // Reload the partitions from the batch.
+ List<Partition> partitions = new ArrayList<>();
+ for (T event : eventsToProcess) {
+ partitions.add(event.getPartitionForBatching());
+ }
+ try {
+ if (baseEvent_ instanceof InsertEvent) {
+ // for insert event, always reload file metadata so that new files
+ // are reflected in HdfsPartition
+ reloadPartitions(partitions, FileMetadataLoadOpts.FORCE_LOAD, getEventDesc(),
+ true);
+ } else {
+ if (!partitionEventsToForceReload.isEmpty()) {
+ // force reload truncated partitions
+ reloadPartitions(partitionEventsToForceReload,
+ FileMetadataLoadOpts.FORCE_LOAD, getEventDesc() + " FOR TRUNCATE",
+ true);
+ }
+ if (!partitions.isEmpty()) {
+ // alter partition event. Reload file metadata of only those partitions
+ // for which sd has changed
+ reloadPartitions(partitions,
+ FileMetadataLoadOpts.LOAD_IF_SD_CHANGED, getEventDesc(), true);
+ }
+ }
+ } catch (CatalogException e) {
+ throw new MetastoreNotificationNeedsInvalidateException(String.format(
+ "Refresh partitions on table %s failed when processing a batch of %s "
+ + "events between event ids %s and %s. "
+ + "Issue an invalidate command to reset the event processor state.",
+ getFullyQualifiedTblName(), getNumberOfEvents(), getFirstEventId(),
+ getLastEventId()), e);
+ }
+ }
+ }
+
+ /**
+ * Gets the event id of the first event in the batch.
+ */
+ private long getFirstEventId() {
+ return batchedEvents_.get(0).getEventId();
+ }
+
+ /**
+ * Gets the event id of the last event in the batch.
+ */
+ private long getLastEventId() {
+ return batchedEvents_.get(batchedEvents_.size()-1).getEventId();
+ }
+
+ @Override
+ protected SelfEventContext getSelfEventContext() {
+ List<List<TPartitionKeyValue>> partitionKeyValues = new ArrayList<>();
+ List<Long> eventIds = new ArrayList<>();
+ // We treat insert event as a special case since the self-event context for an
+ // insert event is generated differently using the eventIds.
+ boolean isInsertEvent = baseEvent_ instanceof InsertEvent;
+ for (T event : batchedEvents_) {
+ partitionKeyValues.add(
+ getTPartitionSpecFromHmsPartition(event.msTbl_,
+ event.getPartitionForBatching()));
+ eventIds.add(event.getEventId());
+ }
+ return new SelfEventContext(dbName_, tblName_, partitionKeyValues,
+ baseEvent_.getPartitionForBatching().getParameters(),
+ isInsertEvent ? eventIds : null);
+ }
+ }
+
+ public static class DropPartitionEvent extends MetastoreTableEvent {
+ private final List<Map<String, String>> droppedPartitions_;
+ public static final String EVENT_TYPE = "DROP_PARTITION";
+
+ /**
+ * Prevent instantiation from outside should use MetastoreEventFactory instead
+ */
+ private DropPartitionEvent(CatalogOpExecutor catalogOpExecutor, Metrics metrics,
+ NotificationEvent event) throws MetastoreNotificationException {
+ super(catalogOpExecutor, metrics, event);
+ Preconditions.checkState(getEventType().equals(MetastoreEventType.DROP_PARTITION));
+ Preconditions.checkNotNull(event.getMessage());
+ DropPartitionMessage dropPartitionMessage =
+ MetastoreEventsProcessor.getMessageDeserializer()
+ .getDropPartitionMessage(event.getMessage());
+ try {
+ msTbl_ = Preconditions.checkNotNull(dropPartitionMessage.getTableObj());
+ droppedPartitions_ = dropPartitionMessage.getPartitions();
+ Preconditions.checkNotNull(droppedPartitions_);
+ } catch (Exception ex) {
+ throw new MetastoreNotificationException(
+ debugString("Could not parse event message. "
+ + "Check if %s is set to true in metastore configuration",
+ MetastoreEventsProcessor.HMS_ADD_THRIFT_OBJECTS_IN_EVENTS_CONFIG_KEY),
+ ex);
+ }
+ }
+
+ public List<Map<String, String>> getDroppedPartitions() {
+ return droppedPartitions_;
+ }
+
+ @Override
+ public void processTableEvent() throws MetastoreNotificationException,
+ CatalogException {
+ // we have seen cases where a add_partition event is generated with empty
+ // partition list (see IMPALA-8547 for details. Make sure that droppedPartitions
+ // list is not empty
+ if (droppedPartitions_.isEmpty()) {
+ metrics_.getCounter(MetastoreEventsProcessor.EVENTS_SKIPPED_METRIC).inc();
+ infoLog("Partition list is empty. Ignoring this event.");
+ return;
+ }
+ try {
+ // Reload the whole table if it's a transactional table or materialized view.
+ // Materialized views are treated as a special case because it's possible to
+ // receive partition event on MVs, but they are regular views in Impala. That
+ // cause problems on the reloading partition logic which expects it to be a
+ // HdfsTable.
+ boolean incrementalRefresh =
+ BackendConfig.INSTANCE.getHMSEventIncrementalRefreshTransactionalTable();
+ if ((AcidUtils.isTransactionalTable(msTbl_.getParameters()) &&
+ !incrementalRefresh) || MetaStoreUtils.isMaterializedViewTable(msTbl_)) {
+ boolean notSkipped = reloadTableFromCatalog(true);
+ if (!notSkipped) {
+ metrics_.getCounter(MetastoreEventsProcessor.EVENTS_SKIPPED_METRIC).inc();
+ }
+ } else {
+ int numPartsRemoved = catalogOpExecutor_
+ .removePartitionsIfNotAddedLater(getEventId(), dbName_, tblName_,
+ droppedPartitions_, "DROP_PARTITION");
+ if (numPartsRemoved > 0) {
+ infoLog("{} partitions dropped from table {}", numPartsRemoved,
+ getFullyQualifiedTblName());
+ metrics_.getCounter(MetastoreEventsProcessor.NUMBER_OF_PARTITIONS_REMOVED)
+ .inc(numPartsRemoved);
+ } else {
+ metrics_.getCounter(MetastoreEventsProcessor.EVENTS_SKIPPED_METRIC).inc();
+ debugLog("Incremented skipped metric to " + metrics_
+ .getCounter(MetastoreEventsProcessor.EVENTS_SKIPPED_METRIC).getCount());
+ }
+ }
+ } catch (CatalogException e) {
+ throw new MetastoreNotificationNeedsInvalidateException(debugString("Failed to "
+ + "drop some partitions from table {} after a drop partitions event. Event "
+ + "processing cannot continue. Issue an invalidate metadata command to "
+ + "reset event processor state.", getFullyQualifiedTblName()), e);
+ }
+ }
+
+ @Override
+ protected SelfEventContext getSelfEventContext() {
+ throw new UnsupportedOperationException("self-event evaluation is not needed for "
+ + "this event type");
+ }
+ }
+
+ /**
+ * Metastore event handler for ALLOC_WRITE_ID_EVENT events. This event is used to keep
+ * track of write ids for partitioned transactional tables.
+ */
+ public static class AllocWriteIdEvent extends MetastoreTableEvent {
+
+ private AllocWriteIdEvent(CatalogOpExecutor catalogOpExecutor,
+ Metrics metrics, NotificationEvent event) throws MetastoreNotificationException {
+ super(catalogOpExecutor, metrics, event);
+ throw new UnsupportedOperationException("AllocWriteIdEvent is not supported.");
+ }
+
+ @Override
+ protected void processTableEvent() throws MetastoreNotificationException {
+ throw new UnsupportedOperationException("AllocWriteIdEvent is not supported.");
+ }
+
+ @Override
+ protected SelfEventContext getSelfEventContext() {
+ throw new UnsupportedOperationException("self-event evaluation is not needed for "
+ + "this event type");
+ }
+
+ @Override
+ protected boolean isEventProcessingDisabled() {
+ throw new UnsupportedOperationException("AllocWriteIdEvent is not supported.");
+ }
+ }
+
+ /**
+ * Metastore event handler for Reload events. A reload event can be generated by
+ * refresh table/partition or invalidate table event. Handles reload events at both
+ * table and partition scopes (If applicable).
+ */
+ public static class ReloadEvent extends MetastoreTableEvent {
+
+ // The partition for this reload event. Null if the table is unpartitioned
+ private Partition reloadPartition_;
+
+ // if isRefresh_ is set to true then it is refresh query, else it is invalidate query
+ private boolean isRefresh_;
+
+ /**
+ * Prevent instantiation from outside should use MetastoreEventFactory instead
+ */
+ @VisibleForTesting
+ ReloadEvent(CatalogOpExecutor catalogOpExecutor, Metrics metrics,
+ NotificationEvent event) throws MetastoreNotificationException {
+ super(catalogOpExecutor, metrics, event);
+ Preconditions.checkArgument(MetastoreEventType.RELOAD.equals(getEventType()));
+ try {
+ Map<String, Object> updatedFields =
+ MetastoreShim.getFieldsFromReloadEvent(event);
+ msTbl_ = (org.apache.hadoop.hive.metastore.api.Table)Preconditions.checkNotNull(
+ updatedFields.get("table"));
+ reloadPartition_ = (Partition)updatedFields.get("partition");
+ isRefresh_ = (boolean)updatedFields.get("isRefresh");
+ } catch (Exception e) {
+ throw new MetastoreNotificationException(debugString("Unable to "
+ + "parse reload message"), e);
+ }
+ }
+
+ @Override
+ public SelfEventContext getSelfEventContext() {
+ throw new UnsupportedOperationException("Self-event evaluation is unnecessary for"
+ + " this event type");
+ }
+
+ @Override
+ public void processTableEvent() throws MetastoreNotificationException {
+ if (isOlderEvent()) {
+ metrics_.getCounter(MetastoreEventsProcessor.EVENTS_SKIPPED_METRIC)
+ .inc(getNumberOfEvents());
+ infoLog("Incremented events skipped counter to {}",
+ metrics_.getCounter(MetastoreEventsProcessor.EVENTS_SKIPPED_METRIC)
+ .getCount());
+ return;
+ }
+ if (isRefresh_) {
+ if (reloadPartition_ != null) {
+ processPartitionReload();
+ } else {
+ processTableReload();
+ }
+ } else {
+ processTableInvalidate();
+ }
+ }
+
+ private boolean isOlderEvent() {
+ org.apache.impala.catalog.Table tbl = catalog_.getTableNoThrow(dbName_, tblName_);
+ if (tbl == null || tbl instanceof IncompleteTable) { return false; }
+ // Always check the lastRefreshEventId on the table first for table level refresh
+ boolean canSkip = tbl.getLastRefreshEventId() >= getEventId();
+ try {
+ if (!canSkip && reloadPartition_ != null) {
+ tbl.takeReadLock();
+ canSkip = catalog_.isPartitionLoadedAfterEvent(dbName_, tblName_,
+ reloadPartition_, getEventId());
+ }
+ return canSkip;
+ } finally {
+ if (tbl.isReadLockedByCurrentThread()) {
+ tbl.releaseReadLock();
+ }
+ }
+ }
+
+ /**
+ * Process partition reload
+ */
+ private void processPartitionReload() throws MetastoreNotificationException {
+ // For partitioned table, refresh the partition only.
+ Preconditions.checkNotNull(reloadPartition_);
+ try {
+ // Ignore event if table or database is not in catalog. Throw exception if
+ // refresh fails. If the partition does not exist in metastore the reload
+ // method below removes it from the catalog
+ // forcing file metadata reload so that new files (due to refresh) are reflected
+ // HdfsPartition
+ reloadPartitions(Arrays.asList(reloadPartition_),
+ FileMetadataLoadOpts.FORCE_LOAD, getEventDesc(), false);
+ } catch (CatalogException e) {
+ throw new MetastoreNotificationNeedsInvalidateException(debugString("Refresh "
+ + "partition on table {} partition {} failed. Event processing cannot "
+ + "continue. Issue an invalidate metadata command to reset the event "
+ + "processor state.", getFullyQualifiedTblName(),
+ Joiner.on(',').join(reloadPartition_.getValues())), e);
+ }
+ }
+
+ /**
+ * Process unpartitioned table reload
+ */
+ private void processTableReload() throws MetastoreNotificationException {
+ // For non-partitioned tables, refresh the whole table.
+ Preconditions.checkState(reloadPartition_ == null);
+ try {
+ // we always treat the table as non-transactional so all the files are reloaded
+ boolean notSkipped = reloadTableFromCatalog(false);
+ if (!notSkipped) {
+ metrics_.getCounter(MetastoreEventsProcessor.EVENTS_SKIPPED_METRIC).inc();
+ }
+ } catch (CatalogException e) {
+ throw new MetastoreNotificationNeedsInvalidateException(
+ debugString("Refresh table {} failed. Event processing "
+ + "cannot continue. Issue an invalidate metadata " +
+ "command to reset the event processor state.",
+ getFullyQualifiedTblName()), e);
+ }
+ }
+
+ private void processTableInvalidate() throws MetastoreNotificationException {
+ Reference<Boolean> tblWasRemoved = new Reference<>();
+ Reference<Boolean> dbWasAdded = new Reference<>();
+ org.apache.impala.catalog.Table tbl = null;
+ try {
+ tbl = catalog_.getTable(dbName_, tblName_);
+ if (tbl == null) {
+ infoLog("Skipping on table {}.{} since it does not exist in cache", dbName_,
+ tblName_);
+ metrics_.getCounter(MetastoreEventsProcessor.EVENTS_SKIPPED_METRIC).inc();
+ return ;
+ }
+ if (tbl instanceof IncompleteTable) {
+ infoLog("Skipping on an incomplete table {}", tbl.getFullName());
+ metrics_.getCounter(MetastoreEventsProcessor.EVENTS_SKIPPED_METRIC).inc();
+ return ;
+ }
+ } catch (DatabaseNotFoundException e) {
+ infoLog("Skipping on table {} because db {} not found in cache", tblName_,
+ dbName_);
+ metrics_.getCounter(MetastoreEventsProcessor.EVENTS_SKIPPED_METRIC).inc();
+ return ;
+ }
+ catalog_.invalidateTable(tbl.getTableName().toThrift(),
+ tblWasRemoved, dbWasAdded, NoOpEventSequence.INSTANCE, getEventId());
+ LOG.info("Table " + tbl.getFullName() + " is invalidated from catalog cache");
+ }
+ }
+
+ /**
+ * Metastore event handler for ABORT_TXN events. Handles abort event for transactional
+ * tables.
+ */
+ public static class AbortTxnEvent extends MetastoreEvent {
+ public static final String EVENT_TYPE = "ABORT_TXN";
+
+ AbortTxnEvent(CatalogOpExecutor catalogOpExecutor, Metrics metrics,
+ NotificationEvent event) {
+ super(catalogOpExecutor, metrics, event);
+ throw new UnsupportedOperationException("AbortTxnEvent is not supported.");
+ }
+
+ public Set<TableWriteId> getTableWriteIds() {
+ throw new UnsupportedOperationException("AbortTxnEvent is not supported.");
+ }
+
+ @Override
+ protected void process() throws MetastoreNotificationException {
+ throw new UnsupportedOperationException("AbortTxnEvent is not supported.");
+ }
+
+ @Override
+ protected boolean onFailure(Exception e) {
+ throw new UnsupportedOperationException("AbortTxnEvent is not supported.");
+ }
+
+ private void addAbortedWriteIdsToTables(Set<TableWriteId> tableWriteIds)
+ throws CatalogException {
+ throw new UnsupportedOperationException("AbortTxnEvent is not supported.");
+ }
+
+ @Override
+ protected boolean isEventProcessingDisabled() {
+ throw new UnsupportedOperationException("AbortTxnEvent is not supported.");
+ }
+
+ @Override
+ protected SelfEventContext getSelfEventContext() {
+ throw new UnsupportedOperationException("Self-event evaluation is not needed for "
+ + "this event type");
+ }
+
+ /*
+ Not skipping the event since there can be multiple tables involved. The actual
+ processing of event would skip or process the event on a table by table basis
+ */
+ @Override
+ protected boolean shouldSkipWhenSyncingToLatestEventId() {
+ throw new UnsupportedOperationException("AbortTxnEvent is not supported.");
+ }
+ }
+
+ /**
+ * Pseudo abort transaction event handles abort processing for a table
+ */
+ public static class PseudoAbortTxnEvent extends DerivedMetastoreTableEvent {
+
+ PseudoAbortTxnEvent(DerivedMetastoreEventContext context, String dbName,
+ String tableName, List<Long> writeIds) {
+ super(context);
+ throw new UnsupportedOperationException("PseudoAbortTxnEvent is not supported.");
+ }
+
+ @Override
+ protected void processTableEvent() throws MetastoreNotificationException {
+ throw new UnsupportedOperationException("PseudoAbortTxnEvent is not supported.");
+ }
+
+ @Override
+ protected boolean shouldSkipWhenSyncingToLatestEventId() {
+ throw new UnsupportedOperationException("PseudoAbortTxnEvent is not supported.");
+ }
+
+ @Override
+ protected boolean isEventProcessingDisabled() {
+ throw new UnsupportedOperationException("PseudoAbortTxnEvent is not supported.");
+ }
+
+ @Override
+ protected SelfEventContext getSelfEventContext() {
+ throw new UnsupportedOperationException(
+ "Self-event evaluation is not needed for this event type");
+ }
+
+ @Override
+ public String getEventDesc() {
+ throw new UnsupportedOperationException("PseudoAbortTxnEvent is not supported.");
+ }
+ }
+
+ /**
+ * Metastore event handler for COMMIT_COMPACTION_EVENT events. Handles
+ * COMMIT_COMPACTION_EVENT event for transactional tables.
+ */
+ public static class CommitCompactionEvent extends MetastoreTableEvent {
+
+ CommitCompactionEvent(CatalogOpExecutor catalogOpExecutor, Metrics metrics,
+ NotificationEvent event) throws MetastoreNotificationException {
+ super(catalogOpExecutor, metrics, event);
+ throw new UnsupportedOperationException("CommitCompactionEvent is not supported.");
+ }
+
+ @Override
+ protected void processTableEvent() throws MetastoreNotificationException {
+ throw new UnsupportedOperationException("CommitCompactionEvent is not supported.");
+ }
+
+ @Override
+ protected SelfEventContext getSelfEventContext() {
+ throw new UnsupportedOperationException("Self-event evaluation is not needed for "
+ + "this event type");
+ }
+
+ @Override
+ protected boolean isEventProcessingDisabled() {
+ throw new UnsupportedOperationException("CommitCompactionEvent is not supported.");
+ }
+ }
+
+ /**
+ * An event type which is ignored. Useful for unsupported metastore event types
+ */
+ public static class IgnoredEvent extends MetastoreEvent {
+
+ /**
+ * Prevent instantiation from outside should use MetastoreEventFactory instead
+ */
+ private IgnoredEvent(
+ CatalogOpExecutor catalogOpExecutor, Metrics metrics, NotificationEvent event) {
+ super(catalogOpExecutor, metrics, event);
+ }
+
+ @Override
+ public void process() {
+ debugLog(
+ "Ignoring unknown event type " + metastoreNotificationEvent_.getEventType());
+ }
+
+ @Override
+ protected boolean isEventProcessingDisabled() {
+ return false;
+ }
+
+ @Override
+ protected boolean shouldSkipWhenSyncingToLatestEventId() {
+ return false;
+ }
+
+ @Override
+ protected SelfEventContext getSelfEventContext() {
+ throw new UnsupportedOperationException("Self-event evaluation is not needed for "
+ + "this event type");
+ }
+
+ @Override
+ protected boolean onFailure(Exception e) {
+ return false;
+ }
+ }
+}
diff --git a/fe/src/compat-apache-hive-2/java/org/apache/impala/compat/HiveEsriGeospatialBuiltins.java b/fe/src/compat-apache-hive-2/java/org/apache/impala/compat/HiveEsriGeospatialBuiltins.java
new file mode 100644
index 000000000..90ac7db46
--- /dev/null
+++ b/fe/src/compat-apache-hive-2/java/org/apache/impala/compat/HiveEsriGeospatialBuiltins.java
@@ -0,0 +1,29 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.impala.compat;
+
+import org.apache.impala.catalog.Db;
+
+public class HiveEsriGeospatialBuiltins {
+ /**
+ * Apache Hive 2 does not include geospatial functions
+ */
+ public static void initBuiltins(Db db) {
+
+ }
+}
diff --git a/fe/src/compat-apache-hive-2/java/org/apache/impala/compat/MetastoreShim.java b/fe/src/compat-apache-hive-2/java/org/apache/impala/compat/MetastoreShim.java
new file mode 100644
index 000000000..2e4a83374
--- /dev/null
+++ b/fe/src/compat-apache-hive-2/java/org/apache/impala/compat/MetastoreShim.java
@@ -0,0 +1,965 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.impala.compat;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Stopwatch;
+import com.google.common.collect.Lists;
+
+import java.lang.reflect.Proxy;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.lang3.math.NumberUtils;
+import org.apache.commons.lang3.reflect.FieldUtils;
+import org.apache.hadoop.hive.common.FileUtils;
+import org.apache.hadoop.hive.common.ValidTxnList;
+import org.apache.hadoop.hive.common.ValidWriteIdList;
+import org.apache.hadoop.hive.metastore.IMetaStoreClient;
+import org.apache.hadoop.hive.metastore.RetryingMetaStoreClient;
+import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
+import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.api.FireEventRequest;
+import org.apache.hadoop.hive.metastore.api.FireEventRequestData;
+import org.apache.hadoop.hive.metastore.api.InsertEventRequestData;
+import org.apache.hadoop.hive.metastore.api.InvalidInputException;
+import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
+import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
+import org.apache.hadoop.hive.metastore.api.NotificationEvent;
+import org.apache.hadoop.hive.metastore.api.NotificationEventRequest;
+import org.apache.hadoop.hive.metastore.api.NotificationEventResponse;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore;
+import org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore.Client;
+import org.apache.hadoop.hive.metastore.messaging.AlterTableMessage;
+import org.apache.hadoop.hive.metastore.messaging.EventMessage;
+import org.apache.hadoop.hive.metastore.messaging.MessageDeserializer;
+import org.apache.hadoop.hive.metastore.messaging.MessageFactory;
+import org.apache.hadoop.hive.metastore.messaging.json.JSONDropDatabaseMessage;
+import org.apache.hadoop.hive.metastore.messaging.json.JSONMessageFactory;
+import org.apache.hadoop.hive.metastore.txn.TxnUtils;
+import org.apache.impala.catalog.CatalogException;
+import org.apache.impala.catalog.CatalogServiceCatalog;
+import org.apache.impala.catalog.DataSource;
+import org.apache.impala.catalog.Db;
+import org.apache.impala.catalog.HdfsPartition;
+import org.apache.impala.catalog.HdfsTable;
+import org.apache.impala.catalog.Hive2MetastoreShimBase;
+import org.apache.impala.catalog.MetaStoreClientPool;
+import org.apache.impala.catalog.MetaStoreClientPool.MetaStoreClient;
+import org.apache.impala.catalog.events.MetastoreEvents.DerivedMetastoreEventContext;
+import org.apache.impala.catalog.events.MetastoreEvents.DerivedMetastoreTableEvent;
+import org.apache.impala.catalog.events.MetastoreEvents.MetastoreEvent;
+import org.apache.impala.catalog.events.MetastoreEventsProcessor.MetaDataFilter;
+import org.apache.impala.catalog.events.MetastoreNotificationException;
+import org.apache.impala.catalog.events.SelfEventContext;
+import org.apache.impala.catalog.local.MetaProvider.PartitionMetadata;
+import org.apache.impala.catalog.local.MetaProvider.PartitionRef;
+import org.apache.impala.catalog.local.MetaProvider.TableMetaRef;
+import org.apache.impala.catalog.metastore.ICatalogMetastoreServer;
+import org.apache.impala.common.ImpalaRuntimeException;
+import org.apache.impala.common.Metrics;
+import org.apache.impala.service.CatalogOpExecutor;
+import org.apache.impala.util.AcidUtils;
+import org.apache.impala.util.AcidUtils.TblTransaction;
+import org.apache.impala.util.MetaStoreUtil.TableInsertEventInfo;
+import org.apache.thrift.TException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.impala.util.HiveMetadataFormatUtils.formatOutput;
+
+/**
+ * A wrapper around some of Hive's Metastore API's to abstract away differences
+ * between major versions of different Hive publishers. This implements the shimmed
+ * methods for Apache Hive 2.
+ */
+public class MetastoreShim extends Hive2MetastoreShimBase {
+ private static final Logger LOG = LoggerFactory.getLogger(MetastoreShim.class);
+
+ public static final byte ACCESSTYPE_NONE = (byte) 1;
+ public static final byte ACCESSTYPE_READONLY = (byte) 2;
+ public static final byte ACCESSTYPE_WRITEONLY = (byte) 4;
+ public static final byte ACCESSTYPE_READWRITE = (byte) 8;
+
+ private static final String ACCESSTYPE = "accessType";
+ private static final String WRITEID = "writeId";
+ private static final String ID = "id";
+ private static final String MANAGEDLOCATIONURI = "managedLocationUri";
+
+ private static final String CONNECTORREAD = "CONNECTORREAD";
+ private static final String CONNECTORWRITE = "CONNECTORWRITE";
+
+ private static List<String> processorCapabilities = Lists.newArrayList();
+
+ /**
+ * Wrapper around IMetaStoreClient.alter_table with validWriteIds as a param.
+ */
+ public static void alterTableWithTransaction(IMetaStoreClient client,
+ Table tbl, TblTransaction tblTxn)
+ throws ImpalaRuntimeException {
+ throw new UnsupportedOperationException(
+ "alterTableWithTransaction is not supported.");
+ }
+
+ /**
+ * Wrapper around IMetaStoreClient.alter_partitions with transaction information
+ */
+ public static void alterPartitionsWithTransaction(IMetaStoreClient client,
+ String dbName, String tblName, List<Partition> partitions, TblTransaction tblTxn
+ ) throws InvalidOperationException, MetaException, TException {
+ throw new UnsupportedOperationException(
+ "alterPartitionsWithTransaction is not supported.");
+ }
+
+ /**
+ * Wrapper around IMetaStoreClient.getTableColumnStatistics() to deal with added
+ * arguments.
+ */
+ public static List<ColumnStatisticsObj> getTableColumnStatistics(
+ IMetaStoreClient client, String dbName, String tableName, List<String> colNames)
+ throws NoSuchObjectException, MetaException, TException {
+ return client.getTableColumnStatistics(dbName, tableName, colNames);
+ }
+
+ /**
+ * Wrapper around IMetaStoreClient.deleteTableColumnStatistics() to deal with added
+ * arguments.
+ */
+ public static boolean deleteTableColumnStatistics(IMetaStoreClient client,
+ String dbName, String tableName, String colName)
+ throws NoSuchObjectException, MetaException, InvalidObjectException, TException,
+ InvalidInputException {
+ return client.deleteTableColumnStatistics(dbName, tableName, colName);
+ }
+
+ /**
+ * Wrapper around ColumnStatistics c'tor to deal with the added engine property.
+ */
+ public static ColumnStatistics createNewHiveColStats() {
+ ColumnStatistics colStats = new ColumnStatistics();
+ return colStats;
+ }
+
+ /**
+ * Wrapper method which returns HMS-3 Message factory in case Impala is
+ * building against Apache Hive-3
+ */
+ public static MessageDeserializer getMessageDeserializer() {
+ return MessageFactory.getInstance().getDeserializer();
+ }
+
+ /**
+ * Wrapper around FileUtils.makePartName to deal with package relocation in Hive 3.
+ * This method uses the metastore's FileUtils method instead of one from hive-exec
+ * @param partitionColNames
+ * @param values
+ * @return
+ */
+ public static String makePartName(List<String> partitionColNames, List<String> values) {
+ return FileUtils.makePartName(partitionColNames, values);
+ }
+
+ /**
+ * Wrapper around HMS-3 message serializer
+ * @param message
+ * @return serialized string to use used in the NotificationEvent's message field
+ */
+ @VisibleForTesting
+ public static String serializeEventMessage(EventMessage message) {
+ return message.toString();
+ }
+
+ /**
+ * Get valid write ids from HMS for the acid table
+ * @param client the client to access HMS
+ * @param tableFullName the name for the table
+ * @return ValidWriteIdList object
+ */
+ public static ValidWriteIdList fetchValidWriteIds(IMetaStoreClient client,
+ String tableFullName) throws TException {
+ throw new UnsupportedOperationException(
+ "fetchValidWriteIds is not supported.");
+ }
+
+ /**
+ * Wrapper around HMS Partition object to get writeID
+ * WriteID is introduced in ACID 2
+ * It is used to detect changes of the partition
+ */
+ public static long getWriteIdFromMSPartition(Partition partition) {
+ Preconditions.checkNotNull(partition);
+ return -1L;
+ }
+
+ /**
+ * Wrapper around HMS Partition object to set writeID
+ * WriteID is introduced in ACID 2
+ * It is used to detect changes of the partition
+ */
+ public static void setWriteIdToMSPartition(Partition partition, long writeId) {
+ Preconditions.checkNotNull(partition);
+ partition.getParameters().put(WRITEID, String.valueOf(writeId));
+ }
+
+ /**
+ * Wrapper around HMS Table object to get writeID
+ * Per table writeId is introduced in ACID 2
+ * It is used to detect changes of the table
+ */
+ public static long getWriteIdFromMSTable(Table msTbl) {
+ Preconditions.checkNotNull(msTbl);
+ return -1L;
+ }
+
+ /**
+ * Set impala capabilities to check the accessType of the table created by hive
+ * Impala supports:
+ * - external table read/write
+ * - insert-only Acid table read
+ * - virtual view read
+ * - materialized view read
+ */
+ public static synchronized void setHiveClientCapabilities() {
+ throw new UnsupportedOperationException("setHiveClientCapabilities");
+ }
+
+ /**
+ * Check if a table has a capability
+ * @param msTbl hms table
+ * @param requiredCapability hive access types or combination of them
+ * @return true if the table has the capability
+ */
+ public static boolean hasTableCapability(Table msTbl, byte requiredCapability) {
+ throw new UnsupportedOperationException("hasTableCapability");
+ }
+
+ /**
+ * Get Access type in string
+ * @param msTbl hms table
+ * @return the string represents the table access type.
+ */
+ public static String getTableAccessType(Table msTbl) {
+ throw new UnsupportedOperationException("getTableAccessType");
+ }
+
+ /**
+ * Set table access type. This is useful for hms Table object constructed for create
+ * table statement. For example, to create a table, we need Read/Write capabilities
+ * not default 0(not defined)
+ */
+ public static void setTableAccessType(Table msTbl, byte accessType) {
+ throw new UnsupportedOperationException("setTableAccessType");
+ }
+
+ /**
+ * CDP Hive-3 only function
+ */
+ public static void setTableColumnStatsTransactional(IMetaStoreClient client,
+ Table msTbl, ColumnStatistics colStats, TblTransaction tblTxn)
+ throws ImpalaRuntimeException {
+ throw new UnsupportedOperationException(
+ "setTableColumnStatsTransactional is not supported.");
+ }
+
+ /**
+ * Fire insert events for table and partition.
+ * In case of any exception, we just log the failure of firing insert events.
+ */
+ public static List<Long> fireInsertEvents(MetaStoreClient msClient,
+ TableInsertEventInfo insertEventInfo, String dbName, String tableName) {
+ return Collections.emptyList();
+ }
+
+ /**
+ * CDP Hive-3 only function
+ * Fires a listener event of the type ACID_WRITE on a transactional table in metastore.
+ * This event is polled by other external systems to detect insert operations into
+ * ACID tables.
+ * @throws TException in case of errors during HMS API call.
+ */
+ private static void fireInsertTransactionalEventHelper(
+ IMetaStoreClient hiveClient, TableInsertEventInfo insertEventInfo, String dbName,
+ String tableName) throws TException {
+ throw new UnsupportedOperationException(
+ "fireInsertTransactionalEventHelper is not supported.");
+ }
+
+ /**
+ * Fires an insert event to HMS notification log. In Hive-3 for partitioned table,
+ * all partition insert events will be fired by a bulk API.
+ *
+ * @param msClient Metastore client,
+ * @param insertEventDataList A list of insert event info encapsulating the information
+ * needed to fire insert events.
+ * @param insertEventPartValList The partition list corresponding to
+ * insertEventDataList, used by Apache Hive 3
+ * @param dbName
+ * @param tableName
+ * @return a list of eventIds for the insert events
+ */
+ @VisibleForTesting
+ public static List<Long> fireInsertEventHelper(IMetaStoreClient msClient,
+ List<InsertEventRequestData> insertEventDataList,
+ List<List<String>> insertEventPartValList, String dbName, String tableName)
+ throws TException {
+ Preconditions.checkNotNull(msClient);
+ Preconditions.checkNotNull(dbName);
+ Preconditions.checkNotNull(tableName);
+ Preconditions.checkState(!insertEventDataList.isEmpty(), "Atleast one insert event "
+ + "info must be provided.");
+ Preconditions.checkState(insertEventDataList.size() == insertEventPartValList.size());
+ LOG.debug(String.format(
+ "Firing %s insert event(s) for %s.%s", insertEventDataList.size(), dbName,
+ tableName));
+ for (int i = 0; i < insertEventDataList.size(); i++) {
+ InsertEventRequestData insertEventData = insertEventDataList.get(i);
+ List<String> partitionVals = insertEventPartValList.get(i);
+ FireEventRequestData data = new FireEventRequestData();
+ FireEventRequest rqst = new FireEventRequest(true, data);
+ rqst.setDbName(dbName);
+ rqst.setTableName(tableName);
+ if (partitionVals != null && !partitionVals.isEmpty()) {
+ rqst.setPartitionVals(partitionVals);
+ }
+ data.setInsertData(insertEventData);
+ msClient.fireListenerEvent(rqst);
+ }
+ //TODO: IMPALA-8632: Add support for self-event detection for insert events
+ return Collections.EMPTY_LIST;
+ }
+
+ /**
+ * CDP Hive-3 only function.
+ */
+ @VisibleForTesting
+ public static List<Long> fireReloadEventHelper(MetaStoreClient msClient,
+ boolean isRefresh, List<List<String>> partVals, String dbName, String tableName,
+ Map<String, String> selfEventParams) throws TException {
+ throw new UnsupportedOperationException("Reload event is not supported.");
+ }
+
+ /**
+ * CDP Hive-3 only function.
+ */
+ public static Map<String, Object> getFieldsFromReloadEvent(NotificationEvent event)
+ throws MetastoreNotificationException {
+ throw new UnsupportedOperationException("Reload event is not supported.");
+ }
+
+ /**
+ * CDP Hive-3 only function.
+ */
+ public static String getPartitionNameFromCommitCompactionEvent(
+ NotificationEvent event) {
+ throw new UnsupportedOperationException("CommitCompaction event is not supported.");
+ }
+
+ /**
+ * Use thrift API directly instead of HiveMetastoreClient#getNextNotification because
+ * the HMS client can throw an IllegalStateException when there is a gap between the
+ * eventIds returned.
+ *
+ * @see org.apache.hadoop.hive.metastore.HiveMetaStoreClient#getNextNotification
+ * @param msClient Metastore client
+ * @param eventRequest Notification event request
+ * @param eventTypeSkipList unused
+ * @return NotificationEventResponse
+ * @throws TException
+ */
+ public static NotificationEventResponse getNextNotification(IMetaStoreClient msClient,
+ NotificationEventRequest eventRequest, List<String> eventTypeSkipList)
+ throws TException {
+ return getThriftClient(msClient).get_next_notification(eventRequest);
+ }
+
+ private static ThriftHiveMetastore.Client getThriftClient(IMetaStoreClient msClient)
+ throws MetaException {
+ try {
+ // The HMS client does not expose the getThriftClient function, which is obtained
+ // by reflection here.
+ if (Proxy.isProxyClass(msClient.getClass())) {
+ RetryingMetaStoreClient handler =
+ (RetryingMetaStoreClient) Proxy.getInvocationHandler(msClient);
+ msClient = (IMetaStoreClient) FieldUtils.readField(handler, "base",
+ true);
+ }
+ Object client = FieldUtils.readField(msClient, "client", true);
+ if (client == null) {
+ throw new MetaException("Client is not initialized");
+ }
+ if (!(client instanceof ThriftHiveMetastore.Client)) {
+ throw new MetaException("getThriftClient is only supported in remote metastore "
+ + "mode.");
+ }
+ return (Client) client;
+ } catch (IllegalAccessException e) {
+ throw new MetaException("getThriftClient() fail: " + e.getMessage());
+ }
+ }
+
+ /**
+ * Wrapper around Database.setManagedLocationUri() to deal with added arguments.
+ */
+ public static void setManagedLocationUri(Database db, String managedLocation) {
+ db.getParameters().put(MANAGEDLOCATIONURI, managedLocation);
+ }
+
+ /**
+ * Wrapper around HMS Database object to get managedLocationUri.
+ */
+ public static String getManagedLocationUri(Database db) {
+ if (db.getParameters().containsKey(MANAGEDLOCATIONURI)) {
+ return db.getParameters().get(MANAGEDLOCATIONURI);
+ }
+ return null;
+ }
+
+ /**
+ * Set the default table path for a new table.
+ */
+ public static void setTableLocation(Db db, Table tbl) throws ImpalaRuntimeException {
+ Preconditions.checkNotNull(db);
+ Preconditions.checkNotNull(tbl);
+ // Set a valid location of this table using the same rules as the cdp metastore,
+ // unless the user specified a path.
+ try {
+ if (tbl.getSd().getLocation() == null || tbl.getSd().getLocation().isEmpty()) {
+ tbl.getSd().setLocation(
+ MetastoreShim.getPathForNewTable(db.getMetaStoreDb(), tbl));
+ }
+ } catch (MetaException e) {
+ throw new ImpalaRuntimeException("setTableLocation", e);
+ }
+ }
+
+ /**
+ * Wrapper around HMS Table object to get table id.
+ */
+ public static long getTableId(Table tbl) {
+ return NumberUtils.toLong(tbl.getParameters().get(ID), -1);
+ }
+
+ /**
+ * Wrapper around JSONDropDatabaseMessage to get database.
+ */
+ public static Database getDatabaseObject(JSONDropDatabaseMessage dropDatabaseMessage) {
+ Database database = new Database();
+ database.setName(dropDatabaseMessage.getDB());
+ return database;
+ }
+
+ /**
+ * Wrapper around IMetaStoreClient.truncateTable() to deal with added arguments.
+ */
+ public static void truncateTable(IMetaStoreClient msClient, String dbName,
+ String tableName, List<String> partNames,
+ String validWriteIds, long writeId) throws TException {
+ throw new UnsupportedOperationException(
+ "truncateTable is not supported.");
+ }
+
+ /**
+ * Wrapper around IMetaStoreClient.listPartitions() to deal with added arguments.
+ */
+ public static List<Partition> getPartitions(IMetaStoreClient msClient,
+ String testDbName, String testTblName) throws TException {
+ return msClient.listPartitions(testDbName, testTblName, (short) -1);
+ }
+
+ /**
+ * CDP Hive-3 only function.
+ * For compatibility, fireInsertEventHelper() added the partitionValList parameter
+ */
+ public static void setPartitionVal(InsertEventRequestData insertEventRequestData,
+ List<String> partVals) {
+ }
+
+ /**
+ * CDP Hive-3 only function.
+ */
+ public static void addToSubDirectoryList(InsertEventRequestData insertEventRequestData,
+ String acidDirPath) {
+ throw new UnsupportedOperationException("addToSubDirectoryList is not supported.");
+ }
+
+ /**
+ * CDP Hive-3 only function.
+ */
+ public static List<HdfsPartition.Builder> getPartitionsForRefreshingFileMetadata(
+ CatalogServiceCatalog catalog, HdfsTable hdfsTable) throws CatalogException {
+ throw new UnsupportedOperationException(
+ "getPartitionsForRefreshingFileMetadata is not supported.");
+ }
+
+ /**
+ * CDP Hive-3 only function.
+ */
+ public static Map<String, Long> getLatestCompactions(MetaStoreClient client,
+ String dbName, String tableName, List<String> partitionNames,
+ String unPartitionedName, long lastCompactionId) throws TException {
+ throw new UnsupportedOperationException("getLatestCompactions is not supported.");
+ }
+
+ /**
+ * CDP Hive-3 only function.
+ */
+ public static List<PartitionRef> checkLatestCompaction(MetaStoreClientPool msClientPool,
+ String dbName, String tableName, TableMetaRef table,
+ Map<PartitionRef, PartitionMetadata> metas, String unPartitionedName)
+ throws TException {
+ throw new UnsupportedOperationException("checkLatestCompaction is not supported.");
+ }
+
+ /**
+ * CDP Hive-3 only function.
+ */
+ public static ICatalogMetastoreServer getCatalogMetastoreServer(
+ CatalogOpExecutor catalogOpExecutor) {
+ throw new UnsupportedOperationException(
+ "getCatalogMetastoreServer is not supported.");
+ }
+
+ /**
+ * CDP Hive-3 only function.
+ */
+ public static class CommitTxnEvent extends MetastoreEvent {
+ public static final String EVENT_TYPE = "COMMIT_TXN";
+
+ public CommitTxnEvent(CatalogOpExecutor catalogOpExecutor, Metrics metrics,
+ NotificationEvent event) {
+ super(catalogOpExecutor, metrics, event);
+ throw new UnsupportedOperationException("CommitTxnEvent is not supported.");
+ }
+
+ @Override
+ protected void process() throws MetastoreNotificationException {
+
+ }
+
+ @Override
+ protected boolean onFailure(Exception e) {
+ return false;
+ }
+
+ @Override
+ protected boolean isEventProcessingDisabled() {
+ return false;
+ }
+
+ @Override
+ protected SelfEventContext getSelfEventContext() {
+ return null;
+ }
+
+ @Override
+ protected boolean shouldSkipWhenSyncingToLatestEventId() {
+ return false;
+ }
+ }
+
+ /**
+ * CDP Hive-3 only function. This is a dummy implementation till the CommitTxnEvent
+ * dummy implementation class defined in this file becomes actual implementation. Need
+ * to change when CommitTxnEvent implementation is supported with IMPALA-13285.
+ */
+ public static class PseudoCommitTxnEvent extends DerivedMetastoreTableEvent {
+ PseudoCommitTxnEvent(DerivedMetastoreEventContext context, String dbName,
+ String tableName, boolean isPartitioned, boolean isMaterializedView,
+ List<Long> writeIds, List<Partition> partitions) {
+ super(context);
+ throw new UnsupportedOperationException("PseudoCommitTxnEvent is not supported.");
+ }
+
+ @Override
+ protected void processTableEvent() {
+ }
+
+ @Override
+ protected boolean isEventProcessingDisabled() {
+ return false;
+ }
+
+ @Override
+ protected SelfEventContext getSelfEventContext() {
+ return null;
+ }
+
+ @Override
+ protected boolean shouldSkipWhenSyncingToLatestEventId() {
+ return false;
+ }
+
+ @Override
+ protected boolean onFailure(Exception e) {
+ return false;
+ }
+ }
+
+ /**
+ * Get Access type in byte from table property
+ * @param msTbl hms table
+ * @return the table access type.
+ */
+ private static byte getAccessType(Table msTbl) {
+ Preconditions.checkNotNull(msTbl);
+ byte accessType = ACCESSTYPE_NONE;
+ Map<String, String> params = msTbl.getParameters();
+ if (params.containsKey(ACCESSTYPE)) {
+ String accessTypeStr = msTbl.getParameters().get(ACCESSTYPE);
+ accessType = accessTypeStr.getBytes()[0];
+ } else { // Table not created by Impala
+ if (!capabilitiestSet_) setHiveClientCapabilities();
+ // The following logic comes from hive's MetastoreDefaultTransformer.transform()
+ String tableType = msTbl.getTableType();
+ String tCapabilities = params.get(CatalogOpExecutor.CAPABILITIES_KEY);
+ int numBuckets = msTbl.isSetSd() ? msTbl.getSd().getNumBuckets() : 0;
+ boolean isBucketed = numBuckets > 0;
+
+ LOG.info("Table " + msTbl.getTableName() + ",#bucket=" + numBuckets + ",isBucketed:"
+ + isBucketed + ",tableType=" + tableType + ",tableCapabilities="
+ + tCapabilities);
+
+ // if the table has no tCapabilities
+ if (tCapabilities == null) {
+ LOG.debug("Table has no specific required capabilities");
+
+ switch (tableType) {
+ case "EXTERNAL_TABLE":
+ if (numBuckets > 0) {
+ if (processorCapabilities.contains(HIVEBUCKET2)) {
+ LOG.debug("External bucketed table with HB2 capability:RW");
+ accessType = ACCESSTYPE_READWRITE;
+ } else {
+ LOG.debug("External bucketed table without HB2 capability:RO");
+ accessType = ACCESSTYPE_READONLY;
+ }
+ } else { // Unbucketed
+ if (processorCapabilities.contains(EXTWRITE) && processorCapabilities
+ .contains(EXTREAD)) {
+ LOG.debug("External unbucketed table with EXTREAD/WRITE capability:RW");
+ accessType = ACCESSTYPE_READWRITE;
+ } else if (processorCapabilities.contains(EXTREAD)) {
+ LOG.debug("External unbucketed table with EXTREAD capability:RO");
+ accessType = ACCESSTYPE_READONLY;
+ } else {
+ LOG.debug(
+ "External unbucketed table without EXTREAD/WRITE capability:NONE");
+ accessType = ACCESSTYPE_NONE;
+ }
+ }
+ break;
+ case "MANAGED_TABLE":
+ String txnal = params.get(AcidUtils.TABLE_IS_TRANSACTIONAL);
+ if (txnal == null || txnal
+ .equalsIgnoreCase("FALSE")) { // non-ACID MANAGED table
+ LOG.debug("Managed non-acid table:RW");
+ accessType = ACCESSTYPE_READWRITE;
+ }
+
+ if (txnal != null && txnal.equalsIgnoreCase("TRUE")) { // ACID table
+ String txntype = params.get(AcidUtils.TABLE_TRANSACTIONAL_PROPERTIES);
+ if (txntype != null && txntype
+ .equalsIgnoreCase("insert_only")) { // MICRO_MANAGED Tables
+ // MGD table is insert only, not full ACID
+ if (processorCapabilities.contains(HIVEMANAGEDINSERTWRITE)
+ || processorCapabilities.contains(CONNECTORWRITE)) {
+ LOG.debug("Managed acid table with INSERTWRITE or CONNECTORWRITE "
+ + "capability:RW");
+ // clients have RW access to INSERT-ONLY ACID tables
+ accessType = ACCESSTYPE_READWRITE;
+ LOG.info("Processor has one of the write capabilities on insert-only, "
+ + "granting RW");
+ } else if (processorCapabilities.contains(HIVEMANAGEDINSERTREAD)
+ || processorCapabilities.contains(CONNECTORREAD)) {
+ LOG.debug("Managed acid table with INSERTREAD or CONNECTORREAD "
+ + "capability:RO");
+ // clients have RO access to INSERT-ONLY ACID tables
+ accessType = ACCESSTYPE_READONLY;
+ LOG.info("Processor has one of the read capabilities on insert-only, "
+ + "granting RO");
+ } else {
+ // clients have NO access to INSERT-ONLY ACID tables
+ accessType = ACCESSTYPE_NONE;
+ LOG.info("Processor has no read or write capabilities on insert-only, "
+ + "NO access");
+ }
+ } else { // FULL ACID MANAGED TABLE
+ if (processorCapabilities.contains(HIVEFULLACIDWRITE)
+ || processorCapabilities.contains(CONNECTORWRITE)) {
+ LOG.debug(
+ "Full acid table with ACIDWRITE or CONNECTORWRITE capability:RW");
+ // clients have RW access to IUD ACID tables
+ accessType = ACCESSTYPE_READWRITE;
+ } else if (processorCapabilities.contains(HIVEFULLACIDREAD)
+ || processorCapabilities.contains(CONNECTORREAD)) {
+ LOG.debug(
+ "Full acid table with ACIDREAD or CONNECTORREAD capability:RO");
+ // clients have RO access to IUD ACID tables
+ accessType = ACCESSTYPE_READONLY;
+ } else {
+ LOG.debug("Full acid table without ACIDREAD/WRITE or "
+ + "CONNECTORREAD/WRITE capability:NONE");
+ // clients have NO access to IUD ACID tables
+ accessType = ACCESSTYPE_NONE;
+ }
+ }
+ }
+ break;
+ case "VIRTUAL_VIEW":
+ if (processorCapabilities.contains(HIVESQL) ||
+ processorCapabilities.contains(CONNECTORREAD)) {
+ accessType = ACCESSTYPE_READONLY;
+ } else {
+ accessType = ACCESSTYPE_NONE;
+ }
+ break;
+ case "MATERIALIZED_VIEW":
+ if ((processorCapabilities.contains(CONNECTORREAD) ||
+ processorCapabilities.contains(HIVEFULLACIDREAD)) && processorCapabilities
+ .contains(HIVEMQT)) {
+ LOG.info(
+ "Processor has one of the READ abilities and HIVEMQT, AccessType=RO");
+ accessType = ACCESSTYPE_READONLY;
+ } else {
+ LOG.info("Processor has no READ abilities or HIVEMQT, AccessType=None");
+ accessType = ACCESSTYPE_NONE;
+ }
+ break;
+ default:
+ accessType = ACCESSTYPE_NONE;
+ break;
+ }
+ return accessType;
+ }
+
+ // WITH CAPABLITIES ON TABLE
+ tCapabilities = tCapabilities.replaceAll("\\s", "")
+ .toUpperCase(); // remove spaces between tCapabilities + toUppercase
+ List<String> requiredCapabilities = Arrays.asList(tCapabilities.split(","));
+ switch (tableType) {
+ case "EXTERNAL_TABLE":
+ if (processorCapabilities.containsAll(requiredCapabilities)) {
+ // AccessType is RW
+ LOG.info(
+ "Abilities for match: Table type=" + tableType + ",accesstype is RW");
+ accessType = ACCESSTYPE_READWRITE;
+ break;
+ }
+
+ if (requiredCapabilities.contains(EXTWRITE) && processorCapabilities
+ .contains(EXTWRITE)) {
+ if (!isBucketed) {
+ LOG.info("EXTWRITE Matches, accessType=" + ACCESSTYPE_READWRITE);
+ accessType = ACCESSTYPE_READWRITE;
+ return accessType;
+ }
+ }
+
+ if (requiredCapabilities.contains(EXTREAD) && processorCapabilities
+ .contains(EXTREAD)) {
+ LOG.info("EXTREAD Matches, accessType=" + ACCESSTYPE_READONLY);
+ accessType = ACCESSTYPE_READONLY;
+ } else {
+ LOG.debug("No matches, accessType=" + ACCESSTYPE_NONE);
+ accessType = ACCESSTYPE_NONE;
+ }
+ break;
+ case "MANAGED_TABLE":
+ if (processorCapabilities.size() == 0) { // processor has no capabilities
+ LOG.info("Client has no capabilities for type " + tableType
+ + ",accesstype is NONE");
+ accessType = ACCESSTYPE_NONE;
+ return accessType;
+ }
+
+ if (processorCapabilities.containsAll(requiredCapabilities)) {
+ // AccessType is RW
+ LOG.info(
+ "Abilities for match: Table type=" + tableType + ",accesstype is RW");
+ accessType = ACCESSTYPE_READWRITE;
+ return accessType;
+ }
+
+ String txnal = params.get(AcidUtils.TABLE_IS_TRANSACTIONAL);
+ if (txnal == null || txnal
+ .equalsIgnoreCase("FALSE")) { // non-ACID MANAGED table
+ LOG.info("Table is non ACID, accesstype is RO");
+ accessType = ACCESSTYPE_READONLY;
+ return accessType;
+ }
+
+ if (txnal != null && txnal.equalsIgnoreCase("TRUE")) { // ACID table
+ String txntype = params.get(AcidUtils.TABLE_TRANSACTIONAL_PROPERTIES);
+ List<String> hintList = new ArrayList<>();
+ if (txntype != null && txntype
+ .equalsIgnoreCase("insert_only")) { // MICRO_MANAGED Tables
+ LOG.info("Table is INSERTONLY ACID");
+ // MGD table is insert only, not full ACID
+ if (processorCapabilities.containsAll(getWrites(requiredCapabilities))
+ // contains all writes on table
+ || processorCapabilities.contains(HIVEFULLACIDWRITE)) {
+ LOG.info("Processor has all writes or full acid write, access is RW");
+ // clients have RW access to INSERT-ONLY ACID tables
+ accessType = ACCESSTYPE_READWRITE;
+ return accessType;
+ }
+
+ if (processorCapabilities.contains(CONNECTORWRITE)) {
+ LOG.debug("Managed acid table with CONNECTORWRITE capability:RW");
+ // clients have RW access to INSERT-ONLY ACID tables with CONNWRITE
+ accessType = ACCESSTYPE_READWRITE;
+ return accessType;
+ } else if (processorCapabilities.containsAll(getReads(requiredCapabilities))
+ || processorCapabilities.contains(HIVEMANAGEDINSERTREAD)) {
+ LOG.debug("Managed acid table with MANAGEDREAD capability:RO");
+ accessType = ACCESSTYPE_READONLY;
+ return accessType;
+ } else if (processorCapabilities.contains(CONNECTORREAD)) {
+ LOG.debug("Managed acid table with CONNECTORREAD capability:RO");
+ accessType = ACCESSTYPE_READONLY;
+ return accessType;
+ } else {
+ LOG.debug("Managed acid table without any READ capability:NONE");
+ accessType = ACCESSTYPE_NONE;
+ return accessType;
+ }
+ } else { // MANAGED FULL ACID TABLES
+ LOG.info("Table is FULLACID");
+ if (processorCapabilities.containsAll(getWrites(requiredCapabilities))
+ // contains all writes on table
+ || processorCapabilities.contains(HIVEFULLACIDWRITE)) {
+ LOG.info("Processor has all writes or atleast " + HIVEFULLACIDWRITE
+ + ", access is RW");
+ // clients have RW access to ACID tables
+ accessType = ACCESSTYPE_READWRITE;
+ return accessType;
+ }
+
+ if (processorCapabilities.contains(CONNECTORWRITE)) {
+ LOG.debug("Full acid table with CONNECTORWRITE capability:RW");
+ // clients have RW access to IUD ACID tables
+ accessType = ACCESSTYPE_READWRITE;
+ return accessType;
+ } else if (processorCapabilities.contains(HIVEFULLACIDREAD)
+ || (processorCapabilities.contains(CONNECTORREAD))) {
+ LOG.debug("Full acid table with CONNECTORREAD/ACIDREAD capability:RO");
+ // clients have RO access to IUD ACID tables
+ accessType = ACCESSTYPE_READONLY;
+ return accessType;
+ } else {
+ LOG.debug("Full acid table without READ capability:RO");
+ // clients have NO access to IUD ACID tables
+ accessType = ACCESSTYPE_NONE;
+ return accessType;
+ }
+ }
+ }
+ break;
+ case "VIRTUAL_VIEW":
+ case "MATERIALIZED_VIEW":
+ if (processorCapabilities.containsAll(requiredCapabilities)) {
+ accessType = ACCESSTYPE_READONLY;
+ } else {
+ accessType = ACCESSTYPE_NONE;
+ }
+ break;
+ default:
+ accessType = ACCESSTYPE_NONE;
+ break;
+ }
+ }
+ return accessType;
+ }
+
+ private static List<String> getWrites(List<String> capabilities) {
+ List<String> writes = new ArrayList<>();
+ for (String capability : capabilities) {
+ if (capability.toUpperCase().endsWith("WRITE") ||
+ capability.toUpperCase().endsWith("STATS") ||
+ capability.toUpperCase().endsWith("INVALIDATE")) {
+ writes.add(capability);
+ }
+ }
+ return writes;
+ }
+
+ private static List<String> getReads(List<String> capabilities) {
+ List<String> reads = new ArrayList<>();
+ for (String capability : capabilities) {
+ if (capability.toUpperCase().endsWith("READ") ||
+ capability.toUpperCase().endsWith("SQL")) {
+ reads.add(capability);
+ }
+ }
+ return reads;
+ }
+
+ public static void getMaterializedViewInfo(StringBuilder tableInfo,
+ Table tbl, boolean isOutputPadded) {
+ formatOutput("View Original Text:", tbl.getViewOriginalText(), tableInfo);
+ formatOutput("View Expanded Text:", tbl.getViewExpandedText(), tableInfo);
+ formatOutput("Rewrite Enabled:", tbl.isRewriteEnabled() ? "Yes" : "No", tableInfo);
+ // TODO: IMPALA-11815: hive metastore doesn't privide api to judge whether the
+ // materialized view is outdated for rewriting, so set the flag to "Unknown"
+ formatOutput("Outdated for Rewriting:", "Unknown", tableInfo);
+ }
+
+ public static void createDataSource(IMetaStoreClient client, DataSource dataSource)
+ throws InvalidObjectException, AlreadyExistsException, MetaException, TException {
+ // noop for unsupported operation IMetaStoreClient.createDataConnector().
+ }
+
+ public static void dropDataSource(IMetaStoreClient client, String name,
+ boolean ifExists) throws NoSuchObjectException, InvalidOperationException,
+ MetaException, TException {
+ // noop for unsupported operation IMetaStoreClient.dropDataConnector().
+ }
+
+ public static Map<String, DataSource> loadAllDataSources(IMetaStoreClient client)
+ throws MetaException, TException {
+ // Unsupported operation IMetaStoreClient.getAllDataConnectorNames() and
+ // IMetaStoreClient.getDataConnector().
+ return null;
+ }
+
+ public static void setNotificationEventRequestWithFilter(
+ NotificationEventRequest eventRequest, MetaDataFilter metaDataFilter) {
+ // noop for non-existent fields of NotificationEventRequest
+ }
+
+ /**
+ * Gets the list of pseudo commit transaction events. Not supported for Apache Hive-3.
+ * @param event Commit transaction event
+ * @return
+ */
+ public static List<PseudoCommitTxnEvent> getPseudoCommitTxnEvents(
+ CommitTxnEvent event) {
+ throw new UnsupportedOperationException("CommitTxnEvent is not supported.");
+ }
+}
diff --git a/fe/src/compat-apache-hive-2/java/org/apache/thrift/TConfiguration.java b/fe/src/compat-apache-hive-2/java/org/apache/thrift/TConfiguration.java
new file mode 100644
index 000000000..9dd039f06
--- /dev/null
+++ b/fe/src/compat-apache-hive-2/java/org/apache/thrift/TConfiguration.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.thrift;
+
+public class TConfiguration {
+ public static final int DEFAULT_MAX_MESSAGE_SIZE = 100 * 1024 * 1024;
+ // this value is used consistently across all Thrift libraries
+ public static final int DEFAULT_MAX_FRAME_SIZE = 16384000;
+ public static final int DEFAULT_RECURSION_DEPTH = 64;
+
+ private int maxMessageSize;
+ private int maxFrameSize;
+ private int recursionLimit;
+
+ public TConfiguration() {
+ this(DEFAULT_MAX_MESSAGE_SIZE, DEFAULT_MAX_FRAME_SIZE, DEFAULT_RECURSION_DEPTH);
+ }
+ public TConfiguration(int maxMessageSize, int maxFrameSize, int recursionLimit) {
+ this.maxFrameSize = maxFrameSize;
+ this.maxMessageSize = maxMessageSize;
+ this.recursionLimit = recursionLimit;
+ }
+
+ public int getMaxMessageSize() { return maxMessageSize; }
+
+ public int getMaxFrameSize() { return maxFrameSize; }
+
+ public int getRecursionLimit() { return recursionLimit; }
+
+ public void setMaxMessageSize(int maxMessageSize) {
+ this.maxMessageSize = maxMessageSize;
+ }
+
+ public void setMaxFrameSize(int maxFrameSize) { this.maxFrameSize = maxFrameSize; }
+
+ public void setRecursionLimit(int recursionLimit) {
+ this.recursionLimit = recursionLimit;
+ }
+
+ public static final TConfiguration DEFAULT = new Builder().build();
+
+ public static TConfiguration.Builder custom() { return new Builder(); }
+
+ public static class Builder {
+ private int maxMessageSize;
+ private int maxFrameSize;
+ private int recursionLimit;
+
+ Builder() {
+ super();
+ this.maxFrameSize = DEFAULT_MAX_FRAME_SIZE;
+ this.maxMessageSize = DEFAULT_MAX_MESSAGE_SIZE;
+ this.recursionLimit = DEFAULT_RECURSION_DEPTH;
+ }
+
+ public Builder setMaxMessageSize(int maxMessageSize) {
+ this.maxMessageSize = maxMessageSize;
+ return this;
+ }
+
+ public Builder setMaxFrameSize(int maxFrameSize) {
+ this.maxFrameSize = maxFrameSize;
+ return this;
+ }
+
+ public Builder setRecursionLimit(int recursionLimit) {
+ this.recursionLimit = recursionLimit;
+ return this;
+ }
+
+ public TConfiguration build() {
+ return new TConfiguration(maxMessageSize, maxFrameSize, recursionLimit);
+ }
+ }
+}
diff --git a/fe/src/compat-apache-hive-2/java/org/apache/thrift/transport/TEndpointTransport.java b/fe/src/compat-apache-hive-2/java/org/apache/thrift/transport/TEndpointTransport.java
new file mode 100644
index 000000000..c4e6454a4
--- /dev/null
+++ b/fe/src/compat-apache-hive-2/java/org/apache/thrift/transport/TEndpointTransport.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.thrift.transport;
+
+import org.apache.thrift.TConfiguration;
+
+import java.util.Objects;
+
+public abstract class TEndpointTransport extends TTransport {
+ protected long getMaxMessageSize() { return getConfiguration().getMaxMessageSize(); }
+
+ protected long knownMessageSize;
+ protected long remainingMessageSize;
+
+ private TConfiguration _configuration;
+ public TConfiguration getConfiguration() { return _configuration; }
+
+ public TEndpointTransport(TConfiguration config) throws TTransportException {
+ _configuration = Objects.isNull(config) ? new TConfiguration() : config;
+
+ resetConsumedMessageSize(-1);
+ }
+
+ /**
+ * Resets RemainingMessageSize to the configured maximum
+ * @param newSize
+ */
+ protected void resetConsumedMessageSize(long newSize) throws TTransportException {
+ // full reset
+ if (newSize < 0) {
+ knownMessageSize = getMaxMessageSize();
+ remainingMessageSize = getMaxMessageSize();
+ return;
+ }
+
+ // update only: message size can shrink, but not grow
+ if (newSize > knownMessageSize)
+ throw new TTransportException(
+ TTransportException.END_OF_FILE, "MaxMessageSize reached");
+
+ knownMessageSize = newSize;
+ remainingMessageSize = newSize;
+ }
+
+ /**
+ * Updates RemainingMessageSize to reflect then known real message size (e.g. framed
+ * transport). Will throw if we already consumed too many bytes or if the new size is
+ * larger than allowed.
+ * @param size
+ */
+ public void updateKnownMessageSize(long size) throws TTransportException {
+ long consumed = knownMessageSize - remainingMessageSize;
+ resetConsumedMessageSize(size == 0 ? -1 : size);
+ countConsumedMessageBytes(consumed);
+ }
+
+ /**
+ * Throws if there are not enough bytes in the input stream to satisfy a read of
+ * numBytes bytes of data
+ * @param numBytes
+ */
+ public void checkReadBytesAvailable(long numBytes) throws TTransportException {
+ if (remainingMessageSize < numBytes)
+ throw new TTransportException(
+ TTransportException.END_OF_FILE, "MaxMessageSize reached");
+ }
+
+ /**
+ * Consumes numBytes from the RemainingMessageSize.
+ * @param numBytes
+ */
+ protected void countConsumedMessageBytes(long numBytes) throws TTransportException {
+ if (remainingMessageSize >= numBytes) {
+ remainingMessageSize -= numBytes;
+ } else {
+ remainingMessageSize = 0;
+ throw new TTransportException(
+ TTransportException.END_OF_FILE, "MaxMessageSize reached");
+ }
+ }
+}
diff --git a/fe/src/main/java/org/apache/impala/analysis/ConvertTableToIcebergStmt.java b/fe/src/main/java/org/apache/impala/analysis/ConvertTableToIcebergStmt.java
index e59309758..998db8a36 100644
--- a/fe/src/main/java/org/apache/impala/analysis/ConvertTableToIcebergStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/ConvertTableToIcebergStmt.java
@@ -25,7 +25,6 @@ import java.util.Collections;
import java.util.List;
import java.util.Map;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
-import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
import org.apache.impala.analysis.QueryStringBuilder.Create;
import org.apache.impala.analysis.QueryStringBuilder.Drop;
import org.apache.impala.analysis.QueryStringBuilder.Invalidate;
@@ -39,6 +38,7 @@ import org.apache.impala.catalog.FeTable;
import org.apache.impala.catalog.IcebergTable;
import org.apache.impala.catalog.Table;
import org.apache.impala.common.AnalysisException;
+import org.apache.impala.compat.MetastoreShim;
import org.apache.impala.thrift.THdfsFileFormat;
import org.apache.impala.thrift.TIcebergCatalog;
import org.apache.impala.thrift.TConvertTableRequest;
@@ -108,9 +108,9 @@ public class ConvertTableToIcebergStmt extends StatementBase implements SingleTa
"CONVERT TO ICEBERG is not supported for transactional tables");
}
- if (!MetaStoreUtils.isExternalTable(table.getMetaStoreTable())) {
+ if (!MetastoreShim.isExternalTable(table.getMetaStoreTable())) {
throw new AnalysisException(
- "CONVERT TO ICEBERG is not supported for managed tables");
+ "CONVERT TO ICEBERG is not supported for managed tables");
}
StorageDescriptor sd = table.getMetaStoreTable().getSd();
diff --git a/fe/src/main/java/org/apache/impala/catalog/ColumnStats.java b/fe/src/main/java/org/apache/impala/catalog/ColumnStats.java
index de41af094..a506f06f4 100644
--- a/fe/src/main/java/org/apache/impala/catalog/ColumnStats.java
+++ b/fe/src/main/java/org/apache/impala/catalog/ColumnStats.java
@@ -38,6 +38,7 @@ import org.apache.impala.analysis.LiteralExpr;
import org.apache.impala.analysis.DateLiteral;
import org.apache.impala.analysis.NumericLiteral;
import org.apache.impala.analysis.SlotRef;
+import org.apache.impala.compat.MetastoreShim;
import org.apache.impala.util.MetaStoreUtil;
import org.apache.impala.thrift.TColumnStats;
import org.apache.impala.thrift.TColumnValue;
@@ -933,11 +934,11 @@ public class ColumnStats {
Decimal highValue = null;
ScalarType colTypeScalar = (ScalarType) colType;
if (isLowValueSet && colStats.low_value.isSetDecimal_val()) {
- lowValue = new Decimal((short) colTypeScalar.decimalScale(),
+ lowValue = MetastoreShim.newDecimal((short) colTypeScalar.decimalScale(),
colStats.low_value.bufferForDecimal_val());
}
if (isHighValueSet && colStats.high_value.isSetDecimal_val()) {
- highValue = new Decimal((short) colTypeScalar.decimalScale(),
+ highValue = MetastoreShim.newDecimal((short) colTypeScalar.decimalScale(),
colStats.high_value.bufferForDecimal_val());
}
updateLowAndHighForHiveColumnStatsData(lowValue, highValue, decimalStatsData);
diff --git a/fe/src/main/java/org/apache/impala/catalog/FeCatalogUtils.java b/fe/src/main/java/org/apache/impala/catalog/FeCatalogUtils.java
index aa62ac352..4bac9a157 100644
--- a/fe/src/main/java/org/apache/impala/catalog/FeCatalogUtils.java
+++ b/fe/src/main/java/org/apache/impala/catalog/FeCatalogUtils.java
@@ -46,6 +46,7 @@ import org.apache.impala.catalog.local.LocalView;
import org.apache.impala.catalog.local.MetaProvider;
import org.apache.impala.catalog.local.MultiMetaProvider;
import org.apache.impala.common.ImpalaException;
+import org.apache.impala.common.ImpalaRuntimeException;
import org.apache.impala.common.NotImplementedException;
import org.apache.impala.service.BackendConfig;
import org.apache.impala.thrift.TCatalogObject;
@@ -278,6 +279,21 @@ public abstract class FeCatalogUtils {
return FileUtils.makePartName(partitionKeys, partitionValues);
}
+ public static String getPartitionName(List<String> partitionKeys,
+ Map<String, String> partitionColNameToValue) throws CatalogException {
+ List<String> partitionValues = new ArrayList<>();
+ for (String partitionColName : partitionKeys) {
+ String partitionValue = partitionColNameToValue.get(partitionColName);
+ if (partitionValue == null) {
+ String errorMessage = String.format("Can't find column %s in %s",
+ partitionColName, partitionColNameToValue);
+ throw new CatalogException(errorMessage);
+ }
+ partitionValues.add(partitionValue);
+ }
+ return FileUtils.makePartName(partitionKeys, partitionValues);
+ }
+
/**
* Return the set of all file formats used in the collection of partitions.
*/
diff --git a/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java b/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
index ef0b50cb9..9c0e0d03f 100644
--- a/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
@@ -50,7 +50,7 @@ import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.PrimaryKeysRequest;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
-import org.apache.hadoop.hive.metastore.utils.FileUtils;
+import org.apache.hadoop.hive.common.FileUtils;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.impala.analysis.Expr;
import org.apache.impala.analysis.LiteralExpr;
diff --git a/fe/src/main/java/org/apache/impala/catalog/Hive3MetastoreShimBase.java b/fe/src/main/java/org/apache/impala/catalog/Hive3MetastoreShimBase.java
index 447636ea1..10ce28c79 100644
--- a/fe/src/main/java/org/apache/impala/catalog/Hive3MetastoreShimBase.java
+++ b/fe/src/main/java/org/apache/impala/catalog/Hive3MetastoreShimBase.java
@@ -21,6 +21,7 @@ import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
+import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.BitSet;
@@ -40,7 +41,9 @@ import org.apache.hadoop.hive.metastore.LockRequestBuilder;
import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.Warehouse;
import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.api.Decimal;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.InsertEventRequestData;
import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
import org.apache.hadoop.hive.metastore.api.LockComponent;
import org.apache.hadoop.hive.metastore.api.LockRequest;
@@ -50,6 +53,7 @@ import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.NoSuchLockException;
import org.apache.hadoop.hive.metastore.api.NoSuchTxnException;
import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.PrincipalType;
import org.apache.hadoop.hive.metastore.api.SQLForeignKey;
import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey;
import org.apache.hadoop.hive.metastore.api.Table;
@@ -77,6 +81,7 @@ import org.apache.impala.thrift.TValidWriteIdList;
import org.apache.impala.util.AcidUtils;
import org.apache.impala.util.AcidUtils.TblTransaction;
import org.apache.impala.util.HiveMetadataFormatUtils;
+import org.apache.impala.util.MetaStoreUtil;
import org.apache.thrift.TException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -113,6 +118,9 @@ public class Hive3MetastoreShimBase {
// Not final, as it makes sense to set it based on role + instance, see IMPALA-8853.
public static String TRANSACTION_USER_ID = "Impala";
+ public static String HIVE_METASTORE_WAREHOUSE_EXTERNAL =
+ HiveConf.ConfVars.HIVE_METASTORE_WAREHOUSE_EXTERNAL.varname;
+
/**
* Initializes and returns a TblTransaction object for table 'tbl'. Opens a new
* transaction if txnId is not valid.
@@ -881,4 +889,33 @@ public class Hive3MetastoreShimBase {
return isTruncate;
}
}
+
+ /**
+ * Wrapper around MetaStoreUtils.isExternalTable
+ */
+ public static boolean isExternalTable(Table table) {
+ return MetaStoreUtils.isExternalTable(table);
+ }
+
+ public static Decimal newDecimal(short scale, ByteBuffer unscaled) {
+ return new Decimal(scale, unscaled);
+ }
+
+ public static PrincipalType getTableOwnerType(Table table) {
+ return table.getOwnerType();
+ }
+
+ public static void setInsertEventRequestDataReplace(
+ InsertEventRequestData insertEventRequestData, boolean isInsertOverwrite) {
+ insertEventRequestData.setReplace(isInsertOverwrite);
+ }
+
+ public static void setTableOwnerType(Table table, PrincipalType principalType) {
+ table.setOwnerType(principalType);
+ }
+
+ public static String getMetastoreConfigValue(
+ IMetaStoreClient client, String config, String defaultVal) throws TException {
+ return MetaStoreUtil.getMetastoreConfigValue(client, config, defaultVal);
+ }
}
diff --git a/fe/src/main/java/org/apache/impala/catalog/Table.java b/fe/src/main/java/org/apache/impala/catalog/Table.java
index ef7808076..33934c8e9 100644
--- a/fe/src/main/java/org/apache/impala/catalog/Table.java
+++ b/fe/src/main/java/org/apache/impala/catalog/Table.java
@@ -954,7 +954,8 @@ public abstract class Table extends CatalogObjectImpl implements FeTable {
LOG.warn("Owner of {} is unknown due to table is unloaded", getFullName());
return null;
}
- return msTable_.getOwnerType() == PrincipalType.USER ? msTable_.getOwner() : null;
+ return MetastoreShim.getTableOwnerType(msTable_) == PrincipalType.USER ?
+ msTable_.getOwner() : null;
}
public void setMetaStoreTable(org.apache.hadoop.hive.metastore.api.Table msTbl) {
diff --git a/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEventsProcessor.java b/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEventsProcessor.java
index 09cb877b0..8807d9e82 100644
--- a/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEventsProcessor.java
+++ b/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEventsProcessor.java
@@ -939,7 +939,7 @@ public class MetastoreEventsProcessor implements ExternalEventsProcessor {
throws TException {
try (MetaStoreClient metaStoreClient = catalog_.getMetaStoreClient()) {
IMetaStoreClient iMetaStoreClient = metaStoreClient.getHiveClient();
- return MetaStoreUtil.getMetastoreConfigValue(iMetaStoreClient, config, defaultVal);
+ return MetastoreShim.getMetastoreConfigValue(iMetaStoreClient, config, defaultVal);
}
}
diff --git a/fe/src/main/java/org/apache/impala/catalog/local/LocalTable.java b/fe/src/main/java/org/apache/impala/catalog/local/LocalTable.java
index ea74b0ca4..e1db906bd 100644
--- a/fe/src/main/java/org/apache/impala/catalog/local/LocalTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/local/LocalTable.java
@@ -237,7 +237,8 @@ abstract class LocalTable implements FeTable {
LOG.warn("Owner of {} is unknown due to msTable is unloaded", getFullName());
return null;
}
- return msTable_.getOwnerType() == PrincipalType.USER ? msTable_.getOwner() : null;
+ return MetastoreShim.getTableOwnerType(msTable_) == PrincipalType.USER ?
+ msTable_.getOwner() : null;
}
@Override
diff --git a/fe/src/main/java/org/apache/impala/catalog/paimon/PaimonUtil.java b/fe/src/main/java/org/apache/impala/catalog/paimon/PaimonUtil.java
index df3284009..afe04f221 100644
--- a/fe/src/main/java/org/apache/impala/catalog/paimon/PaimonUtil.java
+++ b/fe/src/main/java/org/apache/impala/catalog/paimon/PaimonUtil.java
@@ -255,9 +255,14 @@ public class PaimonUtil {
}
case HIVE_CATALOG: {
try {
- String location = isExternal ?
- hiveConf_.get(HiveConf.ConfVars.HIVE_METASTORE_WAREHOUSE_EXTERNAL.varname) :
- hiveConf_.get(HiveConf.ConfVars.METASTOREWAREHOUSE.varname);
+ String location = "";
+ if (MetastoreShim.getMajorVersion() == 2) {
+ location = hiveConf_.get(HiveConf.ConfVars.METASTOREWAREHOUSE.varname);
+ } else {
+ location = isExternal ?
+ hiveConf_.get(MetastoreShim.HIVE_METASTORE_WAREHOUSE_EXTERNAL) :
+ hiveConf_.get(HiveConf.ConfVars.METASTOREWAREHOUSE.varname);
+ }
Path path = new Path(location);
Options catalogOptions = new Options();
catalogOptions.set(CatalogOptions.WAREHOUSE, location);
diff --git a/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java b/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
index 8cf929946..a8984499d 100644
--- a/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
+++ b/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
@@ -3382,8 +3382,8 @@ public class CatalogOpExecutor {
authzManager_.updateTableOwnerPrivilege(params.server_name,
table.getDb().getName(), table.getName(),
table.getMetaStoreTable().getOwner(),
- table.getMetaStoreTable().getOwnerType(), /* newOwner */ null,
- /* newOwnerType */ null, resp);
+ MetastoreShim.getTableOwnerType(table.getMetaStoreTable()),
+ /* newOwner */ null, /* newOwnerType */ null, resp);
}
}
} finally {
@@ -3699,8 +3699,8 @@ public class CatalogOpExecutor {
// We will issue an HMS API call. Register in-flight event before we do.
modification.registerInflightEvent();
String dbName = Preconditions.checkNotNull(hdfsTable.getDb()).getName();
- client.getHiveClient()
- .truncateTable(dbName, hdfsTable.getName(), null);
+ MetastoreShim.truncateTable(client.getHiveClient(), dbName,
+ hdfsTable.getName(), null, null, -1L);
catalogTimeline.markEvent("Truncated table in Metastore");
LOG.trace("Time elapsed after truncating table {} using HMS API: {} msec",
hdfsTable.getFullName(), sw.elapsed(TimeUnit.MILLISECONDS));
@@ -4187,8 +4187,8 @@ public class CatalogOpExecutor {
if (authzConfig_.isEnabled()) {
authzManager_.updateTableOwnerPrivilege(serverName, msTable.getDbName(),
msTable.getTableName(), /* oldOwner */ null,
- /* oldOwnerType */ null, msTable.getOwner(), msTable.getOwnerType(),
- response);
+ /* oldOwnerType */ null, msTable.getOwner(),
+ MetastoreShim.getTableOwnerType(msTable), response);
}
} finally {
getMetastoreDdlLock().unlock();
@@ -6714,9 +6714,10 @@ public class CatalogOpExecutor {
InProgressTableModification modification) throws ImpalaException {
org.apache.hadoop.hive.metastore.api.Table msTbl = tbl.getMetaStoreTable().deepCopy();
String oldOwner = msTbl.getOwner();
- PrincipalType oldOwnerType = msTbl.getOwnerType();
+ PrincipalType oldOwnerType = MetastoreShim.getTableOwnerType(msTbl);
msTbl.setOwner(params.owner_name);
- msTbl.setOwnerType(PrincipalType.valueOf(params.owner_type.name()));
+ MetastoreShim.setTableOwnerType(msTbl,
+ PrincipalType.valueOf(params.owner_type.name()));
// A KuduTable is synchronized if it is a managed KuduTable, or an external table
// with the property of 'external.table.purge' being true.
@@ -6737,7 +6738,7 @@ public class CatalogOpExecutor {
if (authzConfig_.isEnabled()) {
authzManager_.updateTableOwnerPrivilege(params.server_name, msTbl.getDbName(),
msTbl.getTableName(), oldOwner, oldOwnerType, msTbl.getOwner(),
- msTbl.getOwnerType(), response);
+ MetastoreShim.getTableOwnerType(msTbl), response);
}
}
@@ -8213,7 +8214,8 @@ public class CatalogOpExecutor {
new ArrayList<>(newFiles.size()));
boolean isTransactional = AcidUtils.isTransactionalTable(tbl);
MetastoreShim.setPartitionVal(insertEventRequestData, partVals);
- insertEventRequestData.setReplace(isInsertOverwrite);
+ MetastoreShim.setInsertEventRequestDataReplace(insertEventRequestData,
+ isInsertOverwrite);
for (FileMetadata metadata : newFiles) {
insertEventRequestData.addToFilesAdded(metadata.filename);
insertEventRequestData.addToFilesAddedChecksum(metadata.getChecksum());
diff --git a/fe/src/main/java/org/apache/impala/util/HiveMetadataFormatUtils.java b/fe/src/main/java/org/apache/impala/util/HiveMetadataFormatUtils.java
index 2afb69884..b06446156 100644
--- a/fe/src/main/java/org/apache/impala/util/HiveMetadataFormatUtils.java
+++ b/fe/src/main/java/org/apache/impala/util/HiveMetadataFormatUtils.java
@@ -543,7 +543,8 @@ public class HiveMetadataFormatUtils {
boolean isOutputPadded) {
formatOutput("Database:", tbl.getDbName(), tableInfo);
formatOutput("OwnerType:",
- (tbl.getOwnerType() != null) ? tbl.getOwnerType().name() : "null", tableInfo);
+ (MetastoreShim.getTableOwnerType(tbl) != null) ?
+ MetastoreShim.getTableOwnerType(tbl).name() : "null", tableInfo);
formatOutput("Owner:", tbl.getOwner(), tableInfo);
formatOutput("CreateTime:", formatDate(tbl.getCreateTime()), tableInfo);
formatOutput("LastAccessTime:", formatDate(tbl.getLastAccessTime()), tableInfo);
diff --git a/java/calcite-planner/pom.xml b/java/calcite-planner/pom.xml
index 88a5e3d81..eca41b271 100644
--- a/java/calcite-planner/pom.xml
+++ b/java/calcite-planner/pom.xml
@@ -229,4 +229,37 @@ under the License.
</plugin>
</plugins>
</build>
+
+ <profiles>
+ <profile>
+ <id>apache-hive-2</id>
+ <activation>
+ <property>
+ <name>env.IMPALA_HIVE_DIST_TYPE</name>
+ <value>apache-hive-2</value>
+ </property>
+ </activation>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>3.11.0</version>
+ <executions>
+ <execution>
+ <id>default-testCompile</id>
+ <phase>test-compile</phase>
+ <goals>
+ <goal>testCompile</goal>
+ </goals>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
</project>
diff --git a/java/impala-package/pom.xml b/java/impala-package/pom.xml
index e256340e4..03b5740e6 100644
--- a/java/impala-package/pom.xml
+++ b/java/impala-package/pom.xml
@@ -76,4 +76,41 @@ under the License.
</plugin>
</plugins>
</build>
+
+ <profiles>
+ <profile>
+ <id>hive-major-version-2</id>
+ <activation>
+ <property>
+ <name>env.IMPALA_HIVE_MAJOR_VERSION</name>
+ <value>2</value>
+ </property>
+ </activation>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <version>1.8</version>
+ <executions>
+ <execution>
+ <id>copy-hive-metastore-2-dep</id>
+ <phase>package</phase>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <configuration>
+ <target>
+ <echo message="Copying hive-metastore-2.x.jar"/>
+ <copy file="${hive.home}/lib/hive-metastore-${hive.version}.jar"
+ tofile="${project.build.directory}/dependency/hive-metastore-${hive.version}.jar"/>
+ </target>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
</project>
diff --git a/java/pom.xml b/java/pom.xml
index a85cd2db5..196b1e6d5 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -40,6 +40,7 @@ under the License.
<hive.storage.api.version>${env.IMPALA_HIVE_STORAGE_API_VERSION}</hive.storage.api.version>
<hive.major.version>${env.IMPALA_HIVE_MAJOR_VERSION}</hive.major.version>
<hive.dist.type>${env.IMPALA_HIVE_DIST_TYPE}</hive.dist.type>
+ <hive.home>${env.HIVE_HOME}</hive.home>
<hudi.version>${env.IMPALA_HUDI_VERSION}</hudi.version>
<paimon.version>${env.IMPALA_PAIMON_VERSION}</paimon.version>
<arrow.version>${env.IMPALA_ARROW_VERSION}</arrow.version>
@@ -240,6 +241,21 @@ under the License.
<enabled>false</enabled>
</releases>
</repository>
+ <repository>
+ <!--
+ When compiling with Apache Hive 2.x, it is necessary to get the
+ pentaho-aggdesigner-algorithm dependency. This Maven repository address from Alibaba
+ Cloud can be used to stably get it.
+ -->
+ <id>spring-plugins-release</id>
+ <url>https://maven.aliyun.com/repository/spring-plugin</url>
+ <releases>
+ <enabled>true</enabled>
+ </releases>
+ <snapshots>
+ <enabled>false</enabled>
+ </snapshots>
+ </repository>
</repositories>
<pluginRepositories>
<pluginRepository>
diff --git a/java/shaded-deps/hive-exec/pom.xml b/java/shaded-deps/hive-exec/pom.xml
index 8c4ec479b..cf9e57abf 100644
--- a/java/shaded-deps/hive-exec/pom.xml
+++ b/java/shaded-deps/hive-exec/pom.xml
@@ -40,6 +40,61 @@ the same dependencies
<artifactId>hive-exec</artifactId>
</dependency>
</dependencies>
+
+ <!-- For different major versions of Hive, some classes have differences
+ in namespaces. -->
+ <profiles>
+ <profile>
+ <id>hive-major-version-3</id>
+ <activation>
+ <property>
+ <name>env.IMPALA_HIVE_MAJOR_VERSION</name>
+ <value>3</value>
+ </property>
+ </activation>
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-shade-plugin</artifactId>
+ <configuration>
+ <relocations>
+ <relocation>
+ <pattern>com.google</pattern>
+ <shadedPattern>hiveexec.com.google</shadedPattern>
+ </relocation>
+ </relocations>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+
+ <profile>
+ <id>hive-major-version-2</id>
+ <activation>
+ <property>
+ <name>env.IMPALA_HIVE_MAJOR_VERSION</name>
+ <value>2</value>
+ </property>
+ </activation>
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-shade-plugin</artifactId>
+ <configuration>
+ <relocations>
+ <relocation>
+ <pattern>org.apache.hive.com.google</pattern>
+ <shadedPattern>hiveexec.com.google</shadedPattern>
+ </relocation>
+ </relocations>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
+
<build>
<plugins>
<plugin>
@@ -51,10 +106,6 @@ the same dependencies
</includes>
</artifactSet>
<relocations>
- <relocation>
- <pattern>com.google</pattern>
- <shadedPattern>hiveexec.com.google</shadedPattern>
- </relocation>
<relocation>
<pattern>org.joda.time</pattern>
<shadedPattern>hiveexec.org.joda.time</shadedPattern>
diff --git a/testdata/bin/patch_hive2.sh b/testdata/bin/patch_hive2.sh
new file mode 100755
index 000000000..b88284ec9
--- /dev/null
+++ b/testdata/bin/patch_hive2.sh
@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -euo pipefail
+. $IMPALA_HOME/bin/report_build_error.sh
+setup_report_build_error
+
+if [[ "${USE_APACHE_HIVE_2}" != true ]]; then
+ exit 0
+fi
+
+# Rebuild tag
+HIVE_REBUILD=${HIVE_REBUILD-false}
+# Cache applied patches
+PATCHED_CACHE_FILE="$HIVE_SRC_DIR/.patched"
+if [ ! -f "$PATCHED_CACHE_FILE" ]; then touch "$PATCHED_CACHE_FILE"; fi
+# Apache Hive patch dir
+HIVE_PARCH_DIR="${IMPALA_HOME}/testdata/cluster/hive2"
+
+# Apply the patch and save the patch name to .patched
+function apply_patch {
+ p="$1"
+ status=1
+ while IFS= read -r line
+ do
+ if [ "$line" == "$p" ]; then
+ status=0
+ break
+ fi
+ done < $PATCHED_CACHE_FILE
+ if [ $status = "1" ] ;then
+ echo "Apply patch: $p"
+ patch -p1 < ${HIVE_PARCH_DIR}/$p
+ echo $p >> $PATCHED_CACHE_FILE
+ HIVE_REBUILD=true
+ fi
+}
+
+# 1. Apply patches
+pushd "$HIVE_SRC_DIR"
+for file in `ls ${HIVE_PARCH_DIR}/patch*.diff | sort`
+do
+ p=$(basename $file)
+ apply_patch $p
+done
+
+# 2. Repackage the hive submodules affected by the patch
+if [[ "${HIVE_REBUILD}" = "true" ]]; then
+ echo "Repackage the hive-metastore module"
+ ${IMPALA_HOME}/bin/mvn-quiet.sh -pl metastore clean package \
+ -Dmaven.test.skip
+ cp $HIVE_SRC_DIR/metastore/target/hive-metastore-${APACHE_HIVE_2_VERSION}.jar \
+ $HIVE_HOME/lib/
+fi
+popd
diff --git a/testdata/cluster/hive2/patch0-HIVE-18755.diff b/testdata/cluster/hive2/patch0-HIVE-18755.diff
new file mode 100644
index 000000000..7fdfff53e
--- /dev/null
+++ b/testdata/cluster/hive2/patch0-HIVE-18755.diff
@@ -0,0 +1,306 @@
+From 92a3a88870623dad3b8fa1e02d5e855ae55dfcff Mon Sep 17 00:00:00 2001
+From: ttttttz <24...@qq.com>
+Date: Mon, 7 Jul 2025 01:56:58 +0800
+Subject: [PATCH] Add the field catName to the NotificationEvent
+
+---
+ metastore/if/hive_metastore.thrift | 1 +
+ .../hive/metastore/api/NotificationEvent.java | 116 +++++++++++++++++-
+ 2 files changed, 112 insertions(+), 5 deletions(-)
+
+diff --git a/metastore/if/hive_metastore.thrift b/metastore/if/hive_metastore.thrift
+index 8a66bbcc70..bf33ae1954 100755
+--- a/metastore/if/hive_metastore.thrift
++++ b/metastore/if/hive_metastore.thrift
+@@ -820,6 +820,7 @@ struct NotificationEvent {
+ 5: optional string tableName,
+ 6: required string message,
+ 7: optional string messageFormat,
++ 8: optional string catName,
+ }
+
+ struct NotificationEventResponse {
+diff --git a/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/NotificationEvent.java b/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/NotificationEvent.java
+index 8e0fb40bcc..e0e1cd4dc5 100644
+--- a/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/NotificationEvent.java
++++ b/metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/NotificationEvent.java
+@@ -35,7 +35,7 @@
+
+ @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"})
+ @Generated(value = "Autogenerated by Thrift Compiler (0.9.3)")
+-public class NotificationEvent implements org.apache.thrift.TBase<NotificationEvent, NotificationEvent._Fields>, java.io.Serializable, Cloneable, Comparable<NotificationEvent> {
++@org.apache.hadoop.classification.InterfaceAudience.Public @org.apache.hadoop.classification.InterfaceStability.Stable public class NotificationEvent implements org.apache.thrift.TBase<NotificationEvent, NotificationEvent._Fields>, java.io.Serializable, Cloneable, Comparable<NotificationEvent> {
+ private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("NotificationEvent");
+
+ private static final org.apache.thrift.protocol.TField EVENT_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("eventId", org.apache.thrift.protocol.TType.I64, (short)1);
+@@ -45,6 +45,7 @@
+ private static final org.apache.thrift.protocol.TField TABLE_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("tableName", org.apache.thrift.protocol.TType.STRING, (short)5);
+ private static final org.apache.thrift.protocol.TField MESSAGE_FIELD_DESC = new org.apache.thrift.protocol.TField("message", org.apache.thrift.protocol.TType.STRING, (short)6);
+ private static final org.apache.thrift.protocol.TField MESSAGE_FORMAT_FIELD_DESC = new org.apache.thrift.protocol.TField("messageFormat", org.apache.thrift.protocol.TType.STRING, (short)7);
++ private static final org.apache.thrift.protocol.TField CAT_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("catName", org.apache.thrift.protocol.TType.STRING, (short)8);
+
+ private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
+ static {
+@@ -59,6 +60,7 @@
+ private String tableName; // optional
+ private String message; // required
+ private String messageFormat; // optional
++ private String catName; // optional
+
+ /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
+ public enum _Fields implements org.apache.thrift.TFieldIdEnum {
+@@ -68,7 +70,8 @@
+ DB_NAME((short)4, "dbName"),
+ TABLE_NAME((short)5, "tableName"),
+ MESSAGE((short)6, "message"),
+- MESSAGE_FORMAT((short)7, "messageFormat");
++ MESSAGE_FORMAT((short)7, "messageFormat"),
++ CAT_NAME((short)8, "catName");
+
+ private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
+
+@@ -97,6 +100,8 @@ public static _Fields findByThriftId(int fieldId) {
+ return MESSAGE;
+ case 7: // MESSAGE_FORMAT
+ return MESSAGE_FORMAT;
++ case 8: // CAT_NAME
++ return CAT_NAME;
+ default:
+ return null;
+ }
+@@ -140,7 +145,7 @@ public String getFieldName() {
+ private static final int __EVENTID_ISSET_ID = 0;
+ private static final int __EVENTTIME_ISSET_ID = 1;
+ private byte __isset_bitfield = 0;
+- private static final _Fields optionals[] = {_Fields.DB_NAME,_Fields.TABLE_NAME,_Fields.MESSAGE_FORMAT};
++ private static final _Fields optionals[] = {_Fields.DB_NAME,_Fields.TABLE_NAME,_Fields.MESSAGE_FORMAT,_Fields.CAT_NAME};
+ public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
+ static {
+ Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
+@@ -158,6 +163,8 @@ public String getFieldName() {
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
+ tmpMap.put(_Fields.MESSAGE_FORMAT, new org.apache.thrift.meta_data.FieldMetaData("messageFormat", org.apache.thrift.TFieldRequirementType.OPTIONAL,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
++ tmpMap.put(_Fields.CAT_NAME, new org.apache.thrift.meta_data.FieldMetaData("catName", org.apache.thrift.TFieldRequirementType.OPTIONAL,
++ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
+ metaDataMap = Collections.unmodifiableMap(tmpMap);
+ org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(NotificationEvent.class, metaDataMap);
+ }
+@@ -202,6 +209,9 @@ public NotificationEvent(NotificationEvent other) {
+ if (other.isSetMessageFormat()) {
+ this.messageFormat = other.messageFormat;
+ }
++ if (other.isSetCatName()) {
++ this.catName = other.catName;
++ }
+ }
+
+ public NotificationEvent deepCopy() {
+@@ -219,6 +229,7 @@ public void clear() {
+ this.tableName = null;
+ this.message = null;
+ this.messageFormat = null;
++ this.catName = null;
+ }
+
+ public long getEventId() {
+@@ -380,6 +391,29 @@ public void setMessageFormatIsSet(boolean value) {
+ }
+ }
+
++ public String getCatName() {
++ return this.catName;
++ }
++
++ public void setCatName(String catName) {
++ this.catName = catName;
++ }
++
++ public void unsetCatName() {
++ this.catName = null;
++ }
++
++ /** Returns true if field catName is set (has been assigned a value) and false otherwise */
++ public boolean isSetCatName() {
++ return this.catName != null;
++ }
++
++ public void setCatNameIsSet(boolean value) {
++ if (!value) {
++ this.catName = null;
++ }
++ }
++
+ public void setFieldValue(_Fields field, Object value) {
+ switch (field) {
+ case EVENT_ID:
+@@ -438,6 +472,14 @@ public void setFieldValue(_Fields field, Object value) {
+ }
+ break;
+
++ case CAT_NAME:
++ if (value == null) {
++ unsetCatName();
++ } else {
++ setCatName((String)value);
++ }
++ break;
++
+ }
+ }
+
+@@ -464,6 +506,9 @@ public Object getFieldValue(_Fields field) {
+ case MESSAGE_FORMAT:
+ return getMessageFormat();
+
++ case CAT_NAME:
++ return getCatName();
++
+ }
+ throw new IllegalStateException();
+ }
+@@ -489,6 +534,8 @@ public boolean isSet(_Fields field) {
+ return isSetMessage();
+ case MESSAGE_FORMAT:
+ return isSetMessageFormat();
++ case CAT_NAME:
++ return isSetCatName();
+ }
+ throw new IllegalStateException();
+ }
+@@ -569,6 +616,15 @@ public boolean equals(NotificationEvent that) {
+ return false;
+ }
+
++ boolean this_present_catName = true && this.isSetCatName();
++ boolean that_present_catName = true && that.isSetCatName();
++ if (this_present_catName || that_present_catName) {
++ if (!(this_present_catName && that_present_catName))
++ return false;
++ if (!this.catName.equals(that.catName))
++ return false;
++ }
++
+ return true;
+ }
+
+@@ -611,6 +667,11 @@ public int hashCode() {
+ if (present_messageFormat)
+ list.add(messageFormat);
+
++ boolean present_catName = true && (isSetCatName());
++ list.add(present_catName);
++ if (present_catName)
++ list.add(catName);
++
+ return list.hashCode();
+ }
+
+@@ -692,6 +753,16 @@ public int compareTo(NotificationEvent other) {
+ return lastComparison;
+ }
+ }
++ lastComparison = Boolean.valueOf(isSetCatName()).compareTo(other.isSetCatName());
++ if (lastComparison != 0) {
++ return lastComparison;
++ }
++ if (isSetCatName()) {
++ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.catName, other.catName);
++ if (lastComparison != 0) {
++ return lastComparison;
++ }
++ }
+ return 0;
+ }
+
+@@ -765,6 +836,16 @@ public String toString() {
+ }
+ first = false;
+ }
++ if (isSetCatName()) {
++ if (!first) sb.append(", ");
++ sb.append("catName:");
++ if (this.catName == null) {
++ sb.append("null");
++ } else {
++ sb.append(this.catName);
++ }
++ first = false;
++ }
+ sb.append(")");
+ return sb.toString();
+ }
+@@ -882,6 +963,14 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, NotificationEvent s
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
++ case 8: // CAT_NAME
++ if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
++ struct.catName = iprot.readString();
++ struct.setCatNameIsSet(true);
++ } else {
++ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
++ }
++ break;
+ default:
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+@@ -932,6 +1021,13 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, NotificationEvent
+ oprot.writeFieldEnd();
+ }
+ }
++ if (struct.catName != null) {
++ if (struct.isSetCatName()) {
++ oprot.writeFieldBegin(CAT_NAME_FIELD_DESC);
++ oprot.writeString(struct.catName);
++ oprot.writeFieldEnd();
++ }
++ }
+ oprot.writeFieldStop();
+ oprot.writeStructEnd();
+ }
+@@ -963,7 +1059,10 @@ public void write(org.apache.thrift.protocol.TProtocol prot, NotificationEvent s
+ if (struct.isSetMessageFormat()) {
+ optionals.set(2);
+ }
+- oprot.writeBitSet(optionals, 3);
++ if (struct.isSetCatName()) {
++ optionals.set(3);
++ }
++ oprot.writeBitSet(optionals, 4);
+ if (struct.isSetDbName()) {
+ oprot.writeString(struct.dbName);
+ }
+@@ -973,6 +1072,9 @@ public void write(org.apache.thrift.protocol.TProtocol prot, NotificationEvent s
+ if (struct.isSetMessageFormat()) {
+ oprot.writeString(struct.messageFormat);
+ }
++ if (struct.isSetCatName()) {
++ oprot.writeString(struct.catName);
++ }
+ }
+
+ @Override
+@@ -986,7 +1088,7 @@ public void read(org.apache.thrift.protocol.TProtocol prot, NotificationEvent st
+ struct.setEventTypeIsSet(true);
+ struct.message = iprot.readString();
+ struct.setMessageIsSet(true);
+- BitSet incoming = iprot.readBitSet(3);
++ BitSet incoming = iprot.readBitSet(4);
+ if (incoming.get(0)) {
+ struct.dbName = iprot.readString();
+ struct.setDbNameIsSet(true);
+@@ -999,6 +1101,10 @@ public void read(org.apache.thrift.protocol.TProtocol prot, NotificationEvent st
+ struct.messageFormat = iprot.readString();
+ struct.setMessageFormatIsSet(true);
+ }
++ if (incoming.get(3)) {
++ struct.catName = iprot.readString();
++ struct.setCatNameIsSet(true);
++ }
+ }
+ }
+
+--
+2.32.1 (Apple Git-133)
+
diff --git a/tests/common/environ.py b/tests/common/environ.py
index 54ae8f093..f15ba4727 100644
--- a/tests/common/environ.py
+++ b/tests/common/environ.py
@@ -95,7 +95,8 @@ if impala_iceberg_version:
# If we can't parse the version, default to format version 1
pass
-IS_APACHE_HIVE = os.environ.get("USE_APACHE_HIVE_3", False) == 'true'
+IS_APACHE_HIVE = (os.environ.get("USE_APACHE_HIVE_3", False) == 'true'
+ or os.environ.get("USE_APACHE_HIVE_2", False) == 'true')
# Resolve any symlinks in the path.
impalad_basedir = \