You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by jo...@apache.org on 2022/10/19 16:23:10 UTC
[impala] branch master updated (73e2e0a58 -> 83c5e6e40)
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
from 73e2e0a58 IMPALA-11657: Ignore git-reset failures in build-all-flag-combinations.sh
new ad438e7e3 IMPALA-11581: ALTER TABLE RENAME TO doesn't update transient_lastDdlTime
new 22e5ca3d0 IMPALA-11667: Clean up Java dependency exclusions
new 83c5e6e40 IMPALA-11670: Upgrade components, add envvars for override
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
bin/impala-config.sh | 16 ++-
fe/pom.xml | 118 +----------------
.../apache/impala/service/CatalogOpExecutor.java | 1 +
java/TableFlattener/pom.xml | 10 --
java/datagenerator/pom.xml | 36 ------
java/executor-deps/pom.xml | 35 ------
java/ext-data-source/sample/pom.xml | 1 -
java/ext-data-source/test/pom.xml | 1 -
java/pom.xml | 140 ++++++++++++++++++---
java/shaded-deps/hive-exec/pom.xml | 13 --
java/test-hive-udfs/pom.xml | 19 ---
java/yarn-extras/pom.xml | 24 ----
tests/metadata/test_last_ddl_time_update.py | 117 +++++++++++------
13 files changed, 220 insertions(+), 311 deletions(-)
[impala] 03/03: IMPALA-11670: Upgrade components, add envvars for override
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 83c5e6e4098d8ed75de09a7e228d6ef10de2ee12
Author: Michael Smith <mi...@cloudera.com>
AuthorDate: Mon Oct 17 15:09:08 2022 -0700
IMPALA-11670: Upgrade components, add envvars for override
Upgrades guava to 31.1-jre and jackson-databind to 2.13.4.2 to address
CVEs. Adds environment variables for commonly-updated components so they
can be customized via the branch-specific impala-config-branch.sh in a
way that allows both to be updated regularly without merge conflicts.
Also updates httpcomponents.httpcore to 4.4.14 to be consistent with
other httpcomponents libraries included transitively.
Change-Id: I1c2c4481ca3f498abf302aa05361d950b1ed1216
Reviewed-on: http://gerrit.cloudera.org:8080/19147
Reviewed-by: Joe McDonnell <jo...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
bin/impala-config.sh | 16 +++++++++++++++-
java/pom.xml | 33 ++++++++++++++++++---------------
2 files changed, 33 insertions(+), 16 deletions(-)
diff --git a/bin/impala-config.sh b/bin/impala-config.sh
index 4aef3bcf7..718984009 100755
--- a/bin/impala-config.sh
+++ b/bin/impala-config.sh
@@ -221,10 +221,24 @@ export APACHE_OZONE_VERSION=1.2.1
export ARCH_NAME=$(uname -p)
+# Java dependencies that are not also runtime components. Declaring versions here allows
+# other branches to override them in impala-config-branch.sh for cleaner patches.
+export IMPALA_BOUNCY_CASTLE_VERSION=1.68
+export IMPALA_COS_VERSION=3.1.0-8.0.8
+export IMPALA_DERBY_VERSION=10.14.2.0
+export IMPALA_GUAVA_VERSION=31.1-jre
export IMPALA_HUDI_VERSION=0.5.0-incubating
+export IMPALA_HTTP_CORE_VERSION=4.4.14
+export IMPALA_JACKSON_DATABIND_VERSION=2.13.4.2
+export IMPALA_JSON_SMART_VERSION=2.4.7
+export IMPALA_JUNIT_VERSION=4.12
export IMPALA_KITE_VERSION=1.1.0
export IMPALA_ORC_JAVA_VERSION=1.7.6
-export IMPALA_COS_VERSION=3.1.0-8.0.8
+export IMPALA_PAC4J_VERSION=4.5.5
+export IMPALA_RELOAD4j_VERSION=1.2.22
+export IMPALA_SLF4J_VERSION=2.0.3
+export IMPALA_SPRINGFRAMEWORK_VERSION=5.3.20
+export IMPALA_XMLSEC_VERSION=2.2.3
# When Impala is building docker images on Redhat-based distributions,
# it is useful to be able to customize the base image. Some users will
diff --git a/java/pom.xml b/java/pom.xml
index b4e75a17f..5a76a72f4 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -25,6 +25,11 @@ under the License.
<packaging>pom</packaging>
<name>Apache Impala Parent POM</name>
+ <!--
+ When adding new properties to control library versions, also add them to
+ bin/impala-config.sh as environment variables so other branches can easily override
+ them in bin/impala-config-branch.sh for cleaner patches.
+ -->
<properties>
<surefire.reports.dir>${env.IMPALA_LOGS_DIR}/fe_tests</surefire.reports.dir>
<jacoco.skip>true</jacoco.skip>
@@ -53,30 +58,28 @@ under the License.
<impala.query.event.hook.api.version>${project.version}</impala.query.event.hook.api.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<kudu.version>${env.IMPALA_KUDU_VERSION}</kudu.version>
- <commons-io.version>2.6</commons-io.version>
- <slf4j.version>2.0.3</slf4j.version>
- <reload4j.version>1.2.22</reload4j.version>
- <junit.version>4.12</junit.version>
+ <slf4j.version>${env.IMPALA_SLF4J_VERSION}</slf4j.version>
+ <reload4j.version>${env.IMPALA_RELOAD4j_VERSION}</reload4j.version>
+ <junit.version>${env.IMPALA_JUNIT_VERSION}</junit.version>
<!-- Beware compatibility requirements with Thrift and
KMS; see IMPALA-4210. -->
- <httpcomponents.core.version>4.4.9</httpcomponents.core.version>
+ <httpcomponents.core.version>${env.IMPALA_HTTP_CORE_VERSION}</httpcomponents.core.version>
<yarn-extras.version>${project.version}</yarn-extras.version>
<eclipse.output.directory>eclipse-classes</eclipse.output.directory>
<!-- hive-exec seems to leak this version of guava onto our classpath,
so it's important to depend on the same one -->
- <guava.version>28.1-jre</guava.version>
- <derby.version>10.14.2.0</derby.version>
- <jackson-databind.version>2.12.6.1</jackson-databind.version>
+ <guava.version>${env.IMPALA_GUAVA_VERSION}</guava.version>
+ <derby.version>${env.IMPALA_DERBY_VERSION}</derby.version>
+ <jackson-databind.version>${env.IMPALA_JACKSON_DATABIND_VERSION}</jackson-databind.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<iceberg.version>${env.IMPALA_ICEBERG_VERSION}</iceberg.version>
- <pac4j.version>4.5.5</pac4j.version>
+ <pac4j.version>${env.IMPALA_PAC4J_VERSION}</pac4j.version>
<!-- xmlsec, bcprov-jdk15on and springframework are not used by Impala directly,
- but they are needed by pac4j. This uses a newer xmlsec/spring to address CVEs,
- but bcprov-jdk15on matches the versions from pac4j 4.5.5. -->
- <xmlsec.version>2.2.3</xmlsec.version>
- <bouncy-castle.version>1.68</bouncy-castle.version>
- <springframework.version>5.3.20</springframework.version>
- <json-smart.version>2.4.7</json-smart.version>
+ but they are needed by pac4j. This uses newer versions to address CVEs. -->
+ <xmlsec.version>${env.IMPALA_XMLSEC_VERSION}</xmlsec.version>
+ <bouncy-castle.version>${env.IMPALA_BOUNCY_CASTLE_VERSION}</bouncy-castle.version>
+ <springframework.version>${env.IMPALA_SPRINGFRAMEWORK_VERSION}</springframework.version>
+ <json-smart.version>${env.IMPALA_JSON_SMART_VERSION}</json-smart.version>
</properties>
<repositories>
[impala] 02/03: IMPALA-11667: Clean up Java dependency exclusions
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 22e5ca3d0a891373251a4f08f3c3824491336e34
Author: Michael Smith <mi...@cloudera.com>
AuthorDate: Mon Oct 17 11:38:14 2022 -0700
IMPALA-11667: Clean up Java dependency exclusions
Use dependencyManagement to simplify Java dependencies by directly
controlling versions of transitive dependencies instead of using
exclusions and direct inclusion.
Dependency management specifies versions authoritatively, so redundant
version declarations are also removed.
Change-Id: I424a175135855dcbd38ae432ea111cca5f562633
Reviewed-on: http://gerrit.cloudera.org:8080/19146
Reviewed-by: Joe McDonnell <jo...@cloudera.com>
Tested-by: Joe McDonnell <jo...@cloudera.com>
---
fe/pom.xml | 118 +-----------------------------------
java/TableFlattener/pom.xml | 10 ---
java/datagenerator/pom.xml | 36 -----------
java/executor-deps/pom.xml | 35 -----------
java/ext-data-source/sample/pom.xml | 1 -
java/ext-data-source/test/pom.xml | 1 -
java/pom.xml | 111 ++++++++++++++++++++++++++++++++-
java/shaded-deps/hive-exec/pom.xml | 13 ----
java/test-hive-udfs/pom.xml | 19 ------
java/yarn-extras/pom.xml | 24 --------
10 files changed, 112 insertions(+), 256 deletions(-)
diff --git a/fe/pom.xml b/fe/pom.xml
index a2f59c912..2cc39e6be 100644
--- a/fe/pom.xml
+++ b/fe/pom.xml
@@ -33,13 +33,6 @@ under the License.
<name>Apache Impala Query Engine Frontend</name>
<dependencies>
- <!-- Pin json-smart dependency. -->
- <dependency>
- <groupId>net.minidev</groupId>
- <artifactId>json-smart</artifactId>
- <version>${json-smart.version}</version>
- </dependency>
-
<dependency>
<groupId>org.apache.impala</groupId>
<artifactId>impala-executor-deps</artifactId>
@@ -64,15 +57,6 @@ under the License.
<artifactId>hadoop-auth</artifactId>
<version>${hadoop.version}</version>
<exclusions>
- <!-- Exclude json-smart to pin the version -->
- <exclusion>
- <groupId>net.minidev</groupId>
- <artifactId>json-smart</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
@@ -90,14 +74,6 @@ under the License.
<groupId>org.apache.httpcomponents</groupId>
<artifactId>fluent-hc</artifactId>
</exclusion>
- <exclusion>
- <groupId>org.apache.parquet</groupId>
- <artifactId>parquet-avro</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.avro</groupId>
- <artifactId>*</artifactId>
- </exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>*</artifactId>
@@ -122,11 +98,6 @@ under the License.
<!-- Ranger jar also includes solr artifacts which we not exclude here since
they are used during the instantiation of RangerAuthorizationFactory
(See IMPALA-10644 for more details. -->
- <!-- Exclude json-smart to pin the version -->
- <exclusion>
- <groupId>net.minidev</groupId>
- <artifactId>json-smart</artifactId>
- </exclusion>
<!-- We need to exclude ranger-plugins-audit here, or the build will fail
to resolve dependencies for cruise-control-metrics-reporter, which is
unneeded and not uploaded to toolchain. -->
@@ -179,11 +150,6 @@ under the License.
We do not explicitly exclude all artifacts in org.eclipse.jetty because
jetty-client is needed.
-->
- <!-- Exclude json-smart to pin the version -->
- <exclusion>
- <groupId>net.minidev</groupId>
- <artifactId>json-smart</artifactId>
- </exclusion>
<!-- We exclude hive-exec which this jar includes because hive-exec
brings with it tons of other unnecessary dependencies including thrift 0.9.3
which can cause build failures.
@@ -239,7 +205,6 @@ under the License.
<dependency>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
- <version>${avro.version}</version>
</dependency>
<dependency>
@@ -280,22 +245,6 @@ under the License.
<groupId>org.apache.thrift</groupId>
<artifactId>libthrift</artifactId>
<version>${thrift.version}</version>
- <!-- httpcore overriden below -->
- <exclusions>
- <exclusion>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpcore</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <!-- Override httpcore from libthrift to a later maintenance version. Noticed
- certain SPNEGO auth errors when connecting to kerberized HTTP endpoints
- using v4.4.1 pulled by libthrift. -->
- <dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpcore</artifactId>
- <version>${httpcomponents.core.version}</version>
</dependency>
<dependency>
@@ -323,22 +272,9 @@ under the License.
<version>1.2</version>
</dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- <version>${slf4j.version}</version>
- </dependency>
-
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-reload4j</artifactId>
- <version>${slf4j.version}</version>
- </dependency>
-
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <version>${guava.version}</version>
</dependency>
<dependency>
@@ -484,11 +420,6 @@ under the License.
<groupId>io.netty</groupId>
<artifactId>*</artifactId>
</exclusion>
- <!-- Exclude json-smart to pin the version -->
- <exclusion>
- <groupId>net.minidev</groupId>
- <artifactId>json-smart</artifactId>
- </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -511,11 +442,6 @@ under the License.
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-1.2-api</artifactId>
</exclusion>
- <!-- Exclude json-smart to pin the version -->
- <exclusion>
- <groupId>net.minidev</groupId>
- <artifactId>json-smart</artifactId>
- </exclusion>
<exclusion>
<groupId>org.apache.hive</groupId>
<artifactId>hive-serde</artifactId>
@@ -584,16 +510,6 @@ under the License.
<groupId>org.javassist</groupId>
<artifactId>javassist</artifactId>
</exclusion>
- <exclusion>
- <!-- The dependency included in pac4j is old and has known CVEs.
- We exclude it from here and add a separate dependency down below.-->
- <groupId>org.bouncycastle</groupId>
- <artifactId>org.bouncycastle</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.santuario</groupId>
- <artifactId>xmlsec</artifactId>
- </exclusion>
<exclusion>
<groupId>commons-collections</groupId>
<artifactId>commons-collections</artifactId>
@@ -608,20 +524,9 @@ under the License.
</exclusion>
</exclusions>
</dependency>
- <dependency>
- <groupId>org.bouncycastle</groupId>
- <artifactId>bcprov-jdk15on</artifactId>
- <version>${bcprov-jdk15on.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.santuario</groupId>
- <artifactId>xmlsec</artifactId>
- <version>${xmlsec.version}</version>
- </dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-core</artifactId>
- <version>${springframework.version}</version>
</dependency>
</dependencies>
@@ -863,10 +768,10 @@ under the License.
<exclude>ant-contrib:*</exclude>
<exclude>org.apache.ant:*</exclude>
<exclude>org.eclipse.jetty:*</exclude>
- <!-- We use reload4j, avoid all log4j 1.x implementations -->
+ <!-- We use reload4j, exclude log4j 1.x and 2.x -->
<exclude>org.apache.logging.log4j:log4j-slf4j-impl</exclude>
<exclude>org.apache.logging.log4j:log4j-1.2-api</exclude>
- <exclude>log4j:log4j</exclude>
+ <exclude>log4j:*</exclude>
<exclude>org.slf4j:slf4j-log4j12</exclude>
<!-- IMPALA-9108: Avoid pulling in leveldbjni, which is unneeded. -->
<exclude>org.fusesource.leveldbjni:*</exclude>
@@ -1163,23 +1068,4 @@ under the License.
</properties>
</profile>
</profiles>
-
- <dependencyManagement>
- <dependencies>
- <!--
- Pin org.glassfish:javax.el explicitly.
-
- HBase depends on this indirectly, and it's configured with
- a range of versions. This causes Maven to talk to all configured
- repositories, leading both to a lot of chattiness, and also
- failures if one of the repositories is unavailable.
- -->
- <dependency>
- <groupId>org.glassfish</groupId>
- <artifactId>javax.el</artifactId>
- <version>3.0.1-b08</version>
- </dependency>
-
- </dependencies>
- </dependencyManagement>
</project>
diff --git a/java/TableFlattener/pom.xml b/java/TableFlattener/pom.xml
index 439ce6bef..d1195ae3d 100644
--- a/java/TableFlattener/pom.xml
+++ b/java/TableFlattener/pom.xml
@@ -49,25 +49,15 @@
<artifactId>hadoop-client</artifactId>
<version>${hadoop.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro</artifactId>
- <version>${avro.version}</version>
- </dependency>
<dependency>
<groupId>org.apache.parquet</groupId>
<artifactId>parquet-avro</artifactId>
- <version>${parquet.version}</version>
</dependency>
<dependency>
<groupId>org.kitesdk</groupId>
<artifactId>kite-data-core</artifactId>
<version>${kite.version}</version>
<exclusions>
- <exclusion>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro</artifactId>
- </exclusion>
<exclusion>
<groupId>com.twitter</groupId>
<artifactId>parquet-avro</artifactId>
diff --git a/java/datagenerator/pom.xml b/java/datagenerator/pom.xml
index 9b58b5051..16b9c1128 100644
--- a/java/datagenerator/pom.xml
+++ b/java/datagenerator/pom.xml
@@ -33,30 +33,11 @@ under the License.
<name>Test Data Generator</name>
<dependencies>
- <!-- Force json-smart dependency.
- See https://issues.apache.org/jira/browse/HADOOP-14903 -->
- <dependency>
- <groupId>net.minidev</groupId>
- <artifactId>json-smart</artifactId>
- <version>${json-smart.version}</version>
- </dependency>
-
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <version>${guava.version}</version>
- </dependency>
-
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
<exclusions>
- <!-- https://issues.apache.org/jira/browse/HADOOP-14903 -->
- <exclusion>
- <groupId>net.minidev</groupId>
- <artifactId>json-smart</artifactId>
- </exclusion>
<exclusion>
<groupId>org.eclipse.jetty</groupId>
<artifactId>*</artifactId>
@@ -77,22 +58,9 @@ under the License.
</exclusions>
</dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- <version>${slf4j.version}</version>
- </dependency>
-
<dependency>
<groupId>org.apache.parquet</groupId>
<artifactId>parquet-avro</artifactId>
- <version>${parquet.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro</artifactId>
- <version>${avro.version}</version>
</dependency>
<dependency>
@@ -100,10 +68,6 @@ under the License.
<artifactId>kite-data-core</artifactId>
<version>${kite.version}</version>
<exclusions>
- <exclusion>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro</artifactId>
- </exclusion>
<exclusion>
<groupId>com.twitter</groupId>
<artifactId>parquet-avro</artifactId>
diff --git a/java/executor-deps/pom.xml b/java/executor-deps/pom.xml
index e187a8562..232e8ef1f 100644
--- a/java/executor-deps/pom.xml
+++ b/java/executor-deps/pom.xml
@@ -44,14 +44,6 @@ under the License.
<name>Impala Executor Dependencies</name>
<dependencies>
-
- <!-- Pin json-smart dependency. -->
- <dependency>
- <groupId>net.minidev</groupId>
- <artifactId>json-smart</artifactId>
- <version>${json-smart.version}</version>
- </dependency>
-
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs-client</artifactId>
@@ -63,11 +55,6 @@ under the License.
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
<exclusions>
- <!-- Exclude json-smart to pin the version -->
- <exclusion>
- <groupId>net.minidev</groupId>
- <artifactId>json-smart</artifactId>
- </exclusion>
<exclusion>
<groupId>org.eclipse.jetty</groupId>
<artifactId>*</artifactId>
@@ -85,10 +72,6 @@ under the License.
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-servlet</artifactId>
</exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
@@ -117,10 +100,6 @@ under the License.
<groupId>org.eclipse.jetty</groupId>
<artifactId>*</artifactId>
</exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
@@ -150,25 +129,11 @@ under the License.
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
<version>${hbase.version}</version>
- <exclusions>
- <!-- Exclude json-smart to pin the version -->
- <exclusion>
- <groupId>net.minidev</groupId>
- <artifactId>json-smart</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-common</artifactId>
- <exclusions>
- <!-- Exclude json-smart to pin the version -->
- <exclusion>
- <groupId>net.minidev</groupId>
- <artifactId>json-smart</artifactId>
- </exclusion>
- </exclusions>
<version>${hbase.version}</version>
</dependency>
diff --git a/java/ext-data-source/sample/pom.xml b/java/ext-data-source/sample/pom.xml
index 8e1ec9744..a0470bbbe 100644
--- a/java/ext-data-source/sample/pom.xml
+++ b/java/ext-data-source/sample/pom.xml
@@ -40,7 +40,6 @@
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
- <version>${guava.version}</version>
</dependency>
</dependencies>
diff --git a/java/ext-data-source/test/pom.xml b/java/ext-data-source/test/pom.xml
index 2e1f9fe33..ab705312b 100644
--- a/java/ext-data-source/test/pom.xml
+++ b/java/ext-data-source/test/pom.xml
@@ -40,7 +40,6 @@
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
- <version>${guava.version}</version>
</dependency>
</dependencies>
diff --git a/java/pom.xml b/java/pom.xml
index c734a21bb..b4e75a17f 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -55,6 +55,7 @@ under the License.
<kudu.version>${env.IMPALA_KUDU_VERSION}</kudu.version>
<commons-io.version>2.6</commons-io.version>
<slf4j.version>2.0.3</slf4j.version>
+ <reload4j.version>1.2.22</reload4j.version>
<junit.version>4.12</junit.version>
<!-- Beware compatibility requirements with Thrift and
KMS; see IMPALA-4210. -->
@@ -73,7 +74,7 @@ under the License.
but they are needed by pac4j. This uses a newer xmlsec/spring to address CVEs,
but bcprov-jdk15on matches the versions from pac4j 4.5.5. -->
<xmlsec.version>2.2.3</xmlsec.version>
- <bcprov-jdk15on.version>1.68</bcprov-jdk15on.version>
+ <bouncy-castle.version>1.68</bouncy-castle.version>
<springframework.version>5.3.20</springframework.version>
<json-smart.version>2.4.7</json-smart.version>
</properties>
@@ -251,6 +252,114 @@ under the License.
</pluginRepository>
</pluginRepositories>
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>${guava.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>net.minidev</groupId>
+ <artifactId>json-smart</artifactId>
+ <version>${json-smart.version}</version>
+ </dependency>
+
+ <!--
+ Pin org.glassfish:javax.el explicitly.
+
+ HBase depends on this indirectly, and it's configured with
+ a range of versions. This causes Maven to talk to all configured
+ repositories, leading both to a lot of chattiness, and also
+ failures if one of the repositories is unavailable.
+ -->
+ <dependency>
+ <groupId>org.glassfish</groupId>
+ <artifactId>javax.el</artifactId>
+ <version>3.0.1-b08</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro</artifactId>
+ <version>${avro.version}</version>
+ </dependency>
+
+ <!-- Override httpcore from libthrift to a later maintenance version. Noticed
+ certain SPNEGO auth errors when connecting to kerberized HTTP endpoints
+ using v4.4.1 pulled by libthrift. -->
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpcore</artifactId>
+ <version>${httpcomponents.core.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.parquet</groupId>
+ <artifactId>parquet-avro</artifactId>
+ <version>${parquet.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.santuario</groupId>
+ <artifactId>xmlsec</artifactId>
+ <version>${xmlsec.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.bouncycastle</groupId>
+ <artifactId>bcprov-jdk15on</artifactId>
+ <version>${bouncy-castle.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.bouncycastle</groupId>
+ <artifactId>bcpkix-jdk15on</artifactId>
+ <version>${bouncy-castle.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.pac4j</groupId>
+ <artifactId>pac4j-core</artifactId>
+ <version>${pac4j.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>ch.qos.reload4j</groupId>
+ <artifactId>reload4j</artifactId>
+ <version>${reload4j.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>jcl-over-slf4j</artifactId>
+ <version>${slf4j.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>${slf4j.version}</version>
+ </dependency>
+ <!-- Later versions of slf4j-log4j12 map to slf4j-reload4j. This is the simplest
+ way to ensure any instances of slf4j-log4j12 are mapped to slf4j-reload4j. -->
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ <version>${slf4j.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
+ <version>${slf4j.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-core</artifactId>
+ <version>${springframework.version}</version>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+
<modules>
<module>datagenerator</module>
<module>executor-deps</module>
diff --git a/java/shaded-deps/hive-exec/pom.xml b/java/shaded-deps/hive-exec/pom.xml
index 499852614..1b157ae91 100644
--- a/java/shaded-deps/hive-exec/pom.xml
+++ b/java/shaded-deps/hive-exec/pom.xml
@@ -35,14 +35,6 @@ the same dependencies
<packaging>jar</packaging>
<dependencies>
- <!-- Pin json-smart dependency.
- (this is not included in the shaded jar) -->
- <dependency>
- <groupId>net.minidev</groupId>
- <artifactId>json-smart</artifactId>
- <version>${json-smart.version}</version>
- </dependency>
-
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-exec</artifactId>
@@ -52,11 +44,6 @@ the same dependencies
<groupId>org.apache.hive.shims</groupId>
<artifactId>hive-shims-0.20</artifactId>
</exclusion>
- <!-- Exclude json-smart to pin the version -->
- <exclusion>
- <groupId>net.minidev</groupId>
- <artifactId>json-smart</artifactId>
- </exclusion>
<exclusion>
<groupId>org.apache.atlas</groupId>
<artifactId>*</artifactId>
diff --git a/java/test-hive-udfs/pom.xml b/java/test-hive-udfs/pom.xml
index c319acbfa..59f7b4b22 100644
--- a/java/test-hive-udfs/pom.xml
+++ b/java/test-hive-udfs/pom.xml
@@ -37,35 +37,16 @@ under the License.
</properties>
<dependencies>
- <!-- Force json-smart dependency.
- See https://issues.apache.org/jira/browse/HADOOP-14903 -->
- <dependency>
- <groupId>net.minidev</groupId>
- <artifactId>json-smart</artifactId>
- <version>${json-smart.version}</version>
- </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
- <exclusions>
- <!-- https://issues.apache.org/jira/browse/HADOOP-14903 -->
- <exclusion>
- <groupId>net.minidev</groupId>
- <artifactId>json-smart</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-exec</artifactId>
<version>${hive.version}</version>
<exclusions>
- <!-- https://issues.apache.org/jira/browse/HADOOP-14903 -->
- <exclusion>
- <groupId>net.minidev</groupId>
- <artifactId>json-smart</artifactId>
- </exclusion>
<exclusion>
<groupId>org.apache.hive.shims</groupId>
<artifactId>hive-shims-0.20</artifactId>
diff --git a/java/yarn-extras/pom.xml b/java/yarn-extras/pom.xml
index 368273afc..1d3b7924b 100644
--- a/java/yarn-extras/pom.xml
+++ b/java/yarn-extras/pom.xml
@@ -32,30 +32,11 @@
<url>.</url>
<dependencies>
- <!-- Force json-smart dependency.
- See https://issues.apache.org/jira/browse/HADOOP-14903 -->
- <dependency>
- <groupId>net.minidev</groupId>
- <artifactId>json-smart</artifactId>
- <version>${json-smart.version}</version>
- </dependency>
-
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <version>${guava.version}</version>
- </dependency>
-
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
<exclusions>
- <!-- https://issues.apache.org/jira/browse/HADOOP-14903 -->
- <exclusion>
- <groupId>net.minidev</groupId>
- <artifactId>json-smart</artifactId>
- </exclusion>
<exclusion>
<groupId>org.eclipse.jetty</groupId>
<artifactId>*</artifactId>
@@ -87,11 +68,6 @@
<artifactId>hadoop-yarn-common</artifactId>
<version>${hadoop.version}</version>
<exclusions>
- <!-- https://issues.apache.org/jira/browse/HADOOP-14903 -->
- <exclusion>
- <groupId>net.minidev</groupId>
- <artifactId>json-smart</artifactId>
- </exclusion>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-server</artifactId>
[impala] 01/03: IMPALA-11581: ALTER TABLE RENAME TO doesn't update transient_lastDdlTime
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit ad438e7e3cc89a7e8511fcfe67c13411de987007
Author: Daniel Becker <da...@cloudera.com>
AuthorDate: Wed Oct 12 17:59:42 2022 +0200
IMPALA-11581: ALTER TABLE RENAME TO doesn't update transient_lastDdlTime
The following statements behave differently when executed via Hive or
Impala:
CREATE TABLE rename_from (i int);
ALTER TABLE rename_from RENAME TO rename_to;
Hive updates transient_lastDdlTime while Impala leaves it unchanged.
This patch fixes the behaviour of Impala so that it also updates
transient_lastDdlTime.
Testing:
- Added a test in test_last_ddl_time_update.py that checks that
transient_lastDdlTime is updated on rename. Refactored the class a
bit so that the new test fits in easier.
Change-Id: Ib550feaebbad9cf6c9b34ab046293968b157a50c
Reviewed-on: http://gerrit.cloudera.org:8080/19137
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
.../apache/impala/service/CatalogOpExecutor.java | 1 +
tests/metadata/test_last_ddl_time_update.py | 117 +++++++++++++--------
2 files changed, 77 insertions(+), 41 deletions(-)
diff --git a/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java b/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
index 04e525c6d..23712a844 100755
--- a/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
+++ b/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
@@ -5036,6 +5036,7 @@ public class CatalogOpExecutor {
boolean needsHmsAlterTable = !isSynchronizedTable || !integratedHmsTable;
if (needsHmsAlterTable) {
try (MetaStoreClient msClient = catalog_.getMetaStoreClient()) {
+ Table.updateTimestampProperty(msTbl, Table.TBL_PROP_LAST_DDL_TIME);
msClient.getHiveClient().alter_table(
tableName.getDb(), tableName.getTbl(), msTbl);
} catch (TException e) {
diff --git a/tests/metadata/test_last_ddl_time_update.py b/tests/metadata/test_last_ddl_time_update.py
index e9b48fc88..36a4db358 100644
--- a/tests/metadata/test_last_ddl_time_update.py
+++ b/tests/metadata/test_last_ddl_time_update.py
@@ -49,7 +49,7 @@ class TestLastDdlTimeUpdate(ImpalaTestSuite):
self.test_suite = test_suite
self.db_name = db_name
self.tbl_name = tbl_name
- self.fq_tbl_name = "%s.%s" % (self.db_name, self.tbl_name)
+ self.fq_tbl_name = self._generate_fq_tbl_name(self.db_name, self.tbl_name)
def expect_no_time_change(self, query):
"""Neither transient_lastDdlTime or impala.lastComputeStatsTime should be
@@ -72,34 +72,40 @@ class TestLastDdlTimeUpdate(ImpalaTestSuite):
"""
self.run_test(query, False, True)
- def run_test(self, query, expect_changed_ddl_time, expect_changed_stats_time):
+ def expect_ddl_time_change_on_rename(self, new_tbl_name):
+ """
+ Checks that after an ALTER TABLE ... RENAME query transient_lastDdlTime is higher on
+ the new table than it was on the old table.
+ """
+ query = "alter table %(TBL)s rename to {}".format(self.db_name + "." + new_tbl_name)
+ self.run_test(query, True, False, new_tbl_name)
+
+ def run_test(self, query, expect_changed_ddl_time, expect_changed_stats_time,
+ new_tbl_name=None):
"""
Runs the query and compares the last ddl/compute stats time before and after
executing the query. If expect_changed_ddl_time or expect_changed_stat_time
is true, then we expect the given table property to be increased, otherwise we
expect it to be unchanged.
+ If the query renames the table, the new table name has to be provided in
+ 'new_tbl_name' so that the new table is checked after the query.
The following strings are substituted in the query: "%(TBL)s" and "%(WAREHOUSE)s"
"""
-
- HIVE_LAST_DDL_TIME_PARAM_KEY = "transient_lastDdlTime"
- LAST_COMPUTE_STATS_TIME_KEY = "impala.lastComputeStatsTime"
-
# Get timestamps before executing query.
- table = self.test_suite.hive_client.get_table(self.db_name, self.tbl_name)
- assert table is not None
- beforeDdlTime = table.parameters[HIVE_LAST_DDL_TIME_PARAM_KEY]
- beforeStatsTime = table.parameters[LAST_COMPUTE_STATS_TIME_KEY]
+ (beforeDdlTime, beforeStatsTime) = self._get_ddl_and_stats_time(
+ self.db_name, self.tbl_name)
# HMS uses a seconds granularity on the last ddl time - sleeping 1100 ms should be
# enough to ensure that the new timestamps are strictly greater than the old ones.
time.sleep (1.1)
self.test_suite.execute_query(query %
{'TBL': self.fq_tbl_name, 'WAREHOUSE': WAREHOUSE})
+ if new_tbl_name is not None:
+ self._update_name(new_tbl_name)
# Get timestamps after executing query.
- table = self.test_suite.hive_client.get_table(self.db_name, self.tbl_name)
- afterDdlTime = table.parameters[HIVE_LAST_DDL_TIME_PARAM_KEY]
- afterStatsTime = table.parameters[LAST_COMPUTE_STATS_TIME_KEY]
+ (afterDdlTime, afterStatsTime) = self._get_ddl_and_stats_time(
+ self.db_name, self.tbl_name)
if expect_changed_ddl_time:
# check that the new ddlTime is strictly greater than the old one.
@@ -113,17 +119,46 @@ class TestLastDdlTimeUpdate(ImpalaTestSuite):
else:
assert long(afterStatsTime) == long(beforeStatsTime)
- def test_alter(self, vector, unique_database):
- TBL_NAME = "alter_test_tbl"
- FQ_TBL_NAME = unique_database + "." + TBL_NAME
+ def _update_name(self, new_tbl_name):
+ """"
+ Update the name, for example because of a table rename ddl. The database name does
+ not change.
+ """
+ self.tbl_name = new_tbl_name
+ self.fq_tbl_name = self._generate_fq_tbl_name(self.db_name, self.tbl_name)
- self.execute_query("create external table %s (i int) "
- "partitioned by (j int, s string)" % FQ_TBL_NAME)
+ def _generate_fq_tbl_name(self, db_name, tbl_name):
+ return "%s.%s" % (db_name, tbl_name)
+
+ def _get_ddl_and_stats_time(self, db_name, tbl_name):
+ HIVE_LAST_DDL_TIME_PARAM_KEY = "transient_lastDdlTime"
+ LAST_COMPUTE_STATS_TIME_KEY = "impala.lastComputeStatsTime"
+
+ table = self.test_suite.hive_client.get_table(db_name, tbl_name)
+ assert table is not None
+ ddlTime = table.parameters[HIVE_LAST_DDL_TIME_PARAM_KEY]
+ statsTime = table.parameters[LAST_COMPUTE_STATS_TIME_KEY]
+ return (ddlTime, statsTime)
+
+ def _create_table(self, fq_tbl_name, is_kudu):
+ if is_kudu:
+ self.execute_query("create table %s (i int primary key) "
+ "partition by hash(i) partitions 3 stored as kudu" % fq_tbl_name)
+ else:
+ self.execute_query("create external table %s (i int) "
+ "partitioned by (j int, s string)" % fq_tbl_name)
+
+ def _create_and_init_test_helper(self, unique_database, tbl_name, is_kudu):
+ helper = TestLastDdlTimeUpdate.TestHelper(self, unique_database, tbl_name)
+ self._create_table(helper.fq_tbl_name, is_kudu)
# compute statistics to fill table property impala.lastComputeStatsTime
- self.execute_query("compute stats %s" % FQ_TBL_NAME)
+ self.execute_query("compute stats %s" % helper.fq_tbl_name)
+ return helper
- h = TestLastDdlTimeUpdate.TestHelper(self, unique_database, TBL_NAME)
+ def test_alter(self, vector, unique_database):
+ TBL_NAME = "alter_test_tbl"
+ h = self._create_and_init_test_helper(unique_database, TBL_NAME, False)
# add/drop partitions
h.expect_no_time_change("alter table %(TBL)s add partition (j=1, s='2012')")
@@ -140,8 +175,8 @@ class TestLastDdlTimeUpdate(ImpalaTestSuite):
self.run_common_test_cases(h)
# prepare for incremental statistics tests
- self.execute_query("drop stats %s" % FQ_TBL_NAME)
- self.execute_query("alter table %s add partition (j=1, s='2012')" % FQ_TBL_NAME)
+ self.execute_query("drop stats %s" % h.fq_tbl_name)
+ self.execute_query("alter table %s add partition (j=1, s='2012')" % h.fq_tbl_name)
# compute incremental statistics
h.expect_stat_time_change("compute incremental stats %(TBL)s")
@@ -154,7 +189,7 @@ class TestLastDdlTimeUpdate(ImpalaTestSuite):
# prepare for table sample statistics tests
self.execute_query(
"alter table %s set tblproperties ('impala.enable.stats.extrapolation'='true')"
- % FQ_TBL_NAME)
+ % h.fq_tbl_name)
# compute sampled statistics
h.expect_stat_time_change("compute stats %(TBL)s tablesample system(20)")
@@ -162,14 +197,7 @@ class TestLastDdlTimeUpdate(ImpalaTestSuite):
def test_insert(self, vector, unique_database):
TBL_NAME = "insert_test_tbl"
- FQ_TBL_NAME = unique_database + "." + TBL_NAME
- self.execute_query("create external table %s (i int) "
- "partitioned by (j int, s string)" % FQ_TBL_NAME)
-
- # initialize compute stats time
- self.execute_query("compute stats %s" % FQ_TBL_NAME)
-
- h = TestLastDdlTimeUpdate.TestHelper(self, unique_database, TBL_NAME)
+ h = self._create_and_init_test_helper(unique_database, TBL_NAME, False)
# static partition insert
h.expect_no_time_change("insert into %(TBL)s partition(j=1, s='2012') select 10")
@@ -183,22 +211,29 @@ class TestLastDdlTimeUpdate(ImpalaTestSuite):
# dynamic partition insert modifying an existing partition
h.expect_no_time_change("insert into %(TBL)s partition(j, s) select 20, 1, '2012'")
- def test_kudu(self, vector, unique_database):
+ def test_kudu_alter_and_insert(self, vector, unique_database):
TBL_NAME = "kudu_test_tbl"
- FQ_TBL_NAME = unique_database + "." + TBL_NAME
- self.execute_query("create table %s (i int primary key) "
- "partition by hash(i) partitions 3 stored as kudu" % FQ_TBL_NAME)
-
- # initialize last compute stats time
- self.execute_query("compute stats %s" % FQ_TBL_NAME)
-
- h = TestLastDdlTimeUpdate.TestHelper(self, unique_database, TBL_NAME)
+ h = self._create_and_init_test_helper(unique_database, TBL_NAME, True)
# insert
- h.expect_no_time_change("insert into %s values (1)" % FQ_TBL_NAME)
+ h.expect_no_time_change("insert into %s values (1)" % h.fq_tbl_name)
self.run_common_test_cases(h)
+ def test_rename(self, vector, unique_database):
+ # Test non-Kudu table
+ OLD_TBL_NAME = "rename_from_test_tbl"
+ NEW_TBL_NAME = "rename_to_test_tbl"
+
+ h = self._create_and_init_test_helper(unique_database, OLD_TBL_NAME, False)
+ h.expect_ddl_time_change_on_rename(NEW_TBL_NAME)
+
+ # Test Kudu table
+ OLD_KUDU_TBL_NAME = "kudu_rename_from_test_tbl"
+ NEW_KUDU_TBL_NAME = "kudu_rename_to_test_tbl"
+ h = self._create_and_init_test_helper(unique_database, OLD_KUDU_TBL_NAME, True)
+ h.expect_ddl_time_change_on_rename(NEW_KUDU_TBL_NAME)
+
# Tests that should behave the same with HDFS and Kudu tables.
def run_common_test_cases(self, test_helper):
h = test_helper