You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by cs...@apache.org on 2026/09/23 12:04:23 UTC
(impala) branch master updated: IMPALA-15367: Bump thrift to 0.24.0 - java parts
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
The following commit(s) were added to refs/heads/master by this push:
new 88b903cf5 IMPALA-15367: Bump thrift to 0.24.0 - java parts
88b903cf5 is described below
commit 88b903cf591b95d68223afaa78ef433da20ddb99
Author: Csaba Ringhofer <cs...@cloudera.com>
AuthorDate: Fri Sep 11 12:58:49 2026 +0200
IMPALA-15367: Bump thrift to 0.24.0 - java parts
Metastore thrift classes are still pulled in from Hive
and are generated with Thrift 0.16.0, which seems genarally
compatible with 0.24.0 runtime.
Thrift 0.24.0 moved from HttpClient 4 to 5, which required
workarounds in some tests:
- JdbcTest: xfail hs2-http tests until Hive also uses Thrift 0.24
- TestCaseLoaderTest: xfail until Hive also uses Thrift 0.24
- LdapHS2Test: don't compile when using Apache Hive, where Impala
uses Thrift 0.11.0
The xfails can be removed once Impala gets a cdp dependency with
Hive that uses Thrift 0.24.0 (IMPALA-15384).
org.apache.httpcomponents has both versions 4 and 5 in Impala,
which is normal as they use different group and arfifact id. There
are many dependencies that use 4 besides old Thrift.
Change-Id: I9e15704c9db4b802e5c0ca7fd3e652aa37ee1f2f
Assisted-By: Claude Opus 4.8
Reviewed-on: http://gerrit.cloudera.org:8080/24820
Reviewed-by: Michael Smith <mi...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Reviewed-by: Quanlong Huang <hu...@gmail.com>
---
bin/impala-config.sh | 6 ++--
fe/pom.xml | 7 +++++
.../apache/impala/customcluster/LdapHS2Test.java | 22 +++++++++------
.../apache/impala/planner/TestCaseLoaderTest.java | 8 ++++++
.../org/apache/impala/service/JdbcTestBase.java | 16 +++++++++++
java/pom.xml | 33 ++++++++++++++++++++--
6 files changed, 79 insertions(+), 13 deletions(-)
diff --git a/bin/impala-config.sh b/bin/impala-config.sh
index 745d91fec..3625041e1 100755
--- a/bin/impala-config.sh
+++ b/bin/impala-config.sh
@@ -286,6 +286,8 @@ export IMPALA_DERBY_VERSION=10.14.2.0
export IMPALA_GUAVA_VERSION=32.1.2-jre
export IMPALA_HUDI_VERSION=0.5.0-incubating
export IMPALA_HTTP_CORE_VERSION=4.4.14
+export IMPALA_HTTPCLIENT5_VERSION=5.5
+export IMPALA_HTTPCORE5_VERSION=5.3.4
export IMPALA_JACKSON_VERSION=2.19.4
export IMPALA_JACKSON_DATABIND_VERSION=2.19.4
export IMPALA_JSON_SMART_VERSION=2.6.0
@@ -493,8 +495,8 @@ if $USE_APACHE_HIVE_3 || $USE_APACHE_HIVE_2; then
export IMPALA_THRIFT_POM_VERSION=0.11.0
export IMPALA_THRIFT_JAVA_VERSION=${IMPALA_THRIFT_POM_VERSION}-p5
else
- export IMPALA_THRIFT_POM_VERSION=0.16.0
- export IMPALA_THRIFT_JAVA_VERSION=${IMPALA_THRIFT_POM_VERSION}-p7
+ export IMPALA_THRIFT_POM_VERSION=0.24.0
+ export IMPALA_THRIFT_JAVA_VERSION=${IMPALA_THRIFT_POM_VERSION}-p3
fi
unset IMPALA_THRIFT_JAVA_URL
export IMPALA_THRIFT_PY_VERSION=0.16.0-p7
diff --git a/fe/pom.xml b/fe/pom.xml
index 0eff40d5d..130d502ad 100644
--- a/fe/pom.xml
+++ b/fe/pom.xml
@@ -1522,6 +1522,13 @@ under the License.
</excludes>
<testExcludes>
<testExclude>**/org/apache/impala/testutil/Catalog*.java</testExclude>
+ <!-- LdapHS2Test.testHiveserver2SharedCookie uses the HttpClient 5 API,
+ only available with Thrift 0.24. Apache Hive builds pin Thrift to
+ 0.11 (which uses HttpClient 4, see IMPALA-11801), so the test does
+ not compile here. LdapWebserverTest calls LdapHS2Test's static
+ helpers, so it must be excluded alongside it. -->
+ <testExclude>**/org/apache/impala/customcluster/LdapHS2Test.java</testExclude>
+ <testExclude>**/org/apache/impala/customcluster/LdapWebserverTest.java</testExclude>
</testExcludes>
</configuration>
</plugin>
diff --git a/fe/src/test/java/org/apache/impala/customcluster/LdapHS2Test.java b/fe/src/test/java/org/apache/impala/customcluster/LdapHS2Test.java
index 06ea5be55..9cb9724a7 100644
--- a/fe/src/test/java/org/apache/impala/customcluster/LdapHS2Test.java
+++ b/fe/src/test/java/org/apache/impala/customcluster/LdapHS2Test.java
@@ -38,11 +38,12 @@ import org.apache.directory.server.annotations.CreateTransport;
import org.apache.directory.server.core.annotations.ApplyLdifFiles;
import org.apache.directory.server.core.integ.CreateLdapServerRule;
import org.apache.hive.service.rpc.thrift.*;
-import org.apache.http.HttpResponse;
-import org.apache.http.HttpResponseInterceptor;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClients;
-import org.apache.http.protocol.HttpContext;
+import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
+import org.apache.hc.client5.http.impl.classic.HttpClients;
+import org.apache.hc.core5.http.EntityDetails;
+import org.apache.hc.core5.http.HttpResponse;
+import org.apache.hc.core5.http.HttpResponseInterceptor;
+import org.apache.hc.core5.http.protocol.HttpContext;
import org.apache.impala.testutil.WebClient;
import org.apache.thrift.transport.THttpClient;
import org.apache.thrift.protocol.TBinaryProtocol;
@@ -1014,7 +1015,8 @@ public class LdapHS2Test {
// so we can store a cookie from an anonymous function.
final String[] cookie = new String[1];
HttpResponseInterceptor cookieSaver = new HttpResponseInterceptor() {
- @Override public void process(HttpResponse response, HttpContext context) {
+ @Override public void process(
+ HttpResponse response, EntityDetails entity, HttpContext context) {
if (response.containsHeader("Set-Cookie")) {
cookie[0] = response.getFirstHeader("Set-Cookie").getValue();
} else {
@@ -1024,7 +1026,7 @@ public class LdapHS2Test {
};
try (CloseableHttpClient clientImpl =
- HttpClients.custom().addInterceptorFirst(cookieSaver).build();
+ HttpClients.custom().addResponseInterceptorFirst(cookieSaver).build();
THttpClient transport = new THttpClient("http://localhost:28000", clientImpl)) {
Map<String, String> headers = new HashMap<String, String>();
// Authenticate as 'Test1Ldap' with password '12345'
@@ -1091,6 +1093,10 @@ public class LdapHS2Test {
throw new Exception("Expected failure due to changed cookie secret.");
}
}, 20, 100);
- assertEquals("HTTP Response code: 401", respMessage);
+ // Thrift 0.24's THttpClient prefixes the message with the wrapped exception type
+ // (e.g. "java.io.IOException: HTTP Response code: 401"), so match the suffix rather
+ // than the whole string.
+ assertTrue("Unexpected message: " + respMessage,
+ respMessage.contains("HTTP Response code: 401"));
}
}
diff --git a/fe/src/test/java/org/apache/impala/planner/TestCaseLoaderTest.java b/fe/src/test/java/org/apache/impala/planner/TestCaseLoaderTest.java
index b026474bc..37dd90667 100644
--- a/fe/src/test/java/org/apache/impala/planner/TestCaseLoaderTest.java
+++ b/fe/src/test/java/org/apache/impala/planner/TestCaseLoaderTest.java
@@ -27,6 +27,7 @@ import org.apache.impala.testutil.PlannerTestCaseLoader;
import org.apache.impala.util.PatternMatcher;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.junit.Ignore;
import org.junit.Test;
import java.util.List;
@@ -46,7 +47,14 @@ public class TestCaseLoaderTest {
* Randomly picks 10 testcase files from TESTCASE_DATA_DIR and loads them into a
* clean Catalog and makes sure that the query statement from the testcase can be
* planned correctly without any errors.
+ *
+ * TODO(IMPALA-15384): This test spins up an embedded HiveMetaStore which is compiled
+ * against Thrift 0.16. Hive's TServlet extends javax.servlet.http.HttpServlet,
+ * but Thrift 0.24's TServlet extends jakarta.servlet.http.HttpServlet.
+ * I don't see a good way to fix this, disabling till there is Hive with Thrift 0.24.
*/
+ @Ignore("IMPALA-15384: embedded HiveMetaStore incompatible with Thrift 0.24 (Hive on "
+ + "Thrift 0.16)")
@Test
public void testTestCaseImport() throws Exception {
FileStatus[] testCaseFiles = FileSystemUtil.getFileSystemForPath(TESTCASE_DATA_DIR)
diff --git a/fe/src/test/java/org/apache/impala/service/JdbcTestBase.java b/fe/src/test/java/org/apache/impala/service/JdbcTestBase.java
index 47442bf8d..c2774694b 100644
--- a/fe/src/test/java/org/apache/impala/service/JdbcTestBase.java
+++ b/fe/src/test/java/org/apache/impala/service/JdbcTestBase.java
@@ -31,6 +31,8 @@ import org.apache.impala.analysis.Parser;
import org.apache.impala.analysis.StatementBase;
import org.apache.impala.testutil.ImpalaJdbcClient;
import org.junit.After;
+import org.junit.Assume;
+import org.junit.Before;
import org.junit.runners.Parameterized;
import org.junit.runner.RunWith;
@@ -54,6 +56,20 @@ public abstract class JdbcTestBase {
return new String[] {"binary", "http"};
}
+ // TODO(IMPALA-15384): The HTTP JDBC transport goes through the Hive JDBC driver, which
+ // was built against Thrift 0.16 and calls
+ // THttpClient(String, org.apache.http.client.HttpClient) (Apache HttpClient 4). Thrift
+ // 0.24 replaced that constructor with an HttpClient 5 signature, so every HTTP case
+ // fails with NoSuchMethodError. Skip the HTTP cases until IMPALA-15384 provides Hive
+ // dependencies built against Thrift 0.24; the binary transport is unaffected.
+ @Before
+ public void assumeTransportSupported() {
+ Assume.assumeFalse(
+ "HTTP JDBC transport needs the Hive JDBC driver built against Thrift 0.24; "
+ + "skipping until Hive dependencies are updated (IMPALA-15384).",
+ "http".equals(connectionType_));
+ }
+
/**
* Closes 'con_'. Any subclasses that specify their own 'After' will need to call this
* function there.
diff --git a/java/pom.xml b/java/pom.xml
index eafe33718..4f7a07c3f 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -66,6 +66,9 @@ under the License.
<!-- Beware compatibility requirements with Thrift and
KMS; see IMPALA-4210. -->
<httpcomponents.core.version>${env.IMPALA_HTTP_CORE_VERSION}</httpcomponents.core.version>
+ <!-- HttpComponents 5.x, distinct from the v4 httpcomponents.core.version above. -->
+ <httpclient5.version>${env.IMPALA_HTTPCLIENT5_VERSION}</httpclient5.version>
+ <httpcore5.version>${env.IMPALA_HTTPCORE5_VERSION}</httpcore5.version>
<yarn-extras.version>${project.version}</yarn-extras.version>
<eclipse.output.directory>eclipse-classes</eclipse.output.directory>
<!-- hive-exec seems to leak this version of guava onto our classpath,
@@ -305,15 +308,33 @@ under the License.
<version>${thrift.version}</version>
</dependency>
- <!-- Override httpcore from libthrift to a later maintenance version. Noticed
- certain SPNEGO auth errors when connecting to kerberized HTTP endpoints
- using v4.4.1 pulled by libthrift. -->
+ <!-- Override httpcore (v4), pulled by various dependencies, including pac4j, to a
+ later maintenance version. Noticed certain SPNEGO auth errors when connecting to
+ kerberized HTTP endpoints using older v4.4.x versions. -->
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcore</artifactId>
<version>${httpcomponents.core.version}</version>
</dependency>
+ <!-- libthrift 0.24 pulls in httpclient5 5.2.1 / httpcore5 5.2, but the Iceberg
+ REST catalog client requires httpclient5 >= 5.4. Pin to a newer version. -->
+ <dependency>
+ <groupId>org.apache.httpcomponents.client5</groupId>
+ <artifactId>httpclient5</artifactId>
+ <version>${httpclient5.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents.core5</groupId>
+ <artifactId>httpcore5</artifactId>
+ <version>${httpcore5.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents.core5</groupId>
+ <artifactId>httpcore5-h2</artifactId>
+ <version>${httpcore5.version}</version>
+ </dependency>
+
<dependency>
<groupId>org.apache.parquet</groupId>
<artifactId>parquet-avro</artifactId>
@@ -477,6 +498,12 @@ under the License.
<groupId>org.bouncycastle</groupId>
<artifactId>*</artifactId>
</exclusion>
+ <!-- Apache Hive 3.1.3's hive-exec is built against Hadoop 3.1.0 and drags in
+ hadoop-yarn-registry:3.1.0, which brings the banned jersey-server:1.19. -->
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-registry</artifactId>
+ </exclusion>
</exclusions>
</dependency>