You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mb...@apache.org on 2026/09/02 01:39:11 UTC
(asterixdb) 02/06: [NO ISSUE][MISC] Advance awssdk to 2.54.7 and hadoop to 3.4.3
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 1601ff1cc7651308c9e429c58b7843c769afc226
Author: Michael Blow <mi...@couchbase.com>
AuthorDate: Mon Aug 31 08:46:47 2026 -0400
[NO ISSUE][MISC] Advance awssdk to 2.54.7 and hadoop to 3.4.3
- software.amazon.awssdk
- awsjavasdk.version: 2.29.52 -> 2.54.7
- org.apache.hadoop
- hadoop.version: 3.4.2 -> 3.4.3
The two have to move together: hadoop 3.4.3's S3A ChecksumSupport reads
ChecksumAlgorithm.CRC64_NVME, which awssdk added in 2.30, and
initialising that class against an older sdk throws NoSuchFieldError on
the thread compiling the query, which halts the JVM.
Since 2.30 the sdk computes a CRC32 on uploads by default and frames the
body as aws-chunked. The mock server the external dataset tests run
against does not parse that framing: the put appears to succeed and the
object reads back empty, so the test client now asks for checksums only
where the operation requires them, as it did before.
The sdk's service modules have also changed sync http client, from the
apache-client adapter to apache5-client. hadoop-aws builds its client
through software.amazon.awssdk.http.apache, so apache-client is now
declared where hadoop-aws is, and apache5-client is excluded rather than
carried alongside it.
AwsS3ExternalDatasetTest passes.
Ext-ref: MB-73268
Co-Authored-By: Claude Opus 5 <no...@anthropic.com>
Change-Id: I93ceb4930a15f642c6c7485e328a317d00b201f1
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/21645
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mb...@apache.org>
---
.../aws/AwsS3ExternalDatasetTest.java | 8 ++++++-
asterixdb/asterix-external-data/pom.xml | 6 ++++++
asterixdb/pom.xml | 25 ++++++++++++++++------
hyracks-fullstack/pom.xml | 11 +++++-----
4 files changed, 36 insertions(+), 14 deletions(-)
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
index 2971342850..330d27b92a 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
@@ -64,6 +64,8 @@ import org.junit.runners.Parameterized.Parameters;
import io.findify.s3mock.S3Mock;
import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider;
+import software.amazon.awssdk.core.checksums.RequestChecksumCalculation;
+import software.amazon.awssdk.core.checksums.ResponseChecksumValidation;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3Client;
@@ -200,7 +202,11 @@ public class AwsS3ExternalDatasetTest {
S3ClientBuilder builder = S3Client.builder();
URI endpoint = URI.create(MOCK_SERVER_HOSTNAME); // endpoint pointing to S3 mock server
builder.region(Region.of(MOCK_SERVER_REGION)).credentialsProvider(AnonymousCredentialsProvider.create())
- .endpointOverride(endpoint);
+ .endpointOverride(endpoint)
+ // the mock server does not understand the aws-chunked framing used when the sdk computes
+ // upload checksums (its default since 2.30); objects uploaded that way read back empty
+ .requestChecksumCalculation(RequestChecksumCalculation.WHEN_REQUIRED)
+ .responseChecksumValidation(ResponseChecksumValidation.WHEN_REQUIRED);
client = builder.build();
client.createBucket(CreateBucketRequest.builder().bucket(PLAYGROUND_CONTAINER).build());
client.createBucket(CreateBucketRequest.builder().bucket(FIXED_DATA_CONTAINER).build());
diff --git a/asterixdb/asterix-external-data/pom.xml b/asterixdb/asterix-external-data/pom.xml
index cea070bab3..ab2a0e8578 100644
--- a/asterixdb/asterix-external-data/pom.xml
+++ b/asterixdb/asterix-external-data/pom.xml
@@ -516,6 +516,12 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-aws</artifactId>
</dependency>
+ <!-- hadoop-aws builds its sync client through software.amazon.awssdk.http.apache;
+ the sdk's own service modules no longer bring that adapter in -->
+ <dependency>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>apache-client</artifactId>
+ </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-azure</artifactId>
diff --git a/asterixdb/pom.xml b/asterixdb/pom.xml
index b856a6c0bb..a6c92e19c0 100644
--- a/asterixdb/pom.xml
+++ b/asterixdb/pom.xml
@@ -85,16 +85,15 @@
<!-- Versions under dependencymanagement or used in many projects via properties -->
<algebricks.version>0.3.8.3-SNAPSHOT</algebricks.version>
<hyracks.version>0.3.8.3-SNAPSHOT</hyracks.version>
- <!-- hadoop's S3A ChecksumSupport reads the awssdk it was built against: 3.4.3 wants
- CHECKSUM_ALGORITHM CRC64_NVME, added in awssdk 2.30, and initialising that class
- against an older SDK throws NoSuchFieldError on the thread compiling the query,
- which halts the JVM. awssdk cannot move to 2.30+ while these tests run against
- io.findify S3Mock, which mishandles the newer SDK's default upload checksums. -->
+ <!-- hadoop's S3A initialises against the awssdk it was built against (3.4.3 reads
+ ChecksumAlgorithm.CRC64_NVME, added in awssdk 2.30), so the two must move
+ together: advancing hadoop alone throws NoSuchFieldError on the thread
+ compiling the query, which halts the JVM. -->
<!-- @pinned-with: ${awsjavasdk.version} -->
- <hadoop.version>3.4.2</hadoop.version>
+ <hadoop.version>3.4.3</hadoop.version>
<jacoco.version>0.7.6.201602180812</jacoco.version>
<log4j.version>2.25.5</log4j.version>
- <awsjavasdk.version>2.29.52</awsjavasdk.version>
+ <awsjavasdk.version>2.54.7</awsjavasdk.version>
<s3mock.version>0.2.6</s3mock.version>
<parquet.version>1.17.1</parquet.version> <!-- NOTICE: please update transitives from parquet below on any change -->
<hadoop-awsjavasdk.version>1.12.797</hadoop-awsjavasdk.version>
@@ -1539,8 +1538,20 @@
<groupId>io.netty</groupId>
<artifactId>netty-transport-classes-epoll</artifactId>
</exclusion>
+ <!-- the service modules ship the apache5 adapter as their sync client, but
+ hadoop's S3A builds its client through software.amazon.awssdk.http.apache,
+ so apache-client below is the adapter that has to be on the classpath -->
+ <exclusion>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>apache5-client</artifactId>
+ </exclusion>
</exclusions>
</dependency>
+ <dependency>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>apache-client</artifactId>
+ <version>${awsjavasdk.version}</version>
+ </dependency>
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>regions</artifactId>
diff --git a/hyracks-fullstack/pom.xml b/hyracks-fullstack/pom.xml
index 1eb5b43938..b6f86ae914 100644
--- a/hyracks-fullstack/pom.xml
+++ b/hyracks-fullstack/pom.xml
@@ -69,13 +69,12 @@
<test.includes>${global.test.includes}</test.includes>
<test.excludes>${global.test.excludes}</test.excludes>
<!-- Versions under dependencymanagement or used in many projects via properties -->
- <!-- hadoop's S3A ChecksumSupport reads the awssdk it was built against: 3.4.3 wants
- CHECKSUM_ALGORITHM CRC64_NVME, added in awssdk 2.30, and initialising that class
- against an older SDK throws NoSuchFieldError on the thread compiling the query,
- which halts the JVM. awssdk cannot move to 2.30+ while these tests run against
- io.findify S3Mock, which mishandles the newer SDK's default upload checksums. -->
+ <!-- hadoop's S3A initialises against the awssdk it was built against (3.4.3 reads
+ ChecksumAlgorithm.CRC64_NVME, added in awssdk 2.30), so the two must move
+ together: advancing hadoop alone throws NoSuchFieldError on the thread
+ compiling the query, which halts the JVM. -->
<!-- @pinned-with: ${awsjavasdk.version} -->
- <hadoop.version>3.4.2</hadoop.version>
+ <hadoop.version>3.4.3</hadoop.version>
<jacoco.version>0.7.6.201602180812</jacoco.version>
<log4j.version>2.25.5</log4j.version>
<snappy.version>1.1.10.8</snappy.version>