You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/10/12 17:44:59 UTC
(datafusion-comet) 01/01: test: run the S3 suites against an available MinIO image (#6887)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-6887-38f394e2707814020cbf057510e25e85f07e47a8
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
commit 907c0e074c4e34812b3ae808736db28ecb173d6e
Author: dustin <dw...@users.noreply.github.com>
AuthorDate: Mon Oct 12 17:44:34 2026 +0000
test: run the S3 suites against an available MinIO image (#6887)
* test: run the S3 suites against an available MinIO image
Docker Hub no longer serves minio/minio, so every suite built on
CometS3TestBase aborted in beforeAll. Use cgr.dev/chainguard/minio,
declared as a substitute for minio/minio so MinIOContainer accepts it.
The suites also relied on an AWS region from the environment. Give the
bucket setup client an explicit region, and add client.region to the
REST catalog vended properties, since Iceberg's JVM S3FileIO reads that
key rather than s3.region.
* test: pin the MinIO test image by digest
Chainguard rebuilds the latest tag from its MinIO fork, and newer
releases tighten request checks, so a rebuild could break the S3 suites
with no change on our side. Pin the multi-arch index digest of
RELEASE.2026-09-22T19-25-18Z so every run uses the same server.
---
spark/src/test/scala/org/apache/comet/CometS3TestBase.scala | 10 +++++++++-
.../test/scala/org/apache/comet/IcebergReadFromS3Suite.scala | 2 ++
.../apache/comet/cloud/s3/CometS3CredentialBridgeSuite.scala | 1 +
3 files changed, 12 insertions(+), 1 deletion(-)
diff --git a/spark/src/test/scala/org/apache/comet/CometS3TestBase.scala b/spark/src/test/scala/org/apache/comet/CometS3TestBase.scala
index d51f0ccedc..a15e3f75be 100644
--- a/spark/src/test/scala/org/apache/comet/CometS3TestBase.scala
+++ b/spark/src/test/scala/org/apache/comet/CometS3TestBase.scala
@@ -34,6 +34,7 @@ import org.apache.spark.sql.execution.SparkPlan
import org.apache.comet.CometSparkSessionExtensions.isSpark42Plus
import software.amazon.awssdk.auth.credentials.{AwsBasicCredentials, StaticCredentialsProvider}
+import software.amazon.awssdk.regions.Region
import software.amazon.awssdk.services.s3.S3Client
import software.amazon.awssdk.services.s3.model.{CreateBucketRequest, HeadBucketRequest}
@@ -45,8 +46,14 @@ trait CometS3TestBase extends CometTestBase {
protected def testBucketName: String
+ // Docker Hub no longer serves minio/minio. This is Chainguard's build of MinIO
+ // RELEASE.2026-09-22T19-25-18Z, pinned by its multi-arch index digest.
+ private val minioImage = "cgr.dev/chainguard/minio@sha256:" +
+ "dc455cdd4db6e390b1b26514f15ff37681a777bf8dab41aac445a0259f751e56"
+
override def beforeAll(): Unit = {
- minioContainer = new MinIOContainer(DockerImageName.parse("minio/minio:latest"))
+ minioContainer = new MinIOContainer(
+ DockerImageName.parse(minioImage).asCompatibleSubstituteFor("minio/minio"))
.withUserName(userName)
.withPassword(password)
minioContainer.start()
@@ -120,6 +127,7 @@ trait CometS3TestBase extends CometTestBase {
val s3Client = S3Client
.builder()
.endpointOverride(URI.create(minioContainer.getS3URL))
+ .region(Region.US_EAST_1)
.credentialsProvider(StaticCredentialsProvider.create(credentials))
.forcePathStyle(true)
.build()
diff --git a/spark/src/test/scala/org/apache/comet/IcebergReadFromS3Suite.scala b/spark/src/test/scala/org/apache/comet/IcebergReadFromS3Suite.scala
index 1271fa841e..168bd0b49f 100644
--- a/spark/src/test/scala/org/apache/comet/IcebergReadFromS3Suite.scala
+++ b/spark/src/test/scala/org/apache/comet/IcebergReadFromS3Suite.scala
@@ -247,6 +247,7 @@ class IcebergReadFromS3Suite extends CometS3TestBase with RESTCatalogHelper {
"s3.secret-access-key" -> "WRONG_SECRET_KEY",
"s3.endpoint" -> minioContainer.getS3URL,
"s3.region" -> "us-east-1",
+ "client.region" -> "us-east-1",
"s3.path-style-access" -> "true")
val warehouse = s"s3a://$testBucketName/warehouse-bad-creds"
@@ -280,6 +281,7 @@ class IcebergReadFromS3Suite extends CometS3TestBase with RESTCatalogHelper {
"s3.secret-access-key" -> password,
"s3.endpoint" -> minioContainer.getS3URL,
"s3.region" -> "us-east-1",
+ "client.region" -> "us-east-1",
"s3.path-style-access" -> "true")
val warehouse = s"s3a://$testBucketName/warehouse-vending"
diff --git a/spark/src/test/scala/org/apache/comet/cloud/s3/CometS3CredentialBridgeSuite.scala b/spark/src/test/scala/org/apache/comet/cloud/s3/CometS3CredentialBridgeSuite.scala
index 8df9b5bb90..f3bc317c29 100644
--- a/spark/src/test/scala/org/apache/comet/cloud/s3/CometS3CredentialBridgeSuite.scala
+++ b/spark/src/test/scala/org/apache/comet/cloud/s3/CometS3CredentialBridgeSuite.scala
@@ -151,6 +151,7 @@ class CometS3CredentialBridgeSuite
"s3.secret-access-key" -> password,
"s3.endpoint" -> minioContainer.getS3URL,
"s3.region" -> "us-east-1",
+ "client.region" -> "us-east-1",
"s3.path-style-access" -> "true",
"comet.test-tenant-id" -> "tenant-A")
val warehouse = s"s3a://$testBucketName/warehouse-rest-spi"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org