You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/10/12 17:44:59 UTC

(datafusion-comet) 01/01: test: run the S3 suites against an available MinIO image (#6887)

This is an automated email from the ASF dual-hosted git repository.

github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-6887-38f394e2707814020cbf057510e25e85f07e47a8
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git

commit 907c0e074c4e34812b3ae808736db28ecb173d6e
Author: dustin <dw...@users.noreply.github.com>
AuthorDate: Mon Oct 12 17:44:34 2026 +0000

    test: run the S3 suites against an available MinIO image (#6887)
    
    * test: run the S3 suites against an available MinIO image
    
    Docker Hub no longer serves minio/minio, so every suite built on
    CometS3TestBase aborted in beforeAll. Use cgr.dev/chainguard/minio,
    declared as a substitute for minio/minio so MinIOContainer accepts it.
    
    The suites also relied on an AWS region from the environment. Give the
    bucket setup client an explicit region, and add client.region to the
    REST catalog vended properties, since Iceberg's JVM S3FileIO reads that
    key rather than s3.region.
    
    * test: pin the MinIO test image by digest
    
    Chainguard rebuilds the latest tag from its MinIO fork, and newer
    releases tighten request checks, so a rebuild could break the S3 suites
    with no change on our side. Pin the multi-arch index digest of
    RELEASE.2026-09-22T19-25-18Z so every run uses the same server.
---
 spark/src/test/scala/org/apache/comet/CometS3TestBase.scala    | 10 +++++++++-
 .../test/scala/org/apache/comet/IcebergReadFromS3Suite.scala   |  2 ++
 .../apache/comet/cloud/s3/CometS3CredentialBridgeSuite.scala   |  1 +
 3 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/spark/src/test/scala/org/apache/comet/CometS3TestBase.scala b/spark/src/test/scala/org/apache/comet/CometS3TestBase.scala
index d51f0ccedc..a15e3f75be 100644
--- a/spark/src/test/scala/org/apache/comet/CometS3TestBase.scala
+++ b/spark/src/test/scala/org/apache/comet/CometS3TestBase.scala
@@ -34,6 +34,7 @@ import org.apache.spark.sql.execution.SparkPlan
 import org.apache.comet.CometSparkSessionExtensions.isSpark42Plus
 
 import software.amazon.awssdk.auth.credentials.{AwsBasicCredentials, StaticCredentialsProvider}
+import software.amazon.awssdk.regions.Region
 import software.amazon.awssdk.services.s3.S3Client
 import software.amazon.awssdk.services.s3.model.{CreateBucketRequest, HeadBucketRequest}
 
@@ -45,8 +46,14 @@ trait CometS3TestBase extends CometTestBase {
 
   protected def testBucketName: String
 
+  // Docker Hub no longer serves minio/minio. This is Chainguard's build of MinIO
+  // RELEASE.2026-09-22T19-25-18Z, pinned by its multi-arch index digest.
+  private val minioImage = "cgr.dev/chainguard/minio@sha256:" +
+    "dc455cdd4db6e390b1b26514f15ff37681a777bf8dab41aac445a0259f751e56"
+
   override def beforeAll(): Unit = {
-    minioContainer = new MinIOContainer(DockerImageName.parse("minio/minio:latest"))
+    minioContainer = new MinIOContainer(
+      DockerImageName.parse(minioImage).asCompatibleSubstituteFor("minio/minio"))
       .withUserName(userName)
       .withPassword(password)
     minioContainer.start()
@@ -120,6 +127,7 @@ trait CometS3TestBase extends CometTestBase {
     val s3Client = S3Client
       .builder()
       .endpointOverride(URI.create(minioContainer.getS3URL))
+      .region(Region.US_EAST_1)
       .credentialsProvider(StaticCredentialsProvider.create(credentials))
       .forcePathStyle(true)
       .build()
diff --git a/spark/src/test/scala/org/apache/comet/IcebergReadFromS3Suite.scala b/spark/src/test/scala/org/apache/comet/IcebergReadFromS3Suite.scala
index 1271fa841e..168bd0b49f 100644
--- a/spark/src/test/scala/org/apache/comet/IcebergReadFromS3Suite.scala
+++ b/spark/src/test/scala/org/apache/comet/IcebergReadFromS3Suite.scala
@@ -247,6 +247,7 @@ class IcebergReadFromS3Suite extends CometS3TestBase with RESTCatalogHelper {
       "s3.secret-access-key" -> "WRONG_SECRET_KEY",
       "s3.endpoint" -> minioContainer.getS3URL,
       "s3.region" -> "us-east-1",
+      "client.region" -> "us-east-1",
       "s3.path-style-access" -> "true")
     val warehouse = s"s3a://$testBucketName/warehouse-bad-creds"
 
@@ -280,6 +281,7 @@ class IcebergReadFromS3Suite extends CometS3TestBase with RESTCatalogHelper {
       "s3.secret-access-key" -> password,
       "s3.endpoint" -> minioContainer.getS3URL,
       "s3.region" -> "us-east-1",
+      "client.region" -> "us-east-1",
       "s3.path-style-access" -> "true")
     val warehouse = s"s3a://$testBucketName/warehouse-vending"
 
diff --git a/spark/src/test/scala/org/apache/comet/cloud/s3/CometS3CredentialBridgeSuite.scala b/spark/src/test/scala/org/apache/comet/cloud/s3/CometS3CredentialBridgeSuite.scala
index 8df9b5bb90..f3bc317c29 100644
--- a/spark/src/test/scala/org/apache/comet/cloud/s3/CometS3CredentialBridgeSuite.scala
+++ b/spark/src/test/scala/org/apache/comet/cloud/s3/CometS3CredentialBridgeSuite.scala
@@ -151,6 +151,7 @@ class CometS3CredentialBridgeSuite
       "s3.secret-access-key" -> password,
       "s3.endpoint" -> minioContainer.getS3URL,
       "s3.region" -> "us-east-1",
+      "client.region" -> "us-east-1",
       "s3.path-style-access" -> "true",
       "comet.test-tenant-id" -> "tenant-A")
     val warehouse = s"s3a://$testBucketName/warehouse-rest-spi"


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org