(flink) 01/02: [FLINK-40083][tests] Launch the SeaweedFS S3 test backend via 'weed mini'

[ma...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

MartijnVisser pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git

commit 97ad29b8ed436c72098afa2f1d01c8b336e4d32e
Author: Chris Lu <ch...@gmail.com>
AuthorDate: Wed Sep 16 15:58:13 2026 -0700

    [FLINK-40083][tests] Launch the SeaweedFS S3 test backend via 'weed mini'
    
    weed mini pre-creates the bucket (-bucket) and emits its readiness
    banner after the enabled components are up. Both the Java test container
    and the S3 end-to-end script now use mini instead of creating the bucket
    client-side. The Java container also verifies that the requested bucket
    exists before tests proceed.
    
    This drops the startup-503 retry workaround, the explicit aws s3 mb
    step, and the /healthz poll in favour of the mini readiness banner.
    
    Generated-by: Claude Code (claude-opus-4-8)
    Generated-by: GitHub Copilot CLI 1.0.83
    Generated-by: Codex (GPT-5)
---
 .../test-scripts/common_s3_seaweedfs.sh            | 12 +++++------
 .../flink/fs/s3/common/SeaweedFsTestContainer.java | 24 +++++++++-------------
 2 files changed, 15 insertions(+), 21 deletions(-)

diff --git a/flink-end-to-end-tests/test-scripts/common_s3_seaweedfs.sh b/flink-end-to-end-tests/test-scripts/common_s3_seaweedfs.sh
index a8060aa8d14..9da0fce0e49 100644
--- a/flink-end-to-end-tests/test-scripts/common_s3_seaweedfs.sh
+++ b/flink-end-to-end-tests/test-scripts/common_s3_seaweedfs.sh
@@ -48,10 +48,10 @@ function s3_start {
     -P \
     -e "AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID" -e "AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY" \
     chrislusf/seaweedfs:4.34 \
-    server \
-    -s3 \
+    mini \
     -s3.port=8333 \
-    -dir=/data)
+    -dir=/data \
+    -bucket="$IT_CASE_S3_BUCKET")
   while [[ "$(docker inspect -f {{.State.Running}} "$SEAWEEDFS_CONTAINER_ID")" -ne "true" ]]; do
     sleep 0.1
   done
@@ -59,7 +59,8 @@ function s3_start {
   echo "Started seaweedfs @ $S3_ENDPOINT"
   on_exit s3_stop
 
-  while [[ "$(curl -s -o /dev/null -w '%{http_code}' "$S3_ENDPOINT/healthz")" != "200" ]]; do
+  # mini pre-creates the bucket and reports readiness only once all components are up.
+  while ! docker logs "$SEAWEEDFS_CONTAINER_ID" 2>&1 | grep "All enabled components are running and ready to use"; do
     sleep 0.1
   done
   echo "Seaweedfs S3 gateway is up @ $S3_ENDPOINT"
@@ -123,7 +124,4 @@ function s3_setup_with_provider {
 
 source "$(dirname "$0")"/common_s3_operations.sh
 
-# SeaweedFS does not expose on-disk directories as S3 buckets,
-# so the bucket must be created explicitly
-aws_cli s3 mb "s3://$IT_CASE_S3_BUCKET"
 aws_cli s3 cp "/hostdir/test-data/words" "$S3_TEST_DATA_WORDS_URI"
diff --git a/flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/SeaweedFsTestContainer.java b/flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/SeaweedFsTestContainer.java
index a1d8ceed187..dfc557286ca 100644
--- a/flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/SeaweedFsTestContainer.java
+++ b/flink-filesystems/flink-s3-fs-base/src/test/java/org/apache/flink/fs/s3/common/SeaweedFsTestContainer.java
@@ -30,7 +30,7 @@ import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.AmazonS3Client;
 import com.github.dockerjava.api.command.InspectContainerResponse;
 import org.testcontainers.containers.GenericContainer;
-import org.testcontainers.containers.wait.strategy.HttpWaitStrategy;
+import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy;
 import org.testcontainers.utility.Base58;
 
 import java.io.File;
@@ -48,7 +48,6 @@ class SeaweedFsTestContainer extends GenericContainer<SeaweedFsTestContainer> {
     private static final String AWS_SECRET_ACCESS_KEY = "AWS_SECRET_ACCESS_KEY";
 
     private static final String DEFAULT_STORAGE_DIRECTORY = "/data";
-    private static final String HEALTH_ENDPOINT = "/healthz";
 
     private final String accessKey;
     private final String secretKey;
@@ -71,21 +70,22 @@ class SeaweedFsTestContainer extends GenericContainer<SeaweedFsTestContainer> {
         withEnv(AWS_ACCESS_KEY_ID, this.accessKey);
         withEnv(AWS_SECRET_ACCESS_KEY, this.secretKey);
         withCommand(
-                "server", "-s3", "-s3.port=" + DEFAULT_PORT, "-dir=" + DEFAULT_STORAGE_DIRECTORY);
+                "mini",
+                "-s3.port=" + DEFAULT_PORT,
+                "-dir=" + DEFAULT_STORAGE_DIRECTORY,
+                "-bucket=" + defaultBucketName);
         setWaitStrategy(
-                new HttpWaitStrategy()
-                        .forPort(DEFAULT_PORT)
-                        .forPath(HEALTH_ENDPOINT)
+                new LogMessageWaitStrategy()
+                        .withRegEx("(?s).*All enabled components are running and ready to use.*")
                         .withStartupTimeout(Duration.ofMinutes(2)));
-        // Very rarely, a 503 status will be returned continuously while the container is
-        // starting up, slipping past the AmazonS3 client's default retry strategy.
-        withStartupAttempts(3);
     }
 
     @Override
     protected void containerIsStarted(InspectContainerResponse containerInfo) {
         super.containerIsStarted(containerInfo);
-        createDefaultBucket();
+        Preconditions.checkState(
+                getClient().doesBucketExistV2(defaultBucketName),
+                "SeaweedFS did not create the requested default bucket.");
     }
 
     private static String randomString(String prefix, int length) {
@@ -129,10 +129,6 @@ class SeaweedFsTestContainer extends GenericContainer<SeaweedFsTestContainer> {
         FileSystem.initialize(config, null);
     }
 
-    private void createDefaultBucket() {
-        getClient().createBucket(defaultBucketName);
-    }
-
     /** Returns the internally used default bucket. */
     public String getDefaultBucketName() {
         return defaultBucketName;