You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ma...@apache.org on 2026/06/06 19:03:06 UTC

(flink) branch release-1.20 updated: [FLINK-39865][ci] Replace abandoned travis-ci artifacts uploader with AWS CLI for nightly S3 upload

This is an automated email from the ASF dual-hosted git repository.

MartijnVisser pushed a commit to branch release-1.20
in repository https://gitbox.apache.org/repos/asf/flink.git


The following commit(s) were added to refs/heads/release-1.20 by this push:
     new 9f6da3ba910 [FLINK-39865][ci] Replace abandoned travis-ci artifacts uploader with AWS CLI for nightly S3 upload
9f6da3ba910 is described below

commit 9f6da3ba910105b81ec651ef99edc2399f97ddc6
Author: Martijn Visser <29...@users.noreply.github.com>
AuthorDate: Fri Jun 5 13:19:00 2026 +0200

    [FLINK-39865][ci] Replace abandoned travis-ci artifacts uploader with AWS CLI for nightly S3 upload
    
    The nightly "Upload artifacts to S3" step installed the travis-ci/artifacts
    uploader via `curl ... | bash`. That installer downloads its binary from a
    hardcoded travis-ci S3 bucket which now returns HTTP 403, so the XML error
    page was saved as the `artifacts` binary and executed, breaking every nightly
    run since 2026-06-02. The travis-ci/artifacts project is abandoned.
    
    Replace it with the maintained AWS CLI (`aws s3 cp --recursive`), installing
    AWS CLI v2 in the build container when absent. Credentials are now read from
    the environment (AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY) instead of being
    passed as command-line arguments, so they no longer appear in the `set -x`
    trace. The default (private) ACL and bucket-root upload layout are preserved.
    
    Generated-by: Claude Code (Opus 4.8)
    (cherry picked from commit 5c0e9a8b45fbb264cdbc48640aa4d890d97fb41f)
---
 tools/azure-pipelines/build-nightly-dist.yml |  6 ++--
 tools/ci/deploy_nightly_to_s3.sh             | 42 +++++++++++++++-------------
 2 files changed, 26 insertions(+), 22 deletions(-)

diff --git a/tools/azure-pipelines/build-nightly-dist.yml b/tools/azure-pipelines/build-nightly-dist.yml
index 21420361106..01082f1ccda 100644
--- a/tools/azure-pipelines/build-nightly-dist.yml
+++ b/tools/azure-pipelines/build-nightly-dist.yml
@@ -59,8 +59,10 @@ jobs:
             upload_to_s3 ./tools/releasing/release
         env:
           ARTIFACTS_S3_BUCKET: $(ARTIFACTS_S3_BUCKET)
-          ARTIFACTS_AWS_ACCESS_KEY_ID: $(ARTIFACTS_AWS_ACCESS_KEY_ID)
-          ARTIFACTS_AWS_SECRET_ACCESS_KEY: $(ARTIFACTS_AWS_SECRET_ACCESS_KEY)
+          # Mapped to the names the AWS CLI reads natively, so credentials never
+          # appear on a command line (and thus never in the `set -x` trace).
+          AWS_ACCESS_KEY_ID: $(ARTIFACTS_AWS_ACCESS_KEY_ID)
+          AWS_SECRET_ACCESS_KEY: $(ARTIFACTS_AWS_SECRET_ACCESS_KEY)
       # Activate this to publish the binary release as a pipeline artifact on Azure
       #- task: PublishPipelineArtifact@1
       #  displayName: Upload snapshot binary release
diff --git a/tools/ci/deploy_nightly_to_s3.sh b/tools/ci/deploy_nightly_to_s3.sh
index ca373bdea83..b6d1101ffaf 100755
--- a/tools/ci/deploy_nightly_to_s3.sh
+++ b/tools/ci/deploy_nightly_to_s3.sh
@@ -21,23 +21,25 @@
 set -e -x
 
 
-function upload_to_s3() {	
-	local FILES_DIR=$1	
-
-	echo "Installing artifacts deployment script"	
-	export ARTIFACTS_DEST="$HOME/bin/artifacts"	
-	curl -sL https://raw.githubusercontent.com/travis-ci/artifacts/master/install | bash	
-	PATH="$(dirname "$ARTIFACTS_DEST"):$PATH"	
-
-	echo "Uploading contents of $FILES_DIR to S3:"	
-
-
-	artifacts upload \
-		  --bucket $ARTIFACTS_S3_BUCKET \
-		  --key $ARTIFACTS_AWS_ACCESS_KEY_ID \
-		  --secret $ARTIFACTS_AWS_SECRET_ACCESS_KEY \
-		  --target-paths / $FILES_DIR
-
-}	
-
-
+function upload_to_s3() {
+	local FILES_DIR=$1
+
+	# The AWS CLI reads credentials from the environment (AWS_ACCESS_KEY_ID /
+	# AWS_SECRET_ACCESS_KEY), so they are never placed on a command line and
+	# cannot leak through the `set -x` trace enabled above.
+	if ! command -v aws >/dev/null 2>&1; then
+		echo "Installing AWS CLI v2"
+		local AWS_TMP_DIR
+		AWS_TMP_DIR=$(mktemp -d)
+		curl -fsSL "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "$AWS_TMP_DIR/awscliv2.zip"
+		unzip -q "$AWS_TMP_DIR/awscliv2.zip" -d "$AWS_TMP_DIR"
+		"$AWS_TMP_DIR/aws/install" --bin-dir "$HOME/bin" --install-dir "$HOME/aws-cli" --update
+		PATH="$HOME/bin:$PATH"
+	fi
+
+	echo "Uploading contents of $FILES_DIR to S3:"
+
+	# Mirrors the previous uploader's behaviour: copy the directory contents to
+	# the bucket root using the bucket's default (private) ACL.
+	aws s3 cp "$FILES_DIR" "s3://$ARTIFACTS_S3_BUCKET/" --recursive --no-progress
+}