You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by fa...@apache.org on 2026/05/05 03:00:29 UTC

(hadoop) branch af/hadoop-19877 created (now ac00cb2176e)

This is an automated email from the ASF dual-hosted git repository.

ajfabbri pushed a change to branch af/hadoop-19877
in repository https://gitbox.apache.org/repos/asf/hadoop.git


      at ac00cb2176e HADOOP-19877: ci: initial S3A integration test CI

This branch includes the following new commits:

     new e365c69e6ae HADOOP-19877: ci: create composite actions for creating builder image
     new ac00cb2176e HADOOP-19877: ci: initial S3A integration test CI

The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.



---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


(hadoop) 01/02: HADOOP-19877: ci: create composite actions for creating builder image

Posted by fa...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

ajfabbri pushed a commit to branch af/hadoop-19877
in repository https://gitbox.apache.org/repos/asf/hadoop.git

commit e365c69e6ae02fa9284c31c61a1cd5fbff130341
Author: Aaron Fabbri <fa...@apache.org>
AuthorDate: Mon May 4 19:25:16 2026 -0700

    HADOOP-19877: ci: create composite actions for creating builder image
---
 .github/actions/build_image/action.yml     | 62 ++++++++++++++++++++++++++++++
 .github/actions/build_image_url/action.yml | 27 +++++++++++++
 .github/workflows/tmpl_build_and_test.yml  |  1 +
 3 files changed, 90 insertions(+)

diff --git a/.github/actions/build_image/action.yml b/.github/actions/build_image/action.yml
new file mode 100644
index 00000000000..538df067705
--- /dev/null
+++ b/.github/actions/build_image/action.yml
@@ -0,0 +1,62 @@
+name: Create hadoop build image
+
+inputs:
+  branch:
+    description: Git branch to use.
+    required: true
+  os:
+    description: Runner OS name
+    required: true
+  build_image_url:
+    description: URL of build (infra) image.
+    required: true
+
+outputs:
+  uid:
+    description: User ID this action ran as.
+
+runs:
+  using: composite
+  steps:
+    - name: Login to GitHub Container Registry
+      uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ github.token }}
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
+    - name: Build and push ${{ inputs.os }} base build image for ${{ inputs.branch }}
+      id: docker_build_base
+      uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0
+      with:
+        context: ./dev-support/docker/
+        file: ./dev-support/docker/Dockerfile_${{ inputs.os }}
+        push: true
+        tags: ${{ inputs.build_image_url }}-base
+        cache-from: type=registry,ref=ghcr.io/apache/hadoop/gha-build-${{ inputs.os }}-image-cache:${{ inputs.branch }}
+    - name: User-specific Dockerfile
+      shell: bash
+      run: |
+        USER_ID=$(id -u "${USER}")
+        GROUP_ID=$(id -g "${USER}")
+        cat > /tmp/Dockerfile.gha <<UserSpecificDocker
+        FROM ${{ inputs.build_image_url }}-base
+        RUN rm -f /var/log/faillog /var/log/lastlog
+        RUN groupadd --non-unique -g ${GROUP_ID} ${USER}
+        RUN useradd -g ${GROUP_ID} -u ${USER_ID} -k /root -m ${USER}
+        RUN echo "${USER} ALL=NOPASSWD: ALL" > "/etc/sudoers.d/hadoop-build-${USER_ID}"
+        UserSpecificDocker
+    - name: Build and push ${{ inputs.os }} build image for ${{ inputs.branch }}
+      id: docker_build_gha
+      uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0
+      with:
+        context: ./dev-support/docker/
+        file: /tmp/Dockerfile.gha
+        push: true
+        tags: ${{ inputs.build_image_url }}
+    - name: Set up Outputs
+      id: variables
+      shell: bash
+      run: |
+        echo "uid=$(id -u "${USER}")" >> $GITHUB_OUTPUT
diff --git a/.github/actions/build_image_url/action.yml b/.github/actions/build_image_url/action.yml
new file mode 100644
index 00000000000..4c870d77697
--- /dev/null
+++ b/.github/actions/build_image_url/action.yml
@@ -0,0 +1,27 @@
+name: Get Build Image URL
+description:
+
+inputs:
+  branch:
+    description: Git branch to use.
+    required: true
+  os:
+    description: Operating system to run the build on
+    required: true
+
+outputs:
+  build_image_url:
+    description: URL of build (infra) image.
+    value: ${{ steps.compute.outputs.image_url }}
+
+runs:
+  using: composite
+  steps:
+    - id: compute
+      shell: bash
+      run: |
+        # Convert to lowercase to meet Docker repo name requirement
+        REPO_OWNER=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]')
+        echo "build_image_url=ghcr.io/${REPO_OWNER}/gha-build-${{ inputs.os }}:${{ inputs.branch }}-${{ github.run_id }}" \
+        >> $GITHUB_OUTPUT
+
diff --git a/.github/workflows/tmpl_build_and_test.yml b/.github/workflows/tmpl_build_and_test.yml
index 1062c3e3f3d..373a79a6a99 100644
--- a/.github/workflows/tmpl_build_and_test.yml
+++ b/.github/workflows/tmpl_build_and_test.yml
@@ -71,6 +71,7 @@ env:
     -Dmaven.test.failure.ignore=false
 
 jobs:
+  # TODO: use common composite actions in .github/actions/
   precondition:
     name: Preparation
     runs-on: ubuntu-24.04


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


(hadoop) 02/02: HADOOP-19877: ci: initial S3A integration test CI

Posted by fa...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

ajfabbri pushed a commit to branch af/hadoop-19877
in repository https://gitbox.apache.org/repos/asf/hadoop.git

commit ac00cb2176e97f987adeeb039c2765b5888c5b10
Author: Aaron Fabbri <fa...@apache.org>
AuthorDate: Mon May 4 19:59:58 2026 -0700

    HADOOP-19877: ci: initial S3A integration test CI
---
 .github/workflows/cloud_aws.yml      |  32 +++++++++
 .github/workflows/tmpl_cloud_aws.yml | 135 +++++++++++++++++++++++++++++++++++
 2 files changed, 167 insertions(+)

diff --git a/.github/workflows/cloud_aws.yml b/.github/workflows/cloud_aws.yml
new file mode 100644
index 00000000000..88381b5d357
--- /dev/null
+++ b/.github/workflows/cloud_aws.yml
@@ -0,0 +1,32 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: "Cloud - AWS integration tests"
+
+on:
+  pull_request:
+    paths:
+      - 'hadoop-tools/hadoop-aws/**'
+
+jobs:
+  run-aws-integration:
+    permissions:
+      packages: write
+    name: Run
+    uses: ./.github/workflows/tmpl_cloud_aws.yml
diff --git a/.github/workflows/tmpl_cloud_aws.yml b/.github/workflows/tmpl_cloud_aws.yml
new file mode 100644
index 00000000000..abb0789bf47
--- /dev/null
+++ b/.github/workflows/tmpl_cloud_aws.yml
@@ -0,0 +1,135 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name: AWS Integration
+
+on:
+  workflow_call:
+    inputs:
+      java:
+        required: false
+        type: string
+        default: 17
+      branch:
+        required: false
+        type: string
+        description: Branch to run the build against
+        default: trunk
+      os:
+        required: false
+        type: string
+        description: Operating system to run the build on
+        default: ubuntu_24
+
+# Security: Default to minimal permissions for workflow.
+permissions: {}
+
+concurrency:
+  group: >-
+    cloud-aws
+    ${{ github.workflow }}
+    ${{ github.repository == 'apache/hadoop' && github.run_id || github.ref }}
+    ${{ inputs.java }}
+    ${{ inputs.branch }}
+    ${{ inputs.os }}
+  cancel-in-progress: true
+
+env:
+  BUCKET_NAME: hadoop-ci
+
+jobs:
+  precondition:
+    runs-on: ubuntu-24.04
+    outputs:
+      build_image_url: ${{ steps.img.outputs.build_image_url }}
+    steps:
+      - uses: actions/checkout@v6
+      - uses: ./.github/actions/build_image_url
+        id: img
+        with:
+          branch: ${{ inputs.branch }}
+          os: ${{ inputs.os }}
+
+  build-image:
+    runs-on: ubuntu-24.04
+    needs: [ precondition ]
+    permissions:
+      packages: write
+    outputs:
+      uid: ${{ steps.build_img.outputs.uid }}
+    steps:
+      - uses: actions/checkout@v6
+      - uses: ./.github/actions/build_image
+        id: build_img
+        with:
+          branch: ${{ inputs.branch }}
+          os: ${{ inputs.os }}
+          build_image_url: ${{ needs.precondition.outputs.build_image_url }}
+
+  test:
+    name: S3A Integration Tests (Java ${{ inputs.java }})
+    needs: [ precondition, build-image ]
+    runs-on: ubuntu-24.04
+    container:
+      image: ${{ needs.precondition.outputs.build_image_url }}
+      options: --user ${{ needs.build-image.outputs.uid }}
+    steps:
+      - uses: actions/checkout@v6
+      - name: Setup JDK ${{ inputs.java }}
+        uses: actions/setup-java@v5
+        with:
+          distribution: zulu
+          java-version: ${{ inputs.java }}
+          cache: 'maven'
+      - name: Localstack setup
+        # Security: 0.3.2 is an immutable commit, see:
+        # https://github.com/localstack/setup-localstack/commit/7c8a0cb3405bc58be4c8f763f812aa000bc46303
+        uses: LocalStack/setup-localstack@0.3.2
+        with:
+          image-tag: 'latest'
+          install-awslocal: 'true'
+        env:
+          AWS_REGION: us-west-2
+          LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_CI_KEY }}
+          AWS_ACCESS_KEY_ID: test
+          AWS_SECRET_ACCESS_KEY: test
+
+      - name: Maven Build
+        shell: bash
+        run: ./mvnw install -B -DskipTests -am -pl hadoop-tools/hadoop-aws
+      - name: Create localstack s3 bucket
+        shell: bash
+        run: awslocal s3 mb s3://${{ env.BUCKET_NAME }}
+
+      - name: S3A Integration Tests
+        shell: bash
+        env:
+          AWS_ACCESS_KEY_ID: test
+          AWS_SECRET_ACCESS_KEY: test
+          AWS_REGION: us-west-2
+        run: |
+          ./mvnw verify -B -pl hadoop-tools/hadoop-aws \
+            -DskipITs=false \
+            -Dtest=none \
+            -Dscale=true \
+            -Dfs.s3a.access.key=test \
+            -Dfs.s3a.secret.key=test \
+            -Dfs.s3a.endpoint=http://localhost:4566 \
+            -Dfs.s3a.connection.ssl.enabled=false \
+            -Dfs.s3a.path.style.access=true \
+            -Dfs.s3a.endpoint.region=us-west-2 \
+            -Dtest.fs.s3a.name=s3a://${{ env.BUCKET_NAME }}/


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org