[airflow] 01/01: Move constraints generation to separate job in CI workflow

[po...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

potiuk pushed a commit to branch move-early-constrints-generation-to-separate-steps
in repository https://gitbox.apache.org/repos/asf/airflow.git

commit 2a638d209288221aef8cf122f6bc609b00933e2e
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Mon Oct 16 16:49:08 2023 +0200

    Move constraints generation to separate job in CI workflow
    
    We generate constraints for preview right after images are build, in
    order to be able to see the output and to diagnose which dependencies
    have been updated later in the process.
    
    So far we were doing it in CI workflow and When constraints generation
    failed however (for example because pip backtracking takes a log ot
    time), it faied build image workflow and did not allow tests to
    complete.
    
    This PR extracts constraint generation to CI separate job which
    does not block tests from running in case constraints generation
    fails or times out.
    
    It also moves out the steps from the composite action which allows
    to better see which step failed and allows to see the constraints
    used in each job more easily.
---
 .github/actions/build-ci-images/action.yml | 24 --------------
 .github/workflows/ci.yml                   | 51 ++++++++++++++++++++++++++++++
 2 files changed, 51 insertions(+), 24 deletions(-)

diff --git a/.github/actions/build-ci-images/action.yml b/.github/actions/build-ci-images/action.yml
index 81825f9a79..e93b2f4e71 100644
--- a/.github/actions/build-ci-images/action.yml
+++ b/.github/actions/build-ci-images/action.yml
@@ -34,30 +34,6 @@ runs:
     - name: "Build & Push AMD64 CI images ${{ env.IMAGE_TAG }} ${{ env.PYTHON_VERSIONS }}"
       shell: bash
       run: breeze ci-image build --push --tag-as-latest --run-in-parallel --upgrade-on-failure
-    - name: "Generate source constraints"
-      shell: bash
-      run: >
-        breeze release-management generate-constraints --run-in-parallel
-        --airflow-constraints-mode constraints-source-providers
-#    - name: "Generate PyPI constraints"
-#      shell: bash
-#      run: >
-#        breeze release-management generate-constraints --run-in-parallel
-#        --airflow-constraints-mode constraints
-#      if: ${{ inputs.build-provider-packages != 'true' }}
-    - name: "Print dependency upgrade summary"
-      shell: bash
-      run: |
-        for PYTHON_VERSION in ${{ env.PYTHON_VERSIONS }}; do
-          echo "Summarizing Python $PYTHON_VERSION"
-          cat "files/constraints-${PYTHON_VERSION}/*.md" >> $GITHUB_STEP_SUMMARY || true
-        done
-    - name: "Upload constraint artifacts"
-      uses: actions/upload-artifact@v3
-      with:
-        name: constraints
-        path: ./files/constraints-*/constraints-*.txt
-        retention-days: 7
     - name: "Fix ownership"
       shell: bash
       run: breeze ci fix-ownership
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index ca44ad3199..e242254e66 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -326,6 +326,57 @@ jobs:
           DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
           BUILD_TIMEOUT_MINUTES: 70
 
+  preview-constraints:
+    permissions:
+      contents: read
+    timeout-minutes: 20
+    continue-on-error: true
+    name: >
+      Preview constraints
+      ${{needs.build-info.outputs.all-python-versions-list-as-string}}
+    runs-on: "${{needs.build-info.outputs.runs-on}}"
+    needs: [build-info, build-ci-images]
+    if: needs.build-info.outputs.upgrade-to-newer-dependencies != "false"
+    env:
+      RUNS_ON: "${{ needs.build-info.outputs.runs-on }}"
+      PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}}
+    steps:
+      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+        uses: actions/checkout@v3
+        with:
+          persist-credentials: false
+      - name: >
+          Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
+        uses: ./.github/actions/prepare_breeze_and_image
+      - name: "Source constraints"
+        shell: bash
+        run: >
+          breeze release-management generate-constraints --run-in-parallel
+          --airflow-constraints-mode constraints-source-providers || true
+        env:
+          DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
+      - name: "PyPI constraints"
+        shell: bash
+        timeout-minutes: 10
+        run: >
+          breeze release-management generate-constraints --run-in-parallel
+          --airflow-constraints-mode constraints || true
+        env:
+          DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
+      - name: "Dependency upgrade summary"
+        shell: bash
+        run: |
+          for PYTHON_VERSION in ${{ env.PYTHON_VERSIONS }}; do
+            echo "Summarizing Python $PYTHON_VERSION"
+            cat "files/constraints-${PYTHON_VERSION}/*.md" >> $GITHUB_STEP_SUMMARY || true
+          done
+      - name: "Upload constraint artifacts"
+        uses: actions/upload-artifact@v3
+        with:
+          name: constraints
+          path: ./files/constraints-*/constraints-*.txt
+          retention-days: 7
+
   build-prod-images:
     timeout-minutes: 80
     name: >