You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2023/03/05 10:26:29 UTC
[airflow] branch main updated: Separate CI Job to run Pytest collection check (#29923)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 30b2e6c185 Separate CI Job to run Pytest collection check (#29923)
30b2e6c185 is described below
commit 30b2e6c185305a56f9fd43683f1176f01fe4e3f6
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Sun Mar 5 11:26:21 2023 +0100
Separate CI Job to run Pytest collection check (#29923)
Before we attempt to run tests in parallel, we quickly check once
if Pytest collection works. This is in order to avoid costly
parallel test execution if that makes no sense to initialize all
the parallel machines. This check used to be done in "Wait for
CI Inages" step, but running it there has the undesireable
side effect that it is not obvious that it's the collection
that fails, also it prevents other jobs (for example
static checks and docs building) from running. This means that
the contributor does not get all the feedback that could be
given immediately.
This PR separates the collection into separate job and only
makes "test" jobs depend on it - all the other jobs that need
CI image depend on "wait for CI image" one and should continue
running even if pytest collection fails.
CI diagrams are also updated to reflect a bit better optionality
and parallelism of the CI jobs.
---
.github/workflows/ci.yml | 46 ++++++++---
CI.rst | 2 +
CI_DIAGRAMS.md | 207 +++++++++++++++++++++++++++--------------------
3 files changed, 158 insertions(+), 97 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 689dcb5318..64a95a8705 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -540,8 +540,6 @@ jobs:
env:
PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }}
DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
- - name: "Tests Pytest collection: ${{matrix.python-version}}"
- run: breeze shell "python /opt/airflow/scripts/in_container/test_pytest_collection.py"
- name: "Fix ownership"
run: breeze ci fix-ownership
if: always()
@@ -834,13 +832,43 @@ jobs:
- name: "Post Helm Tests"
uses: ./.github/actions/post_tests
+ test-pytest-collection:
+ timeout-minutes: 5
+ name: "Test Pytest collection"
+ runs-on: "${{needs.build-info.outputs.runs-on}}"
+ needs: [build-info, wait-for-ci-images]
+ if: needs.build-info.outputs.image-build == 'true'
+ env:
+ RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
+ BACKEND: sqlite
+ PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}"
+ steps:
+ - name: Cleanup repo
+ run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
+ - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+ uses: actions/checkout@v3
+ with:
+ persist-credentials: false
+ - name: "Install Breeze"
+ uses: ./.github/actions/breeze
+ - name: Wait for CI images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG }}
+ id: wait-for-images
+ run: breeze ci-image pull --wait-for-image --tag-as-latest
+ env:
+ DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
+ - name: "Tests Pytest collection"
+ run: breeze shell "python /opt/airflow/scripts/in_container/test_pytest_collection.py"
+ - name: "Fix ownership"
+ run: breeze ci fix-ownership
+ if: always()
+
tests-postgres:
timeout-minutes: 130
name: >
Postgres${{matrix.postgres-version}},Py${{matrix.python-version}}:
${{needs.build-info.outputs.test-types}}
runs-on: "${{needs.build-info.outputs.runs-on}}"
- needs: [build-info, wait-for-ci-images]
+ needs: [build-info, test-pytest-collection]
strategy:
matrix:
python-version: "${{fromJson(needs.build-info.outputs.python-versions)}}"
@@ -884,7 +912,7 @@ jobs:
name: >
MySQL${{matrix.mysql-version}}, Py${{matrix.python-version}}: ${{needs.build-info.outputs.test-types}}
runs-on: "${{needs.build-info.outputs.runs-on}}"
- needs: [build-info, wait-for-ci-images]
+ needs: [build-info, test-pytest-collection]
strategy:
matrix:
python-version: "${{fromJson(needs.build-info.outputs.python-versions)}}"
@@ -928,7 +956,7 @@ jobs:
name: >
MSSQL${{matrix.mssql-version}}, Py${{matrix.python-version}}: ${{needs.build-info.outputs.test-types}}
runs-on: "${{needs.build-info.outputs.runs-on}}"
- needs: [build-info, wait-for-ci-images]
+ needs: [build-info, test-pytest-collection]
strategy:
matrix:
python-version: "${{fromJson(needs.build-info.outputs.python-versions)}}"
@@ -970,7 +998,7 @@ jobs:
name: >
Sqlite Py${{matrix.python-version}}: ${{needs.build-info.outputs.test-types}}
runs-on: "${{needs.build-info.outputs.runs-on}}"
- needs: [build-info, wait-for-ci-images]
+ needs: [build-info, test-pytest-collection]
strategy:
matrix:
python-version: ${{ fromJson(needs.build-info.outputs.python-versions) }}
@@ -1011,7 +1039,7 @@ jobs:
timeout-minutes: 130
name: Integration Tests Postgres
runs-on: "${{needs.build-info.outputs.runs-on}}"
- needs: [build-info, wait-for-ci-images]
+ needs: [build-info, test-pytest-collection]
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
TEST_TYPES: "${{needs.build-info.outputs.test-types}}"
@@ -1071,7 +1099,7 @@ jobs:
timeout-minutes: 130
name: Integration Tests MySQL
runs-on: "${{needs.build-info.outputs.runs-on}}"
- needs: [build-info, wait-for-ci-images]
+ needs: [build-info, test-pytest-collection]
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
TEST_TYPES: "${{needs.build-info.outputs.test-types}}"
@@ -1112,7 +1140,7 @@ jobs:
name: "Quarantined tests"
runs-on: "${{needs.build-info.outputs.runs-on}}"
continue-on-error: true
- needs: [build-info, wait-for-ci-images]
+ needs: [build-info, test-pytest-collection]
env:
RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
TEST_TYPES: "Quarantined"
diff --git a/CI.rst b/CI.rst
index 8f9862f835..b0b5240423 100644
--- a/CI.rst
+++ b/CI.rst
@@ -382,6 +382,8 @@ This workflow is a regular workflow that performs all checks of Airflow code.
+---------------------------------+----------------------------------------------------------+----------+----------+-----------+-------------------+
| Build docs | Builds documentation | Yes | Yes | Yes | Yes |
+---------------------------------+----------------------------------------------------------+----------+----------+-----------+-------------------+
+| Test Pytest collection | Tests if pytest collection works | Yes | Yes | Yes | Yes |
++---------------------------------+----------------------------------------------------------+----------+----------+-----------+-------------------+
| Tests | Run the Pytest unit tests (Backend/Python matrix) | Yes | Yes | Yes | Yes (8) |
+---------------------------------+----------------------------------------------------------+----------+----------+-----------+-------------------+
| Integration tests | Runs integration tests (Postgres/Mysql) | Yes | Yes | Yes | Yes (9) |
diff --git a/CI_DIAGRAMS.md b/CI_DIAGRAMS.md
index 4fc7724c15..a47c62cc70 100644
--- a/CI_DIAGRAMS.md
+++ b/CI_DIAGRAMS.md
@@ -47,6 +47,10 @@ sequenceDiagram
Note over Tests: Test examples<br>PROD image building
and
Note over Tests: Test git clone on Windows
+ and
+ opt
+ Note over Tests: Run basic <br>static checks
+ end
end
Build Images ->> GitHub Registry: Push CI Images<br>[COMMIT_SHA]
loop Wait for CI images
@@ -58,39 +62,49 @@ sequenceDiagram
Note over Build Images: Build PROD Images<br>[COMMIT_SHA]
and
opt
+ GitHub Registry ->> Tests: Pull CI Images<br>[COMMIT_SHA]
Note over Tests: Run static checks
end
and
opt
- Note over Tests: Run basic <br>static checks
- end
- and
- opt
+ GitHub Registry ->> Tests: Pull CI Images<br>[COMMIT_SHA]
Note over Tests: Build docs
end
and
opt
- Note over Tests: Unit Tests
- end
- and
- opt
- Note over Tests: Integration Tests
- end
- and
- opt
- Note over Tests: Quarantined Tests
+ GitHub Registry ->> Tests: Pull CI Images<br>[COMMIT_SHA]
+ Note over Tests: Test Pytest collection<br>[COMMIT_SHA]
+ par
+ opt
+ GitHub Registry ->> Tests: Pull CI Images<br>[COMMIT_SHA]
+ Note over Tests: Unit Tests<br>Python/DB matrix
+ end
+ and
+ opt
+ GitHub Registry ->> Tests: Pull CI Images<br>[COMMIT_SHA]
+ Note over Tests: Integration Tests
+ end
+ and
+ opt
+ GitHub Registry ->> Tests: Pull CI Images<br>[COMMIT_SHA]
+ Note over Tests: Quarantined Tests
+ end
+ end
end
and
opt
- Note over Tests: Test provider <br>packages build
+ GitHub Registry ->> Tests: Pull CI Images<br>[COMMIT_SHA]
+ Note over Tests: Test provider <br>packages build
end
and
opt
- Note over Tests: Test airflow <br>packages build
+ GitHub Registry ->> Tests: Pull CI Images<br>[COMMIT_SHA]
+ Note over Tests: Test airflow <br>packages build
end
and
opt
- Note over Tests: Helm tests
+ GitHub Registry ->> Tests: Pull CI Images<br>[COMMIT_SHA]
+ Note over Tests: Helm tests
end
end
Note over Tests: Summarize Warnings
@@ -102,14 +116,17 @@ sequenceDiagram
Note over Tests: Verify PROD Image<br>[COMMIT_SHA]
par
opt
+ GitHub Registry ->> Tests: Pull PROD Images<br>[COMMIT_SHA]
Note over Tests: Run Kubernetes <br>tests
end
and
opt
+ GitHub Registry ->> Tests: Pull PROD Images<br>[COMMIT_SHA]
Note over Tests: Run docker-compose <br>tests
end
end
opt
+ GitHub Registry ->> Tests: Pull CI Images<br>[COMMIT_SHA]
Note over Tests: Generate constraints
end
Note over Tests: Build ARM CI images
@@ -145,6 +162,10 @@ sequenceDiagram
Note over Tests: Test examples<br>PROD image building
and
Note over Tests: Test git clone on Windows
+ and
+ opt
+ Note over Tests: Run basic <br>static checks
+ end
end
Tests ->> GitHub Registry: Push CI Images<br>[COMMIT_SHA]
GitHub Registry ->> Tests: Pull CI Images<br>[COMMIT_SHA]
@@ -153,39 +174,49 @@ sequenceDiagram
Note over Tests: Build PROD Images<br>[COMMIT_SHA]
and
opt
+ GitHub Registry ->> Tests: Pull CI Images<br>[COMMIT_SHA]
Note over Tests: Run static checks
end
and
opt
- Note over Tests: Run basic <br>static checks
- end
- and
- opt
+ GitHub Registry ->> Tests: Pull CI Images<br>[COMMIT_SHA]
Note over Tests: Build docs
end
and
opt
- Note over Tests: Unit Tests
+ GitHub Registry ->> Tests: Pull CI Images<br>[COMMIT_SHA]
+ Note over Tests: Test Pytest collection<br>[COMMIT_SHA]
+ par
+ opt
+ GitHub Registry ->> Tests: Pull CI Images<br>[COMMIT_SHA]
+ Note over Tests: Unit Tests<br>Python/DB matrix
+ end
+ and
+ opt
+ GitHub Registry ->> Tests: Pull CI Images<br>[COMMIT_SHA]
+ Note over Tests: Integration Tests
+ end
+ and
+ opt
+ GitHub Registry ->> Tests: Pull CI Images<br>[COMMIT_SHA]
+ Note over Tests: Quarantined Tests
+ end
+ end
end
and
opt
- Note over Tests: Integration Tests
- end
- and
- opt
- Note over Tests: Quarantined Tests
- end
- and
- opt
- Note over Tests: Test provider <br>packages build
+ GitHub Registry ->> Tests: Pull CI Images<br>[COMMIT_SHA]
+ Note over Tests: Test provider <br>packages build
end
and
opt
- Note over Tests: Test airflow <br>packages build
+ GitHub Registry ->> Tests: Pull CI Images<br>[COMMIT_SHA]
+ Note over Tests: Test airflow <br>packages build
end
and
opt
- Note over Tests: Helm tests
+ GitHub Registry ->> Tests: Pull CI Images<br>[COMMIT_SHA]
+ Note over Tests: Helm tests
end
end
Note over Tests: Summarize Warnings
@@ -194,14 +225,17 @@ sequenceDiagram
Note over Tests: Verify PROD Image<br>[COMMIT_SHA]
par
opt
+ GitHub Registry ->> Tests: Pull PROD Images<br>[COMMIT_SHA]
Note over Tests: Run Kubernetes <br>tests
end
and
opt
+ GitHub Registry ->> Tests: Pull PROD Images<br>[COMMIT_SHA]
Note over Tests: Run docker-compose <br>tests
end
end
opt
+ GitHub Registry ->> Tests: Pull CI Images<br>[COMMIT_SHA]
Note over Tests: Generate constraints
end
Note over Tests: Build ARM CI images
@@ -241,36 +275,36 @@ sequenceDiagram
par
Note over Tests: Build PROD Images<br>[COMMIT_SHA]
and
- opt
- Note over Tests: Run static checks
- end
- and
- opt
- Note over Tests: Run basic <br>static checks
- end
- and
- opt
- Note over Tests: Build docs
- end
+ GitHub Registry ->> Tests: Pull CI Images<br>[COMMIT_SHA]
+ Note over Tests: Run static checks
and
- opt
- Note over Tests: Unit Tests
- end
+ GitHub Registry ->> Tests: Pull CI Images<br>[COMMIT_SHA]
+ Note over Tests: Build docs
and
- opt
+ GitHub Registry ->> Tests: Pull CI Images<br>[COMMIT_SHA]
+ Note over Tests: Test Pytest collection<br>[COMMIT_SHA]
+ par
+ GitHub Registry ->> Tests: Pull CI Images<br>[COMMIT_SHA]
+ Note over Tests: Unit Tests<br>Python/DB matrix
+ and
+ GitHub Registry ->> Tests: Pull CI Images<br>[COMMIT_SHA]
Note over Tests: Integration Tests
+ and
+ GitHub Registry ->> Tests: Pull CI Images<br>[COMMIT_SHA]
+ Note over Tests: Quarantined Tests
end
and
opt
- Note over Tests: Test provider <br>packages build
+ GitHub Registry ->> Tests: Pull CI Images<br>[COMMIT_SHA]
+ Note over Tests: Test provider <br>packages build
end
and
- opt
- Note over Tests: Test airflow <br>packages build
- end
+ GitHub Registry ->> Tests: Pull CI Images<br>[COMMIT_SHA]
+ Note over Tests: Test airflow <br>packages build
and
opt
- Note over Tests: Helm tests
+ GitHub Registry ->> Tests: Pull CI Images<br>[COMMIT_SHA]
+ Note over Tests: Helm tests
end
end
Note over Tests: Summarize Warnings
@@ -278,20 +312,21 @@ sequenceDiagram
GitHub Registry ->> Tests: Pull PROD Image<br>[COMMIT_SHA]
Note over Tests: Verify PROD Image<br>[COMMIT_SHA]
par
- opt
- Note over Tests: Run Kubernetes <br>tests
- end
+ GitHub Registry ->> Tests: Pull PROD Image<br>[COMMIT_SHA]
+ Note over Tests: Run Kubernetes <br>tests
and
- opt
- Note over Tests: Run docker-compose <br>tests
- end
+ GitHub Registry ->> Tests: Pull PROD Image<br>[COMMIT_SHA]
+ Note over Tests: Run docker-compose <br>tests
end
+ GitHub Registry ->> Tests: Pull CI Images<br>[COMMIT_SHA]
Note over Tests: Generate constraints
Tests ->> Airflow Repo: Push constraints if changed
Note over Tests: Build CI Images<br>[latest]<br>Use latest constraints
Tests ->> GitHub Registry: Push CI Image<br>[latest]
Note over Tests: Build PROD Images<br>[latest]<br>Use latest constraints
Tests ->> GitHub Registry: Push PROD Image<br>[latest]
+ Note over Tests: Build ARM CI images
+ Tests ->> GitHub Registry: Push ARM CI Image cache
Tests -->> Airflow Repo: Status update
deactivate Airflow Repo
deactivate Tests
@@ -326,51 +361,46 @@ sequenceDiagram
GitHub Registry ->> Tests: Pull PROD Images<br>[latest]
Note over Tests: Build PROD Images<br>[COMMIT_SHA]
and
- opt
- Note over Tests: Run static checks
- end
- and
- opt
- Note over Tests: Run basic <br>static checks
- end
- and
- opt
- Note over Tests: Build docs
- end
+ GitHub Registry ->> Tests: Pull CI Images<br>[COMMIT_SHA]
+ Note over Tests: Run static checks
and
- opt
- Note over Tests: Unit Tests
- end
+ GitHub Registry ->> Tests: Pull CI Images<br>[COMMIT_SHA]
+ Note over Tests: Build docs
and
- opt
+ GitHub Registry ->> Tests: Pull CI Images<br>[COMMIT_SHA]
+ Note over Tests: Test Pytest collection<br>[COMMIT_SHA]
+ par
+ GitHub Registry ->> Tests: Pull CI Images<br>[COMMIT_SHA]
+ Note over Tests: Unit Tests<br>Python/DB matrix
+ and
+ GitHub Registry ->> Tests: Pull CI Images<br>[COMMIT_SHA]
Note over Tests: Integration Tests
+ and
+ GitHub Registry ->> Tests: Pull CI Images<br>[COMMIT_SHA]
+ Note over Tests: Quarantined Tests
end
and
- opt
- Note over Tests: Test provider <br>packages build
- end
+ GitHub Registry ->> Tests: Pull CI Images<br>[COMMIT_SHA]
+ Note over Tests: Test provider <br>packages build
and
- opt
- Note over Tests: Test airflow <br>packages build
- end
+ GitHub Registry ->> Tests: Pull CI Images<br>[COMMIT_SHA]
+ Note over Tests: Test airflow <br>packages build
and
- opt
- Note over Tests: Helm tests
- end
+ GitHub Registry ->> Tests: Pull CI Images<br>[COMMIT_SHA]
+ Note over Tests: Helm tests
end
Note over Tests: Summarize Warnings
Tests ->> GitHub Registry: Push PROD Images<br>[COMMIT_SHA]
GitHub Registry ->> Tests: Pull PROD Image<br>[COMMIT_SHA]
Note over Tests: Verify PROD Image<br>[COMMIT_SHA]
par
- opt
- Note over Tests: Run Kubernetes <br>tests
- end
+ GitHub Registry ->> Tests: Pull PROD Image<br>[COMMIT_SHA]
+ Note over Tests: Run Kubernetes <br>tests
and
- opt
- Note over Tests: Run docker-compose <br>tests
- end
+ GitHub Registry ->> Tests: Pull PROD Image<br>[COMMIT_SHA]
+ Note over Tests: Run docker-compose <br>tests
end
+ GitHub Registry ->> Tests: Pull CI Images<br>[COMMIT_SHA]
Note over Tests: Generate constraints
Tests ->> Airflow Repo: Push constraints if changed
Note over Tests: Build CI Images<br>[latest]<br>Use latest constraints
@@ -378,6 +408,7 @@ sequenceDiagram
Note over Tests: Build PROD Images<br>[latest]<br>Use latest constraints
Tests ->> GitHub Registry: Push PROD Image cache + latest
Note over Tests: Build ARM CI images
+ Tests ->> GitHub Registry: Push ARM CI Image cache
Tests -->> Airflow Repo: Status update
deactivate Airflow Repo
deactivate Tests