You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by hu...@apache.org on 2024/08/25 17:01:45 UTC
(airflow) branch test_arc_3 created (now 959f9dcebe)
This is an automated email from the ASF dual-hosted git repository.
husseinawala pushed a change to branch test_arc_3
in repository https://gitbox.apache.org/repos/asf/airflow.git
at 959f9dcebe Test GHA ARC
This branch includes the following new commits:
new 959f9dcebe Test GHA ARC
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(airflow) 01/01: Test GHA ARC
Posted by hu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
husseinawala pushed a commit to branch test_arc_3
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 959f9dcebec7201319eb5f4232ccb5ea96145ac3
Author: hussein-awala <hu...@awala.fr>
AuthorDate: Sun Aug 25 18:35:17 2024 +0200
Test GHA ARC
---
.github/workflows/basic-tests.yml | 16 +++----
.github/workflows/build-images.yml | 2 -
.github/workflows/ci.yml | 4 +-
dev/breeze/doc/ci/04_selective_checks.md | 2 -
dev/breeze/src/airflow_breeze/global_constants.py | 6 +--
.../src/airflow_breeze/utils/selective_checks.py | 18 +-------
dev/breeze/tests/test_pr_info.py | 12 ++----
dev/breeze/tests/test_selective_checks.py | 49 +++-------------------
8 files changed, 20 insertions(+), 89 deletions(-)
diff --git a/.github/workflows/basic-tests.yml b/.github/workflows/basic-tests.yml
index f5b1d54084..f171d5cce8 100644
--- a/.github/workflows/basic-tests.yml
+++ b/.github/workflows/basic-tests.yml
@@ -20,8 +20,8 @@ name: Basic tests
on: # yamllint disable-line rule:truthy
workflow_call:
inputs:
- runs-on-as-json-public:
- description: "The array of labels (in json form) determining public runners."
+ runs-on-as-json-default:
+ description: "The array of labels (in json form) determining default runners."
required: true
type: string
run-www-tests:
@@ -62,7 +62,7 @@ jobs:
run-breeze-tests:
timeout-minutes: 10
name: Breeze unit tests
- runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
+ runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }}
steps:
- name: "Cleanup repo"
shell: bash
@@ -86,7 +86,7 @@ jobs:
tests-www:
timeout-minutes: 10
name: React WWW tests
- runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
+ runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }}
if: inputs.run-www-tests == 'true'
steps:
- name: "Cleanup repo"
@@ -115,7 +115,7 @@ jobs:
test-openapi-client:
timeout-minutes: 10
name: "Test OpenAPI client"
- runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
+ runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }}
if: inputs.needs-api-codegen == 'true'
steps:
- name: "Cleanup repo"
@@ -206,7 +206,7 @@ jobs:
static-checks-basic-checks-only:
timeout-minutes: 30
name: "Static checks: basic checks only"
- runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
+ runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }}
if: inputs.basic-checks-only == 'true'
steps:
- name: "Cleanup repo"
@@ -273,7 +273,7 @@ jobs:
upgrade-check:
timeout-minutes: 45
name: "Upgrade checks"
- runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
+ runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }}
env:
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
if: inputs.canary-run == 'true' && inputs.latest-versions-only != 'true'
@@ -341,7 +341,7 @@ jobs:
test-airflow-release-commands:
timeout-minutes: 80
name: "Test Airflow release commands"
- runs-on: ${{ fromJSON(inputs.runs-on-as-json-public) }}
+ runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }}
env:
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
IMAGE_TAG: ${{ inputs.image-tag }}
diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml
index 1256fd2f0d..9205b0568d 100644
--- a/.github/workflows/build-images.yml
+++ b/.github/workflows/build-images.yml
@@ -76,8 +76,6 @@ jobs:
is-airflow-runner: ${{ steps.selective-checks.outputs.is-airflow-runner }}
is-amd-runner: ${{ steps.selective-checks.outputs.is-amd-runner }}
is-arm-runner: ${{ steps.selective-checks.outputs.is-arm-runner }}
- is-vm-runner: ${{ steps.selective-checks.outputs.is-vm-runner }}
- is-k8s-runner: ${{ steps.selective-checks.outputs.is-k8s-runner }}
chicken-egg-providers: ${{ steps.selective-checks.outputs.chicken-egg-providers }}
target-commit-sha: "${{steps.discover-pr-merge-commit.outputs.target-commit-sha ||
github.event.pull_request.head.sha ||
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 11cb390d32..2e74efd842 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -116,8 +116,6 @@ jobs:
is-airflow-runner: ${{ steps.selective-checks.outputs.is-airflow-runner }}
is-amd-runner: ${{ steps.selective-checks.outputs.is-amd-runner }}
is-arm-runner: ${{ steps.selective-checks.outputs.is-arm-runner }}
- is-vm-runner: ${{ steps.selective-checks.outputs.is-vm-runner }}
- is-k8s-runner: ${{ steps.selective-checks.outputs.is-k8s-runner }}
latest-versions-only: ${{ steps.selective-checks.outputs.latest-versions-only }}
chicken-egg-providers: ${{ steps.selective-checks.outputs.chicken-egg-providers }}
has-migrations: ${{ steps.selective-checks.outputs.has-migrations }}
@@ -167,7 +165,7 @@ jobs:
needs: [build-info]
uses: ./.github/workflows/basic-tests.yml
with:
- runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }}
+ runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }}
run-www-tests: ${{needs.build-info.outputs.run-www-tests}}
needs-api-codegen: ${{needs.build-info.outputs.needs-api-codegen}}
default-python-version: ${{needs.build-info.outputs.default-python-version}}
diff --git a/dev/breeze/doc/ci/04_selective_checks.md b/dev/breeze/doc/ci/04_selective_checks.md
index 819633d4c5..86b8fff5f8 100644
--- a/dev/breeze/doc/ci/04_selective_checks.md
+++ b/dev/breeze/doc/ci/04_selective_checks.md
@@ -194,9 +194,7 @@ Github Actions to pass the list of parameters to a command to execute
| is-amd-runner | Whether runner used is an AMD one | true | |
| is-arm-runner | Whether runner used is an ARM one | false | |
| is-committer-build | Whether the build is triggered by a committer | false | |
-| is-k8s-runner | Whether the build runs on our k8s infrastructure | false | |
| is-self-hosted-runner | Whether the runner is self-hosted | false | |
-| is-vm-runner | Whether the runner uses VM to run | true | |
| kind-version | Which Kind version to use for tests | v0.16.0 | |
| kubernetes-combos-list-as-string | All combinations of Python version and Kubernetes version to use for tests as space-separated string | 3.8-v1.25.2 3.9-v1.26.4 | * |
| kubernetes-versions | All Kubernetes versions to use for tests as JSON array | ['v1.25.2'] | |
diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py
index 46b0c3d521..47b465de8e 100644
--- a/dev/breeze/src/airflow_breeze/global_constants.py
+++ b/dev/breeze/src/airflow_breeze/global_constants.py
@@ -32,12 +32,8 @@ from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT
RUNS_ON_PUBLIC_RUNNER = '["ubuntu-22.04"]'
# we should get more sophisticated logic here in the future, but for now we just check if
# we use self airflow, vm-based, amd hosted runner as a default
-# TODO: temporarily we need to switch to public runners to avoid issues with self-hosted runners
-RUNS_ON_SELF_HOSTED_RUNNER = '["ubuntu-22.04"]'
+RUNS_ON_SELF_HOSTED_RUNNER = '["self-hosted", "airflow-runner", "eks-runner", "X64", "small"]'
RUNS_ON_SELF_HOSTED_ASF_RUNNER = '["self-hosted", "asf-runner"]'
-# TODO: when we have it properly set-up with labels we should change it to
-# RUNS_ON_SELF_HOSTED_RUNNER = '["self-hosted", "airflow-runner", "vm-runner", "X64"]'
-# RUNS_ON_SELF_HOSTED_RUNNER = '["self-hosted", "Linux", "X64"]'
SELF_HOSTED_RUNNERS_CPU_COUNT = 8
ANSWER = ""
diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
index 224e76c251..209155ee81 100644
--- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py
+++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
@@ -1154,7 +1154,7 @@ class SelectiveChecks:
@cached_property
def runs_on_as_json_docs_build(self) -> str:
if self._is_canary_run():
- return RUNS_ON_SELF_HOSTED_ASF_RUNNER
+ return RUNS_ON_SELF_HOSTED_RUNNER
else:
return RUNS_ON_PUBLIC_RUNNER
@@ -1181,7 +1181,7 @@ class SelectiveChecks:
# TODO: when we have it properly set-up with labels we should just check for
# "airflow-runner" presence in runs_on
runs_on_array = json.loads(self.runs_on_as_json_default)
- return "Linux" in runs_on_array and "X64" in runs_on_array and "self-hosted" in runs_on_array
+ return "self-hosted" in runs_on_array and "airflow-runner" in runs_on_array
@cached_property
def is_amd_runner(self) -> bool:
@@ -1221,20 +1221,6 @@ class SelectiveChecks:
]
)
- @cached_property
- def is_vm_runner(self) -> bool:
- """Whether the runner is VM runner (managed by airflow)."""
- # TODO: when we have it properly set-up with labels we should just check for
- # "airflow-runner" presence in runs_on
- return self.is_airflow_runner
-
- @cached_property
- def is_k8s_runner(self) -> bool:
- """Whether the runner is K8s runner (managed by airflow)."""
- # TODO: when we have it properly set-up with labels we should just check for
- # "k8s-runner" presence in runs_on
- return False
-
@cached_property
def has_migrations(self) -> bool:
return any([file.startswith("airflow/migrations/") for file in self._files])
diff --git a/dev/breeze/tests/test_pr_info.py b/dev/breeze/tests/test_pr_info.py
index 1c874e5c4d..292cef262a 100644
--- a/dev/breeze/tests/test_pr_info.py
+++ b/dev/breeze/tests/test_pr_info.py
@@ -81,9 +81,7 @@ def test_runs_on_self_hosted():
assert wi.head_repo == "apache/airflow"
assert wi.event_name == "pull_request"
assert wi.pr_number == 1234
- # TODO: fix it when we fix self-hosted runners
- assert wi.get_runs_on() == '["ubuntu-22.04"]'
- # assert wi.get_runs_on() == '["self-hosted", "Linux", "X64"]'
+ assert wi.get_runs_on() == '["self-hosted", "airflow-runner", "eks-runner", "X64", "small"]'
assert wi.is_canary_run() == "false"
assert wi.run_coverage() == "false"
@@ -125,9 +123,7 @@ def test_runs_on_push_other_branch():
assert wi.head_repo == "apache/airflow"
assert wi.event_name == "push"
assert wi.pr_number is None
- # TODO: revert it when we fix self-hosted runners
- assert wi.get_runs_on() == '["ubuntu-22.04"]'
- # assert wi.get_runs_on() == '["self-hosted", "Linux", "X64"]'
+ assert wi.get_runs_on() == '["self-hosted", "airflow-runner", "eks-runner", "X64", "small"]'
assert wi.is_canary_run() == "false"
assert wi.run_coverage() == "false"
@@ -141,8 +137,6 @@ def test_runs_on_push_v_test_branch():
assert wi.head_repo == "apache/airflow"
assert wi.event_name == "push"
assert wi.pr_number is None
- # TODO: revert it when we fix self-hosted runners
- assert wi.get_runs_on() == '["ubuntu-22.04"]'
- # assert wi.get_runs_on() == '["self-hosted", "Linux", "X64"]'
+ assert wi.get_runs_on() == '["self-hosted", "airflow-runner", "eks-runner", "X64", "small"]'
assert wi.is_canary_run() == "true"
assert wi.run_coverage() == "false"
diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py
index 09327e18ef..6434ce7a7d 100644
--- a/dev/breeze/tests/test_selective_checks.py
+++ b/dev/breeze/tests/test_selective_checks.py
@@ -1752,7 +1752,7 @@ def test_helm_tests_trigger_ci_build(files: tuple[str, ...], expected_outputs: d
(
"github_event, github_actor, github_repository, pr_labels, "
"github_context_dict, runs_on_as_json_default, runs_on_as_docs_build, is_self_hosted_runner, "
- "is_airflow_runner, is_amd_runner, is_arm_runner, is_vm_runner, is_k8s_runner, exception"
+ "is_airflow_runner, is_amd_runner, is_arm_runner, exception"
),
[
pytest.param(
@@ -1761,20 +1761,11 @@ def test_helm_tests_trigger_ci_build(files: tuple[str, ...], expected_outputs: d
"apache/airflow",
(),
dict(),
- # TODO: revert it when we fix self-hosted runners
- '["ubuntu-22.04"]',
- '["self-hosted", "asf-runner"]',
- # '["self-hosted", "Linux", "X64"]',
- # TODO: revert it when we fix self-hosted runners
- "false",
- "false",
- # "true",
- # "true",
+ '["self-hosted", "airflow-runner", "eks-runner", "X64", "small"]',
+ '["self-hosted", "airflow-runner", "eks-runner", "X64", "small"]',
+ "true",
+ "true",
"true",
- "false",
- # TODO: revert it when we fix self-hosted runners
- "false",
- # "true",
"false",
False,
id="Push event",
@@ -1791,8 +1782,6 @@ def test_helm_tests_trigger_ci_build(files: tuple[str, ...], expected_outputs: d
"false",
"true",
"false",
- "false",
- "false",
False,
id="Push event for private repo",
),
@@ -1808,8 +1797,6 @@ def test_helm_tests_trigger_ci_build(files: tuple[str, ...], expected_outputs: d
"false",
"true",
"false",
- "false",
- "false",
False,
id="Pull request",
),
@@ -1825,8 +1812,6 @@ def test_helm_tests_trigger_ci_build(files: tuple[str, ...], expected_outputs: d
"false",
"true",
"false",
- "false",
- "false",
False,
id="Pull request private repo",
),
@@ -1842,8 +1827,6 @@ def test_helm_tests_trigger_ci_build(files: tuple[str, ...], expected_outputs: d
"false",
"true",
"false",
- "false",
- "false",
False,
id="Pull request committer",
),
@@ -1859,8 +1842,6 @@ def test_helm_tests_trigger_ci_build(files: tuple[str, ...], expected_outputs: d
"false",
"true",
"false",
- "false",
- "false",
False,
id="Pull request committer pr non-committer",
),
@@ -1876,8 +1857,6 @@ def test_helm_tests_trigger_ci_build(files: tuple[str, ...], expected_outputs: d
"false",
"true",
"false",
- "false",
- "false",
False,
id="Pull request private repo committer",
),
@@ -1893,8 +1872,6 @@ def test_helm_tests_trigger_ci_build(files: tuple[str, ...], expected_outputs: d
"false",
"true",
"false",
- "false",
- "false",
False,
id="Pull request target",
),
@@ -1910,8 +1887,6 @@ def test_helm_tests_trigger_ci_build(files: tuple[str, ...], expected_outputs: d
"false",
"true",
"false",
- "false",
- "false",
False,
id="Pull request target private repo",
),
@@ -1927,8 +1902,6 @@ def test_helm_tests_trigger_ci_build(files: tuple[str, ...], expected_outputs: d
"false",
"true",
"false",
- "false",
- "false",
False,
id="Pull request target committer",
),
@@ -1944,8 +1917,6 @@ def test_helm_tests_trigger_ci_build(files: tuple[str, ...], expected_outputs: d
"false",
"true",
"false",
- "false",
- "false",
False,
id="Pull request target committer pr non-committer",
),
@@ -1961,8 +1932,6 @@ def test_helm_tests_trigger_ci_build(files: tuple[str, ...], expected_outputs: d
"false",
"true",
"false",
- "false",
- "false",
False,
id="Pull request targe private repo committer",
),
@@ -1978,8 +1947,6 @@ def test_helm_tests_trigger_ci_build(files: tuple[str, ...], expected_outputs: d
"false",
"true",
"false",
- "false",
- "false",
True,
id="Pull request by non committer with 'use self-hosted runners' label.",
),
@@ -1995,8 +1962,6 @@ def test_helm_tests_trigger_ci_build(files: tuple[str, ...], expected_outputs: d
"false",
"true",
"false",
- "false",
- "false",
False,
id="Pull request by committer with 'use public runners' label.",
),
@@ -2014,8 +1979,6 @@ def test_runs_on(
is_airflow_runner: str,
is_amd_runner: str,
is_arm_runner: str,
- is_vm_runner: str,
- is_k8s_runner: str,
exception: bool,
):
def get_output() -> str:
@@ -2043,8 +2006,6 @@ def test_runs_on(
assert_outputs_are_printed({"is-airflow-runner": is_airflow_runner}, str(stderr))
assert_outputs_are_printed({"is-amd-runner": is_amd_runner}, str(stderr))
assert_outputs_are_printed({"is-arm-runner": is_arm_runner}, str(stderr))
- assert_outputs_are_printed({"is-vm-runner": is_vm_runner}, str(stderr))
- assert_outputs_are_printed({"is-k8s-runner": is_k8s_runner}, str(stderr))
@pytest.mark.parametrize(