You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2024/07/30 17:47:33 UTC
(airflow) branch main updated: Build documentation in canary runs using ASF self-hosted AMD runners (#41057)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new a3d3479bb9 Build documentation in canary runs using ASF self-hosted AMD runners (#41057)
a3d3479bb9 is described below
commit a3d3479bb96ab870bdfcb9785398a07b63052a5d
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Tue Jul 30 19:47:27 2024 +0200
Build documentation in canary runs using ASF self-hosted AMD runners (#41057)
The runners have more disk space which is needed to checkout and
build full documentation.
Related: #40925
---
.github/workflows/ci.yml | 3 +
.github/workflows/static-checks-mypy-docs.yml | 86 ++++++++++++----------
dev/breeze/src/airflow_breeze/global_constants.py | 1 +
.../src/airflow_breeze/utils/selective_checks.py | 22 +++++-
dev/breeze/tests/test_selective_checks.py | 20 ++++-
5 files changed, 88 insertions(+), 44 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 0bff966cc6..68aa51bf86 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -108,8 +108,10 @@ jobs:
helm-test-packages: ${{ steps.selective-checks.outputs.helm-test-packages }}
debug-resources: ${{ steps.selective-checks.outputs.debug-resources }}
runs-on-as-json-default: ${{ steps.selective-checks.outputs.runs-on-as-json-default }}
+ runs-on-as-json-docs-build: ${{ steps.selective-checks.outputs.runs-on-as-json-docs-build }}
runs-on-as-json-public: ${{ steps.selective-checks.outputs.runs-on-as-json-public }}
runs-on-as-json-self-hosted: ${{ steps.selective-checks.outputs.runs-on-as-json-self-hosted }}
+ runs-on-as-json-self-hosted-asf: ${{ steps.selective-checks.outputs.runs-on-as-json-self-hosted-asf }}
is-self-hosted-runner: ${{ steps.selective-checks.outputs.is-self-hosted-runner }}
is-airflow-runner: ${{ steps.selective-checks.outputs.is-airflow-runner }}
is-amd-runner: ${{ steps.selective-checks.outputs.is-amd-runner }}
@@ -285,6 +287,7 @@ jobs:
secrets: inherit
with:
runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }}
+ runs-on-as-json-docs-build: ${{ needs.build-info.outputs.runs-on-as-json-docs-build }}
image-tag: ${{ needs.build-info.outputs.image-tag }}
needs-mypy: ${{ needs.build-info.outputs.needs-mypy }}
mypy-folders: ${{ needs.build-info.outputs.mypy-folders }}
diff --git a/.github/workflows/static-checks-mypy-docs.yml b/.github/workflows/static-checks-mypy-docs.yml
index 0b057e6fbf..cb782e2955 100644
--- a/.github/workflows/static-checks-mypy-docs.yml
+++ b/.github/workflows/static-checks-mypy-docs.yml
@@ -24,6 +24,10 @@ on: # yamllint disable-line rule:truthy
description: "The array of labels (in json form) determining default runner used for the build."
required: true
type: string
+ runs-on-as-json-docs-build:
+ description: "The array of labels (in json form) determining the labels used for docs build."
+ required: true
+ type: string
image-tag:
description: "Tag to set for the image"
required: true
@@ -175,9 +179,11 @@ jobs:
INCLUDE_MYPY_VOLUME: "false"
build-docs:
- timeout-minutes: 60
+ timeout-minutes: 150
name: "Build documentation"
- runs-on: ${{ fromJSON(inputs.runs-on-as-json-default) }}
+ # For canary runs we need to push documentation to AWS S3 and preparing it takes a lot of space
+ # So we should use self-hosted ASF runners for this
+ runs-on: ${{ fromJSON(inputs.runs-on-as-json-docs-build) }}
strategy:
fail-fast: false
matrix:
@@ -214,41 +220,41 @@ jobs:
- name: "Building docs with ${{ matrix.flag }} flag"
run: >
breeze build-docs ${{ inputs.docs-list-as-string }} ${{ matrix.flag }}
- # - name: "Clone airflow-site"
- # run: >
- # git clone https://github.com/apache/airflow-site.git ${GITHUB_WORKSPACE}/airflow-site &&
- # echo "AIRFLOW_SITE_DIRECTORY=${GITHUB_WORKSPACE}/airflow-site" >> "$GITHUB_ENV"
- # if: inputs.canary-run == 'true' && matrix.flag == '--docs-only'
- # - name: "Publish docs"
- # run: >
- # breeze release-management publish-docs --override-versioned --run-in-parallel
- # ${{ inputs.docs-list-as-string }}
- # if: inputs.canary-run == 'true' && matrix.flag == '--docs-only'
- # - name: "Generate back references for providers"
- # run: breeze release-management add-back-references all-providers
- # if: inputs.canary-run == 'true' && matrix.flag == '--docs-only'
- # - name: "Generate back references for apache-airflow"
- # run: breeze release-management add-back-references apache-airflow
- # if: inputs.canary-run == 'true' && matrix.flag == '--docs-only'
- # - name: "Generate back references for docker-stack"
- # run: breeze release-management add-back-references docker-stack
- # if: inputs.canary-run == 'true' && matrix.flag == '--docs-only'
- # - name: "Generate back references for helm-chart"
- # run: breeze release-management add-back-references helm-chart
- # if: inputs.canary-run == 'true' && matrix.flag == '--docs-only'
- # - name: Configure AWS credentials
- # uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1
- # if: >
- # inputs.canary-run == 'true' &&
- # inputs.branch == 'main' &&
- # matrix.flag == '--docs-only'
- # with:
- # aws-access-key-id: ${{ secrets.DOCS_AWS_ACCESS_KEY_ID }}
- # aws-secret-access-key: ${{ secrets.DOCS_AWS_SECRET_ACCESS_KEY }}
- # aws-region: eu-central-1
- # - name: "Upload documentation to AWS S3"
- # if: >
- # inputs.canary-run == 'true' &&
- # inputs.branch == 'main' &&
- # matrix.flag == '--docs-only'
- # run: aws s3 sync --delete ./files/documentation s3://apache-airflow-docs
+ - name: "Clone airflow-site"
+ run: >
+ git clone https://github.com/apache/airflow-site.git ${GITHUB_WORKSPACE}/airflow-site &&
+ echo "AIRFLOW_SITE_DIRECTORY=${GITHUB_WORKSPACE}/airflow-site" >> "$GITHUB_ENV"
+ if: inputs.canary-run == 'true' && matrix.flag == '--docs-only'
+ - name: "Publish docs"
+ run: >
+ breeze release-management publish-docs --override-versioned --run-in-parallel
+ ${{ inputs.docs-list-as-string }}
+ if: inputs.canary-run == 'true' && matrix.flag == '--docs-only'
+ - name: "Generate back references for providers"
+ run: breeze release-management add-back-references all-providers
+ if: inputs.canary-run == 'true' && matrix.flag == '--docs-only'
+ - name: "Generate back references for apache-airflow"
+ run: breeze release-management add-back-references apache-airflow
+ if: inputs.canary-run == 'true' && matrix.flag == '--docs-only'
+ - name: "Generate back references for docker-stack"
+ run: breeze release-management add-back-references docker-stack
+ if: inputs.canary-run == 'true' && matrix.flag == '--docs-only'
+ - name: "Generate back references for helm-chart"
+ run: breeze release-management add-back-references helm-chart
+ if: inputs.canary-run == 'true' && matrix.flag == '--docs-only'
+ - name: Configure AWS credentials
+ uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1
+ if: >
+ inputs.canary-run == 'true' &&
+ inputs.branch == 'main' &&
+ matrix.flag == '--docs-only'
+ with:
+ aws-access-key-id: ${{ secrets.DOCS_AWS_ACCESS_KEY_ID }}
+ aws-secret-access-key: ${{ secrets.DOCS_AWS_SECRET_ACCESS_KEY }}
+ aws-region: eu-central-1
+ - name: "Upload documentation to AWS S3"
+ if: >
+ inputs.canary-run == 'true' &&
+ inputs.branch == 'main' &&
+ matrix.flag == '--docs-only'
+ run: aws s3 sync --delete ./files/documentation s3://apache-airflow-docs
diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py
index 94b64d621c..bfa0a812dd 100644
--- a/dev/breeze/src/airflow_breeze/global_constants.py
+++ b/dev/breeze/src/airflow_breeze/global_constants.py
@@ -34,6 +34,7 @@ RUNS_ON_PUBLIC_RUNNER = '["ubuntu-22.04"]'
# we use self airflow, vm-based, amd hosted runner as a default
# TODO: temporarily we need to switch to public runners to avoid issues with self-hosted runners
RUNS_ON_SELF_HOSTED_RUNNER = '["ubuntu-22.04"]'
+RUNS_ON_SELF_HOSTED_ASF_RUNNER = '["self-hosted", "asf-runner"]'
# TODO: when we have it properly set-up with labels we should change it to
# RUNS_ON_SELF_HOSTED_RUNNER = '["self-hosted", "airflow-runner", "vm-runner", "X64"]'
# RUNS_ON_SELF_HOSTED_RUNNER = '["self-hosted", "Linux", "X64"]'
diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
index 070f669bd4..62f5a20abc 100644
--- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py
+++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
@@ -44,6 +44,7 @@ from airflow_breeze.global_constants import (
HELM_VERSION,
KIND_VERSION,
RUNS_ON_PUBLIC_RUNNER,
+ RUNS_ON_SELF_HOSTED_ASF_RUNNER,
RUNS_ON_SELF_HOSTED_RUNNER,
TESTABLE_INTEGRATIONS,
GithubEvents,
@@ -67,6 +68,7 @@ from airflow_breeze.utils.provider_dependencies import DEPENDENCIES, get_related
from airflow_breeze.utils.run_utils import run_command
ALL_VERSIONS_LABEL = "all versions"
+CANARY_LABEL = "canary"
DEBUG_CI_RESOURCES_LABEL = "debug ci resources"
DEFAULT_VERSIONS_ONLY_LABEL = "default versions only"
DISABLE_IMAGE_CACHE_LABEL = "disable image cache"
@@ -1102,8 +1104,7 @@ class SelectiveChecks:
@cached_property
def runs_on_as_json_default(self) -> str:
if self._github_repository == APACHE_AIRFLOW_GITHUB_REPOSITORY:
- if self._github_event in [GithubEvents.SCHEDULE, GithubEvents.PUSH]:
- # Canary and Scheduled runs
+ if self._is_canary_run():
return RUNS_ON_SELF_HOSTED_RUNNER
if self._pr_labels and USE_PUBLIC_RUNNERS_LABEL in self._pr_labels:
# Forced public runners
@@ -1142,6 +1143,17 @@ class SelectiveChecks:
def runs_on_as_json_self_hosted(self) -> str:
return RUNS_ON_SELF_HOSTED_RUNNER
+ @cached_property
+ def runs_on_as_json_self_hosted_asf(self) -> str:
+ return RUNS_ON_SELF_HOSTED_ASF_RUNNER
+
+ @cached_property
+ def runs_on_as_json_docs_build(self) -> str:
+ if self._is_canary_run():
+ return RUNS_ON_SELF_HOSTED_ASF_RUNNER
+ else:
+ return RUNS_ON_PUBLIC_RUNNER
+
@cached_property
def runs_on_as_json_public(self) -> str:
return RUNS_ON_PUBLIC_RUNNER
@@ -1303,3 +1315,9 @@ class SelectiveChecks:
get_related_providers(provider, upstream_dependencies=True, downstream_dependencies=True)
)
return sorted(all_providers)
+
+ def _is_canary_run(self):
+ return (
+ self._github_event in [GithubEvents.SCHEDULE, GithubEvents.PUSH]
+ and self._github_repository == APACHE_AIRFLOW_GITHUB_REPOSITORY
+ ) or CANARY_LABEL in self._pr_labels
diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py
index f7e712b821..7c0ca94094 100644
--- a/dev/breeze/tests/test_selective_checks.py
+++ b/dev/breeze/tests/test_selective_checks.py
@@ -1748,7 +1748,7 @@ def test_helm_tests_trigger_ci_build(files: tuple[str, ...], expected_outputs: d
@pytest.mark.parametrize(
(
"github_event, github_actor, github_repository, pr_labels, "
- "github_context_dict, runs_on_as_json_default, is_self_hosted_runner, "
+ "github_context_dict, runs_on_as_json_default, runs_on_as_docs_build, is_self_hosted_runner, "
"is_airflow_runner, is_amd_runner, is_arm_runner, is_vm_runner, is_k8s_runner, exception"
),
[
@@ -1760,6 +1760,7 @@ def test_helm_tests_trigger_ci_build(files: tuple[str, ...], expected_outputs: d
dict(),
# TODO: revert it when we fix self-hosted runners
'["ubuntu-22.04"]',
+ '["self-hosted", "asf-runner"]',
# '["self-hosted", "Linux", "X64"]',
# TODO: revert it when we fix self-hosted runners
"false",
@@ -1782,6 +1783,7 @@ def test_helm_tests_trigger_ci_build(files: tuple[str, ...], expected_outputs: d
(),
dict(),
'["ubuntu-22.04"]',
+ '["ubuntu-22.04"]',
"false",
"false",
"true",
@@ -1798,6 +1800,7 @@ def test_helm_tests_trigger_ci_build(files: tuple[str, ...], expected_outputs: d
(),
dict(),
'["ubuntu-22.04"]',
+ '["ubuntu-22.04"]',
"false",
"false",
"true",
@@ -1814,6 +1817,7 @@ def test_helm_tests_trigger_ci_build(files: tuple[str, ...], expected_outputs: d
(),
dict(),
'["ubuntu-22.04"]',
+ '["ubuntu-22.04"]',
"false",
"false",
"true",
@@ -1830,6 +1834,7 @@ def test_helm_tests_trigger_ci_build(files: tuple[str, ...], expected_outputs: d
(),
dict(),
'["ubuntu-22.04"]',
+ '["ubuntu-22.04"]',
"false",
"false",
"true",
@@ -1846,6 +1851,7 @@ def test_helm_tests_trigger_ci_build(files: tuple[str, ...], expected_outputs: d
(),
dict(event=dict(pull_request=dict(user=dict(login="user")))),
'["ubuntu-22.04"]',
+ '["ubuntu-22.04"]',
"false",
"false",
"true",
@@ -1862,6 +1868,7 @@ def test_helm_tests_trigger_ci_build(files: tuple[str, ...], expected_outputs: d
(),
dict(),
'["ubuntu-22.04"]',
+ '["ubuntu-22.04"]',
"false",
"false",
"true",
@@ -1878,6 +1885,7 @@ def test_helm_tests_trigger_ci_build(files: tuple[str, ...], expected_outputs: d
(),
dict(),
'["ubuntu-22.04"]',
+ '["ubuntu-22.04"]',
"false",
"false",
"true",
@@ -1894,6 +1902,7 @@ def test_helm_tests_trigger_ci_build(files: tuple[str, ...], expected_outputs: d
(),
dict(),
'["ubuntu-22.04"]',
+ '["ubuntu-22.04"]',
"false",
"false",
"true",
@@ -1910,6 +1919,7 @@ def test_helm_tests_trigger_ci_build(files: tuple[str, ...], expected_outputs: d
[],
dict(),
'["ubuntu-22.04"]',
+ '["ubuntu-22.04"]',
"false",
"false",
"true",
@@ -1926,6 +1936,7 @@ def test_helm_tests_trigger_ci_build(files: tuple[str, ...], expected_outputs: d
(),
dict(event=dict(pull_request=dict(user=dict(login="user")))),
'["ubuntu-22.04"]',
+ '["ubuntu-22.04"]',
"false",
"false",
"true",
@@ -1942,6 +1953,7 @@ def test_helm_tests_trigger_ci_build(files: tuple[str, ...], expected_outputs: d
(),
dict(),
'["ubuntu-22.04"]',
+ '["ubuntu-22.04"]',
"false",
"false",
"true",
@@ -1958,6 +1970,7 @@ def test_helm_tests_trigger_ci_build(files: tuple[str, ...], expected_outputs: d
("use self-hosted runners",),
dict(),
'["ubuntu-22.04"]',
+ '["ubuntu-22.04"]',
"false",
"false",
"true",
@@ -1974,6 +1987,7 @@ def test_helm_tests_trigger_ci_build(files: tuple[str, ...], expected_outputs: d
("use public runners",),
dict(),
'["ubuntu-22.04"]',
+ '["ubuntu-22.04"]',
"false",
"false",
"true",
@@ -1991,7 +2005,8 @@ def test_runs_on(
github_repository: str,
pr_labels: tuple[str, ...],
github_context_dict: dict[str, Any],
- runs_on_as_json_default,
+ runs_on_as_json_default: str,
+ runs_on_as_docs_build: str,
is_self_hosted_runner: str,
is_airflow_runner: str,
is_amd_runner: str,
@@ -2020,6 +2035,7 @@ def test_runs_on(
else:
stderr = get_output()
assert_outputs_are_printed({"runs-on-as-json-default": runs_on_as_json_default}, str(stderr))
+ assert_outputs_are_printed({"runs-on-as-json-docs-build": runs_on_as_docs_build}, str(stderr))
assert_outputs_are_printed({"is-self-hosted-runner": is_self_hosted_runner}, str(stderr))
assert_outputs_are_printed({"is-airflow-runner": is_airflow_runner}, str(stderr))
assert_outputs_are_printed({"is-amd-runner": is_amd_runner}, str(stderr))