You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2024/02/13 00:57:48 UTC
(airflow) 35/44: Follow up after moving hatch_build.py to source root (#37341)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v2-8-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 444c6b0f5ced95263dea804a955fc877271efaf9
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Sun Feb 11 20:15:25 2024 +0100
Follow up after moving hatch_build.py to source root (#37341)
Turns out that #37340 was not enough to make airflow build from
git url - we also need to move provider lists.
(cherry picked from commit 1e4a2d5c765b0f98cab3017389b3a03d86e38649)
---
.dockerignore | 2 ++
.github/actions/build-prod-images/action.yml | 2 +-
.github/workflows/ci.yml | 2 +-
Dockerfile.ci | 2 +-
...re_installed_providers.txt => airflow_pre_installed_providers.txt | 0
dev/README_RELEASE_AIRFLOW.md | 2 +-
dev/breeze/src/airflow_breeze/utils/docker_command_utils.py | 1 +
dev/refresh_images.sh | 2 +-
docker_tests/test_prod_image.py | 5 ++---
hatch_build.py | 3 +--
...age_installed_providers.txt => prod_image_installed_providers.txt | 0
scripts/ci/docker-compose/local.yml | 3 +++
scripts/ci/pre_commit/pre_commit_sort_installed_providers.py | 5 ++---
13 files changed, 16 insertions(+), 13 deletions(-)
diff --git a/.dockerignore b/.dockerignore
index ae3496d802..e7431de857 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -54,6 +54,8 @@
!empty
!Dockerfile
!hatch_build.py
+!prod_image_installed_providers.txt
+!airflow_pre_installed_providers.txt
# This folder is for you if you want to add any packages to the docker context when you build your own
# docker image. most of other files and any new folder you add will be excluded by default
diff --git a/.github/actions/build-prod-images/action.yml b/.github/actions/build-prod-images/action.yml
index 141a2cf9b1..47f3934566 100644
--- a/.github/actions/build-prod-images/action.yml
+++ b/.github/actions/build-prod-images/action.yml
@@ -41,7 +41,7 @@ runs:
shell: bash
run: >
breeze release-management prepare-provider-packages
- --package-list-file ./dev/prod_image_installed_providers.txt
+ --package-list-file ./prod_image_installed_providers.txt
--package-format wheel --version-suffix-for-pypi dev0
if: ${{ inputs.build-provider-packages == 'true' }}
- name: "Prepare chicken-eggs provider packages"
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 8d1a8bfb2e..5dac5ce49f 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -2441,7 +2441,7 @@ jobs:
- name: "Prepare providers packages for PROD build"
run: >
breeze release-management prepare-provider-packages
- --package-list-file ./dev/prod_image_installed_providers.txt
+ --package-list-file ./prod_image_installed_providers.txt
--package-format wheel
env:
VERSION_SUFFIX_FOR_PYPI: "dev0"
diff --git a/Dockerfile.ci b/Dockerfile.ci
index 73b7bd5313..5d593b31b6 100644
--- a/Dockerfile.ci
+++ b/Dockerfile.ci
@@ -1178,7 +1178,7 @@ COPY airflow/__init__.py ${AIRFLOW_SOURCES}/airflow/
COPY generated/* ${AIRFLOW_SOURCES}/generated/
COPY constraints/* ${AIRFLOW_SOURCES}/constraints/
COPY LICENSE ${AIRFLOW_SOURCES}/LICENSE
-COPY dev/airflow_pre_installed_providers.txt ${AIRFLOW_SOURCES}/dev/airflow_pre_installed_providers.txt
+COPY airflow_pre_installed_providers.txt ${AIRFLOW_SOURCES}/
COPY hatch_build.py ${AIRFLOW_SOURCES}/
COPY --from=scripts install_airflow.sh /scripts/docker/
diff --git a/dev/airflow_pre_installed_providers.txt b/airflow_pre_installed_providers.txt
similarity index 100%
rename from dev/airflow_pre_installed_providers.txt
rename to airflow_pre_installed_providers.txt
diff --git a/dev/README_RELEASE_AIRFLOW.md b/dev/README_RELEASE_AIRFLOW.md
index efcd9b1cd6..2748543cf2 100644
--- a/dev/README_RELEASE_AIRFLOW.md
+++ b/dev/README_RELEASE_AIRFLOW.md
@@ -778,7 +778,7 @@ git push -f apache constraints-X.Y.Z
that have >= ``X.Y.0`` in the corresponding provider.yaml file.
-3. In case the provider should also be installed in the image (it is part of ``dev/prod_image_installed_providers.txt``)
+3. In case the provider should also be installed in the image (it is part of ``prod_image_installed_providers.txt``)
it should also be added at this moment to ``Dockerfile`` to the list of default extras in the line with ``AIRFLOW_EXTRAS``:
```Dockerfile
diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
index 3307866454..b6f2cc37c2 100644
--- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
+++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py
@@ -98,6 +98,7 @@ VOLUMES_FOR_SELECTED_MOUNTS = [
("kubernetes_tests", "/opt/airflow/kubernetes_tests"),
("docker_tests", "/opt/airflow/docker_tests"),
("chart", "/opt/airflow/chart"),
+ ("hatch_build.py", "/opt/airflow/hatch_build.py"),
]
diff --git a/dev/refresh_images.sh b/dev/refresh_images.sh
index a6d4e9a06f..26bf00dd72 100755
--- a/dev/refresh_images.sh
+++ b/dev/refresh_images.sh
@@ -36,7 +36,7 @@ breeze ci-image build \
rm -fv ./dist/* ./docker-context-files/*
breeze release-management prepare-provider-packages \
- --package-list-file ./dev/prod_image_installed_providers.txt \
+ --package-list-file ./prod_image_installed_providers.txt \
--package-format wheel \
--version-suffix-for-pypi dev0
diff --git a/docker_tests/test_prod_image.py b/docker_tests/test_prod_image.py
index 4f6a93de42..ac56cc6eff 100644
--- a/docker_tests/test_prod_image.py
+++ b/docker_tests/test_prod_image.py
@@ -33,9 +33,8 @@ from docker_tests.docker_tests_utils import (
run_python_in_docker,
)
-DEV_DIR_PATH = SOURCE_ROOT / "dev"
-AIRFLOW_PRE_INSTALLED_PROVIDERS_FILE_PATH = DEV_DIR_PATH / "airflow_pre_installed_providers.txt"
-PROD_IMAGE_PROVIDERS_FILE_PATH = DEV_DIR_PATH / "prod_image_installed_providers.txt"
+AIRFLOW_PRE_INSTALLED_PROVIDERS_FILE_PATH = SOURCE_ROOT / "airflow_pre_installed_providers.txt"
+PROD_IMAGE_PROVIDERS_FILE_PATH = SOURCE_ROOT / "prod_image_installed_providers.txt"
AIRFLOW_ROOT_PATH = Path(__file__).parents[2].resolve()
SLIM_IMAGE_PROVIDERS = [
f"apache-airflow-providers-{provider_id.replace('.','-')}"
diff --git a/hatch_build.py b/hatch_build.py
index 844bd39dca..2d6a4ed861 100644
--- a/hatch_build.py
+++ b/hatch_build.py
@@ -34,8 +34,7 @@ log.setLevel(log_level)
AIRFLOW_ROOT_PATH = Path(__file__).parent.resolve()
GENERATED_PROVIDERS_DEPENDENCIES_FILE = AIRFLOW_ROOT_PATH / "generated" / "provider_dependencies.json"
-DEV_DIR_PATH = AIRFLOW_ROOT_PATH / "dev"
-PREINSTALLED_PROVIDERS_FILE = DEV_DIR_PATH / "airflow_pre_installed_providers.txt"
+PREINSTALLED_PROVIDERS_FILE = AIRFLOW_ROOT_PATH / "airflow_pre_installed_providers.txt"
DEPENDENCIES = json.loads(GENERATED_PROVIDERS_DEPENDENCIES_FILE.read_text())
PREINSTALLED_PROVIDER_IDS = [
package.strip()
diff --git a/dev/prod_image_installed_providers.txt b/prod_image_installed_providers.txt
similarity index 100%
rename from dev/prod_image_installed_providers.txt
rename to prod_image_installed_providers.txt
diff --git a/scripts/ci/docker-compose/local.yml b/scripts/ci/docker-compose/local.yml
index 07ec8a89c6..2a55d8733c 100644
--- a/scripts/ci/docker-compose/local.yml
+++ b/scripts/ci/docker-compose/local.yml
@@ -109,4 +109,7 @@ services:
- type: bind
source: ../../../chart
target: /opt/airflow/chart
+ - type: bind
+ source: ../../../hatch_build.py
+ target: /opt/airflow/hatch_build.py
# END automatically generated volumes from VOLUMES_FOR_SELECTED_MOUNTS in docker_command_utils.py
diff --git a/scripts/ci/pre_commit/pre_commit_sort_installed_providers.py b/scripts/ci/pre_commit/pre_commit_sort_installed_providers.py
index fb735569a3..897fddf2ef 100755
--- a/scripts/ci/pre_commit/pre_commit_sort_installed_providers.py
+++ b/scripts/ci/pre_commit/pre_commit_sort_installed_providers.py
@@ -27,7 +27,6 @@ if __name__ not in ("__main__", "__mp_main__"):
AIRFLOW_SOURCES = Path(__file__).parents[3].resolve()
-DEV_DIR_PATH = AIRFLOW_SOURCES / "dev"
def stable_sort(x):
@@ -45,7 +44,7 @@ def sort_file(path: Path):
if __name__ == "__main__":
- prod_image_installed_providers_path = DEV_DIR_PATH / "prod_image_installed_providers.txt"
- airflow_pre_installed_providers_path = DEV_DIR_PATH / "airflow_pre_installed_providers.txt"
+ prod_image_installed_providers_path = AIRFLOW_SOURCES / "prod_image_installed_providers.txt"
+ airflow_pre_installed_providers_path = AIRFLOW_SOURCES / "airflow_pre_installed_providers.txt"
sort_file(prod_image_installed_providers_path)
sort_file(airflow_pre_installed_providers_path)