You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2020/12/12 18:39:18 UTC
[airflow] branch master updated: Install airflow and providers from
dist and verifies them (#13033)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/master by this push:
new abf2a42 Install airflow and providers from dist and verifies them (#13033)
abf2a42 is described below
commit abf2a4264b18e750dbc2eb384a86d08d821dfba4
Author: Jarek Potiuk <ja...@polidea.com>
AuthorDate: Sat Dec 12 19:38:30 2020 +0100
Install airflow and providers from dist and verifies them (#13033)
* Install airflow and providers from dist and verifies them
This check is there to prevent problems similar to those reported
in #13027 and fixed in #13031.
Previously we always built airflow from wheels, only providers were
installed from sdist packages and tested. In this version both
airflow and providers are installed using the same package format
(sdist or wheel).
* Update scripts/in_container/entrypoint_ci.sh
Co-authored-by: Kaxil Naik <ka...@gmail.com>
Co-authored-by: Kaxil Naik <ka...@gmail.com>
---
.github/workflows/ci.yml | 54 ++++++++++------------
BREEZE.rst | 4 +-
Dockerfile | 2 +-
breeze-complete | 1 +
docs/apache-airflow/production-deployment.rst | 1 +
scripts/ci/images/ci_verify_ci_image.sh | 2 +
scripts/ci/images/ci_verify_prod_image.sh | 2 +
scripts/ci/libraries/_build_images.sh | 7 +--
scripts/in_container/_in_container_utils.sh | 19 +++++++-
scripts/in_container/entrypoint_ci.sh | 9 +++-
.../run_install_and_test_provider_packages.sh | 9 ++++
11 files changed, 69 insertions(+), 41 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 1e0cd9d..637c6cf 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -210,6 +210,8 @@ jobs:
- name: "Free space"
run: ./scripts/ci/tools/ci_free_space_on_ci.sh
if: needs.build-info.outputs.waitForImage == 'true'
+ - name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
+ run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- name: "Verify CI image Py${{matrix.python-version}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
run: ./scripts/ci/images/ci_verify_ci_image.sh
if: needs.build-info.outputs.waitForImage == 'true'
@@ -358,16 +360,20 @@ jobs:
prepare-backport-provider-packages:
timeout-minutes: 30
- name: "Backport packages"
+ name: "Backport packages: ${{ matrix.package-format }}"
runs-on: ubuntu-20.04
needs: [build-info, ci-images]
+ strategy:
+ matrix:
+ package-format: ['wheel', 'sdist']
env:
# In this case we want to install airflow from the latest released 1.10 version
# all provider packages are installed from wheels or .tar.gz files
INSTALL_AIRFLOW_VERSION: "1.10.14"
PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
BACKPORT_PACKAGES: "true"
- VERSION_SUFFIX_FOR_SVN: "rc1"
+ VERSION_SUFFIX_FOR_PYPI: "rc1"
+ PACKAGE_FORMAT: ${{ matrix.package-format }}
if: needs.build-info.outputs.image-build == 'true'
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
@@ -382,18 +388,10 @@ jobs:
run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- name: "Prepare provider readmes"
run: ./scripts/ci/provider_packages/ci_prepare_provider_readmes.sh
- - name: "Prepare provider packages"
+ - name: "Prepare provider packages: ${{ matrix.package-format }}"
run: ./scripts/ci/provider_packages/ci_prepare_provider_packages.sh
- env:
- PACKAGE_FORMAT: "both"
- - name: "Install and test provider packages via wheel files"
+ - name: "Install and test provider packages and airflow via ${{ matrix.package-format }} files"
run: ./scripts/ci/provider_packages/ci_install_and_test_provider_packages.sh
- env:
- PACKAGE_FORMAT: "wheel"
- - name: "Install and test provider packages via sdist files"
- run: ./scripts/ci/provider_packages/ci_install_and_test_provider_packages.sh
- env:
- PACKAGE_FORMAT: "sdist"
- name: "Upload provider package artifacts"
uses: actions/upload-artifact@v2
if: always()
@@ -403,7 +401,7 @@ jobs:
retention-days: 7
- name: "Upload readme artifacts"
uses: actions/upload-artifact@v2
- if: always()
+ if: always() && matrix.package-format == 'wheel'
with:
name: airflow-backport-readmes
path: "./files/airflow-readme-*"
@@ -411,15 +409,17 @@ jobs:
prepare-provider-packages:
timeout-minutes: 30
- name: "Provider packages"
+ name: "Provider packages ${{ matrix.package-format }}"
runs-on: ubuntu-20.04
needs: [build-info, ci-images]
env:
- INSTALL_AIRFLOW_VERSION: "wheel"
+ INSTALL_AIRFLOW_VERSION: "${{ matrix.package-format }}"
PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
- VERSION_SUFFIX_FOR_PYPI: "a2"
- VERSION_SUFFIX_FOR_SVN: "a2"
- PACKAGE_FORMAT: "both"
+ VERSION_SUFFIX_FOR_PYPI: "rc1"
+ PACKAGE_FORMAT: ${{ matrix.package-format }}
+ strategy:
+ matrix:
+ package-format: ['wheel', 'sdist']
if: needs.build-info.outputs.image-build == 'true'
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
@@ -434,20 +434,12 @@ jobs:
run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- name: "Prepare provider readmes"
run: ./scripts/ci/provider_packages/ci_prepare_provider_readmes.sh
- - name: "Prepare provider packages"
+ - name: "Prepare provider packages: ${{ matrix.package-format }}"
run: ./scripts/ci/provider_packages/ci_prepare_provider_packages.sh
- env:
- PACKAGE_FORMAT: "both"
- - name: "Prepare airflow package so that it can be installed"
+ - name: "Prepare airflow packages: ${{ matrix.package-format }}"
run: ./scripts/ci/build_airflow/ci_build_airflow_package.sh
- - name: "Install and test provider packages via wheel files"
- run: ./scripts/ci/provider_packages/ci_install_and_test_provider_packages.sh
- env:
- PACKAGE_FORMAT: "wheel"
- - name: "Install and test provider packages via sdist files"
+ - name: "Install and test provider packages and airflow via ${{ matrix.package-format }} files"
run: ./scripts/ci/provider_packages/ci_install_and_test_provider_packages.sh
- env:
- PACKAGE_FORMAT: "sdist"
- name: "Upload provider package artifacts"
uses: actions/upload-artifact@v2
if: always()
@@ -457,7 +449,7 @@ jobs:
retention-days: 7
- name: "Upload readme artifacts"
uses: actions/upload-artifact@v2
- if: always()
+ if: always() && matrix.package-format == 'wheel'
with:
name: airflow-provider-readmes
path: "./files/airflow-readme-*"
@@ -821,6 +813,8 @@ jobs:
- name: "Free space"
run: ./scripts/ci/tools/ci_free_space_on_ci.sh
if: needs.build-info.outputs.waitForImage == 'true'
+ - name: "Prepare PROD Image"
+ run: ./scripts/ci/images/ci_prepare_prod_image_on_ci.sh
- name: "Verify PROD image Py${{matrix.python-version}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
run: ./scripts/ci/images/ci_verify_prod_image.sh
if: needs.build-info.outputs.waitForImage == 'true'
diff --git a/BREEZE.rst b/BREEZE.rst
index 9a5eed6..238b6c0 100644
--- a/BREEZE.rst
+++ b/BREEZE.rst
@@ -1275,7 +1275,7 @@ This is the current syntax for `./breeze <./breeze>`_:
If specified, installs Airflow directly from PIP released version. This happens at
image building time in production image and at container entering time for CI image. One of:
- 1.10.14 1.10.12 1.10.11 1.10.10 1.10.9 none wheel
+ 1.10.14 1.10.12 1.10.11 1.10.10 1.10.9 none wheel sdist
When 'none' is used, you can install airflow from local packages. When building image,
airflow package should be added to 'docker-context-files' and
@@ -2378,7 +2378,7 @@ This is the current syntax for `./breeze <./breeze>`_:
If specified, installs Airflow directly from PIP released version. This happens at
image building time in production image and at container entering time for CI image. One of:
- 1.10.14 1.10.12 1.10.11 1.10.10 1.10.9 none wheel
+ 1.10.14 1.10.12 1.10.11 1.10.10 1.10.9 none wheel sdist
When 'none' is used, you can install airflow from local packages. When building image,
airflow package should be added to 'docker-context-files' and
diff --git a/Dockerfile b/Dockerfile
index af7c07c..968c657 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -255,7 +255,7 @@ RUN if [[ ${INSTALL_MYSQL_CLIENT} != "true" ]]; then \
pip install --user ${ADDITIONAL_PYTHON_DEPS} --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}"; \
fi; \
if [[ ${INSTALL_FROM_DOCKER_CONTEXT_FILES} == "true" ]]; then \
- if ls /docker-context-files/*.whl 1> /dev/null 2>&1; then \
+ if ls /docker-context-files/*.{whl,tar.gz} 1> /dev/null 2>&1; then \
pip install --user --no-deps /docker-context-files/*.{whl,tar.gz}; \
fi ; \
fi; \
diff --git a/breeze-complete b/breeze-complete
index 21b8ab5..6bbb51a 100644
--- a/breeze-complete
+++ b/breeze-complete
@@ -58,6 +58,7 @@ _breeze_allowed_install_airflow_versions=$(cat <<-EOF
1.10.9
none
wheel
+sdist
EOF
)
diff --git a/docs/apache-airflow/production-deployment.rst b/docs/apache-airflow/production-deployment.rst
index 1105b66..3dbf2c3 100644
--- a/docs/apache-airflow/production-deployment.rst
+++ b/docs/apache-airflow/production-deployment.rst
@@ -618,6 +618,7 @@ production image. There are three types of build:
| | | GitHub repository tag or branch or "." to install from sources. |
| | | Note that installing from local sources requires appropriate values of the |
| | | ``AIRFLOW_SOURCES_FROM`` and ``AIRFLOW_SOURCES_TO`` variables as described below. |
+| | | Only used when ``INSTALL_FROM_PYPI`` is set to ``true``. |
+-----------------------------------+------------------------+-----------------------------------------------------------------------------------+
| ``AIRFLOW_INSTALL_VERSION`` | | Optional - might be used for package installation of different Airflow version |
| | | for example"==1.10.14". For consistency, you should also set``AIRFLOW_VERSION`` |
diff --git a/scripts/ci/images/ci_verify_ci_image.sh b/scripts/ci/images/ci_verify_ci_image.sh
index 004eac0..831fd28 100755
--- a/scripts/ci/images/ci_verify_ci_image.sh
+++ b/scripts/ci/images/ci_verify_ci_image.sh
@@ -52,4 +52,6 @@ function pull_ci_image() {
build_images::prepare_ci_build
+pull_ci_image
+
verify_ci_image_dependencies
diff --git a/scripts/ci/images/ci_verify_prod_image.sh b/scripts/ci/images/ci_verify_prod_image.sh
index a0e0d5d..da59242 100755
--- a/scripts/ci/images/ci_verify_prod_image.sh
+++ b/scripts/ci/images/ci_verify_prod_image.sh
@@ -90,6 +90,8 @@ function pull_prod_image() {
build_images::prepare_prod_build
+pull_prod_image
+
verify_prod_image_has_airflow_and_providers
verify_prod_image_dependencies
diff --git a/scripts/ci/libraries/_build_images.sh b/scripts/ci/libraries/_build_images.sh
index f2b30e1..22c0ac5 100644
--- a/scripts/ci/libraries/_build_images.sh
+++ b/scripts/ci/libraries/_build_images.sh
@@ -921,16 +921,11 @@ function build_images::build_prod_images_from_packages() {
# Build necessary provider packages
runs::run_prepare_provider_packages "${INSTALLED_PROVIDERS[@]}"
- mv "${AIRFLOW_SOURCES}/dist/"*.whl "${AIRFLOW_SOURCES}/docker-context-files/"
+ mv "${AIRFLOW_SOURCES}/dist/"* "${AIRFLOW_SOURCES}/docker-context-files/"
# Build apache airflow packages
build_airflow_packages::build_airflow_packages
- # Remove generated tar.gz packages
- rm -f "${AIRFLOW_SOURCES}/dist/"apache-airflow*.tar.gz
-
- # move the packages to docker-context-files folder
- mkdir -pv "${AIRFLOW_SOURCES}/docker-context-files"
mv "${AIRFLOW_SOURCES}/dist/"* "${AIRFLOW_SOURCES}/docker-context-files/"
build_images::build_prod_images
}
diff --git a/scripts/in_container/_in_container_utils.sh b/scripts/in_container/_in_container_utils.sh
index 90311f3..9a0fb75 100644
--- a/scripts/in_container/_in_container_utils.sh
+++ b/scripts/in_container/_in_container_utils.sh
@@ -272,7 +272,7 @@ function install_airflow_from_wheel() {
local extras
extras="${1}"
local airflow_package
- airflow_package=$(find /dist/ -maxdepth 1 -type f -name 'apache_airflow-*.whl')
+ airflow_package=$(find /dist/ -maxdepth 1 -type f -name 'apache_airflow-[0-9]*.whl')
echo
echo "Found package: ${airflow_package}. Installing."
echo
@@ -285,6 +285,23 @@ function install_airflow_from_wheel() {
pip install "${airflow_package}${1}" >"${OUTPUT_PRINTED_ONLY_ON_ERROR}" 2>&1
}
+function install_airflow_from_sdist() {
+ local extras
+ extras="${1}"
+ local airflow_package
+ airflow_package=$(find /dist/ -maxdepth 1 -type f -name 'apache-airflow-[0-9]*.tar.gz')
+ echo
+ echo "Found package: ${airflow_package}. Installing."
+ echo
+ if [[ -z "${airflow_package}" ]]; then
+ >&2 echo
+ >&2 echo "ERROR! Could not find airflow sdist package to install in dist"
+ >&2 echo
+ exit 4
+ fi
+ pip install "${airflow_package}${1}" >"${OUTPUT_PRINTED_ONLY_ON_ERROR}" 2>&1
+}
+
function install_remaining_dependencies() {
pip install apache-beam[gcp] >"${OUTPUT_PRINTED_ONLY_ON_ERROR}" 2>&1
}
diff --git a/scripts/in_container/entrypoint_ci.sh b/scripts/in_container/entrypoint_ci.sh
index ce9fb4e..6e892ce 100755
--- a/scripts/in_container/entrypoint_ci.sh
+++ b/scripts/in_container/entrypoint_ci.sh
@@ -103,7 +103,7 @@ if [[ -z ${INSTALL_AIRFLOW_VERSION=} ]]; then
export PYTHONPATH=${AIRFLOW_SOURCES}
elif [[ ${INSTALL_AIRFLOW_VERSION} == "none" ]]; then
echo
- echo "Skip installing airflow - only install wheel packages that are present locally"
+ echo "Skip installing airflow - only install wheel/tar.gz packages that are present locally"
echo
uninstall_airflow_and_providers
elif [[ ${INSTALL_AIRFLOW_VERSION} == "wheel" ]]; then
@@ -113,6 +113,13 @@ elif [[ ${INSTALL_AIRFLOW_VERSION} == "wheel" ]]; then
uninstall_airflow_and_providers
install_airflow_from_wheel "[all]"
uninstall_providers
+elif [[ ${INSTALL_AIRFLOW_VERSION} == "sdist" ]]; then
+ echo
+ echo "Install airflow from sdist package with [all] extras but uninstalling providers."
+ echo
+ uninstall_airflow_and_providers
+ install_airflow_from_sdist "[all]"
+ uninstall_providers
else
echo
echo "Install airflow from PyPI including [all] extras"
diff --git a/scripts/in_container/run_install_and_test_provider_packages.sh b/scripts/in_container/run_install_and_test_provider_packages.sh
index 02eccbb..b519b6b 100755
--- a/scripts/in_container/run_install_and_test_provider_packages.sh
+++ b/scripts/in_container/run_install_and_test_provider_packages.sh
@@ -53,11 +53,20 @@ elif [[ ${INSTALL_AIRFLOW_VERSION} == "wheel" ]]; then
echo
uninstall_airflow_and_providers
install_airflow_from_wheel "[all]"
+ uninstall_providers
+elif [[ ${INSTALL_AIRFLOW_VERSION} == "sdist" ]]; then
+ echo
+ echo "Install airflow from sdist including [all] extras"
+ echo
+ uninstall_airflow_and_providers
+ install_airflow_from_sdist "[all]"
+ uninstall_providers
else
echo
echo "Install airflow from PyPI including [all] extras"
echo
install_released_airflow_version "${INSTALL_AIRFLOW_VERSION}" "[all]"
+ uninstall_providers
fi
echo