You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2020/06/20 16:53:14 UTC
[airflow] 15/25: Fix broken CI image optimisation (#9313)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v1-10-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 68c95bd850278dc6622df79f6c7dcccf81af5fab
Author: Jarek Potiuk <ja...@polidea.com>
AuthorDate: Tue Jun 16 01:38:55 2020 +0200
Fix broken CI image optimisation (#9313)
The commit 5918efc86a2217caa641a6ada289eee1c21407f8 broke
optimisation of the CI image - using the Apache Airflow
master branch as a base package installation source from PyPI.
This commit restores it including removal of the
obsolete CI_OPTIMISED arg - as now we have a separate
production and CI image and CI image is by default
CI_OPTIMISED
(cherry picked from commit 696e74594f26ec67c1f1330af725db537e97f18e)
---
Dockerfile.ci | 17 +++++------------
IMAGES.rst | 10 ++--------
scripts/ci/libraries/_build_images.sh | 2 --
3 files changed, 7 insertions(+), 22 deletions(-)
diff --git a/Dockerfile.ci b/Dockerfile.ci
index 24ee87d..4c9741b 100644
--- a/Dockerfile.ci
+++ b/Dockerfile.ci
@@ -214,25 +214,18 @@ ENV AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS}${ADDITIONAL_AIRFLOW_EXTRAS:+,}${ADDITIONAL_
RUN echo "Installing with extras: ${AIRFLOW_EXTRAS}."
-ARG AIRFLOW_CONTAINER_CI_OPTIMISED_BUILD="true"
-ENV AIRFLOW_CONTAINER_CI_OPTIMISED_BUILD=${AIRFLOW_CONTAINER_CI_OPTIMISED_BUILD}
-
# By changing the CI build epoch we can force reinstalling Arflow from the current master
# It can also be overwritten manually by setting the AIRFLOW_CI_BUILD_EPOCH environment variable.
ARG AIRFLOW_CI_BUILD_EPOCH="1"
ENV AIRFLOW_CI_BUILD_EPOCH=${AIRFLOW_CI_BUILD_EPOCH}
-# In case of CI-optimised builds we want to pre-install master version of airflow dependencies so that
+# In case of CI builds we want to pre-install master version of airflow dependencies so that
# We do not have to always reinstall it from the scratch.
# This can be reinstalled from latest master by increasing PIP_DEPENDENCIES_EPOCH_NUMBER.
-# And is automatically reinstalled from the scratch every month
-RUN \
- if [[ "${AIRFLOW_CONTAINER_CI_OPTIMISED_BUILD}" == "true" ]]; then \
- pip install \
- "https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz#egg=apache-airflow[${AIRFLOW_EXTRAS}]" \
- --constraint "https://raw.githubusercontent.com/${AIRFLOW_REPO}/${AIRFLOW_BRANCH}/requirements/requirements-python${PYTHON_MAJOR_MINOR_VERSION}.txt" \
- && pip uninstall --yes apache-airflow; \
- fi
+# And is automatically reinstalled from the scratch with every python patch level release
+RUN pip install "https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz#egg=apache-airflow[${AIRFLOW_EXTRAS}]" \
+ --constraint "https://raw.githubusercontent.com/${AIRFLOW_REPO}/${AIRFLOW_BRANCH}/requirements/requirements-python${PYTHON_MAJOR_MINOR_VERSION}.txt" \
+ && pip uninstall --yes apache-airflow
# Link dumb-init for backwards compatibility (so that older images also work)
RUN ln -sf /usr/bin/dumb-init /usr/local/bin/dumb-init
diff --git a/IMAGES.rst b/IMAGES.rst
index 0d4bd8c..3add528 100644
--- a/IMAGES.rst
+++ b/IMAGES.rst
@@ -123,8 +123,8 @@ Technical details of Airflow images
The CI image is used by Breeze as shell image but it is also used during CI build.
The image is single segment image that contains Airflow installation with "all" dependencies installed.
-It is optimised for rebuild speed (``AIRFLOW_CONTAINER_CI_OPTIMISED_BUILD`` flag set to "true").
-It installs PIP dependencies from the current branch first - so that any changes in setup.py do not trigger
+It is optimised for rebuild speed It installs PIP dependencies from the current branch first -
+so that any changes in setup.py do not trigger
reinstalling of all dependencies. There is a second step of installation that re-installs the dependencies
from the latest sources so that we are sure that latest dependencies are installed.
@@ -179,12 +179,6 @@ The following build arguments (``--build-arg`` in docker build command) can be u
| ``CASS_DRIVER_NO_CYTHON`` | ``1`` | if set to 1 no CYTHON compilation is |
| | | done for cassandra driver (much faster) |
+------------------------------------------+------------------------------------------+------------------------------------------+
-| ``AIRFLOW_CONTAINER_CI_OPTIMISED_BUILD`` | ``true`` | if set then PIP dependencies are |
-| | | installed from repo first before they |
-| | | are reinstalled from local sources. This |
-| | | allows for incremental faster builds |
-| | | when requirements change |
-+------------------------------------------+------------------------------------------+------------------------------------------+
| ``AIRFLOW_REPO`` | ``apache/airflow`` | the repository from which PIP |
| | | dependencies are installed (CI |
| | | optimised) |
diff --git a/scripts/ci/libraries/_build_images.sh b/scripts/ci/libraries/_build_images.sh
index 9d2b748..a54d6d8 100644
--- a/scripts/ci/libraries/_build_images.sh
+++ b/scripts/ci/libraries/_build_images.sh
@@ -344,7 +344,6 @@ function prepare_ci_build() {
fi
export THE_IMAGE_TYPE="CI"
export IMAGE_DESCRIPTION="Airflow CI"
- export AIRFLOW_CONTAINER_CI_OPTIMISED_BUILD="true"
export AIRFLOW_EXTRAS="${AIRFLOW_EXTRAS:="${DEFAULT_CI_EXTRAS}"}"
export ADDITIONAL_AIRFLOW_EXTRAS="${ADDITIONAL_AIRFLOW_EXTRAS:=""}"
export ADDITIONAL_PYTHON_DEPS="${ADDITIONAL_PYTHON_DEPS:=""}"
@@ -551,7 +550,6 @@ Docker building ${AIRFLOW_CI_IMAGE}.
--build-arg ADDITIONAL_PYTHON_DEPS="${ADDITIONAL_PYTHON_DEPS}" \
--build-arg ADDITIONAL_DEV_DEPS="${ADDITIONAL_DEV_DEPS}" \
--build-arg ADDITIONAL_RUNTIME_DEPS="${ADDITIONAL_RUNTIME_DEPS}" \
- --build-arg AIRFLOW_CONTAINER_CI_OPTIMISED_BUILD="${AIRFLOW_CONTAINER_CI_OPTIMISED_BUILD}" \
--build-arg UPGRADE_TO_LATEST_REQUIREMENTS="${UPGRADE_TO_LATEST_REQUIREMENTS}" \
"${DOCKER_CACHE_CI_DIRECTIVE[@]}" \
-t "${AIRFLOW_CI_IMAGE}" \