You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2021/08/10 20:36:24 UTC
[airflow] 13/22: Optimizes structure of the Dockerfiles and use
latest tools (#17418)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v2-1-test-jarek-cherry-picks
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 106d9f09852090da21f3dfb744c69e4be5fd90e5
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Wed Aug 4 23:32:12 2021 +0200
Optimizes structure of the Dockerfiles and use latest tools (#17418)
* Remove CONTINUE_ON_PIP_CHECK_FAILURE parameter
This parameter was useful when upgrading new dependencies,
however it is going to be replaced with better approach in the
upcoming image convention change.
* Optimizes structure of the Dockerfiles and use latest tools
This PR optimizes the structure of Dockerfile by moving some
expensive operations before the COPY sources so that
rebuilding image when only few sources change is much faster.
At the same time, we upgrade PIP and HELM chart used to latest
versions and clean-up some parameter inconsistencies.
(cherry picked from commit 94b03f6f43277e7332c25fdc63aedfde605f9773)
---
.github/workflows/build-images.yml | 1 -
BREEZE.rst | 10 +----
Dockerfile | 27 +++++------
Dockerfile.ci | 52 +++++++++++-----------
IMAGES.rst | 6 ---
breeze | 9 ----
breeze-complete | 2 +-
docs/docker-stack/build-arg-ref.rst | 6 ---
scripts/ci/libraries/_build_images.sh | 20 ---------
scripts/ci/libraries/_initialization.sh | 10 ++---
scripts/docker/install_additional_dependencies.sh | 5 +--
scripts/docker/install_airflow.sh | 4 +-
...nstall_airflow_dependencies_from_branch_tip.sh} | 11 ++---
.../docker/install_from_docker_context_files.sh | 2 +-
14 files changed, 55 insertions(+), 110 deletions(-)
diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml
index f29e199..ec8f435 100644
--- a/.github/workflows/build-images.yml
+++ b/.github/workflows/build-images.yml
@@ -148,7 +148,6 @@ jobs:
BACKEND: postgres
PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }}
UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }}
- CONTINUE_ON_PIP_CHECK_FAILURE: "true"
DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }}
CHECK_IF_BASE_PYTHON_IMAGE_UPDATED: >
${{ github.event_name == 'pull_request_target' && 'false' || 'true' }}
diff --git a/BREEZE.rst b/BREEZE.rst
index 90d3f0b..468709b 100644
--- a/BREEZE.rst
+++ b/BREEZE.rst
@@ -1279,9 +1279,6 @@ This is the current syntax for `./breeze <./breeze>`_:
--upgrade-to-newer-dependencies
Upgrades PIP packages to latest versions available without looking at the constraints.
- --continue-on-pip-check-failure
- Continue even if 'pip check' fails.
-
-I, --production-image
Use production image for entering the environment and builds (not for tests).
@@ -2382,9 +2379,9 @@ This is the current syntax for `./breeze <./breeze>`_:
Helm version - only used in case one of kind-cluster commands is used.
One of:
- v3.2.4
+ v3.6.3
- Default: v3.2.4
+ Default: v3.6.3
--executor EXECUTOR
Executor to use in a kubernetes cluster.
@@ -2435,9 +2432,6 @@ This is the current syntax for `./breeze <./breeze>`_:
--upgrade-to-newer-dependencies
Upgrades PIP packages to latest versions available without looking at the constraints.
- --continue-on-pip-check-failure
- Continue even if 'pip check' fails.
-
****************************************************************************************************
Use different Airflow version at runtime in CI image
diff --git a/Dockerfile b/Dockerfile
index 66c9649..210918c 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -44,7 +44,8 @@ ARG AIRFLOW_GID="50000"
ARG PYTHON_BASE_IMAGE="python:3.6-slim-buster"
-ARG AIRFLOW_PIP_VERSION=21.1.2
+ARG AIRFLOW_PIP_VERSION=21.2.2
+ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow"
# By default PIP has progress bar but you can disable it.
ARG PIP_PROGRESS_BAR="on"
@@ -108,12 +109,13 @@ ARG DEV_APT_COMMAND="\
&& curl https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - > /dev/null \
&& echo 'deb https://dl.yarnpkg.com/debian/ stable main' > /etc/apt/sources.list.d/yarn.list"
ARG ADDITIONAL_DEV_APT_COMMAND="echo"
+ARG ADDITIONAL_DEV_APT_ENV=""
ENV DEV_APT_DEPS=${DEV_APT_DEPS} \
ADDITIONAL_DEV_APT_DEPS=${ADDITIONAL_DEV_APT_DEPS} \
DEV_APT_COMMAND=${DEV_APT_COMMAND} \
ADDITIONAL_DEV_APT_COMMAND=${ADDITIONAL_DEV_APT_COMMAND} \
- ADDITIONAL_DEV_APT_ENV=""
+ ADDITIONAL_DEV_APT_ENV=${ADDITIONAL_DEV_APT_ENV}
# Note missing man directories on debian-buster
# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=863199
@@ -216,7 +218,7 @@ ENV AIRFLOW_PRE_CACHED_PIP_PACKAGES=${AIRFLOW_PRE_CACHED_PIP_PACKAGES} \
RUN bash /scripts/docker/install_pip_version.sh; \
if [[ ${AIRFLOW_PRE_CACHED_PIP_PACKAGES} == "true" && \
${UPGRADE_TO_NEWER_DEPENDENCIES} == "false" ]]; then \
- bash /scripts/docker/install_airflow_from_branch_tip.sh; \
+ bash /scripts/docker/install_airflow_dependencies_from_branch_tip.sh; \
fi
COPY ${AIRFLOW_SOURCES_FROM} ${AIRFLOW_SOURCES_TO}
@@ -236,14 +238,11 @@ ARG INSTALL_FROM_PYPI="true"
# * pyjwt<2.0.0: flask-jwt-extended requires it
# * dill<0.3.3 required by apache-beam
ARG EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS="pyjwt<2.0.0 dill<0.3.3 certifi<2021.0.0"
-ARG CONTINUE_ON_PIP_CHECK_FAILURE="false"
-
ENV ADDITIONAL_PYTHON_DEPS=${ADDITIONAL_PYTHON_DEPS} \
INSTALL_FROM_DOCKER_CONTEXT_FILES=${INSTALL_FROM_DOCKER_CONTEXT_FILES} \
INSTALL_FROM_PYPI=${INSTALL_FROM_PYPI} \
- EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS} \
- CONTINUE_ON_PIP_CHECK_FAILURE=${CONTINUE_ON_PIP_CHECK_FAILURE}
+ EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS}
WORKDIR /opt/airflow
@@ -276,7 +275,7 @@ RUN if [[ -f /docker-context-files/requirements.txt ]]; then \
ARG BUILD_ID
ARG COMMIT_SHA
-ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow"
+ARG AIRFLOW_IMAGE_REPOSITORY
ARG AIRFLOW_IMAGE_DATE_CREATED
ENV BUILD_ID=${BUILD_ID} COMMIT_SHA=${COMMIT_SHA}
@@ -293,15 +292,14 @@ LABEL org.apache.airflow.distro="debian" \
org.opencontainers.image.created=${AIRFLOW_IMAGE_DATE_CREATED} \
org.opencontainers.image.authors="dev@airflow.apache.org" \
org.opencontainers.image.url="https://airflow.apache.org" \
- org.opencontainers.image.documentation="https://airflow.apache.org/docs/apache-airflow/stable/production-deployment.html" \
- org.opencontainers.image.source="https://github.com/apache/airflow" \
+ org.opencontainers.image.documentation="https://airflow.apache.org/docs/docker-stack/index.html" \
org.opencontainers.image.version="${AIRFLOW_VERSION}" \
org.opencontainers.image.revision="${COMMIT_SHA}" \
org.opencontainers.image.vendor="Apache Software Foundation" \
org.opencontainers.image.licenses="Apache-2.0" \
org.opencontainers.image.ref.name="airflow-build-image" \
org.opencontainers.image.title="Build Image Segment for Production Airflow Image" \
- org.opencontainers.image.description="Installed Apache Airflow with build-time dependencies"
+ org.opencontainers.image.description="Reference build-time dependencies image for production-ready Apache Airflow image"
##############################################################################################
# This is the actual Airflow image - much smaller than the build one. We copy
@@ -379,7 +377,7 @@ ARG AIRFLOW_HOME
ARG AIRFLOW_INSTALLATION_METHOD="apache-airflow"
ARG BUILD_ID
ARG COMMIT_SHA
-ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow"
+ARG AIRFLOW_IMAGE_REPOSITORY
ARG AIRFLOW_IMAGE_DATE_CREATED
# By default PIP will install everything in ~/.local
ARG PIP_USER="true"
@@ -468,15 +466,14 @@ LABEL org.apache.airflow.distro="debian" \
org.opencontainers.image.created=${AIRFLOW_IMAGE_DATE_CREATED} \
org.opencontainers.image.authors="dev@airflow.apache.org" \
org.opencontainers.image.url="https://airflow.apache.org" \
- org.opencontainers.image.documentation="https://airflow.apache.org/docs/apache-airflow/stable/production-deployment.html" \
- org.opencontainers.image.source="https://github.com/apache/airflow" \
+ org.opencontainers.image.documentation="https://airflow.apache.org/docs/docker-stack/index.html" \
org.opencontainers.image.version="${AIRFLOW_VERSION}" \
org.opencontainers.image.revision="${COMMIT_SHA}" \
org.opencontainers.image.vendor="Apache Software Foundation" \
org.opencontainers.image.licenses="Apache-2.0" \
org.opencontainers.image.ref.name="airflow" \
org.opencontainers.image.title="Production Airflow Image" \
- org.opencontainers.image.description="Installed Apache Airflow"
+ org.opencontainers.image.description="Reference, production-ready Apache Airflow image"
ENTRYPOINT ["/usr/bin/dumb-init", "--", "/entrypoint"]
diff --git a/Dockerfile.ci b/Dockerfile.ci
index 7b1dbf1..8d599db 100644
--- a/Dockerfile.ci
+++ b/Dockerfile.ci
@@ -22,6 +22,8 @@ SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"]
ARG PYTHON_BASE_IMAGE="python:3.6-slim-buster"
ARG AIRFLOW_VERSION="2.1.0.dev0"
+ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow"
+
# By increasing this number we can do force build of all dependencies
ARG DEPENDENCIES_EPOCH_NUMBER="6"
@@ -125,6 +127,15 @@ ARG RUNTIME_APT_DEPS="\
unzip \
vim \
xxd"
+
+# Install Helm
+ARG HELM_VERSION="v3.6.3"
+
+RUN SYSTEM=$(uname -s | tr '[:upper:]' '[:lower:]') \
+ && HELM_URL="https://get.helm.sh/helm-${HELM_VERSION}-${SYSTEM}-amd64.tar.gz" \
+ && curl --location "${HELM_URL}" | tar -xvz -O "${SYSTEM}"-amd64/helm > /usr/local/bin/helm \
+ && chmod +x /usr/local/bin/helm
+
ARG ADDITIONAL_RUNTIME_APT_DEPS=""
ARG RUNTIME_APT_COMMAND=""
ARG ADDITIONAL_RUNTIME_APT_COMMAND=""
@@ -207,7 +218,7 @@ ARG AIRFLOW_PRE_CACHED_PIP_PACKAGES="true"
# By default in the image, we are installing all providers when installing from sources
ARG INSTALL_PROVIDERS_FROM_SOURCES="true"
ARG INSTALL_FROM_PYPI="true"
-ARG AIRFLOW_PIP_VERSION=21.1.2
+ARG AIRFLOW_PIP_VERSION=21.2.2
# Setup PIP
# By default PIP install run without cache to make image smaller
ARG PIP_NO_CACHE_DIR="true"
@@ -270,7 +281,7 @@ ENV EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=${EAGER_UPGRADE_ADDITIONAL_REQUIREMENT
RUN bash /scripts/docker/install_pip_version.sh; \
if [[ ${AIRFLOW_PRE_CACHED_PIP_PACKAGES} == "true" && \
${UPGRADE_TO_NEWER_DEPENDENCIES} == "false" ]]; then \
- bash /scripts/docker/install_airflow_from_branch_tip.sh; \
+ bash /scripts/docker/install_airflow_dependencies_from_branch_tip.sh; \
fi
# Generate random hex dump file so that we can determine whether it's faster to rebuild the image
@@ -299,8 +310,6 @@ COPY setup.cfg ${AIRFLOW_SOURCES}/setup.cfg
COPY airflow/__init__.py ${AIRFLOW_SOURCES}/airflow/__init__.py
-ARG CONTINUE_ON_PIP_CHECK_FAILURE="false"
-
# The goal of this line is to install the dependencies from the most current setup.py from sources
# This will be usually incremental small set of packages in CI optimized build, so it will be very fast
# In non-CI optimized build this will install all dependencies before installing sources.
@@ -325,11 +334,13 @@ RUN chmod a+x /entrypoint
COPY scripts/docker/load.bash /opt/bats/lib/
-# We can copy everything here. The Context is filtered by dockerignore. This makes sure we are not
-# copying over stuff that is accidentally generated or that we do not need (such as egg-info)
-# if you want to add something that is missing and you expect to see it in the image you can
-# add it with ! in .dockerignore next to the airflow, test etc. directories there
-COPY . ${AIRFLOW_SOURCES}/
+# Additional python deps to install
+ARG ADDITIONAL_PYTHON_DEPS=""
+
+RUN bash /scripts/docker/install_pip_version.sh; \
+ if [[ -n "${ADDITIONAL_PYTHON_DEPS}" ]]; then \
+ bash /scripts/docker/install_additional_dependencies.sh; \
+ fi
# Install autocomplete for airflow
RUN if command -v airflow; then \
@@ -339,27 +350,16 @@ RUN if command -v airflow; then \
# Install autocomplete for Kubectl
RUN echo "source /etc/bash_completion" >> ~/.bashrc
-WORKDIR ${AIRFLOW_SOURCES}
-
-# Install Helm
-ARG HELM_VERSION="v3.2.4"
-
-RUN SYSTEM=$(uname -s | tr '[:upper:]' '[:lower:]') \
- && HELM_URL="https://get.helm.sh/helm-${HELM_VERSION}-${SYSTEM}-amd64.tar.gz" \
- && curl --location "${HELM_URL}" | tar -xvz -O "${SYSTEM}"-amd64/helm > /usr/local/bin/helm \
- && chmod +x /usr/local/bin/helm
-
-# Additional python deps to install
-ARG ADDITIONAL_PYTHON_DEPS=""
+# We can copy everything here. The Context is filtered by dockerignore. This makes sure we are not
+# copying over stuff that is accidentally generated or that we do not need (such as egg-info)
+# if you want to add something that is missing and you expect to see it in the image you can
+# add it with ! in .dockerignore next to the airflow, test etc. directories there
+COPY . ${AIRFLOW_SOURCES}/
-RUN bash /scripts/docker/install_pip_version.sh; \
- if [[ -n "${ADDITIONAL_PYTHON_DEPS}" ]]; then \
- bash /scripts/docker/install_additional_dependencies.sh; \
- fi
+WORKDIR ${AIRFLOW_SOURCES}
ARG BUILD_ID
ARG COMMIT_SHA
-ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow"
ARG AIRFLOW_IMAGE_DATE_CREATED
ENV PATH="/files/bin/:/opt/airflow/scripts/in_container/bin/:${HOME}:${PATH}" \
diff --git a/IMAGES.rst b/IMAGES.rst
index 82e6989..bc34e6c 100644
--- a/IMAGES.rst
+++ b/IMAGES.rst
@@ -445,12 +445,6 @@ The following build arguments (``--build-arg`` in docker build command) can be u
| | | upgraded to newer versions matching |
| | | setup.py before installation. |
+------------------------------------------+------------------------------------------+------------------------------------------+
-| ``CONTINUE_ON_PIP_CHECK_FAILURE`` | ``false`` | By default the image will fail if pip |
-| | | check fails for it. This is good for |
-| | | interactive building but on CI the |
-| | | image should be built regardless - we |
-| | | have a separate step to verify image. |
-+------------------------------------------+------------------------------------------+------------------------------------------+
| ``AIRFLOW_PRE_CACHED_PIP_PACKAGES`` | ``true`` | Allows to pre-cache airflow PIP packages |
| | | from the GitHub of Apache Airflow |
| | | This allows to optimize iterations for |
diff --git a/breeze b/breeze
index 7decdf6..1f45ed1 100755
--- a/breeze
+++ b/breeze
@@ -1177,12 +1177,6 @@ function breeze::parse_arguments() {
echo
shift
;;
- --continue-on-pip-check-failure)
- export CONTINUE_ON_PIP_CHECK_FAILURE="true"
- echo "Skip PIP check failure."
- echo
- shift
- ;;
--package-format)
export PACKAGE_FORMAT="${2}"
echo "Selected package type: ${PACKAGE_FORMAT}"
@@ -2400,9 +2394,6 @@ ${FORMATTED_INSTALLATION_METHOD}
--upgrade-to-newer-dependencies
Upgrades PIP packages to latest versions available without looking at the constraints.
---continue-on-pip-check-failure
- Continue even if 'pip check' fails.
-
"
}
diff --git a/breeze-complete b/breeze-complete
index db9e42a..045bb65 100644
--- a/breeze-complete
+++ b/breeze-complete
@@ -29,7 +29,7 @@ _breeze_allowed_integrations="cassandra kerberos mongo openldap pinot rabbitmq r
_breeze_allowed_generate_constraints_modes="source-providers pypi-providers no-providers"
_breeze_allowed_kubernetes_modes="image"
_breeze_allowed_kubernetes_versions="v1.20.2 v1.19.7 v1.18.15"
-_breeze_allowed_helm_versions="v3.2.4"
+_breeze_allowed_helm_versions="v3.6.3"
_breeze_allowed_kind_versions="v0.11.1"
_breeze_allowed_mysql_versions="5.7 8"
_breeze_allowed_postgres_versions="9.6 10 11 12 13"
diff --git a/docs/docker-stack/build-arg-ref.rst b/docs/docker-stack/build-arg-ref.rst
index 8780970..f2507e0 100644
--- a/docs/docker-stack/build-arg-ref.rst
+++ b/docs/docker-stack/build-arg-ref.rst
@@ -79,12 +79,6 @@ for examples of using those arguments.
+------------------------------------------+------------------------------------------+------------------------------------------+
| Build argument | Default value | Description |
+==========================================+==========================================+==========================================+
-| ``CONTINUE_ON_PIP_CHECK_FAILURE`` | ``false`` | By default the image build fails if pip |
-| | | check fails for it. This is good for |
-| | | interactive building but on CI the |
-| | | image should be built regardless - we |
-| | | have a separate step to verify image. |
-+------------------------------------------+------------------------------------------+------------------------------------------+
| ``UPGRADE_TO_NEWER_DEPENDENCIES`` | ``false`` | If set to true, the dependencies are |
| | | upgraded to newer versions matching |
| | | setup.py before installation. |
diff --git a/scripts/ci/libraries/_build_images.sh b/scripts/ci/libraries/_build_images.sh
index ca94e4c..4ead442 100644
--- a/scripts/ci/libraries/_build_images.sh
+++ b/scripts/ci/libraries/_build_images.sh
@@ -669,7 +669,6 @@ Docker building ${AIRFLOW_CI_IMAGE}.
--build-arg ADDITIONAL_RUNTIME_APT_DEPS="${ADDITIONAL_RUNTIME_APT_DEPS}" \
--build-arg ADDITIONAL_RUNTIME_APT_ENV="${ADDITIONAL_RUNTIME_APT_ENV}" \
--build-arg UPGRADE_TO_NEWER_DEPENDENCIES="${UPGRADE_TO_NEWER_DEPENDENCIES}" \
- --build-arg CONTINUE_ON_PIP_CHECK_FAILURE="${CONTINUE_ON_PIP_CHECK_FAILURE}" \
--build-arg CONSTRAINTS_GITHUB_REPOSITORY="${CONSTRAINTS_GITHUB_REPOSITORY}" \
--build-arg AIRFLOW_CONSTRAINTS_REFERENCE="${DEFAULT_CONSTRAINTS_BRANCH}" \
--build-arg AIRFLOW_CONSTRAINTS="${AIRFLOW_CONSTRAINTS}" \
@@ -810,7 +809,6 @@ function build_images::build_prod_images() {
--build-arg INSTALL_FROM_PYPI="${INSTALL_FROM_PYPI}" \
--build-arg INSTALL_FROM_DOCKER_CONTEXT_FILES="${INSTALL_FROM_DOCKER_CONTEXT_FILES}" \
--build-arg UPGRADE_TO_NEWER_DEPENDENCIES="${UPGRADE_TO_NEWER_DEPENDENCIES}" \
- --build-arg CONTINUE_ON_PIP_CHECK_FAILURE="${CONTINUE_ON_PIP_CHECK_FAILURE}" \
--build-arg BUILD_ID="${CI_BUILD_ID}" \
--build-arg COMMIT_SHA="${COMMIT_SHA}" \
--build-arg CONSTRAINTS_GITHUB_REPOSITORY="${CONSTRAINTS_GITHUB_REPOSITORY}" \
@@ -845,7 +843,6 @@ function build_images::build_prod_images() {
--build-arg INSTALL_FROM_PYPI="${INSTALL_FROM_PYPI}" \
--build-arg INSTALL_FROM_DOCKER_CONTEXT_FILES="${INSTALL_FROM_DOCKER_CONTEXT_FILES}" \
--build-arg UPGRADE_TO_NEWER_DEPENDENCIES="${UPGRADE_TO_NEWER_DEPENDENCIES}" \
- --build-arg CONTINUE_ON_PIP_CHECK_FAILURE="${CONTINUE_ON_PIP_CHECK_FAILURE}" \
--build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \
--build-arg AIRFLOW_BRANCH="${AIRFLOW_BRANCH_FOR_PYPI_PRELOADING}" \
--build-arg AIRFLOW_EXTRAS="${AIRFLOW_EXTRAS}" \
@@ -1014,23 +1011,6 @@ ${COLOR_BLUE}
./breeze build-image --production-image --upgrade-to-newer-dependencies --python 3.6
${COLOR_RESET}
-* If you want to build the image regardless if 'pip check' fails for it, you can add
- --continue-on-pip-check-failure flag and enter the image and inspect dependencies.
-
-CI image:
-
-${COLOR_BLUE}
- ./breeze build-image --upgrade-to-newer-dependencies --python 3.6 --continue-on-pip-check-failure
- docker run -it apache/airflow:main-3.6-ci bash
-${COLOR_RESET}
-
-Production image:
-
-${COLOR_BLUE}
- ./breeze build-image --production-image --upgrade-to-newer-dependencies --python 3.6 --continue-on-pip-check-failure
- docker run -it apache/airflow:main-3.6 bash
-${COLOR_RESET}
-
* You will see error messages there telling which requirements are conflicting and which packages caused the
conflict. Add the limitation that caused the conflict to EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS
variable in Dockerfile.ci. Note that the limitations might be different for Dockerfile.ci and Dockerfile
diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh
index cbeb932..2cebe7d 100644
--- a/scripts/ci/libraries/_initialization.sh
+++ b/scripts/ci/libraries/_initialization.sh
@@ -200,7 +200,7 @@ function initialization::initialize_files_for_rebuild_check() {
"scripts/docker/common.sh"
"scripts/docker/install_additional_dependencies.sh"
"scripts/docker/install_airflow.sh"
- "scripts/docker/install_airflow_from_branch_tip.sh"
+ "scripts/docker/install_airflow_dependencies_from_branch_tip.sh"
"scripts/docker/install_from_docker_context_files.sh"
"scripts/docker/install_mysql.sh"
"airflow/www/package.json"
@@ -401,7 +401,7 @@ function initialization::initialize_image_build_variables() {
export INSTALLED_PROVIDERS
export INSTALLED_EXTRAS="async,amazon,celery,cncf.kubernetes,docker,dask,elasticsearch,ftp,grpc,hashicorp,http,imap,ldap,google,microsoft.azure,mysql,postgres,redis,sendgrid,sftp,slack,ssh,statsd,virtualenv"
- AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION:="21.1"}
+ AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION:="21.2.2"}
export AIRFLOW_PIP_VERSION
# We also pin version of wheel used to get consistent builds
@@ -425,9 +425,6 @@ function initialization::initialize_image_build_variables() {
# Installs different airflow version than current from the sources
export INSTALL_AIRFLOW_VERSION=${INSTALL_AIRFLOW_VERSION:=""}
- # Continue on PIP CHECK failure
- export CONTINUE_ON_PIP_CHECK_FAILURE=${CONTINUE_ON_PIP_CHECK_FAILURE:="false"}
-
# Determines if airflow should be installed from a specified reference in GitHub
export INSTALL_AIRFLOW_REFERENCE=${INSTALL_AIRFLOW_REFERENCE:=""}
@@ -482,7 +479,7 @@ function initialization::initialize_kubernetes_variables() {
CURRENT_KIND_VERSIONS+=("v0.11.1")
export CURRENT_KIND_VERSIONS
# Currently supported versions of Helm
- CURRENT_HELM_VERSIONS+=("v3.2.4")
+ CURRENT_HELM_VERSIONS+=("v3.6.3")
export CURRENT_HELM_VERSIONS
# Current executor in chart
CURRENT_EXECUTOR+=("KubernetesExecutor")
@@ -663,7 +660,6 @@ Common image build variables:
INSTALL_FROM_PYPI: '${INSTALL_FROM_PYPI}'
AIRFLOW_PRE_CACHED_PIP_PACKAGES: '${AIRFLOW_PRE_CACHED_PIP_PACKAGES}'
UPGRADE_TO_NEWER_DEPENDENCIES: '${UPGRADE_TO_NEWER_DEPENDENCIES}'
- CONTINUE_ON_PIP_CHECK_FAILURE: '${CONTINUE_ON_PIP_CHECK_FAILURE}'
CHECK_IMAGE_FOR_REBUILD: '${CHECK_IMAGE_FOR_REBUILD}'
AIRFLOW_CONSTRAINTS_LOCATION: '${AIRFLOW_CONSTRAINTS_LOCATION}'
AIRFLOW_CONSTRAINTS_REFERENCE: '${AIRFLOW_CONSTRAINTS_REFERENCE}'
diff --git a/scripts/docker/install_additional_dependencies.sh b/scripts/docker/install_additional_dependencies.sh
index 6c035ae..4f9c05f 100755
--- a/scripts/docker/install_additional_dependencies.sh
+++ b/scripts/docker/install_additional_dependencies.sh
@@ -23,7 +23,6 @@ test -v ADDITIONAL_PYTHON_DEPS
test -v EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS
test -v AIRFLOW_INSTALL_USER_FLAG
test -v AIRFLOW_PIP_VERSION
-test -v CONTINUE_ON_PIP_CHECK_FAILURE
# shellcheck source=scripts/docker/common.sh
. "$( dirname "${BASH_SOURCE[0]}" )/common.sh"
@@ -41,7 +40,7 @@ function install_additional_dependencies() {
${ADDITIONAL_PYTHON_DEPS} ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS}
# make sure correct PIP version is used
pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}"
- pip check || ${CONTINUE_ON_PIP_CHECK_FAILURE}
+ pip check
else
echo
echo Installing additional dependencies upgrading only if needed
@@ -51,7 +50,7 @@ function install_additional_dependencies() {
${ADDITIONAL_PYTHON_DEPS}
# make sure correct PIP version is used
pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}"
- pip check || ${CONTINUE_ON_PIP_CHECK_FAILURE}
+ pip check
fi
}
diff --git a/scripts/docker/install_airflow.sh b/scripts/docker/install_airflow.sh
index 4904027..e2bca4f 100755
--- a/scripts/docker/install_airflow.sh
+++ b/scripts/docker/install_airflow.sh
@@ -60,7 +60,7 @@ function install_airflow() {
# make sure correct PIP version is used
pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}"
- pip check || ${CONTINUE_ON_PIP_CHECK_FAILURE}
+ pip check
else \
echo
echo Installing all packages with constraints and upgrade if needed
@@ -76,7 +76,7 @@ function install_airflow() {
"${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \
# make sure correct PIP version is used
pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}"
- pip check || ${CONTINUE_ON_PIP_CHECK_FAILURE}
+ pip check
fi
}
diff --git a/scripts/docker/install_airflow_from_branch_tip.sh b/scripts/docker/install_airflow_dependencies_from_branch_tip.sh
similarity index 85%
rename from scripts/docker/install_airflow_from_branch_tip.sh
rename to scripts/docker/install_airflow_dependencies_from_branch_tip.sh
index 925a872..61aaa13 100755
--- a/scripts/docker/install_airflow_from_branch_tip.sh
+++ b/scripts/docker/install_airflow_dependencies_from_branch_tip.sh
@@ -30,28 +30,29 @@
. "$( dirname "${BASH_SOURCE[0]}" )/common.sh"
-function install_airflow_from_branch_tip() {
+function install_airflow_dependencies_from_branch_tip() {
echo
echo "Installing airflow from ${AIRFLOW_BRANCH}. It is used to cache dependencies"
echo
if [[ ${INSTALL_MYSQL_CLIENT} != "true" ]]; then
AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS/mysql,}
fi
- # Install latest set of dependencies using constraints
+ # Install latest set of dependencies using constraints. In case constraints were upgraded and there
+ # are conflicts, this might fail, but it should be fixed in the following installation steps
pip install ${AIRFLOW_INSTALL_USER_FLAG} \
"https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz#egg=apache-airflow[${AIRFLOW_EXTRAS}]" \
- --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}"
+ --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" || true
# make sure correct PIP version is used
pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}"
pip freeze | grep apache-airflow-providers | xargs pip uninstall --yes || true
echo
echo Uninstalling just airflow. Dependencies remain.
echo
- pip uninstall --yes apache-airflow
+ pip uninstall --yes apache-airflow || true
}
common::get_airflow_version_specification
common::override_pip_version_if_needed
common::get_constraints_location
-install_airflow_from_branch_tip
+install_airflow_dependencies_from_branch_tip
diff --git a/scripts/docker/install_from_docker_context_files.sh b/scripts/docker/install_from_docker_context_files.sh
index 813d1b0..d8ed6bc 100755
--- a/scripts/docker/install_from_docker_context_files.sh
+++ b/scripts/docker/install_from_docker_context_files.sh
@@ -96,7 +96,7 @@ function install_airflow_and_providers_from_docker_context_files(){
# make sure correct PIP version is left installed
pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}"
- pip check || ${CONTINUE_ON_PIP_CHECK_FAILURE}
+ pip check
}