You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2020/12/12 11:04:03 UTC
[airflow] branch master updated: Changes release image preparation
to use PyPI packages (#12990)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/master by this push:
new db02773 Changes release image preparation to use PyPI packages (#12990)
db02773 is described below
commit db027735a7ee03d2678c8ca8712dfd75de148261
Author: Jarek Potiuk <ja...@polidea.com>
AuthorDate: Sat Dec 12 12:01:58 2020 +0100
Changes release image preparation to use PyPI packages (#12990)
* Changes release image preparation to use PyPI packages
Since we released all teh provider packages to PyPI now in
RC version, we can now change the mechanism to prepare the
production to use released packages in case of tagged builds.
The "branch" production images are still prepared using the
CI images and .whl packages built from sources, but the
release packages are built from officially released PyPI
packages.
Also some corrections and updates were made to the release process:
* the constraint tags when RC candidate is sent should contain
rcn suffix.
* there was missing step about pushing the release tag once the
release is out
* pushing tag to GitHub should be done after the PyPI packages
are uploaded, so that automated image building in DockerHub
can use those packages.
* added a note that in case we will release some provider
packages that depend on the just released airflow version
they shoudl be released after airflow is in PyPI but before
the tag is pushed to GitHub (also to allow the image to be
build automatically from the released packages)
Fixes: #12970
* Update dev/README_RELEASE_AIRFLOW.md
Co-authored-by: Ash Berlin-Taylor <as...@firemirror.com>
* Update dev/README_RELEASE_AIRFLOW.md
Co-authored-by: Ash Berlin-Taylor <as...@firemirror.com>
Co-authored-by: Ash Berlin-Taylor <as...@firemirror.com>
---
BREEZE.rst | 6 +-
Dockerfile | 44 ++++--
Dockerfile.ci | 1 +
IMAGES.rst | 4 +-
breeze-complete | 8 -
dev/README_RELEASE_AIRFLOW.md | 122 +++++++++++++--
docs/apache-airflow/production-deployment.rst | 204 ++++++++++++--------------
docs/spelling_wordlist.txt | 1 +
scripts/ci/images/ci_build_dockerhub.sh | 95 ++++++++----
scripts/ci/libraries/_build_images.sh | 68 +++++----
scripts/ci/libraries/_initialization.sh | 18 ++-
scripts/in_container/_in_container_utils.sh | 7 +-
12 files changed, 364 insertions(+), 214 deletions(-)
diff --git a/BREEZE.rst b/BREEZE.rst
index 1450982..9a5eed6 100644
--- a/BREEZE.rst
+++ b/BREEZE.rst
@@ -1275,8 +1275,7 @@ This is the current syntax for `./breeze <./breeze>`_:
If specified, installs Airflow directly from PIP released version. This happens at
image building time in production image and at container entering time for CI image. One of:
- 1.10.14 1.10.13 1.10.12 1.10.11 1.10.10 1.10.9 1.10.8 1.10.7 1.10.6 1.10.5 1.10.4
- 1.10.3 1.10.2 none wheel
+ 1.10.14 1.10.12 1.10.11 1.10.10 1.10.9 none wheel
When 'none' is used, you can install airflow from local packages. When building image,
airflow package should be added to 'docker-context-files' and
@@ -2379,8 +2378,7 @@ This is the current syntax for `./breeze <./breeze>`_:
If specified, installs Airflow directly from PIP released version. This happens at
image building time in production image and at container entering time for CI image. One of:
- 1.10.14 1.10.13 1.10.12 1.10.11 1.10.10 1.10.9 1.10.8 1.10.7 1.10.6 1.10.5 1.10.4
- 1.10.3 1.10.2 none wheel
+ 1.10.14 1.10.12 1.10.11 1.10.10 1.10.9 none wheel
When 'none' is used, you can install airflow from local packages. When building image,
airflow package should be added to 'docker-context-files' and
diff --git a/Dockerfile b/Dockerfile
index 3a4771a..af7c07c 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -167,15 +167,17 @@ ENV AIRFLOW_CONSTRAINTS_LOCATION=${AIRFLOW_CONSTRAINTS_LOCATION}
ENV PATH=${PATH}:/root/.local/bin
RUN mkdir -p /root/.local/bin
-ARG AIRFLOW_PRE_CACHED_PIP_PACKAGES="true"
-ENV AIRFLOW_PRE_CACHED_PIP_PACKAGES=${AIRFLOW_PRE_CACHED_PIP_PACKAGES}
-
RUN if [[ -f /docker-context-files/.pypirc ]]; then \
cp /docker-context-files/.pypirc /root/.pypirc; \
fi
RUN pip install --upgrade "pip==${PIP_VERSION}"
+# By default we do not use pre-cached packages, but in CI/Breeze environment we override this to speed up
+# builds in case setup.py/setup.cfg changed. This is pure optimisation of CI/Breeze builds.
+ARG AIRFLOW_PRE_CACHED_PIP_PACKAGES="false"
+ENV AIRFLOW_PRE_CACHED_PIP_PACKAGES=${AIRFLOW_PRE_CACHED_PIP_PACKAGES}
+
# In case of Production build image segment we want to pre-install master version of airflow
# dependencies from GitHub so that we do not have to always reinstall it from the scratch.
RUN if [[ ${AIRFLOW_PRE_CACHED_PIP_PACKAGES} == "true" ]]; then \
@@ -188,10 +190,13 @@ RUN if [[ ${AIRFLOW_PRE_CACHED_PIP_PACKAGES} == "true" ]]; then \
&& pip uninstall --yes apache-airflow; \
fi
-ARG AIRFLOW_SOURCES_FROM="."
+# By default we install latest airflow from PyPI so we do not need to copy sources of Airflow
+# but in case of breeze/CI builds we use latest sources and we override those
+# those SOURCES_FROM/TO with "." and "/opt/airflow" respectively
+ARG AIRFLOW_SOURCES_FROM="empty"
ENV AIRFLOW_SOURCES_FROM=${AIRFLOW_SOURCES_FROM}
-ARG AIRFLOW_SOURCES_TO="/opt/airflow"
+ARG AIRFLOW_SOURCES_TO="/empty"
ENV AIRFLOW_SOURCES_TO=${AIRFLOW_SOURCES_TO}
COPY ${AIRFLOW_SOURCES_FROM} ${AIRFLOW_SOURCES_TO}
@@ -199,28 +204,41 @@ COPY ${AIRFLOW_SOURCES_FROM} ${AIRFLOW_SOURCES_TO}
ARG CASS_DRIVER_BUILD_CONCURRENCY
ENV CASS_DRIVER_BUILD_CONCURRENCY=${CASS_DRIVER_BUILD_CONCURRENCY}
+# This is airflow version that is put in the label of the image build
ARG AIRFLOW_VERSION
ENV AIRFLOW_VERSION=${AIRFLOW_VERSION}
ARG ADDITIONAL_PYTHON_DEPS=""
ENV ADDITIONAL_PYTHON_DEPS=${ADDITIONAL_PYTHON_DEPS}
-ARG AIRFLOW_INSTALL_SOURCES="."
-ENV AIRFLOW_INSTALL_SOURCES=${AIRFLOW_INSTALL_SOURCES}
+# Determines the way airflow is installed. By default we install airflow from PyPI `apache-airflow` package
+# But it also can be `.` from local installation or GitHub URL pointing to specific branch or tag
+# Of Airflow. Note That for local source installation you need to have local sources of
+# Airflow checked out together with the Dockerfile and AIRFLOW_SOURCES_FROM and AIRFLOW_SOURCES_TO
+# set to "." and "/opt/airflow" respectively.
+ARG AIRFLOW_INSTALLATION_METHOD="apache-airflow"
+ENV AIRFLOW_INSTALLATION_METHOD=${AIRFLOW_INSTALLATION_METHOD}
+# By default latest released version of airflow is installed (when empty) but this value can be overriden
+# and we can install specific version of airflow this way.
ARG AIRFLOW_INSTALL_VERSION=""
ENV AIRFLOW_INSTALL_VERSION=${AIRFLOW_INSTALL_VERSION}
+# We can seet this value to true in case we want to install .whl .tar.gz packages placed in the
+# docker-context-files folder. This can be done for both - additional packages you want to install
+# and for airflow as well (you have to set INSTALL_FROM_PYPI to false in this case)
ARG INSTALL_FROM_DOCKER_CONTEXT_FILES=""
ENV INSTALL_FROM_DOCKER_CONTEXT_FILES=${INSTALL_FROM_DOCKER_CONTEXT_FILES}
+# By default we install latest airflow from PyPI. You can set it to false if you want to install
+# Airflow from the .whl or .tar.gz packages placed in `docker-context-files` folder.
ARG INSTALL_FROM_PYPI="true"
ENV INSTALL_FROM_PYPI=${INSTALL_FROM_PYPI}
-ARG SLUGIFY_USES_TEXT_UNIDECODE=""
-ENV SLUGIFY_USES_TEXT_UNIDECODE=${SLUGIFY_USES_TEXT_UNIDECODE}
-
-ARG INSTALL_PROVIDERS_FROM_SOURCES="true"
+# By default we install providers from PyPI but in case of Breze build we want to install providers
+# from local sources without the neeed of preparing provider packages upfront. This value is
+# automatically overridden by Breeze scripts.
+ARG INSTALL_PROVIDERS_FROM_SOURCES="false"
ENV INSTALL_PROVIDERS_FROM_SOURCES=${INSTALL_PROVIDERS_FROM_SOURCES}
WORKDIR /opt/airflow
@@ -230,7 +248,7 @@ RUN if [[ ${INSTALL_MYSQL_CLIENT} != "true" ]]; then \
AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS/mysql,}; \
fi; \
if [[ ${INSTALL_FROM_PYPI} == "true" ]]; then \
- pip install --user "${AIRFLOW_INSTALL_SOURCES}[${AIRFLOW_EXTRAS}]${AIRFLOW_INSTALL_VERSION}" \
+ pip install --user "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_INSTALL_VERSION}" \
--constraint "${AIRFLOW_CONSTRAINTS_LOCATION}"; \
fi; \
if [[ -n "${ADDITIONAL_PYTHON_DEPS}" ]]; then \
@@ -273,6 +291,7 @@ LABEL org.apache.airflow.distro="debian" \
org.apache.airflow.module="airflow" \
org.apache.airflow.component="airflow" \
org.apache.airflow.image="airflow-build-image" \
+ org.apache.airflow.version="${AIRFLOW_VERSION}" \
org.apache.airflow.buildImage.buildId=${BUILD_ID} \
org.apache.airflow.buildImage.commitSha=${COMMIT_SHA}
@@ -434,6 +453,7 @@ LABEL org.apache.airflow.distro="debian" \
org.apache.airflow.module="airflow" \
org.apache.airflow.component="airflow" \
org.apache.airflow.image="airflow" \
+ org.apache.airflow.version="${AIRFLOW_VERSION}" \
org.apache.airflow.uid="${AIRFLOW_UID}" \
org.apache.airflow.gid="${AIRFLOW_GID}" \
org.apache.airflow.mainImage.buildId=${BUILD_ID} \
diff --git a/Dockerfile.ci b/Dockerfile.ci
index 923cc82..f998a74 100644
--- a/Dockerfile.ci
+++ b/Dockerfile.ci
@@ -396,6 +396,7 @@ LABEL org.apache.airflow.distro="debian" \
org.apache.airflow.module="airflow" \
org.apache.airflow.component="airflow" \
org.apache.airflow.image="airflow-ci" \
+ org.apache.airflow.version="${AIRFLOW_VERSION}" \
org.apache.airflow.uid="0" \
org.apache.airflow.gid="0" \
org.apache.airflow.buildId=${BUILD_ID} \
diff --git a/IMAGES.rst b/IMAGES.rst
index a538270..9a4bf08 100644
--- a/IMAGES.rst
+++ b/IMAGES.rst
@@ -310,7 +310,7 @@ additional apt dev and runtime dependencies.
docker build . -f Dockerfile.ci \
--build-arg PYTHON_BASE_IMAGE="python:3.7-slim-buster" \
--build-arg PYTHON_MAJOR_MINOR_VERSION=3.7 \
- --build-arg AIRFLOW_INSTALL_SOURCES="apache-airflow" \
+ --build-arg AIRFLOW_INSTALLATION_METHOD="apache-airflow" \
--build-arg AIRFLOW_VERSION="1.10.14" \
--build-arg AIRFLOW_INSTALL_VERSION="==1.10.14" \
--build-arg AIRFLOW_CONSTRAINTS_REFERENCE="constraints-1-10" \
@@ -345,7 +345,7 @@ based on example in `this comment <https://github.com/apache/airflow/issues/8605
docker build . -f Dockerfile.ci \
--build-arg PYTHON_BASE_IMAGE="python:3.7-slim-buster" \
--build-arg PYTHON_MAJOR_MINOR_VERSION=3.7 \
- --build-arg AIRFLOW_INSTALL_SOURCES="apache-airflow" \
+ --build-arg AIRFLOW_INSTALLATION_METHOD="apache-airflow" \
--build-arg AIRFLOW_VERSION="1.10.14" \
--build-arg AIRFLOW_INSTALL_VERSION="==1.10.14" \
--build-arg AIRFLOW_CONSTRAINTS_REFERENCE="constraints-1-10" \
diff --git a/breeze-complete b/breeze-complete
index 1944ba9..21b8ab5 100644
--- a/breeze-complete
+++ b/breeze-complete
@@ -52,18 +52,10 @@ _breeze_allowed_package_formats="wheel sdist both"
_breeze_allowed_install_airflow_versions=$(cat <<-EOF
1.10.14
-1.10.13
1.10.12
1.10.11
1.10.10
1.10.9
-1.10.8
-1.10.7
-1.10.6
-1.10.5
-1.10.4
-1.10.3
-1.10.2
none
wheel
EOF
diff --git a/dev/README_RELEASE_AIRFLOW.md b/dev/README_RELEASE_AIRFLOW.md
index 03d0a2f..2fb1294 100644
--- a/dev/README_RELEASE_AIRFLOW.md
+++ b/dev/README_RELEASE_AIRFLOW.md
@@ -23,6 +23,7 @@
- [Prepare the Apache Airflow Package RC](#prepare-the-apache-airflow-package-rc)
- [Build RC artifacts](#build-rc-artifacts)
- [Prepare PyPI convenience "snapshot" packages](#prepare-pypi-convenience-snapshot-packages)
+ - [\[Optional\] - Manually prepare production Docker Image](#%5Coptional%5C---manually-prepare-production-docker-image)
- [Prepare Vote email on the Apache Airflow release candidate](#prepare-vote-email-on-the-apache-airflow-release-candidate)
- [Verify the release candidate by PMCs](#verify-the-release-candidate-by-pmcs)
- [SVN check](#svn-check)
@@ -35,6 +36,7 @@
- [Publish release to SVN](#publish-release-to-svn)
- [Prepare PyPI "release" packages](#prepare-pypi-release-packages)
- [Update CHANGELOG.md](#update-changelogmd)
+ - [\[Optional\] - Manually prepare production Docker Image](#%5Coptional%5C---manually-prepare-production-docker-image-1)
- [Publish documentation](#publish-documentation)
- [Notify developers of release](#notify-developers-of-release)
- [Update Announcements page](#update-announcements-page)
@@ -112,18 +114,12 @@ The Release Candidate artifacts we vote upon should be the exact ones we vote ag
${AIRFLOW_REPO_ROOT}/dev/sign.sh apache_airflow-${VERSION}-py2.py3-none-any.whl
```
-- Push Tags
-
- ```shell script
- git push origin ${VERSION}
- ```
-
-- Tag & Push latest constraints files
+- Tag & Push latest constraints files. This pushes constraints with rc suffix (this is expected)!
```shell script
git checkout constraints-1-10
- git tag -s "constraints-${VERSION%rc?}"
- git push origin "constraints-${VERSION%rc?}"
+ git tag -s "constraints-${VERSION}"
+ git push origin "constraints-${VERSION}"
```
- Push the artifacts to ASF dev dist repo
@@ -146,7 +142,12 @@ svn commit -m "Add artifacts for Airflow ${VERSION}"
## Prepare PyPI convenience "snapshot" packages
At this point we have the artefact that we vote on, but as a convenience to developers we also want to
-publish "snapshots" of the RC builds to pypi for installing via pip. To do this we need to
+publish "snapshots" of the RC builds to pypi for installing via pip. Also those packages
+are used to build the production docker image in DockerHub, so we need to upload the packages
+before we push the tag to GitHub. Pushing the tag to GitHub automatically triggers image building in
+DockerHub.
+
+To do this we need to
- Build the package:
@@ -178,6 +179,51 @@ https://pypi.python.org/pypi/apache-airflow
It is important to stress that this snapshot should not be named "release", and it
is not supposed to be used by and advertised to the end-users who do not read the devlist.
+- Push Tag for the release candidate
+
+ This step should only be done now and not before, because it triggers an automated build of
+ the production docker image, using the packages that are currently released in PyPI
+ (both airflow and latest provider packages).
+
+ ```shell script
+ git push origin ${VERSION}
+ ```
+
+## \[Optional\] - Manually prepare production Docker Image
+
+Production Docker images should be automatically built in 2-3 hours after the release tag has been
+pushed. If this did not happen - please login to DockerHub and check the status of builds:
+[Build Timeline](https://hub.docker.com/repository/docker/apache/airflow/timeline)
+
+In case you need, you can also build and push the images manually:
+
+Airflow 2+:
+
+```shell script
+export DOCKER_REPO=docker.io/apache/airflow
+for python_version in "3.6" "3.7" "3.8"
+(
+ export DOCKER_TAG=${VERSION}-python${python_version}
+ ./scripts/ci/images/ci_build_dockerhub.sh
+)
+```
+
+This will wipe Breeze cache and docker-context-files in order to make sure the build is "clean".
+
+Airflow 1.10:
+
+```shell script
+for python_version in "2.7" "3.5" "3.6" "3.7" "3.8"
+do
+ ./breeze build-image --production-image --python ${python_version} \
+ --image-tag apache/airflow:${VERSION}-python${python_version} --build-cache-local
+ docker push apache/airflow:${VERSION}-python${python_version}
+done
+docker tag apache/airflow:${VERSION}-python3.6 apache/airflow:${VERSION}
+docker push apache/airflow:${VERSION}
+```
+
+
## Prepare Vote email on the Apache Airflow release candidate
- Use the dev/airflow-jira script to generate a list of Airflow JIRAs that were closed in the release.
@@ -247,6 +293,7 @@ Cheers,
<your name>
```
+
# Verify the release candidate by PMCs
The PMCs should verify the releases in order to make sure the release is following the
@@ -478,7 +525,7 @@ You need to migrate the RC artifacts that passed to this repository:
https://dist.apache.org/repos/dist/release/airflow/
(The migration should include renaming the files so that they no longer have the RC number in their filenames.)
-The best way of doing this is to svn cp between the two repos (this avoids having to upload the binaries again, and gives a clearer history in the svn commit logs):
+The best way of doing this is to svn cp between the two repos (this avoids having to upload the binaries again, and gives a clearer history in the svn commit logs):
```shell script
# First clone the repo
@@ -548,6 +595,59 @@ At this point we release an official package:
- Update CHANGELOG.md with the details, and commit it.
+- Re-Tag & Push the constraints files with the final release version.
+
+ ```shell script
+ git checkout constraints-${RC}
+ git tag -s "constraints-${VERSION}"
+ git push origin "constraints-${VERSION}"
+ ```
+
+- Push Tag for the final version
+
+ This step should only be done now and not before, because it triggers an automated build of
+ the production docker image, using the packages that are currently released in PyPI
+ (both airflow and latest provider packages).
+
+ ```shell script
+ git push origin ${VERSION}
+ ```
+
+## \[Optional\] - Manually prepare production Docker Image
+
+Production Docker images should be automatically built in 2-3 hours after the release tag has been
+pushed. If this did not happen - please login to DockerHub and check the status of builds:
+[Build Timeline](https://hub.docker.com/repository/docker/apache/airflow/timeline)
+
+In case you need, you can also build and push the images manually:
+
+Airflow 2+:
+
+```shell script
+export DOCKER_REPO=docker.io/apache/airflow
+for python_version in "3.6" "3.7" "3.8"
+(
+ export DOCKER_TAG=${VERSION}-python${python_version}
+ ./scripts/ci/images/ci_build_dockerhub.sh
+)
+```
+
+This will wipe Breeze cache and docker-context-files in order to make sure the build is "clean".
+
+
+Airflow 1.10:
+
+```shell script
+for python_version in "2.7" "3.5" "3.6" "3.7" "3.8"
+do
+ ./breeze build-image --production-image --python ${python_version} \
+ --image-tag apache/airflow:${VERSION}-python${python_version} --build-cache-local
+ docker push apache/airflow:${VERSION}-python${python_version}
+done
+docker tag apache/airflow:${VERSION}-python3.6 apache/airflow:${VERSION}
+docker push apache/airflow:${VERSION}
+```
+
## Publish documentation
Documentation is an essential part of the product and should be made available to users.
diff --git a/docs/apache-airflow/production-deployment.rst b/docs/apache-airflow/production-deployment.rst
index b7ff8d1..1105b66 100644
--- a/docs/apache-airflow/production-deployment.rst
+++ b/docs/apache-airflow/production-deployment.rst
@@ -220,7 +220,7 @@ additional apt dev and runtime dependencies.
docker build . \
--build-arg PYTHON_BASE_IMAGE="python:3.7-slim-buster" \
--build-arg PYTHON_MAJOR_MINOR_VERSION=3.7 \
- --build-arg AIRFLOW_INSTALL_SOURCES="apache-airflow" \
+ --build-arg AIRFLOW_INSTALLATION_METHOD="apache-airflow" \
--build-arg AIRFLOW_VERSION="1.10.14" \
--build-arg AIRFLOW_INSTALL_VERSION="==1.10.14" \
--build-arg AIRFLOW_CONSTRAINTS_REFERENCE="constraints-1-10" \
@@ -253,7 +253,7 @@ based on example in `this comment <https://github.com/apache/airflow/issues/8605
docker build . -f Dockerfile \
--build-arg PYTHON_BASE_IMAGE="python:3.7-slim-buster" \
--build-arg PYTHON_MAJOR_MINOR_VERSION=3.7 \
- --build-arg AIRFLOW_INSTALL_SOURCES="apache-airflow" \
+ --build-arg AIRFLOW_INSTALLATION_METHOD="apache-airflow" \
--build-arg AIRFLOW_VERSION="1.10.14" \
--build-arg AIRFLOW_INSTALL_VERSION="==1.10.14" \
--build-arg AIRFLOW_CONSTRAINTS_REFERENCE="constraints-1-10" \
@@ -333,7 +333,7 @@ or
docker build . \
--build-arg PYTHON_BASE_IMAGE="python:3.7-slim-buster" \
--build-arg PYTHON_MAJOR_MINOR_VERSION=3.7 \
- --build-arg AIRFLOW_INSTALL_SOURCES="apache-airflow" \
+ --build-arg AIRFLOW_INSTALLATION_METHOD="apache-airflow" \
--build-arg AIRFLOW_VERSION="1.10.14" \
--build-arg AIRFLOW_INSTALL_VERSION="==1.10.14" \
--build-arg AIRFLOW_CONSTRAINTS_REFERENCE="constraints-1-10" \
@@ -476,18 +476,18 @@ The following build arguments (``--build-arg`` in docker build command) can be u
+------------------------------------------+------------------------------------------+------------------------------------------+
| Build argument | Default value | Description |
+==========================================+==========================================+==========================================+
-| ``PYTHON_BASE_IMAGE`` | ``python:3.6-slim-buster`` | Base python image |
+| ``PYTHON_BASE_IMAGE`` | ``python:3.6-slim-buster`` | Base python image. |
+------------------------------------------+------------------------------------------+------------------------------------------+
| ``PYTHON_MAJOR_MINOR_VERSION`` | ``3.6`` | major/minor version of Python (should |
-| | | match base image) |
+| | | match base image). |
+------------------------------------------+------------------------------------------+------------------------------------------+
-| ``AIRFLOW_VERSION`` | ``2.0.0.dev0`` | version of Airflow |
+| ``AIRFLOW_VERSION`` | ``2.0.0.dev0`` | version of Airflow. |
+------------------------------------------+------------------------------------------+------------------------------------------+
| ``AIRFLOW_REPO`` | ``apache/airflow`` | the repository from which PIP |
-| | | dependencies are pre-installed |
+| | | dependencies are pre-installed. |
+------------------------------------------+------------------------------------------+------------------------------------------+
| ``AIRFLOW_BRANCH`` | ``master`` | the branch from which PIP dependencies |
-| | | are pre-installed initially |
+| | | are pre-installed initially. |
+------------------------------------------+------------------------------------------+------------------------------------------+
| ``AIRFLOW_CONSTRAINTS_LOCATION`` | | If not empty, it will override the |
| | | source of the constraints with the |
@@ -495,101 +495,103 @@ The following build arguments (``--build-arg`` in docker build command) can be u
| | | file has to be in docker context so |
| | | it's best to place such file in |
| | | one of the folders included in |
-| | | .dockerignore |
+| | | .dockerignore. |
+------------------------------------------+------------------------------------------+------------------------------------------+
-| ``AIRFLOW_CONSTRAINTS_REFERENCE`` | ``constraints-master`` | reference (branch or tag) from GitHub |
-| | | repository from which constraints are |
-| | | used. By default it is set to |
-| | | ``constraints-master`` but can be |
-| | | ``constraints-1-10`` for 1.10.* versions |
-| | | or it could point to specific version |
-| | | for example ``constraints-1.10.14`` |
+| ``AIRFLOW_CONSTRAINTS_REFERENCE`` | ``constraints-master`` | Reference (branch or tag) from GitHub |
+| | | where constraints file is taken from |
+| | | It can be ``constraints-master`` but |
+| | | also can be ``constraints-1-10`` for |
+| | | 1.10.* installation. In case of building |
+| | | specific version you want to point it |
+| | | to specific tag, for example |
+| | | ``constraints-1.10.14``. |
+------------------------------------------+------------------------------------------+------------------------------------------+
-| ``INSTALL_PROVIDERS_FROM_SOURCES`` | ``true`` | If set to false and image is built from |
-| | | sources, all provider packages are not |
-| | | installed. By default when building from |
-| | | sources, all provider packages are also |
-| | | installed together with the core airflow |
-| | | package. It has no effect when |
+| ``INSTALL_PROVIDERS_FROM_SOURCES`` | ``false`` | If set to ``true`` and image is built |
+| | | from sources, all provider packages are |
+| | | installed from sources rather than from |
+| | | packages. It has no effect when |
| | | installing from PyPI or GitHub repo. |
+------------------------------------------+------------------------------------------+------------------------------------------+
| ``AIRFLOW_EXTRAS`` | (see Dockerfile) | Default extras with which airflow is |
-| | | installed |
+| | | installed. |
+------------------------------------------+------------------------------------------+------------------------------------------+
| ``INSTALL_FROM_PYPI`` | ``true`` | If set to true, Airflow is installed |
-| | | from pypi. if you want to install |
-| | | Airflow from externally provided binary |
-| | | package you can set it to false, place |
-| | | the package in ``docker-context-files`` |
-| | | and set |
+| | | from PyPI. if you want to install |
+| | | Airflow from self-build package |
+| | | you can set it to false, put package in |
+| | | ``docker-context-files`` and set |
| | | ``INSTALL_FROM_DOCKER_CONTEXT_FILES`` to |
-| | | true. For this you have to also set the |
+| | | ``true``. For this you have to also keep |
| | | ``AIRFLOW_PRE_CACHED_PIP_PACKAGES`` flag |
-| | | to false |
+| | | set to ``false``. |
+------------------------------------------+------------------------------------------+------------------------------------------+
-| ``AIRFLOW_PRE_CACHED_PIP_PACKAGES`` | ``true`` | Allows to pre-cache airflow PIP packages |
+| ``AIRFLOW_PRE_CACHED_PIP_PACKAGES`` | ``false`` | Allows to pre-cache airflow PIP packages |
| | | from the GitHub of Apache Airflow |
| | | This allows to optimize iterations for |
-| | | Image builds and speeds up CI builds |
-| | | But in some corporate environments it |
-| | | might be forbidden to download anything |
-| | | from public repositories. |
+| | | Image builds and speeds up CI builds. |
+------------------------------------------+------------------------------------------+------------------------------------------+
-| ``INSTALL_FROM_DOCKER_CONTEXT_FILES`` | ``false`` | If set to true, Airflow and it's |
-| | | dependencies are installed during build |
-| | | from locally downloaded .whl .tar.gz |
-| | | files placed in the |
-| | | ``docker-context-files``. |
+| ``INSTALL_FROM_DOCKER_CONTEXT_FILES`` | ``false`` | If set to true, Airflow, providers and |
+| | | all dependencies are installed from |
+| | | from locally built/downloaded |
+| | | .whl and .tar.gz files placed in the |
+| | | ``docker-context-files``. In certain |
+| | | corporate environments, this is required |
+| | | to install airflow from such pre-vetted |
+| | | packages rather than from PyPI. For this |
+| | | to work, also set ``INSTALL_FROM_PYPI`` |
+| | | to false. |
+------------------------------------------+------------------------------------------+------------------------------------------+
| ``ADDITIONAL_AIRFLOW_EXTRAS`` | | Optional additional extras with which |
-| | | airflow is installed |
+| | | airflow is installed. |
+------------------------------------------+------------------------------------------+------------------------------------------+
| ``ADDITIONAL_PYTHON_DEPS`` | | Optional python packages to extend |
-| | | the image with some extra dependencies |
+| | | the image with some extra dependencies. |
+------------------------------------------+------------------------------------------+------------------------------------------+
| ``DEV_APT_COMMAND`` | (see Dockerfile) | Dev apt command executed before dev deps |
-| | | are installed in the Build image |
+| | | are installed in the Build image. |
+------------------------------------------+------------------------------------------+------------------------------------------+
| ``ADDITIONAL_DEV_APT_COMMAND`` | | Additional Dev apt command executed |
| | | before dev dep are installed |
-| | | in the Build image. Should start with && |
+| | | in the Build image. Should start with |
+| | | ``&&``. |
+------------------------------------------+------------------------------------------+------------------------------------------+
| ``DEV_APT_DEPS`` | (see Dockerfile) | Dev APT dependencies installed |
-| | | in the Build image |
+| | | in the Build image. |
+------------------------------------------+------------------------------------------+------------------------------------------+
| ``ADDITIONAL_DEV_APT_DEPS`` | | Additional apt dev dependencies |
-| | | installed in the Build image |
+| | | installed in the Build image. |
+------------------------------------------+------------------------------------------+------------------------------------------+
| ``ADDITIONAL_DEV_APT_ENV`` | | Additional env variables defined |
-| | | when installing dev deps |
+| | | when installing dev deps. |
+------------------------------------------+------------------------------------------+------------------------------------------+
| ``RUNTIME_APT_COMMAND`` | (see Dockerfile) | Runtime apt command executed before deps |
-| | | are installed in the Main image |
+| | | are installed in the Main image. |
+------------------------------------------+------------------------------------------+------------------------------------------+
| ``ADDITIONAL_RUNTIME_APT_COMMAND`` | | Additional Runtime apt command executed |
| | | before runtime dep are installed |
-| | | in the Main image. Should start with && |
+| | | in the Main image. Should start with |
+| | | ``&&``. |
+------------------------------------------+------------------------------------------+------------------------------------------+
| ``RUNTIME_APT_DEPS`` | (see Dockerfile) | Runtime APT dependencies installed |
-| | | in the Main image |
+| | | in the Main image. |
+------------------------------------------+------------------------------------------+------------------------------------------+
| ``ADDITIONAL_RUNTIME_APT_DEPS`` | | Additional apt runtime dependencies |
-| | | installed in the Main image |
+| | | installed in the Main image. |
+------------------------------------------+------------------------------------------+------------------------------------------+
| ``ADDITIONAL_RUNTIME_APT_ENV`` | | Additional env variables defined |
-| | | when installing runtime deps |
+| | | when installing runtime deps. |
+------------------------------------------+------------------------------------------+------------------------------------------+
| ``AIRFLOW_HOME`` | ``/opt/airflow`` | Airflow’s HOME (that’s where logs and |
-| | | sqlite databases are stored) |
+| | | sqlite databases are stored). |
+------------------------------------------+------------------------------------------+------------------------------------------+
-| ``AIRFLOW_UID`` | ``50000`` | Airflow user UID |
+| ``AIRFLOW_UID`` | ``50000`` | Airflow user UID. |
+------------------------------------------+------------------------------------------+------------------------------------------+
| ``AIRFLOW_GID`` | ``50000`` | Airflow group GID. Note that most files |
| | | created on behalf of airflow user belong |
| | | to the ``root`` group (0) to keep |
-| | | OpenShift Guidelines compatibility |
+| | | OpenShift Guidelines compatibility. |
+------------------------------------------+------------------------------------------+------------------------------------------+
-| ``AIRFLOW_USER_HOME_DIR`` | ``/home/airflow`` | Home directory of the Airflow user |
+| ``AIRFLOW_USER_HOME_DIR`` | ``/home/airflow`` | Home directory of the Airflow user. |
+------------------------------------------+------------------------------------------+------------------------------------------+
| ``CASS_DRIVER_BUILD_CONCURRENCY`` | ``8`` | Number of processors to use for |
| | | cassandra PIP install (speeds up |
@@ -598,7 +600,7 @@ The following build arguments (``--build-arg`` in docker build command) can be u
+------------------------------------------+------------------------------------------+------------------------------------------+
| ``INSTALL_MYSQL_CLIENT`` | ``true`` | Whether MySQL client should be installed |
| | | The mysql extra is removed from extras |
-| | | if the client is not installed |
+| | | if the client is not installed. |
+------------------------------------------+------------------------------------------+------------------------------------------+
There are build arguments that determine the installation mechanism of Apache Airflow for the
@@ -608,59 +610,33 @@ production image. There are three types of build:
* You can build the image from released PyPi airflow package (used to build the official Docker image)
* You can build the image from any version in GitHub repository(this is used mostly for system testing).
-+-----------------------------------+-----------------------------------+
-| Build argument | What to specify |
-+===================================+===================================+
-| ``AIRFLOW_INSTALL_SOURCES`` | Should point to the sources of |
-| | of Apache Airflow. It can be |
-| | either "." for installation from |
-| | local sources, "apache-airflow" |
-| | for installation from packages |
-| | and URL to installation from |
-| | GitHub repository (see below) |
-| | to install from any GitHub |
-| | version |
-+-----------------------------------+-----------------------------------+
-| ``AIRFLOW_INSTALL_VERSION`` | Optional - might be used for |
-| | package installation case to |
-| | set Airflow version for example |
-| | "==1.10.14". Remember to also |
-| | Set ``AIRFLOW_VERSION`` |
-| | when you use it. |
-+-----------------------------------+-----------------------------------+
-| ``AIRFLOW_CONSTRAINTS_REFERENCE`` | reference (branch or tag) from |
-| | GitHub where constraints file |
-| | is taken from. By default it is |
-| | ``constraints-master`` but can be |
-| | ``constraints-1-10`` for 1.10.* |
-| | constraint or if you want to |
-| | point to specific version |
-| | might be ``constraints-1.10.14`` |
-+-----------------------------------+-----------------------------------+
-| ``SLUGIFY_USES_TEXT_UNIDECODE`` | In case of of installing airflow |
-| | 1.10.2 or 1.10.1 you need to |
-| | set this arg to ``yes``. |
-+-----------------------------------+-----------------------------------+
-| ``AIRFLOW_WWW`` | In case of Airflow 2.0 it should |
-| | be "www", in case of Airflow 1.10 |
-| | series it should be "www_rbac". |
-| | See examples below |
-+-----------------------------------+-----------------------------------+
-| ``AIRFLOW_SOURCES_FROM`` | Sources of Airflow. Set it to |
-| | "empty" to avoid costly |
-| | Docker context copying |
-| | in case of installation from |
-| | the package or from GitHub URL. |
-| | See examples below |
-+-----------------------------------+-----------------------------------+
-| ``AIRFLOW_SOURCES_TO`` | Target for Airflow sources. Set |
-| | to "/empty" to avoid costly |
-| | Docker context copying |
-| | in case of installation from |
-| | the package or from GitHub URL. |
-| | See examples below |
-+-----------------------------------+-----------------------------------+
-
++-----------------------------------+------------------------+-----------------------------------------------------------------------------------+
+| Build argument | Default | What to specify |
++===================================+========================+===================================================================================+
+| ``AIRFLOW_INSTALLATION_METHOD`` | ``apache-airflow`` | Should point to the installation method of Apache Airflow. It can be |
+| | | ``apache-airflow`` for installation from packages and URL to installation from |
+| | | GitHub repository tag or branch or "." to install from sources. |
+| | | Note that installing from local sources requires appropriate values of the |
+| | | ``AIRFLOW_SOURCES_FROM`` and ``AIRFLOW_SOURCES_TO`` variables as described below. |
++-----------------------------------+------------------------+-----------------------------------------------------------------------------------+
+| ``AIRFLOW_INSTALL_VERSION`` | | Optional - might be used for package installation of different Airflow version |
+| | | for example"==1.10.14". For consistency, you should also set``AIRFLOW_VERSION`` |
+| | | to the same value AIRFLOW_VERSION is embedded as label in the image created. |
++-----------------------------------+------------------------+-----------------------------------------------------------------------------------+
+| ``AIRFLOW_CONSTRAINTS_REFERENCE`` | ``constraints-master`` | Reference (branch or tag) from GitHub where constraints file is taken from. |
+| | | It can be ``constraints-master`` but also can be``constraints-1-10`` for |
+| | | 1.10.* installations. In case of building specific version |
+| | | you want to point it to specific tag, for example ``constraints-1.10.14`` |
++-----------------------------------+------------------------+-----------------------------------------------------------------------------------+
+| ``AIRFLOW_WWW`` | ``www`` | In case of Airflow 2.0 it should be "www", in case of Airflow 1.10 |
+| | | series it should be "www_rbac". |
++-----------------------------------+------------------------+-----------------------------------------------------------------------------------+
+| ``AIRFLOW_SOURCES_FROM`` | ``empty`` | Sources of Airflow. Set it to "." when you install airflow from |
+| | | local sources. |
++-----------------------------------+------------------------+-----------------------------------------------------------------------------------+
+| ``AIRFLOW_SOURCES_TO`` | ``/empty`` | Target for Airflow sources. Set to "/opt/airflow" when |
+| | | you want to install airflow from local sources. |
++-----------------------------------+------------------------+-----------------------------------------------------------------------------------+
This builds production image in version 3.6 with default extras from the local sources (master version
of 2.0 currently):
@@ -677,7 +653,7 @@ constraints taken from constraints-1-10-12 branch in GitHub.
docker build . \
--build-arg PYTHON_BASE_IMAGE="python:3.7-slim-buster" \
--build-arg PYTHON_MAJOR_MINOR_VERSION=3.7 \
- --build-arg AIRFLOW_INSTALL_SOURCES="https://github.com/apache/airflow/archive/1.10.14.tar.gz#egg=apache-airflow" \
+ --build-arg AIRFLOW_INSTALLATION_METHOD="https://github.com/apache/airflow/archive/1.10.14.tar.gz#egg=apache-airflow" \
--build-arg AIRFLOW_CONSTRAINTS_REFERENCE="constraints-1-10" \
--build-arg AIRFLOW_BRANCH="v1-10-test" \
--build-arg AIRFLOW_SOURCES_FROM="empty" \
@@ -692,7 +668,7 @@ of v1-10-test branch.
docker build . \
--build-arg PYTHON_BASE_IMAGE="python:3.7-slim-buster" \
--build-arg PYTHON_MAJOR_MINOR_VERSION=3.7 \
- --build-arg AIRFLOW_INSTALL_SOURCES="apache-airflow" \
+ --build-arg AIRFLOW_INSTALLATION_METHOD="apache-airflow" \
--build-arg AIRFLOW_VERSION="1.10.14" \
--build-arg AIRFLOW_INSTALL_VERSION="==1.10.14" \
--build-arg AIRFLOW_BRANCH="v1-10-test" \
@@ -708,7 +684,7 @@ additional python dependencies and pre-installed pip dependencies from 1.10.14 t
docker build . \
--build-arg PYTHON_BASE_IMAGE="python:3.7-slim-buster" \
--build-arg PYTHON_MAJOR_MINOR_VERSION=3.7 \
- --build-arg AIRFLOW_INSTALL_SOURCES="apache-airflow" \
+ --build-arg AIRFLOW_INSTALLATION_METHOD="apache-airflow" \
--build-arg AIRFLOW_VERSION="1.10.14" \
--build-arg AIRFLOW_INSTALL_VERSION="==1.10.14" \
--build-arg AIRFLOW_BRANCH="v1-10-test" \
@@ -726,7 +702,7 @@ additional apt dev and runtime dependencies.
docker build . \
--build-arg PYTHON_BASE_IMAGE="python:3.7-slim-buster" \
--build-arg PYTHON_MAJOR_MINOR_VERSION=3.7 \
- --build-arg AIRFLOW_INSTALL_SOURCES="apache-airflow" \
+ --build-arg AIRFLOW_INSTALLATION_METHOD="apache-airflow" \
--build-arg AIRFLOW_VERSION="1.10.14" \
--build-arg AIRFLOW_INSTALL_VERSION="==1.10.14" \
--build-arg AIRFLOW_CONSTRAINTS_REFERENCE="constraints-1-10" \
diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt
index f97c5fd..5e01212 100644
--- a/docs/spelling_wordlist.txt
+++ b/docs/spelling_wordlist.txt
@@ -443,6 +443,7 @@ artwr
asc
ascii
asciiart
+asctime
ashb
asia
assertEqualIgnoreMultipleSpaces
diff --git a/scripts/ci/images/ci_build_dockerhub.sh b/scripts/ci/images/ci_build_dockerhub.sh
index 3b1ce3d..294ebcc 100755
--- a/scripts/ci/images/ci_build_dockerhub.sh
+++ b/scripts/ci/images/ci_build_dockerhub.sh
@@ -49,46 +49,83 @@ echo "DOCKER_TAG=${DOCKER_TAG}"
echo "Detected PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION}"
echo
-(
- export INSTALL_FROM_PYPI="true"
- export INSTALL_FROM_DOCKER_CONTEXT_FILES="false"
- export INSTALL_PROVIDERS_FROM_SOURCES="true"
- export AIRFLOW_PRE_CACHED_PIP_PACKAGES="true"
- export DOCKER_CACHE="pulled"
- # shellcheck source=scripts/ci/libraries/_script_init.sh
- . "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
-
+if [[ ! "${DOCKER_TAG}" =~ ^[0-9].* ]]; then
echo
- echo "Building and pushing CI image for ${PYTHON_MAJOR_MINOR_VERSION} in a sub-process"
+ echo "Building airflow from branch or non-release tag: ${DOCKER_TAG}"
echo
- rm -rf "${BUILD_CACHE_DIR}"
- build_images::prepare_ci_build
- build_images::rebuild_ci_image_if_needed
- if [[ ! "${DOCKER_TAG}" =~ ^[0-9].* ]]; then
- # Do not push if we are building a tagged version
- push_pull_remove_images::push_ci_images
- fi
-)
+ # Only build and push CI image for the nightly-master, v1-10-test and v2-0-test branches
+ # for tagged releases we build everything from PyPI, so we do not need CI images
+ # For development images, we have to build all packages from current sources because we want to produce
+ # `Latest and greatest` image from those branches. We need to build and push CI image as well as PROD
+ # image but we need to build CI image first, in order to use it to prepare provider packages
+ # The CI image provides an environment where we can reproducibly download the right .whl packages
+ # and build the provider packages and then build the production image using those .whl packages
+ # prepared. This is as close as it can get to production images - everything is build from
+ # packages, but not from PyPI - those packages are built locally using the latest sources!
-(
- export INSTALL_FROM_PYPI="false"
- export INSTALL_FROM_DOCKER_CONTEXT_FILES="true"
+ # Note - we need sub-processes here, because we can run _script_init.sh only once per process
+ # and it determines how to build the image - since we are building two images here
+ # we need to run those in sub-processes
+ (
+ export INSTALL_FROM_PYPI="true"
+ export INSTALL_FROM_DOCKER_CONTEXT_FILES="false"
+ export INSTALL_PROVIDERS_FROM_SOURCES="true"
+ export AIRFLOW_PRE_CACHED_PIP_PACKAGES="true"
+ export DOCKER_CACHE="pulled"
+ # shellcheck source=scripts/ci/libraries/_script_init.sh
+ . "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
+ echo
+ echo "Building and pushing CI image for ${PYTHON_MAJOR_MINOR_VERSION} in a sub-process"
+ echo
+ rm -rf "${BUILD_CACHE_DIR}"
+ rm -rf "${AIRFLOW_SOURCES}/docker-context-files/*"
+ build_images::prepare_ci_build
+ build_images::rebuild_ci_image_if_needed
+ push_pull_remove_images::push_ci_images
+ )
+ (
+ export INSTALL_FROM_PYPI="false"
+ export INSTALL_FROM_DOCKER_CONTEXT_FILES="true"
+ export INSTALL_PROVIDERS_FROM_SOURCES="false"
+ export AIRFLOW_PRE_CACHED_PIP_PACKAGES="false"
+ export DOCKER_CACHE="pulled"
+ # shellcheck source=scripts/ci/libraries/_script_init.sh
+ . "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
+ echo
+ echo "Building and pushing PROD image for ${PYTHON_MAJOR_MINOR_VERSION} in a sub-process"
+ echo
+ rm -rf "${BUILD_CACHE_DIR}"
+ rm -rf "${AIRFLOW_SOURCES}/docker-context-files/*"
+ build_images::prepare_prod_build
+ build_images::build_prod_images_from_packages
+ push_pull_remove_images::push_prod_images
+ )
+else
+ echo
+ echo "Building airflow from release tag: ${DOCKER_TAG}"
+ echo
+ # This is an imaae built from the "release" tag (either RC or final one).
+ # In this case all packages are taken from PyPI rather than from locally built sources
+ export INSTALL_FROM_PYPI="true"
+ export INSTALL_FROM_DOCKER_CONTEXT_FILES="false"
export INSTALL_PROVIDERS_FROM_SOURCES="false"
export AIRFLOW_PRE_CACHED_PIP_PACKAGES="false"
- export DOCKER_CACHE="pulled"
+ export DOCKER_CACHE="local"
+ # Name the image based on the TAG rather than based on the branch name
+ export FORCE_AIRFLOW_PROD_BASE_TAG="${DOCKER_TAG}"
+ export AIRFLOW_CONSTRAINTS_REFERENCE="constraints-${INSTALL_AIRFLOW_VERSION}"
+ export AIRFLOW_SOURCES_FROM="empty"
+ export AIRFLOW_SOURCES_TO="/empty"
+ export INSTALL_AIRFLOW_VERSION="${DOCKER_TAG%-python*}"
- if [[ "${DOCKER_TAG}" =~ ^[0-9].* ]]; then
- # Disable cache and set name of the tag as image name if we are building a tagged version
- export DOCKER_CACHE="disabled"
- export FORCE_AIRFLOW_PROD_BASE_TAG="${DOCKER_TAG}"
- fi
# shellcheck source=scripts/ci/libraries/_script_init.sh
. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
echo
echo "Building and pushing PROD image for ${PYTHON_MAJOR_MINOR_VERSION} in a sub-process"
echo
rm -rf "${BUILD_CACHE_DIR}"
+ rm -rf "${AIRFLOW_SOURCES}/docker-context-files/*"
build_images::prepare_prod_build
- build_images::build_prod_images_from_packages
+ build_images::build_prod_images
push_pull_remove_images::push_prod_images
-)
+fi
diff --git a/scripts/ci/libraries/_build_images.sh b/scripts/ci/libraries/_build_images.sh
index ddecb11..f2b30e1 100644
--- a/scripts/ci/libraries/_build_images.sh
+++ b/scripts/ci/libraries/_build_images.sh
@@ -30,34 +30,53 @@ function build_images::add_build_args_for_remote_install() {
EXTRA_DOCKER_PROD_BUILD_FLAGS+=(
"--build-arg" "AIRFLOW_CONSTRAINTS_REFERENCE=${AIRFLOW_CONSTRAINTS_REFERENCE}"
)
- fi
- if [[ "${AIRFLOW_CONSTRAINTS_LOCATION}" != "" ]]; then
- EXTRA_DOCKER_PROD_BUILD_FLAGS+=(
- "--build-arg" "AIRFLOW_CONSTRAINTS_LOCATION=${AIRFLOW_CONSTRAINTS_LOCATION}"
- )
- fi
- if [[ ${AIRFLOW_VERSION} =~ [^0-9]*1[^0-9]*10[^0-9]([0-9]*) ]]; then
- # All types of references/versions match this regexp for 1.10 series
- # for example v1_10_test, 1.10.10, 1.10.9 etc. ${BASH_REMATCH[1]} matches last
- # minor digit of version and it's length is 0 for v1_10_test, 1 for 1.10.9 and 2 for 1.10.10+
- AIRFLOW_MINOR_VERSION_NUMBER=${BASH_REMATCH[1]}
- if [[ ${#AIRFLOW_MINOR_VERSION_NUMBER} == "0" ]]; then
- # For v1_10_* branches use constraints-1-10 branch
+ else
+ if [[ ${AIRFLOW_VERSION} =~ [^0-9]*1[^0-9]*10[^0-9]([0-9]*) ]]; then
+ # All types of references/versions match this regexp for 1.10 series
+ # for example v1_10_test, 1.10.10, 1.10.9 etc. ${BASH_REMATCH[1]} matches last
+ # minor digit of version and it's length is 0 for v1_10_test, 1 for 1.10.9 and 2 for 1.10.10+
+ AIRFLOW_MINOR_VERSION_NUMBER=${BASH_REMATCH[1]}
+ if [[ ${#AIRFLOW_MINOR_VERSION_NUMBER} == "0" ]]; then
+ # For v1_10_* branches use constraints-1-10 branch
+ EXTRA_DOCKER_PROD_BUILD_FLAGS+=(
+ "--build-arg" "AIRFLOW_CONSTRAINTS_REFERENCE=constraints-1-10"
+ )
+ else
+ EXTRA_DOCKER_PROD_BUILD_FLAGS+=(
+ # For specified minor version of 1.10 or v1 branch use specific reference constraints
+ "--build-arg" "AIRFLOW_CONSTRAINTS_REFERENCE=constraints-${AIRFLOW_VERSION}"
+ )
+ fi
+ elif [[ ${AIRFLOW_VERSION} =~ v?2.* ]]; then
EXTRA_DOCKER_PROD_BUILD_FLAGS+=(
- "--build-arg" "AIRFLOW_CONSTRAINTS_REFERENCE=constraints-1-10"
+ # For specified minor version of 2.0 or v2 branch use specific reference constraints
+ "--build-arg" "AIRFLOW_CONSTRAINTS_REFERENCE=constraints-${AIRFLOW_VERSION}"
)
else
+ # For all other we just get the default constraint branch coming from the _initialization.sh
EXTRA_DOCKER_PROD_BUILD_FLAGS+=(
- # For specified minor version of 1.10 use specific reference constraints
- "--build-arg" "AIRFLOW_CONSTRAINTS_REFERENCE=constraints-${AIRFLOW_VERSION}"
+ "--build-arg" "AIRFLOW_CONSTRAINTS_REFERENCE=${DEFAULT_CONSTRAINTS_BRANCH}"
)
fi
- AIRFLOW_BRANCH_FOR_PYPI_PRELOADING="v1-10-test"
- else
- # For all other (master, 2.0+) we just get the default constraint branch
+ fi
+ if [[ "${AIRFLOW_CONSTRAINTS_LOCATION}" != "" ]]; then
EXTRA_DOCKER_PROD_BUILD_FLAGS+=(
- "--build-arg" "AIRFLOW_CONSTRAINTS_REFERENCE=${DEFAULT_CONSTRAINTS_BRANCH}"
+ "--build-arg" "AIRFLOW_CONSTRAINTS_LOCATION=${AIRFLOW_CONSTRAINTS_LOCATION}"
)
+ fi
+ # Depending on the version built, we choose the right branch for preloading the packages from
+ # If we run build for v1-10-test builds we should choose v1-10-test, for v2-0-test we choose v2-0-test
+ # all other builds when you choose a specific version (1.0 or 2.0 series) should choose stable branch
+ # to preload. For all other builds we use the default branch defined in _initialization.sh
+ if [[ ${AIRFLOW_VERSION} == 'v1-10-test' ]]; then
+ AIRFLOW_BRANCH_FOR_PYPI_PRELOADING="v1-10-test"
+ elif [[ ${AIRFLOW_VERSION} =~ v?1.* ]]; then
+ AIRFLOW_BRANCH_FOR_PYPI_PRELOADING="v1-10-stable"
+ elif [[ ${AIRFLOW_VERSION} == 'v2-0-test' ]]; then
+ AIRFLOW_BRANCH_FOR_PYPI_PRELOADING="v2-0-test"
+ elif [[ ${AIRFLOW_VERSION} =~ v?2.* ]]; then
+ AIRFLOW_BRANCH_FOR_PYPI_PRELOADING="v2-0-stable"
+ else
AIRFLOW_BRANCH_FOR_PYPI_PRELOADING=${DEFAULT_BRANCH}
fi
}
@@ -663,23 +682,18 @@ function build_images::prepare_prod_build() {
if [[ -n "${INSTALL_AIRFLOW_REFERENCE=}" ]]; then
# When --install-airflow-reference is used then the image is build from GitHub tag
EXTRA_DOCKER_PROD_BUILD_FLAGS=(
- "--build-arg" "AIRFLOW_INSTALL_SOURCES=https://github.com/apache/airflow/archive/${INSTALL_AIRFLOW_REFERENCE}.tar.gz#egg=apache-airflow"
+ "--build-arg" "AIRFLOW_INSTALLATION_METHOD=https://github.com/apache/airflow/archive/${INSTALL_AIRFLOW_REFERENCE}.tar.gz#egg=apache-airflow"
)
export AIRFLOW_VERSION="${INSTALL_AIRFLOW_REFERENCE}"
build_images::add_build_args_for_remote_install
elif [[ -n "${INSTALL_AIRFLOW_VERSION=}" ]]; then
# When --install-airflow-version is used then the image is build from PIP package
EXTRA_DOCKER_PROD_BUILD_FLAGS=(
- "--build-arg" "AIRFLOW_INSTALL_SOURCES=apache-airflow"
+ "--build-arg" "AIRFLOW_INSTALLATION_METHOD=apache-airflow"
"--build-arg" "AIRFLOW_INSTALL_VERSION===${INSTALL_AIRFLOW_VERSION}"
"--build-arg" "AIRFLOW_VERSION=${INSTALL_AIRFLOW_VERSION}"
)
export AIRFLOW_VERSION="${INSTALL_AIRFLOW_VERSION}"
- if [[ ${AIRFLOW_VERSION} == "1.10.2" || ${AIRFLOW_VERSION} == "1.10.1" ]]; then
- EXTRA_DOCKER_PROD_BUILD_FLAGS+=(
- "--build-arg" "SLUGIFY_USES_TEXT_UNIDECODE=yes"
- )
- fi
build_images::add_build_args_for_remote_install
else
# When no airflow version/reference is specified, production image is built from local sources
diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh
index fa138e1..a30e186 100644
--- a/scripts/ci/libraries/_initialization.sh
+++ b/scripts/ci/libraries/_initialization.sh
@@ -175,11 +175,25 @@ function initialization::initialize_base_variables() {
export INSTALLED_EXTRAS="async,amazon,celery,cncf.kubernetes,docker,dask,elasticsearch,ftp,grpc,hashicorp,http,imap,google,microsoft.azure,mysql,postgres,redis,sendgrid,sftp,slack,ssh,statsd,virtualenv"
readonly INSTALLED_EXTRAS
- PIP_VERSION="20.2.4"
+ # default version of PIP USED (This has to be < 20.3 until https://github.com/apache/airflow/issues/12838 is solved)
+ PIP_VERSION=${PIP_VERSION:="20.2.4"}
export PIP_VERSION
- WHEEL_VERSION="0.35.1"
+ # We also pin version of wheel used to get consistent builds
+ WHEEL_VERSION=${WHEEL_VERSION:="0.36.1"}
export WHEEL_VERSION
+
+ # Sources by default are installed from local sources when using breeze/ci
+ AIRFLOW_SOURCES_FROM=${AIRFLOW_SOURCES_FROM:="."}
+ export AIRFLOW_SOURCES_FROM
+
+ # They are copied to /opt/airflow by default (breeze and ci)
+ AIRFLOW_SOURCES_TO=${AIRFLOW_SOURCES_TO:="/opt/airflow"}
+ export AIRFLOW_SOURCES_TO
+
+ # And installed from there (breeze and ci)
+ AIRFLOW_INSTALL_VERSION=${AIRFLOW_INSTALL_VERSION:="."}
+ export AIRFLOW_INSTALL_VERSION
}
# Determine current branch
diff --git a/scripts/in_container/_in_container_utils.sh b/scripts/in_container/_in_container_utils.sh
index 822200b..90311f3 100644
--- a/scripts/in_container/_in_container_utils.sh
+++ b/scripts/in_container/_in_container_utils.sh
@@ -327,14 +327,11 @@ function install_released_airflow_version() {
local version="${1}"
local extras="${2}"
echo
- echo "Installing released ${1} version of airflow with extras ${2}"
+ echo "Installing released ${version} version of airflow with extras ${extras}"
echo
- if [[ ${version} == "1.10.2" || ${version} == "1.10.1" ]]; then
- export SLUGIFY_USES_TEXT_UNIDECODE=yes
- fi
rm -rf "${AIRFLOW_SOURCES}"/*.egg-info
- pip install --upgrade "apache-airflow${extras}==${1}" >"${OUTPUT_PRINTED_ONLY_ON_ERROR}" 2>&1
+ pip install --upgrade "apache-airflow${extras}==${version}" >"${OUTPUT_PRINTED_ONLY_ON_ERROR}" 2>&1
}
function install_all_provider_packages_from_wheels() {