You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2021/02/09 11:12:45 UTC
[airflow] branch v2-0-test updated: Restores flexible installation
version, fixes manual tag build process. (#14107)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v2-0-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/v2-0-test by this push:
new 94ae38f Restores flexible installation version, fixes manual tag build process. (#14107)
94ae38f is described below
commit 94ae38f8a996f2d4a8a11cbd1669440c036acba5
Author: Jarek Potiuk <ja...@polidea.com>
AuthorDate: Fri Feb 5 21:49:31 2021 +0100
Restores flexible installation version, fixes manual tag build process. (#14107)
Revert "Fix Commands to install Airflow in docker/install_airflow.sh (#14099)"
This reverts commit 68758b826076e93fadecf599108a4d304dd87ac7.
Also fixes the docker build script that was the reason for original
attempt to fix it.
(cherry picked from commit 212d5cd315fedeef876750446126bc962047d0cb)
---
Dockerfile | 6 +++---
Dockerfile.ci | 2 +-
IMAGES.rst | 4 ++--
docs/apache-airflow/production-deployment.rst | 18 +++++++++---------
scripts/ci/libraries/_build_images.sh | 2 +-
scripts/ci/libraries/_initialization.sh | 6 +++---
scripts/docker/install_airflow.sh | 11 ++++++-----
7 files changed, 25 insertions(+), 24 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index 8e61ada..78f3b20 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -240,9 +240,9 @@ ARG AIRFLOW_INSTALLATION_METHOD="apache-airflow"
ENV AIRFLOW_INSTALLATION_METHOD=${AIRFLOW_INSTALLATION_METHOD}
# By default latest released version of airflow is installed (when empty) but this value can be overridden
-# and we can install specific version of airflow this way.
-ARG AIRFLOW_INSTALL_VERSION=""
-ENV AIRFLOW_INSTALL_VERSION=${AIRFLOW_INSTALL_VERSION}
+# and we can install version according to specification (For example ==2.0.2 or <3.0.0).
+ARG AIRFLOW_VERSION_SPECIFICATION=""
+ENV AIRFLOW_VERSION_SPECIFICATION=${AIRFLOW_VERSION_SPECIFICATION}
# We can seet this value to true in case we want to install .whl .tar.gz packages placed in the
# docker-context-files folder. This can be done for both - additional packages you want to install
diff --git a/Dockerfile.ci b/Dockerfile.ci
index 646adcf..6d6023f 100644
--- a/Dockerfile.ci
+++ b/Dockerfile.ci
@@ -273,7 +273,7 @@ ENV INSTALL_MYSQL_CLIENT="true"
ENV AIRFLOW_INSTALLATION_METHOD="."
ENV AIRFLOW_INSTALL_USER_FLAG=""
ENV AIRFLOW_INSTALL_EDITABLE_FLAG="--editable"
-ENV AIRFLOW_INSTALL_VERSION=""
+ENV AIRFLOW_VERSION_SPECIFICATION=""
# Setup PIP
# By default PIP install run without cache to make image smaller
diff --git a/IMAGES.rst b/IMAGES.rst
index fd1497a..995a46e 100644
--- a/IMAGES.rst
+++ b/IMAGES.rst
@@ -392,7 +392,7 @@ additional apt dev and runtime dependencies.
--build-arg PYTHON_MAJOR_MINOR_VERSION=3.7 \
--build-arg AIRFLOW_INSTALLATION_METHOD="apache-airflow" \
--build-arg AIRFLOW_VERSION="2.0.0" \
- --build-arg AIRFLOW_INSTALL_VERSION="==2.0.0" \
+ --build-arg AIRFLOW_VERSION_SPECIFICATION="==2.0.0" \
--build-arg AIRFLOW_CONSTRAINTS_REFERENCE="constraints-2-0" \
--build-arg AIRFLOW_SOURCES_FROM="empty" \
--build-arg AIRFLOW_SOURCES_TO="/empty" \
@@ -427,7 +427,7 @@ based on example in `this comment <https://github.com/apache/airflow/issues/8605
--build-arg PYTHON_MAJOR_MINOR_VERSION=3.7 \
--build-arg AIRFLOW_INSTALLATION_METHOD="apache-airflow" \
--build-arg AIRFLOW_VERSION="2.0.0" \
- --build-arg AIRFLOW_INSTALL_VERSION="==2.0.0" \
+ --build-arg AIRFLOW_VERSION_SPECIFICATION="==2.0.0" \
--build-arg AIRFLOW_CONSTRAINTS_REFERENCE="constraints-2-0" \
--build-arg AIRFLOW_SOURCES_FROM="empty" \
--build-arg AIRFLOW_SOURCES_TO="/empty" \
diff --git a/docs/apache-airflow/production-deployment.rst b/docs/apache-airflow/production-deployment.rst
index e737fc5..fb4939c 100644
--- a/docs/apache-airflow/production-deployment.rst
+++ b/docs/apache-airflow/production-deployment.rst
@@ -248,7 +248,7 @@ additional apt dev and runtime dependencies.
--build-arg PYTHON_MAJOR_MINOR_VERSION=3.7 \
--build-arg AIRFLOW_INSTALLATION_METHOD="apache-airflow" \
--build-arg AIRFLOW_VERSION="2.0.0" \
- --build-arg AIRFLOW_INSTALL_VERSION="==2.0.0" \
+ --build-arg AIRFLOW_VERSION_SPECIFICATION="==2.0.0" \
--build-arg AIRFLOW_CONSTRAINTS_REFERENCE="constraints-2-0" \
--build-arg AIRFLOW_SOURCES_FROM="empty" \
--build-arg AIRFLOW_SOURCES_TO="/empty" \
@@ -281,7 +281,7 @@ based on example in `this comment <https://github.com/apache/airflow/issues/8605
--build-arg PYTHON_MAJOR_MINOR_VERSION=3.7 \
--build-arg AIRFLOW_INSTALLATION_METHOD="apache-airflow" \
--build-arg AIRFLOW_VERSION="2.0.0" \
- --build-arg AIRFLOW_INSTALL_VERSION="==2.0.0" \
+ --build-arg AIRFLOW_VERSION_SPECIFICATION="==2.0.0" \
--build-arg AIRFLOW_CONSTRAINTS_REFERENCE="constraints-2-0" \
--build-arg AIRFLOW_SOURCES_FROM="empty" \
--build-arg AIRFLOW_SOURCES_TO="/empty" \
@@ -322,7 +322,7 @@ installation as it is using external installation method.
Note that as a prerequisite - you need to have downloaded wheel files. In the example below we
first download such constraint file locally and then use ``pip download`` to get the .whl files needed
but in most likely scenario, those wheel files should be copied from an internal repository of such .whl
-files. Note that ``AIRFLOW_INSTALL_VERSION`` is only there for reference, the apache airflow .whl file
+files. Note that ``AIRFLOW_VERSION_SPECIFICATION`` is only there for reference, the apache airflow .whl file
in the right version is part of the .whl files downloaded.
Note that 'pip download' will only works on Linux host as some of the packages need to be compiled from
@@ -376,7 +376,7 @@ or
--build-arg PYTHON_MAJOR_MINOR_VERSION=3.7 \
--build-arg AIRFLOW_INSTALLATION_METHOD="apache-airflow" \
--build-arg AIRFLOW_VERSION="2.0.0" \
- --build-arg AIRFLOW_INSTALL_VERSION="==2.0.0" \
+ --build-arg AIRFLOW_VERSION_SPECIFICATION="==2.0.0" \
--build-arg AIRFLOW_CONSTRAINTS_REFERENCE="constraints-2-0" \
--build-arg AIRFLOW_SOURCES_FROM="empty" \
--build-arg AIRFLOW_SOURCES_TO="/empty" \
@@ -682,9 +682,9 @@ production image. There are three types of build:
| | | ``AIRFLOW_SOURCES_FROM`` and ``AIRFLOW_SOURCES_TO`` variables as described below. |
| | | Only used when ``INSTALL_FROM_PYPI`` is set to ``true``. |
+-----------------------------------+------------------------+-----------------------------------------------------------------------------------+
-| ``AIRFLOW_INSTALL_VERSION`` | | Optional - might be used for package installation of different Airflow version |
+| ``AIRFLOW_VERSION_SPECIFICATION`` | | Optional - might be used for package installation of different Airflow version |
| | | for example"==2.0.0". For consistency, you should also set``AIRFLOW_VERSION`` |
-| | | to the same value AIRFLOW_VERSION is embedded as label in the image created. |
+| | | to the same value AIRFLOW_VERSION is resolved as label in the image created. |
+-----------------------------------+------------------------+-----------------------------------------------------------------------------------+
| ``AIRFLOW_CONSTRAINTS_REFERENCE`` | ``constraints-master`` | Reference (branch or tag) from GitHub where constraints file is taken from. |
| | | It can be ``constraints-master`` but also can be``constraints-1-10`` for |
@@ -733,7 +733,7 @@ of v1-10-test branch.
--build-arg PYTHON_MAJOR_MINOR_VERSION=3.7 \
--build-arg AIRFLOW_INSTALLATION_METHOD="apache-airflow" \
--build-arg AIRFLOW_VERSION="2.0.0" \
- --build-arg AIRFLOW_INSTALL_VERSION="==2.0.0" \
+ --build-arg AIRFLOW_VERSION_SPECIFICATION="==2.0.0" \
--build-arg AIRFLOW_BRANCH="v1-10-test" \
--build-arg AIRFLOW_CONSTRAINTS_REFERENCE="constraints-2.0.0" \
--build-arg AIRFLOW_SOURCES_FROM="empty" \
@@ -749,7 +749,7 @@ additional python dependencies and pre-installed pip dependencies from 2.0.0 tag
--build-arg PYTHON_MAJOR_MINOR_VERSION=3.7 \
--build-arg AIRFLOW_INSTALLATION_METHOD="apache-airflow" \
--build-arg AIRFLOW_VERSION="2.0.0" \
- --build-arg AIRFLOW_INSTALL_VERSION="==2.0.0" \
+ --build-arg AIRFLOW_VERSION_SPECIFICATION="==2.0.0" \
--build-arg AIRFLOW_BRANCH="v1-10-test" \
--build-arg AIRFLOW_CONSTRAINTS_REFERENCE="constraints-2.0.0" \
--build-arg AIRFLOW_SOURCES_FROM="empty" \
@@ -767,7 +767,7 @@ additional apt dev and runtime dependencies.
--build-arg PYTHON_MAJOR_MINOR_VERSION=3.7 \
--build-arg AIRFLOW_INSTALLATION_METHOD="apache-airflow" \
--build-arg AIRFLOW_VERSION="2.0.0" \
- --build-arg AIRFLOW_INSTALL_VERSION="==2.0.0" \
+ --build-arg AIRFLOW_VERSION_SPECIFICATION="==2.0.0" \
--build-arg AIRFLOW_CONSTRAINTS_REFERENCE="constraints-2-0" \
--build-arg AIRFLOW_SOURCES_FROM="empty" \
--build-arg AIRFLOW_SOURCES_TO="/empty" \
diff --git a/scripts/ci/libraries/_build_images.sh b/scripts/ci/libraries/_build_images.sh
index be82a6a..2c6763c 100644
--- a/scripts/ci/libraries/_build_images.sh
+++ b/scripts/ci/libraries/_build_images.sh
@@ -793,7 +793,7 @@ function build_images::prepare_prod_build() {
# When --install-airflow-version is used then the image is build from PIP package
EXTRA_DOCKER_PROD_BUILD_FLAGS=(
"--build-arg" "AIRFLOW_INSTALLATION_METHOD=apache-airflow"
- "--build-arg" "AIRFLOW_INSTALL_VERSION=${INSTALL_AIRFLOW_VERSION}"
+ "--build-arg" "AIRFLOW_VERSION_SPECIFICATION===${INSTALL_AIRFLOW_VERSION}"
"--build-arg" "AIRFLOW_VERSION=${INSTALL_AIRFLOW_VERSION}"
)
export AIRFLOW_VERSION="${INSTALL_AIRFLOW_VERSION}"
diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh
index e861bf1..fff9f63 100644
--- a/scripts/ci/libraries/_initialization.sh
+++ b/scripts/ci/libraries/_initialization.sh
@@ -388,8 +388,8 @@ function initialization::initialize_image_build_variables() {
export WHEEL_VERSION
# And installed from there (breeze and ci)
- AIRFLOW_INSTALL_VERSION=${AIRFLOW_INSTALL_VERSION:="."}
- export AIRFLOW_INSTALL_VERSION
+ AIRFLOW_VERSION_SPECIFICATION=${AIRFLOW_VERSION_SPECIFICATION:=""}
+ export AIRFLOW_VERSION_SPECIFICATION
# By default no sources are copied to image
AIRFLOW_SOURCES_FROM=${AIRFLOW_SOURCES_FROM:="empty"}
@@ -640,7 +640,7 @@ Common image build variables:
Production image build variables:
AIRFLOW_INSTALLATION_METHOD: '${AIRFLOW_INSTALLATION_METHOD}'
- AIRFLOW_INSTALL_VERSION: '${AIRFLOW_INSTALL_VERSION}'
+ AIRFLOW_VERSION_SPECIFICATION: '${AIRFLOW_VERSION_SPECIFICATION}'
AIRFLOW_SOURCES_FROM: '${AIRFLOW_SOURCES_FROM}'
AIRFLOW_SOURCES_TO: '${AIRFLOW_SOURCES_TO}'
diff --git a/scripts/docker/install_airflow.sh b/scripts/docker/install_airflow.sh
index 91f3860..1518782 100755
--- a/scripts/docker/install_airflow.sh
+++ b/scripts/docker/install_airflow.sh
@@ -20,7 +20,8 @@
# AIRFLOW_INSTALLATION_METHOD - determines where to install airflow form:
# "." - installs airflow from local sources
# "apache-airflow" - installs airflow from PyPI 'apache-airflow' package
-# AIRFLOW_INSTALL_VERSION - optionally specify version to install
+# AIRFLOW_VERSION_SPECIFICATION - optional specification for Airflow version to install (
+# might be ==2.0.2 for example or <3.0.0
# UPGRADE_TO_NEWER_DEPENDENCIES - determines whether eager-upgrade should be performed with the
# dependencies (with EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS added)
#
@@ -54,14 +55,14 @@ function install_airflow() {
echo
# eager upgrade
pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade --upgrade-strategy eager \
- "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_INSTALL_VERSION}" \
+ "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \
${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS}
if [[ -n "${AIRFLOW_INSTALL_EDITABLE_FLAG}" ]]; then
# Remove airflow and reinstall it using editable flag
# We can only do it when we install airflow from sources
pip uninstall apache-airflow --yes
pip install ${AIRFLOW_INSTALL_EDITABLE_FLAG} \
- "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_INSTALL_VERSION}"
+ "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}"
fi
# Work around to install azure-storage-blob
pip uninstall azure-storage azure-storage-blob azure-storage-file --yes
@@ -74,14 +75,14 @@ function install_airflow() {
echo Installing all packages with constraints and upgrade if needed
echo
pip install ${AIRFLOW_INSTALL_USER_FLAG} ${AIRFLOW_INSTALL_EDITABLE_FLAG} \
- "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_INSTALL_VERSION}" \
+ "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \
--constraint "${AIRFLOW_CONSTRAINTS_LOCATION}"
# make sure correct PIP version is used
pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}"
# then upgrade if needed without using constraints to account for new limits in setup.py
pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade --upgrade-strategy only-if-needed \
${AIRFLOW_INSTALL_EDITABLE_FLAG} \
- "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_INSTALL_VERSION}" \
+ "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \
# Work around to install azure-storage-blob
pip uninstall azure-storage azure-storage-blob azure-storage-file --yes
pip install azure-storage-blob azure-storage-file