You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2021/01/02 12:37:37 UTC
[airflow] branch master updated: Add extras when installing prod
image from packages (#13432)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/master by this push:
new 3a73110 Add extras when installing prod image from packages (#13432)
3a73110 is described below
commit 3a731108f530b6f7a90a9465faa0ae1daaa72552
Author: Jarek Potiuk <ja...@polidea.com>
AuthorDate: Sat Jan 2 13:37:29 2021 +0100
Add extras when installing prod image from packages (#13432)
In the latest change #13422 change in the way product images are
prepared removed extras from installed airflow - thus caused
failing production image verification check.
This change restores extras when airflow is installed from packages
---
Dockerfile | 36 ++++++++++++++++++++++++++++--------
Dockerfile.ci | 27 ++++++++++++++++++++-------
2 files changed, 48 insertions(+), 15 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index 81737a1..da033cd 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -265,20 +265,37 @@ RUN if [[ ${INSTALL_MYSQL_CLIENT} != "true" ]]; then \
fi; \
fi; \
if [[ ${INSTALL_FROM_DOCKER_CONTEXT_FILES} == "true" ]]; then \
- reinstalling_apache_airflow_packages=$(ls /docker-context-files/apache?airflow*.{whl,tar.gz} 2>/dev/null || true); \
- # We want to install apache airflow packages with constraints \
- if [[ "${reinstalling_apache_airflow_packages}" != "" ]]; then \
+ # We want to install apache airflow package with extras - that's why it is a separate step \
+ # But we should still install them with all dependencies \
+ reinstalling_apache_airflow_package=$(ls /docker-context-files/apache?airflow?[0-9]*.{whl,tar.gz} 2>/dev/null || true); \
+ if [[ "${reinstalling_apache_airflow_package}" != "" ]]; then \
if [[ "${UPGRADE_TO_NEWER_DEPENDENCIES}" != "false" ]]; then \
pip install --force-reinstall --upgrade --upgrade-strategy eager \
- --user ${reinstalling_apache_airflow_packages}; \
+ --user "${reinstalling_apache_airflow_package}[${AIRFLOW_EXTRAS}]"; \
pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}"; \
else \
+ # We want to install apache airflow package with constraints \
pip install --force-reinstall --upgrade --upgrade-strategy only-if-needed \
- --user ${reinstalling_apache_airflow_packages} --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}"; \
+ --user "${reinstalling_apache_airflow_package}[${AIRFLOW_EXTRAS}]" --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}"; \
pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}"; \
fi; \
fi ; \
- # All the others we want to reinstall as-is, without dependencies \
+ reinstalling_apache_airflow_providers_packages=$(ls /docker-context-files/apache?airflow?providers*.{whl,tar.gz} 2>/dev/null || true); \
+ # We want to install apache airflow package with extras - that's why it is a separate step \
+ # But we should still install them with all dependencies \
+ if [[ "${reinstalling_apache_airflow_providers_packages}" != "" ]]; then \
+ if [[ "${UPGRADE_TO_NEWER_DEPENDENCIES}" != "false" ]]; then \
+ pip install --force-reinstall --upgrade --upgrade-strategy eager \
+ --user ${reinstalling_apache_airflow_providers_packages}; \
+ pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}"; \
+ else \
+ # We want to install apache airflow provider packages with constraints \
+ pip install --force-reinstall --upgrade --upgrade-strategy only-if-needed \
+ --user ${reinstalling_apache_airflow_providers_packages} --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}"; \
+ pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}"; \
+ fi; \
+ fi ; \
+ # All the other packages we want to reinstall as-is, without dependencies \
reinstalling_other_packages=$(ls /docker-context-files/*.{whl,tar.gz} 2>/dev/null | \
grep -v apache_airflow | grep -v apache-airflow || true); \
if [[ "${reinstalling_other_packages}" != "" ]]; then \
@@ -287,8 +304,11 @@ RUN if [[ ${INSTALL_MYSQL_CLIENT} != "true" ]]; then \
fi; \
if [[ -n "${ADDITIONAL_PYTHON_DEPS}" ]]; then \
if [[ "${UPGRADE_TO_NEWER_DEPENDENCIES}" != "false" ]]; then \
- pip install --user ${ADDITIONAL_PYTHON_DEPS} --upgrade --upgrade-strategy eager; \
- pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}"; \
+ pip install --user ${ADDITIONAL_PYTHON_DEPS} \
+ ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS} \
+ --upgrade --upgrade-strategy eager; \
+ pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}"; \
+ pip check || ${CONTINUE_ON_PIP_CHECK_FAILURE}; \
else \
pip install --user ${ADDITIONAL_PYTHON_DEPS} --upgrade --upgrade-strategy only-if-needed \
--constraint "${AIRFLOW_CONSTRAINTS_LOCATION}"; \
diff --git a/Dockerfile.ci b/Dockerfile.ci
index 63360a0..7ac8760 100644
--- a/Dockerfile.ci
+++ b/Dockerfile.ci
@@ -337,26 +337,39 @@ COPY docker-context-files/ /docker-context-files/
# hadolint ignore=SC2086, SC2010
RUN if [[ ${INSTALL_FROM_DOCKER_CONTEXT_FILES} == "true" ]]; then \
- reinstalling_apache_airflow_packages=$(ls /docker-context-files/apache?airflow*.{whl,tar.gz} 2>/dev/null || true); \
- # We want to install apache airflow packages with constraints \
- if [[ "${reinstalling_apache_airflow_packages}" != "" ]]; then \
+ reinstalling_apache_airflow_package=$(ls /docker-context-files/apache?airflow?[0-9]*.{whl,tar.gz} 2>/dev/null || true); \
+ # We want to install apache airflow package with constraints \
+ if [[ "${reinstalling_apache_airflow_package}" != "" ]]; then \
if [[ "${UPGRADE_TO_NEWER_DEPENDENCIES}" != "false" ]]; then \
pip install --force-reinstall --upgrade --upgrade-strategy eager \
- --user ${reinstalling_apache_airflow_packages}; \
+ --user ${reinstalling_apache_airflow_package}[${AIRFLOW_EXTRAS}]; \
pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}"; \
else \
pip install --force-reinstall --upgrade --upgrade-strategy only-if-needed \
- --user ${reinstalling_apache_airflow_packages} --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}"; \
+ --user ${reinstalling_apache_airflow_package}[${AIRFLOW_EXTRAS}] --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}"; \
pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}"; \
fi; \
fi ; \
- # All the others we want to reinstall as-is, without dependencies \
+ reinstalling_apache_airflow_providers_packages=$(ls /docker-context-files/apache?airflow?providers*.{whl,tar.gz} 2>/dev/null || true); \
+ # We want to install apache airflow provider packages with constraints \
+ if [[ "${reinstalling_apache_airflow_providers_packages}" != "" ]]; then \
+ if [[ "${UPGRADE_TO_NEWER_DEPENDENCIES}" != "false" ]]; then \
+ pip install --force-reinstall --upgrade --upgrade-strategy eager \
+ --user ${reinstalling_apache_airflow_providers_packages}; \
+ pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}"; \
+ else \
+ pip install --force-reinstall --upgrade --upgrade-strategy only-if-needed \
+ --user ${reinstalling_apache_airflow_providers_packages} --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}"; \
+ pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}"; \
+ fi; \
+ fi ; \
+ # All the other packages we want to reinstall as-is, without dependencies \
reinstalling_other_packages=$(ls /docker-context-files/*.{whl,tar.gz} 2>/dev/null | \
grep -v apache_airflow | grep -v apache-airflow || true); \
if [[ "${reinstalling_other_packages}" != "" ]]; then \
pip install --force-reinstall --user --no-deps ${reinstalling_other_packages}; \
fi; \
- fi;
+ fi
# Copy all the www/ files we need to compile assets. Done as two separate COPY
# commands so as otherwise it copies the _contents_ of static/ in to www/