You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2020/06/09 09:04:27 UTC
[airflow] 14/36: Additional python extras and deps can be set in
breeze (#9035)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v1-10-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 62c97ed6fa6739fb94797be871b1e703501fadea
Author: Jarek Potiuk <ja...@polidea.com>
AuthorDate: Wed May 27 17:09:11 2020 +0200
Additional python extras and deps can be set in breeze (#9035)
Closes #8604
Closes #8866
(cherry picked from commit 738667082d32d3ef93ec2cd6c3735ff3691ba1cc)
---
BREEZE.rst | 12 ++++++++++++
Dockerfile | 4 ++--
Dockerfile.ci | 3 ++-
breeze | 14 ++++++++++++++
breeze-complete | 1 +
scripts/ci/_utils.sh | 12 +++++++++++-
6 files changed, 42 insertions(+), 4 deletions(-)
diff --git a/BREEZE.rst b/BREEZE.rst
index 44af4e5..262ddcd 100644
--- a/BREEZE.rst
+++ b/BREEZE.rst
@@ -774,6 +774,12 @@ This is the current syntax for `./breeze <./breeze>`_:
async,aws,azure,celery,dask,elasticsearch,gcp,kubernetes,mysql,postgres,redis,slack,
ssh,statsd,virtualenv
+ --additional-extras
+ Additional extras to pass to build images The default is no additional extras.
+
+ --additional-python-deps
+ Additional python dependencies to use when building the images.
+
-C, --force-clean-images
Force build images with cache disabled. This will remove the pulled or build images
and start building images from scratch. This might take a long time.
@@ -1312,6 +1318,12 @@ This is the current syntax for `./breeze <./breeze>`_:
async,aws,azure,celery,dask,elasticsearch,gcp,kubernetes,mysql,postgres,redis,slack,
ssh,statsd,virtualenv
+ --additional-extras
+ Additional extras to pass to build images The default is no additional extras.
+
+ --additional-python-deps
+ Additional python dependencies to use when building the images.
+
-C, --force-clean-images
Force build images with cache disabled. This will remove the pulled or build images
and start building images from scratch. This might take a long time.
diff --git a/Dockerfile b/Dockerfile
index ef3a988..ac27823 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -169,10 +169,10 @@ ARG AIRFLOW_VERSION
ENV AIRFLOW_VERSION=${AIRFLOW_VERSION}
ARG AIRFLOW_EXTRAS
-ARG ADDITIONAL_AIRFLOW_EXTRAS
+ARG ADDITIONAL_AIRFLOW_EXTRAS=""
ENV AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS}${ADDITIONAL_AIRFLOW_EXTRAS:+,}${ADDITIONAL_AIRFLOW_EXTRAS}
-ARG ADDITIONAL_PYTHON_DEPS
+ARG ADDITIONAL_PYTHON_DEPS=""
ENV ADDITIONAL_PYTHON_DEPS=${ADDITIONAL_PYTHON_DEPS}
ARG AIRFLOW_INSTALL_SOURCES="."
diff --git a/Dockerfile.ci b/Dockerfile.ci
index 15f3732..2d107aa 100644
--- a/Dockerfile.ci
+++ b/Dockerfile.ci
@@ -295,7 +295,8 @@ ENV AIRFLOW_BRANCH=${AIRFLOW_BRANCH}
# Airflow Extras installed
ARG AIRFLOW_EXTRAS="all"
-ENV AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS}
+ARG ADDITIONAL_AIRFLOW_EXTRAS=""
+ENV AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS}${ADDITIONAL_AIRFLOW_EXTRAS:+,}${ADDITIONAL_AIRFLOW_EXTRAS}
RUN echo "Installing with extras: ${AIRFLOW_EXTRAS}."
diff --git a/breeze b/breeze
index 7f5958b..b2e3c67 100755
--- a/breeze
+++ b/breeze
@@ -772,6 +772,14 @@ function parse_arguments() {
export AIRFLOW_EXTRAS="${2}"
echo "Extras : ${AIRFLOW_EXTRAS}"
shift 2 ;;
+ --additional-extras)
+ export ADDITIONAL_AIRFLOW_EXTRAS="${2}"
+ echo "Additional extras : ${ADDITIONAL_AIRFLOW_EXTRAS}"
+ shift 2 ;;
+ --additional-python-deps)
+ export ADDITIONAL_PYTHON_DEPS="${2}"
+ echo "Additional python dependencies: ${ADDITIONAL_PYTHON_DEPS}"
+ shift 2 ;;
-D|--dockerhub-user)
export DOCKERHUB_USER="${2}"
echo "Dockerhub user ${DOCKERHUB_USER}"
@@ -1527,6 +1535,12 @@ ${FORMATTED_DEFAULT_CI_EXTRAS}
Production image:
${FORMATTED_DEFAULT_PROD_EXTRAS}
+--additional-extras
+ Additional extras to pass to build images The default is no additional extras.
+
+--additional-python-deps
+ Additional python dependencies to use when building the images.
+
-C, --force-clean-images
Force build images with cache disabled. This will remove the pulled or build images
and start building images from scratch. This might take a long time.
diff --git a/breeze-complete b/breeze-complete
index 3164851..8afe550 100644
--- a/breeze-complete
+++ b/breeze-complete
@@ -99,6 +99,7 @@ verbose assume-yes assume-no assume-quit forward-credentials
force-build-images force-pull-images production-image extras: force-clean-images use-local-cache
dockerhub-user: dockerhub-repo: push-images
postgres-version: mysql-version:
+additional-extras: additional-python-deps:
"
export BREEZE_COMMANDS="
diff --git a/scripts/ci/_utils.sh b/scripts/ci/_utils.sh
index 7bbd2dd..85aeb91 100644
--- a/scripts/ci/_utils.sh
+++ b/scripts/ci/_utils.sh
@@ -1311,6 +1311,8 @@ Docker building ${AIRFLOW_CI_IMAGE}.
--build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \
--build-arg AIRFLOW_BRANCH="${BRANCH_NAME}" \
--build-arg AIRFLOW_EXTRAS="${AIRFLOW_EXTRAS}" \
+ --build-arg ADDITIONAL_AIRFLOW_EXTRAS="${ADDITIONAL_AIRFLOW_EXTRAS}" \
+ --build-arg ADDITIONAL_PYTHON_DEPS="${ADDITIONAL_PYTHON_DEPS}" \
--build-arg AIRFLOW_CONTAINER_CI_OPTIMISED_BUILD="${AIRFLOW_CONTAINER_CI_OPTIMISED_BUILD}" \
--build-arg UPGRADE_TO_LATEST_REQUIREMENTS="${UPGRADE_TO_LATEST_REQUIREMENTS}" \
"${DOCKER_CACHE_CI_DIRECTIVE[@]}" \
@@ -1364,6 +1366,8 @@ function build_prod_image() {
--build-arg PYTHON_MAJOR_MINOR_VERSION="${PYTHON_MAJOR_MINOR_VERSION}" \
--build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \
--build-arg AIRFLOW_EXTRAS="${AIRFLOW_EXTRAS}" \
+ --build-arg ADDITIONAL_AIRFLOW_EXTRAS="${ADDITIONAL_AIRFLOW_EXTRAS}" \
+ --build-arg ADDITIONAL_PYTHON_DEPS="${ADDITIONAL_PYTHON_DEPS}" \
"${DOCKER_CACHE_PROD_BUILD_DIRECTIVE[@]}" \
-t "${AIRFLOW_PROD_BUILD_IMAGE}" \
--target "airflow-build-image" \
@@ -1372,6 +1376,8 @@ function build_prod_image() {
"${EXTRA_DOCKER_PROD_BUILD_FLAGS[@]}" \
--build-arg PYTHON_BASE_IMAGE="${PYTHON_BASE_IMAGE}" \
--build-arg PYTHON_MAJOR_MINOR_VERSION="${PYTHON_MAJOR_MINOR_VERSION}" \
+ --build-arg ADDITIONAL_AIRFLOW_EXTRAS="${ADDITIONAL_AIRFLOW_EXTRAS}" \
+ --build-arg ADDITIONAL_PYTHON_DEPS="${ADDITIONAL_PYTHON_DEPS}" \
--build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \
--build-arg AIRFLOW_EXTRAS="${AIRFLOW_EXTRAS}" \
"${DOCKER_CACHE_PROD_DIRECTIVE[@]}" \
@@ -1473,8 +1479,9 @@ function prepare_ci_build() {
export IMAGE_DESCRIPTION="Airflow CI"
export AIRFLOW_CONTAINER_CI_OPTIMISED_BUILD="true"
export AIRFLOW_EXTRAS="${AIRFLOW_EXTRAS:="${DEFAULT_CI_EXTRAS}"}"
+ export ADDITIONAL_AIRFLOW_EXTRAS="${ADDITIONAL_AIRFLOW_EXTRAS:=""}"
+ export ADDITIONAL_PYTHON_DEPS="${ADDITIONAL_PYTHON_DEPS:=""}"
export AIRFLOW_IMAGE="${AIRFLOW_CI_IMAGE}"
-
go_to_airflow_sources
fix_group_permissions
}
@@ -1532,7 +1539,10 @@ function prepare_prod_build() {
export THE_IMAGE_TYPE="PROD"
export IMAGE_DESCRIPTION="Airflow production"
export AIRFLOW_EXTRAS="${AIRFLOW_EXTRAS:="${DEFAULT_PROD_EXTRAS}"}"
+ export ADDITIONAL_AIRFLOW_EXTRAS="${ADDITIONAL_AIRFLOW_EXTRAS:=""}"
+ export ADDITIONAL_PYTHON_DEPS="${ADDITIONAL_PYTHON_DEPS:=""}"
export AIRFLOW_IMAGE="${AIRFLOW_PROD_IMAGE}"
+
if [[ ${CACHE_REGISTRY=""} != "" ]]; then
echo "${CACHE_REGISTRY_PASSWORD}" | docker login \
--username "${CACHE_REGISTRY_USERNAME}" \