You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2022/01/22 15:49:30 UTC
[airflow] 17/33: Cleaner output for Docker image building (#20747)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v2-2-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit a3be411ea43fbe4d4ecbe7bcfdef5ebad69fb15e
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Mon Jan 10 06:46:09 2022 +0100
Cleaner output for Docker image building (#20747)
There was some "junk" output generated by the scripts that are
used in Airflow image building. The junk has been cleaned up so
that no unnecessary warnings are generated.
This change includes:
* making sure that when everything is fine, there are no
warnnings generated by PROD docker build proces
* making sure that when CI image is build the only remaining
warning is "Using root" - this warning cannot be silenced
https://github.com/pypa/pip/issues/10556 and instead
in CI build we explain in green that this is invalid warning
* the "scripted" steps of docker build have nicely blue headers
that visually separate steps of building the iamge and give
more information on what's going on
* the current way of printing ouput will play very nicely with
BUILDKIT UI where Blue color indicates progress in building
Separated out from #20238
(cherry picked from commit 4d33ebf76cd101b05476c0f1c840f10013bebd5f)
---
Dockerfile | 95 +++++++++++-----------
Dockerfile.ci | 87 +++++++++-----------
scripts/docker/common.sh | 24 ++++--
scripts/docker/compile_www_assets.sh | 16 +++-
scripts/docker/install_additional_dependencies.sh | 11 ++-
scripts/docker/install_airflow.sh | 18 ++--
...install_airflow_dependencies_from_branch_tip.sh | 9 +-
.../docker/install_from_docker_context_files.sh | 10 ++-
scripts/docker/install_mssql.sh | 9 +-
scripts/docker/install_mysql.sh | 9 +-
scripts/docker/install_pip_version.sh | 16 ++--
11 files changed, 173 insertions(+), 131 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index fd58eca..bb80aca 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -47,35 +47,31 @@ ARG PYTHON_BASE_IMAGE="python:3.6-slim-buster"
ARG AIRFLOW_PIP_VERSION=21.3.1
ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow"
+# By default latest released version of airflow is installed (when empty) but this value can be overridden
+# and we can install version according to specification (For example ==2.0.2 or <3.0.0).
+ARG AIRFLOW_VERSION_SPECIFICATION=""
+
# By default PIP has progress bar but you can disable it.
ARG PIP_PROGRESS_BAR="on"
-
##############################################################################################
# This is the build image where we build all dependencies
##############################################################################################
FROM ${PYTHON_BASE_IMAGE} as airflow-build-image
-SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"]
+
+# Nolog bash flag is currently ignored - but you can replace it with
+# xtrace - to show commands executed)
+SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "nounset", "-o", "nolog", "-c"]
ARG PYTHON_BASE_IMAGE
ENV PYTHON_BASE_IMAGE=${PYTHON_BASE_IMAGE} \
DEBIAN_FRONTEND=noninteractive LANGUAGE=C.UTF-8 LANG=C.UTF-8 LC_ALL=C.UTF-8 \
LC_CTYPE=C.UTF-8 LC_MESSAGES=C.UTF-8
-# Install curl and gnupg2 - needed for many other installation steps
-RUN apt-get update \
- && apt-get install -y --no-install-recommends \
- curl \
- gnupg2 \
- && apt-get autoremove -yqq --purge \
- && apt-get clean \
- && rm -rf /var/lib/apt/lists/*
-
ARG DEV_APT_DEPS="\
apt-transport-https \
apt-utils \
build-essential \
ca-certificates \
- gnupg \
dirmngr \
freetds-bin \
freetds-dev \
@@ -103,10 +99,13 @@ ARG DEV_APT_DEPS="\
unixodbc \
unixodbc-dev \
yarn"
+
ARG ADDITIONAL_DEV_APT_DEPS=""
ARG DEV_APT_COMMAND="\
- curl --fail --location https://deb.nodesource.com/setup_14.x | bash - \
- && curl https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - > /dev/null \
+ curl --silent --fail --location https://deb.nodesource.com/setup_14.x | \
+ bash -o pipefail -o errexit -o nolog - \
+ && curl --silent https://dl.yarnpkg.com/debian/pubkey.gpg | \
+ apt-key add - >/dev/null 2>&1\
&& echo 'deb https://dl.yarnpkg.com/debian/ stable main' > /etc/apt/sources.list.d/yarn.list"
ARG ADDITIONAL_DEV_APT_COMMAND="echo"
ARG ADDITIONAL_DEV_APT_ENV=""
@@ -120,11 +119,14 @@ ENV DEV_APT_DEPS=${DEV_APT_DEPS} \
# Note missing man directories on debian-buster
# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=863199
# Install basic and additional apt dependencies
-RUN mkdir -pv /usr/share/man/man1 \
+RUN apt-get update \
+ && apt-get install --no-install-recommends -yqq apt-utils >/dev/null 2>&1 \
+ && apt-get install -y --no-install-recommends curl gnupg2 \
+ && mkdir -pv /usr/share/man/man1 \
&& mkdir -pv /usr/share/man/man7 \
&& export ${ADDITIONAL_DEV_APT_ENV?} \
- && bash -o pipefail -e -u -x -c "${DEV_APT_COMMAND}" \
- && bash -o pipefail -e -u -x -c "${ADDITIONAL_DEV_APT_COMMAND}" \
+ && bash -o pipefail -o errexit -o nounset -o nolog -c "${DEV_APT_COMMAND}" \
+ && bash -o pipefail -o errexit -o nounset -o nolog -c "${ADDITIONAL_DEV_APT_COMMAND}" \
&& apt-get update \
&& apt-get install -y --no-install-recommends \
${DEV_APT_DEPS} \
@@ -153,6 +155,9 @@ ARG PIP_PROGRESS_BAR
ARG AIRFLOW_PRE_CACHED_PIP_PACKAGES="false"
# This is airflow version that is put in the label of the image build
ARG AIRFLOW_VERSION
+# By default latest released version of airflow is installed (when empty) but this value can be overridden
+# and we can install version according to specification (For example ==2.0.2 or <3.0.0).
+ARG AIRFLOW_VERSION_SPECIFICATION
# By default we install providers from PyPI but in case of Breeze build we want to install providers
# from local sources without the need of preparing provider packages upfront. This value is
# automatically overridden by Breeze scripts.
@@ -163,9 +168,6 @@ ARG INSTALL_PROVIDERS_FROM_SOURCES="false"
# Airflow checked out together with the Dockerfile and AIRFLOW_SOURCES_FROM and AIRFLOW_SOURCES_TO
# set to "." and "/opt/airflow" respectively.
ARG AIRFLOW_INSTALLATION_METHOD="apache-airflow"
-# By default latest released version of airflow is installed (when empty) but this value can be overridden
-# and we can install version according to specification (For example ==2.0.2 or <3.0.0).
-ARG AIRFLOW_VERSION_SPECIFICATION=""
# By default we do not upgrade to latest dependencies
ARG UPGRADE_TO_NEWER_DEPENDENCIES="false"
# By default we install latest airflow from PyPI so we do not need to copy sources of Airflow
@@ -201,8 +203,8 @@ ENV INSTALL_MYSQL_CLIENT=${INSTALL_MYSQL_CLIENT} \
COPY scripts/docker/install_mysql.sh scripts/docker/install_mssql.sh /scripts/docker/
-RUN bash ./scripts/docker/install_mysql.sh dev \
- && bash ./scripts/docker/install_mssql.sh
+RUN bash -o pipefail -o errexit -o nounset -o nolog ./scripts/docker/install_mysql.sh dev \
+ && bash -o pipefail -o errexit -o nounset -o nolog ./scripts/docker/install_mssql.sh
ENV PATH=${PATH}:/opt/mssql-tools/bin
COPY docker-context-files /docker-context-files
@@ -237,10 +239,11 @@ ENV AIRFLOW_PRE_CACHED_PIP_PACKAGES=${AIRFLOW_PRE_CACHED_PIP_PACKAGES} \
# the cache is only used when "upgrade to newer dependencies" is not set to automatically
# account for removed dependencies (we do not install them in the first place)
# Upgrade to specific PIP version
-RUN bash /scripts/docker/install_pip_version.sh; \
+RUN bash -o pipefail -o errexit -o nounset -o nolog /scripts/docker/install_pip_version.sh; \
if [[ ${AIRFLOW_PRE_CACHED_PIP_PACKAGES} == "true" && \
${UPGRADE_TO_NEWER_DEPENDENCIES} == "false" ]]; then \
- bash /scripts/docker/install_airflow_dependencies_from_branch_tip.sh; \
+ bash -o pipefail -o errexit -o nounset -o nolog \
+ /scripts/docker/install_airflow_dependencies_from_branch_tip.sh; \
fi
COPY --chown=airflow:0 ${AIRFLOW_SOURCES_FROM} ${AIRFLOW_SOURCES_TO}
@@ -272,15 +275,15 @@ WORKDIR /opt/airflow
RUN if [[ ${AIRFLOW_INSTALLATION_METHOD} == "." ]]; then \
# only compile assets if the prod image is build from sources
# otherwise they are already compiled-in
- bash /scripts/docker/compile_www_assets.sh; \
+ bash -o pipefail -o errexit -o nounset -o nolog /scripts/docker/compile_www_assets.sh; \
fi; \
if [[ ${INSTALL_FROM_DOCKER_CONTEXT_FILES} == "true" ]]; then \
- bash /scripts/docker/install_from_docker_context_files.sh; \
+ bash -o pipefail -o errexit -o nounset -o nolog /scripts/docker/install_from_docker_context_files.sh; \
elif [[ ${INSTALL_FROM_PYPI} == "true" ]]; then \
- bash /scripts/docker/install_airflow.sh; \
+ bash -o pipefail -o errexit -o nounset -o nolog /scripts/docker/install_airflow.sh; \
fi; \
if [[ -n "${ADDITIONAL_PYTHON_DEPS}" ]]; then \
- bash /scripts/docker/install_additional_dependencies.sh; \
+ bash -o pipefail -o errexit -o nounset -o nolog /scripts/docker/install_additional_dependencies.sh; \
fi; \
find "${AIRFLOW_USER_HOME_DIR}/.local/" -name '*.pyc' -print0 | xargs -0 rm -f || true ; \
find "${AIRFLOW_USER_HOME_DIR}/.local/" -type d -name '__pycache__' -print0 | xargs -0 rm -rf || true ; \
@@ -328,7 +331,10 @@ LABEL org.apache.airflow.distro="debian" \
# installed Airflow and all it's dependencies from the build image to make it smaller.
##############################################################################################
FROM ${PYTHON_BASE_IMAGE} as main
-SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"]
+
+# Nolog bash flag is currently ignored - but you can replace it with other flags (for example
+# xtrace - to show commands executed)
+SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "nounset", "-o", "nolog", "-c"]
ARG AIRFLOW_UID
@@ -350,15 +356,6 @@ ENV PYTHON_BASE_IMAGE=${PYTHON_BASE_IMAGE} \
LC_CTYPE=C.UTF-8 LC_MESSAGES=C.UTF-8 \
AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION}
-# Install curl and gnupg2 - needed for many other installation steps
-RUN apt-get update \
- && apt-get install -y --no-install-recommends \
- curl \
- gnupg2 \
- && apt-get autoremove -yqq --purge \
- && apt-get clean \
- && rm -rf /var/lib/apt/lists/*
-
ARG RUNTIME_APT_DEPS="\
apt-transport-https \
apt-utils \
@@ -366,7 +363,6 @@ ARG RUNTIME_APT_DEPS="\
curl \
dumb-init \
freetds-bin \
- gnupg \
gosu \
krb5-user \
ldap-utils \
@@ -400,6 +396,7 @@ ARG BUILD_ID
ARG COMMIT_SHA
ARG AIRFLOW_IMAGE_REPOSITORY
ARG AIRFLOW_IMAGE_DATE_CREATED
+ARG AIRFLOW_VERSION_SPECIFICATION
ENV RUNTIME_APT_DEPS=${RUNTIME_APT_DEPS} \
ADDITIONAL_RUNTIME_APT_DEPS=${ADDITIONAL_RUNTIME_APT_DEPS} \
@@ -414,6 +411,7 @@ ENV RUNTIME_APT_DEPS=${RUNTIME_APT_DEPS} \
PATH="${AIRFLOW_USER_HOME_DIR}/.local/bin:${PATH}" \
GUNICORN_CMD_ARGS="--worker-tmp-dir /dev/shm" \
AIRFLOW_INSTALLATION_METHOD=${AIRFLOW_INSTALLATION_METHOD} \
+ AIRFLOW_VERSION_SPECIFICATION=${AIRFLOW_VERSION_SPECIFICATION} \
BUILD_ID=${BUILD_ID} \
COMMIT_SHA=${COMMIT_SHA} \
# By default PIP installs everything to ~/.local
@@ -422,11 +420,14 @@ ENV RUNTIME_APT_DEPS=${RUNTIME_APT_DEPS} \
# Note missing man directories on debian-buster
# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=863199
# Install basic and additional apt dependencies
-RUN mkdir -pv /usr/share/man/man1 \
+RUN apt-get update \
+ && apt-get install --no-install-recommends -yqq apt-utils >/dev/null 2>&1 \
+ && apt-get install -y --no-install-recommends curl gnupg2 \
+ && mkdir -pv /usr/share/man/man1 \
&& mkdir -pv /usr/share/man/man7 \
&& export ${ADDITIONAL_RUNTIME_APT_ENV?} \
- && bash -o pipefail -e -u -x -c "${RUNTIME_APT_COMMAND}" \
- && bash -o pipefail -e -u -x -c "${ADDITIONAL_RUNTIME_APT_COMMAND}" \
+ && bash -o pipefail -o errexit -o nounset -o nolog -c "${RUNTIME_APT_COMMAND}" \
+ && bash -o pipefail -o errexit -o nounset -o nolog -c "${ADDITIONAL_RUNTIME_APT_COMMAND}" \
&& apt-get update \
&& apt-get install -y --no-install-recommends \
${RUNTIME_APT_DEPS} \
@@ -437,14 +438,15 @@ RUN mkdir -pv /usr/share/man/man1 \
# Only copy install_m(y/s)sql and install_pip_version.sh. We do not need any other scripts in the final image.
COPY scripts/docker/install_mysql.sh /scripts/docker/install_mssql.sh scripts/docker/install_pip_version.sh \
- /scripts/docker/
+ scripts/docker/common.sh /scripts/docker/
# fix permission issue in Azure DevOps when running the scripts
RUN chmod a+x /scripts/docker/install_mysql.sh && \
/scripts/docker/install_mysql.sh prod && \
chmod a+x /scripts/docker/install_mssql.sh && \
/scripts/docker/install_mssql.sh && \
- adduser --quiet "airflow" --uid "${AIRFLOW_UID}" --gid "0" --home "${AIRFLOW_USER_HOME_DIR}" && \
+ adduser --gecos "First Last,RoomNumber,WorkPhone,HomePhone" --disabled-password \
+ --quiet "airflow" --uid "${AIRFLOW_UID}" --gid "0" --home "${AIRFLOW_USER_HOME_DIR}" && \
# Make Airflow files belong to the root group and are accessible. This is to accommodate the guidelines from
# OpenShift https://docs.openshift.com/enterprise/3.0/creating_images/guidelines.html
mkdir -pv "${AIRFLOW_HOME}"; \
@@ -463,8 +465,7 @@ COPY --chown=airflow:0 scripts/in_container/prod/clean-logs.sh /clean-logs
# See https://github.com/apache/airflow/issues/9248
RUN chmod a+x /entrypoint /clean-logs && \
- chmod g=u /etc/passwd && \
- bash /scripts/docker/install_pip_version.sh
+ chmod g=u /etc/passwd
WORKDIR ${AIRFLOW_HOME}
@@ -474,6 +475,8 @@ RUN usermod -g 0 airflow -G 0
USER ${AIRFLOW_UID}
+RUN /scripts/docker/install_pip_version.sh
+
LABEL org.apache.airflow.distro="debian" \
org.apache.airflow.distro.version="buster" \
org.apache.airflow.module="airflow" \
diff --git a/Dockerfile.ci b/Dockerfile.ci
index 0d956c2..62a61f5 100644
--- a/Dockerfile.ci
+++ b/Dockerfile.ci
@@ -18,7 +18,9 @@
ARG PYTHON_BASE_IMAGE="python:3.6-slim-buster"
FROM ${PYTHON_BASE_IMAGE} as main
-SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"]
+# Nolog bash flag is currently ignored - but you can replace it with other flags (for example
+# xtrace - to show commands executed)
+SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "nounset", "-o", "nolog", "-c"]
ARG PYTHON_BASE_IMAGE="python:3.6-slim-buster"
ARG AIRFLOW_VERSION="2.2.0.dev0"
@@ -31,24 +33,17 @@ ARG DEPENDENCIES_EPOCH_NUMBER="6"
ENV PYTHON_BASE_IMAGE=${PYTHON_BASE_IMAGE} AIRFLOW_VERSION=${AIRFLOW_VERSION} \
DEBIAN_FRONTEND=noninteractive LANGUAGE=C.UTF-8 LANG=C.UTF-8 LC_ALL=C.UTF-8 \
LC_CTYPE=C.UTF-8 LC_MESSAGES=C.UTF-8 \
- DEPENDENCIES_EPOCH_NUMBER=${DEPENDENCIES_EPOCH_NUMBER}
+ DEPENDENCIES_EPOCH_NUMBER=${DEPENDENCIES_EPOCH_NUMBER} \
+ INSTALL_MYSQL_CLIENT="true" \
+ INSTALL_MSSQL_CLIENT="true"
# Print versions
RUN echo "Base image: ${PYTHON_BASE_IMAGE}, Airflow version: ${AIRFLOW_VERSION}"
-# Install curl and gnupg2 - needed to download nodejs in the next step
-RUN apt-get update \
- && apt-get install -y --no-install-recommends \
- curl \
- gnupg2 \
- && apt-get autoremove -yqq --purge \
- && apt-get clean \
- && rm -rf /var/lib/apt/lists/*
-
ARG ADDITIONAL_DEV_APT_DEPS=""
ARG DEV_APT_COMMAND="\
- curl --fail --location https://deb.nodesource.com/setup_14.x | bash - \
- && curl https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - > /dev/null \
+ curl --silent --fail --location https://deb.nodesource.com/setup_14.x | bash - \
+ && curl --silent --fail https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - >/dev/null 2>&1 \
&& echo 'deb https://dl.yarnpkg.com/debian/ stable main' > /etc/apt/sources.list.d/yarn.list"
ARG ADDITIONAL_DEV_APT_COMMAND=""
ARG ADDITIONAL_DEV_ENV_VARS=""
@@ -57,16 +52,15 @@ ENV DEV_APT_COMMAND=${DEV_APT_COMMAND} \
ADDITIONAL_DEV_APT_DEPS=${ADDITIONAL_DEV_APT_DEPS} \
ADDITIONAL_DEV_APT_COMMAND=${ADDITIONAL_DEV_APT_COMMAND}
-# As of August 2021, Debian buster-slim does not include Python2 by default and we need it
-# as we still support running Python2 via PythonVirtualenvOperator
-# TODO: Remove python2 when we stop supporting it
-
# Install basic and additional apt dependencies
-RUN mkdir -pv /usr/share/man/man1 \
+RUN apt-get update \
+ && apt-get install --no-install-recommends -yqq apt-utils >/dev/null 2>&1 \
+ && apt-get install -y --no-install-recommends curl gnupg2 \
+ && mkdir -pv /usr/share/man/man1 \
&& mkdir -pv /usr/share/man/man7 \
&& export ${ADDITIONAL_DEV_ENV_VARS?} \
- && bash -o pipefail -e -u -x -c "${DEV_APT_COMMAND}" \
- && bash -o pipefail -e -u -x -c "${ADDITIONAL_DEV_APT_COMMAND}" \
+ && bash -o pipefail -o errexit -o nounset -o nolog -c "${DEV_APT_COMMAND}" \
+ && bash -o pipefail -o errexit -o nounset -o nolog -c "${ADDITIONAL_DEV_APT_COMMAND}" \
&& apt-get update \
&& apt-get install -y --no-install-recommends \
apt-utils \
@@ -101,18 +95,18 @@ RUN mkdir -pv /usr/share/man/man1 \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
-COPY scripts/docker/*.sh /scripts/docker/
-RUN bash /scripts/docker/install_mysql.sh dev \
- && bash /scripts/docker/install_mssql.sh \
- && adduser airflow \
- && echo "airflow:airflow" | chpasswd \
+# Only copy mysql/mssql installation scripts for now - so that changing the other
+# scripts which are needed much later will not invalidate the docker layer here
+COPY scripts/docker/install_mysql.sh scripts/docker/install_mssql.sh /scripts/docker/
+RUN bash -o pipefail -o errexit -o nounset -o nolog /scripts/docker/install_mysql.sh dev \
+ && bash -o pipefail -o errexit -o nounset -o nolog /scripts/docker/install_mssql.sh \
+ && adduser --gecos "First Last,RoomNumber,WorkPhone,HomePhone" --disabled-password \
+ --quiet "airflow" --home "/home/airflow" \
+ && echo -e "airflow\nairflow" | passwd airflow 2>&1 \
&& echo "airflow ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/airflow \
&& chmod 0440 /etc/sudoers.d/airflow
-# The latest buster images do not have libpython 2.7 installed and it is needed
-# To run virtualenv tests with python 2
ARG RUNTIME_APT_DEPS="\
- gnupg \
libgcc-8-dev \
apt-transport-https \
bash-completion \
@@ -138,7 +132,7 @@ ARG HELM_VERSION="v3.6.3"
RUN SYSTEM=$(uname -s | tr '[:upper:]' '[:lower:]') \
&& HELM_URL="https://get.helm.sh/helm-${HELM_VERSION}-${SYSTEM}-amd64.tar.gz" \
- && curl --location "${HELM_URL}" | tar -xvz -O "${SYSTEM}"-amd64/helm > /usr/local/bin/helm \
+ && curl --silent --location "${HELM_URL}" | tar -xz -O "${SYSTEM}"-amd64/helm > /usr/local/bin/helm \
&& chmod +x /usr/local/bin/helm
ARG ADDITIONAL_RUNTIME_APT_DEPS=""
@@ -152,7 +146,6 @@ ARG HOME=/root
ARG AIRFLOW_HOME=/root/airflow
ARG AIRFLOW_SOURCES=/opt/airflow
-
ENV RUNTIME_APT_DEP=${RUNTIME_APT_DEPS} \
ADDITIONAL_RUNTIME_APT_DEPS=${ADDITIONAL_RUNTIME_APT_DEPS} \
RUNTIME_APT_COMMAND=${RUNTIME_APT_COMMAND} \
@@ -168,8 +161,8 @@ RUN mkdir -pv /usr/share/man/man1 \
&& mkdir -pv /usr/share/man/man7 \
&& export ${ADDITIONAL_DEV_APT_ENV?} \
&& export ${ADDITIONAL_RUNTIME_APT_ENV?} \
- && bash -o pipefail -e -u -x -c "${RUNTIME_APT_COMMAND}" \
- && bash -o pipefail -e -u -x -c "${ADDITIONAL_RUNTIME_APT_COMMAND}" \
+ && bash -o pipefail -o errexit -o nounset -o nolog -c "${RUNTIME_APT_COMMAND}" \
+ && bash -o pipefail -o errexit -o nounset -o nolog -c "${ADDITIONAL_RUNTIME_APT_COMMAND}" \
&& apt-get update \
&& apt-get install --no-install-recommends -y \
${RUNTIME_APT_DEPS} \
@@ -177,7 +170,7 @@ RUN mkdir -pv /usr/share/man/man1 \
&& apt-get autoremove -yqq --purge \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/* \
- && curl https://download.docker.com/linux/static/stable/x86_64/docker-${DOCKER_CLI_VERSION}.tgz \
+ && curl --silent "https://download.docker.com/linux/static/stable/x86_64/docker-${DOCKER_CLI_VERSION}.tgz" \
| tar -C /usr/bin --strip-components=1 -xvzf - docker/docker
WORKDIR ${AIRFLOW_SOURCES}
@@ -194,7 +187,7 @@ ARG BATS_FILE_VERSION="0.2.0"
RUN curl -sSL https://github.com/bats-core/bats-core/archive/v${BATS_VERSION}.tar.gz -o /tmp/bats.tgz \
&& tar -zxf /tmp/bats.tgz -C /tmp \
- && /bin/bash /tmp/bats-core-${BATS_VERSION}/install.sh /opt/bats && rm -rf \
+ && bash -o pipefail -o errexit -o nounset -o nolog /tmp/bats-core-${BATS_VERSION}/install.sh /opt/bats && rm -rf \
&& mkdir -p /opt/bats/lib/bats-support \
&& curl -sSL https://github.com/bats-core/bats-support/archive/v${BATS_SUPPORT_VERSION}.tar.gz -o /tmp/bats-support.tgz \
&& tar -zxf /tmp/bats-support.tgz -C /opt/bats/lib/bats-support --strip 1 && rm -rf /tmp/* \
@@ -276,6 +269,8 @@ ARG UPGRADE_TO_NEWER_DEPENDENCIES="false"
ENV EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS} \
UPGRADE_TO_NEWER_DEPENDENCIES=${UPGRADE_TO_NEWER_DEPENDENCIES}
+COPY scripts/docker/*.sh scripts/docker/install_pip_version.sh /scripts/docker/
+
# In case of CI builds we want to pre-install main version of airflow dependencies so that
# We do not have to always reinstall it from the scratch.
# And is automatically reinstalled from the scratch every time patch release of python gets released
@@ -283,10 +278,12 @@ ENV EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=${EAGER_UPGRADE_ADDITIONAL_REQUIREMENT
# are uninstalled, only dependencies remain.
# the cache is only used when "upgrade to newer dependencies" is not set to automatically
# account for removed dependencies (we do not install them in the first place)
-RUN bash /scripts/docker/install_pip_version.sh; \
+RUN echo -e "\n\e[32mThe 'Running pip as the root user' warnings below are not valid but we can't disable them :(\e[0m\n"; \
+ echo -e "\n\e[34mSee https://github.com/pypa/pip/issues/10556 for details.\e[0m\n" ; \
+ bash -o pipefail -o errexit -o nounset -o nolog /scripts/docker/install_pip_version.sh; \
if [[ ${AIRFLOW_PRE_CACHED_PIP_PACKAGES} == "true" && \
${UPGRADE_TO_NEWER_DEPENDENCIES} == "false" ]]; then \
- bash /scripts/docker/install_airflow_dependencies_from_branch_tip.sh; \
+ bash -o pipefail -o errexit -o nounset -o nolog /scripts/docker/install_airflow_dependencies_from_branch_tip.sh; \
fi
# Generate random hex dump file so that we can determine whether it's faster to rebuild the image
@@ -297,13 +294,6 @@ RUN head -c 30 /dev/urandom | xxd -ps >/build-cache-hash
# Link dumb-init for backwards compatibility (so that older images also work)
RUN ln -sf /usr/bin/dumb-init /usr/local/bin/dumb-init
-# Install NPM dependencies here. The NPM dependencies don't change that often and we already have pip
-# installed dependencies in case of CI optimised build, so it is ok to install NPM deps here
-# Rather than after setup.py is added.
-COPY airflow/www/yarn.lock airflow/www/package.json ${AIRFLOW_SOURCES}/airflow/www/
-
-RUN yarn --cwd airflow/www install --frozen-lockfile --no-cache && yarn cache clean
-
# Note! We are copying everything with airflow:airflow user:group even if we use root to run the scripts
# This is fine as root user will be able to use those dirs anyway.
@@ -322,17 +312,16 @@ COPY airflow/__init__.py ${AIRFLOW_SOURCES}/airflow/__init__.py
# But in cron job we will install latest versions matching setup.py to see if there is no breaking change
# and push the constraints if everything is successful
RUN if [[ ${INSTALL_FROM_PYPI} == "true" ]]; then \
- bash /scripts/docker/install_airflow.sh; \
+ bash -o pipefail -o errexit -o nounset -o nolog /scripts/docker/install_airflow.sh; \
fi
# Copy all the www/ files we need to compile assets. Done as two separate COPY
# commands so as otherwise it copies the _contents_ of static/ in to www/
-COPY airflow/www/webpack.config.js ${AIRFLOW_SOURCES}/airflow/www/
+COPY airflow/www/webpack.config.js airflow/www/package.json airflow/www/yarn.lock ${AIRFLOW_SOURCES}/airflow/www/
COPY airflow/www/static ${AIRFLOW_SOURCES}/airflow/www/static/
-COPY airflow/www/compile_assets.sh ${AIRFLOW_SOURCES}/airflow/www/compile_assets.sh
# Package JS/css for production
-RUN bash airflow/www/compile_assets.sh
+RUN bash -o pipefail -o errexit -o nounset -o nolog /scripts/docker/compile_www_assets.sh
COPY scripts/in_container/entrypoint_ci.sh /entrypoint
RUN chmod a+x /entrypoint
@@ -342,9 +331,9 @@ COPY scripts/docker/load.bash /opt/bats/lib/
# Additional python deps to install
ARG ADDITIONAL_PYTHON_DEPS=""
-RUN bash /scripts/docker/install_pip_version.sh; \
+RUN bash -o pipefail -o errexit -o nounset -o nolog /scripts/docker/install_pip_version.sh; \
if [[ -n "${ADDITIONAL_PYTHON_DEPS}" ]]; then \
- bash /scripts/docker/install_additional_dependencies.sh; \
+ bash -o pipefail -o errexit -o nounset -o nolog /scripts/docker/install_additional_dependencies.sh; \
fi
# Install autocomplete for airflow
diff --git a/scripts/docker/common.sh b/scripts/docker/common.sh
index c3c33f7..36097d1 100755
--- a/scripts/docker/common.sh
+++ b/scripts/docker/common.sh
@@ -17,14 +17,22 @@
# under the License.
set -euo pipefail
-: "${INSTALL_MYSQL_CLIENT:?Should be true or false}"
-: "${INSTALL_MSSQL_CLIENT:?Should be true or false}"
-: "${AIRFLOW_REPO:?Should be set}"
-: "${AIRFLOW_BRANCH:?Should be set}"
-: "${AIRFLOW_PIP_VERSION:?Should be set}"
+function common::get_colors() {
+ COLOR_BLUE=$'\e[34m'
+ COLOR_GREEN=$'\e[32m'
+ COLOR_RED=$'\e[31m'
+ COLOR_RESET=$'\e[0m'
+ COLOR_YELLOW=$'\e[33m'
+ export COLOR_BLUE
+ export COLOR_GREEN
+ export COLOR_RED
+ export COLOR_RESET
+ export COLOR_YELLOW
+}
+
function common::get_airflow_version_specification() {
- if [[ -z ${AIRFLOW_VERSION_SPECIFICATION}
+ if [[ -z ${AIRFLOW_VERSION_SPECIFICATION=}
&& -n ${AIRFLOW_VERSION}
&& ${AIRFLOW_INSTALLATION_METHOD} != "." ]]; then
AIRFLOW_VERSION_SPECIFICATION="==${AIRFLOW_VERSION}"
@@ -41,7 +49,7 @@ function common::override_pip_version_if_needed() {
function common::get_constraints_location() {
# auto-detect Airflow-constraint reference and location
- if [[ -z "${AIRFLOW_CONSTRAINTS_REFERENCE}" ]]; then
+ if [[ -z "${AIRFLOW_CONSTRAINTS_REFERENCE=}" ]]; then
if [[ ${AIRFLOW_VERSION} =~ v?2.* ]]; then
AIRFLOW_CONSTRAINTS_REFERENCE=constraints-${AIRFLOW_VERSION}
else
@@ -49,7 +57,7 @@ function common::get_constraints_location() {
fi
fi
- if [[ -z ${AIRFLOW_CONSTRAINTS_LOCATION} ]]; then
+ if [[ -z ${AIRFLOW_CONSTRAINTS_LOCATION=} ]]; then
local constraints_base="https://raw.githubusercontent.com/${CONSTRAINTS_GITHUB_REPOSITORY}/${AIRFLOW_CONSTRAINTS_REFERENCE}"
local python_version
python_version="$(python --version 2>/dev/stdout | cut -d " " -f 2 | cut -d "." -f 1-2)"
diff --git a/scripts/docker/compile_www_assets.sh b/scripts/docker/compile_www_assets.sh
index 5813f1b..b9034be 100755
--- a/scripts/docker/compile_www_assets.sh
+++ b/scripts/docker/compile_www_assets.sh
@@ -18,10 +18,17 @@
# shellcheck disable=SC2086
set -euo pipefail
+BUILD_TYPE=${BUILD_TYPE="prod"}
+
+COLOR_BLUE=$'\e[34m'
+readonly COLOR_BLUE
+COLOR_RESET=$'\e[0m'
+readonly COLOR_RESET
+
# Installs additional dependencies passed as Argument to the Docker build command
function compile_www_assets() {
echo
- echo Compiling WWW assets
+ echo "${COLOR_BLUE}Compiling www assets${COLOR_RESET}"
echo
local md5sum_file
md5sum_file="static/dist/sum.md5"
@@ -29,7 +36,7 @@ function compile_www_assets() {
local www_dir
if [[ ${AIRFLOW_INSTALLATION_METHOD=} == "." ]]; then
# In case we are building from sources in production image, we should build the assets
- www_dir="${AIRFLOW_SOURCES_TO}/airflow/www"
+ www_dir="${AIRFLOW_SOURCES_TO=${AIRFLOW_SOURCES}}/airflow/www"
else
www_dir="$(python -m site --user-site)/airflow/www"
fi
@@ -44,13 +51,14 @@ function compile_www_assets() {
>&2 cat /tmp/out-yarn-install.txt && rm -f /tmp/out-yarn-install.txt
exit 1
fi
- yarn run prod 2>/tmp/out-yarn-run.txt
+ rm -f /tmp/out-yarn-install.txt
+ yarn run "${BUILD_TYPE}" 2>/tmp/out-yarn-run.txt
res=$?
if [[ ${res} != 0 ]]; then
>&2 echo
>&2 echo "Error when running yarn install:"
>&2 echo
- >&2 cat /tmp/out-yarn-run.txt && rm -f /tmp/out-yarn-run.txt
+ >&2 cat /tmp/out-yarn-run.txt && rm -rf /tmp/out-yarn-run.txt
exit 1
fi
rm -f /tmp/out-yarn-run.txt
diff --git a/scripts/docker/install_additional_dependencies.sh b/scripts/docker/install_additional_dependencies.sh
index 3ab4ac7..112ff63 100755
--- a/scripts/docker/install_additional_dependencies.sh
+++ b/scripts/docker/install_additional_dependencies.sh
@@ -33,25 +33,32 @@ set -x
function install_additional_dependencies() {
if [[ "${UPGRADE_TO_NEWER_DEPENDENCIES}" != "false" ]]; then
echo
- echo Installing additional dependencies while upgrading to newer dependencies
+ echo "${COLOR_BLUE}Installing additional dependencies while upgrading to newer dependencies${COLOR_RESET}"
echo
pip install --upgrade --upgrade-strategy eager \
${ADDITIONAL_PYTHON_DEPS} ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS}
# make sure correct PIP version is used
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}"
+ echo
+ echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
+ echo
pip check
else
echo
- echo Installing additional dependencies upgrading only if needed
+ echo "${COLOR_BLUE}Installing additional dependencies upgrading only if needed${COLOR_RESET}"
echo
pip install --upgrade --upgrade-strategy only-if-needed \
${ADDITIONAL_PYTHON_DEPS}
# make sure correct PIP version is used
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}"
+ echo
+ echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
+ echo
pip check
fi
}
+common::get_colors
common::get_airflow_version_specification
common::override_pip_version_if_needed
common::get_constraints_location
diff --git a/scripts/docker/install_airflow.sh b/scripts/docker/install_airflow.sh
index e89e09b..b2ece66 100755
--- a/scripts/docker/install_airflow.sh
+++ b/scripts/docker/install_airflow.sh
@@ -29,13 +29,15 @@
# shellcheck source=scripts/docker/common.sh
. "$( dirname "${BASH_SOURCE[0]}" )/common.sh"
+: "${AIRFLOW_PIP_VERSION:?Should be set}"
+
function install_airflow() {
# Coherence check for editable installation mode.
if [[ ${AIRFLOW_INSTALLATION_METHOD} != "." && \
${AIRFLOW_INSTALL_EDITABLE_FLAG} == "--editable" ]]; then
echo
- echo "ERROR! You can only use --editable flag when installing airflow from sources!"
- echo " Current installation method is '${AIRFLOW_INSTALLATION_METHOD} and should be '.'"
+ echo "${COLOR_RED}ERROR! You can only use --editable flag when installing airflow from sources!${COLOR_RESET}"
+ echo "{COLOR_RED} Current installation method is '${AIRFLOW_INSTALLATION_METHOD} and should be '.'${COLOR_RESET}"
exit 1
fi
# Remove mysql from extras if client is not going to be installed
@@ -44,7 +46,7 @@ function install_airflow() {
fi
if [[ "${UPGRADE_TO_NEWER_DEPENDENCIES}" != "false" ]]; then
echo
- echo Installing all packages with eager upgrade
+ echo "${COLOR_BLUE}Installing all packages with eager upgrade${COLOR_RESET}"
echo
# eager upgrade
pip install --upgrade --upgrade-strategy eager \
@@ -57,13 +59,15 @@ function install_airflow() {
pip install ${AIRFLOW_INSTALL_EDITABLE_FLAG} \
"${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}"
fi
-
# make sure correct PIP version is used
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}"
+ echo
+ echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
+ echo
pip check
else \
echo
- echo Installing all packages with constraints and upgrade if needed
+ echo "${COLOR_BLUE}Installing all packages with constraints and upgrade if needed${COLOR_RESET}"
echo
pip install ${AIRFLOW_INSTALL_EDITABLE_FLAG} \
"${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \
@@ -76,11 +80,15 @@ function install_airflow() {
"${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}"
# make sure correct PIP version is used
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}"
+ echo
+ echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
+ echo
pip check
fi
}
+common::get_colors
common::get_airflow_version_specification
common::override_pip_version_if_needed
common::get_constraints_location
diff --git a/scripts/docker/install_airflow_dependencies_from_branch_tip.sh b/scripts/docker/install_airflow_dependencies_from_branch_tip.sh
index 8ebb619..38577c1 100755
--- a/scripts/docker/install_airflow_dependencies_from_branch_tip.sh
+++ b/scripts/docker/install_airflow_dependencies_from_branch_tip.sh
@@ -29,10 +29,14 @@
# shellcheck source=scripts/docker/common.sh
. "$( dirname "${BASH_SOURCE[0]}" )/common.sh"
+: "${AIRFLOW_REPO:?Should be set}"
+: "${AIRFLOW_BRANCH:?Should be set}"
+: "${INSTALL_MYSQL_CLIENT:?Should be true or false}"
+: "${AIRFLOW_PIP_VERSION:?Should be set}"
function install_airflow_dependencies_from_branch_tip() {
echo
- echo "Installing airflow from ${AIRFLOW_BRANCH}. It is used to cache dependencies"
+ echo "${COLOR_BLUE}Installing airflow from ${AIRFLOW_BRANCH}. It is used to cache dependencies${COLOR_RESET}"
echo
if [[ ${INSTALL_MYSQL_CLIENT} != "true" ]]; then
AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS/mysql,}
@@ -46,11 +50,12 @@ function install_airflow_dependencies_from_branch_tip() {
pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}"
pip freeze | grep apache-airflow-providers | xargs pip uninstall --yes 2>/dev/null || true
echo
- echo Uninstalling just airflow. Dependencies remain.
+ echo "${COLOR_BLUE}Uninstalling just airflow. Dependencies remain. Now target airflow can be reinstalled using mostly cached dependencies${COLOR_RESET}"
echo
pip uninstall --yes apache-airflow || true
}
+common::get_colors
common::get_airflow_version_specification
common::override_pip_version_if_needed
common::get_constraints_location
diff --git a/scripts/docker/install_from_docker_context_files.sh b/scripts/docker/install_from_docker_context_files.sh
index 0afa9b6..1bd78b5 100755
--- a/scripts/docker/install_from_docker_context_files.sh
+++ b/scripts/docker/install_from_docker_context_files.sh
@@ -25,6 +25,8 @@
# shellcheck source=scripts/docker/common.sh
. "$( dirname "${BASH_SOURCE[0]}" )/common.sh"
+: "${AIRFLOW_PIP_VERSION:?Should be set}"
+
function install_airflow_and_providers_from_docker_context_files(){
if [[ ${INSTALL_MYSQL_CLIENT} != "true" ]]; then
AIRFLOW_EXTRAS=${AIRFLOW_EXTRAS/mysql,}
@@ -65,7 +67,7 @@ function install_airflow_and_providers_from_docker_context_files(){
if [[ "${UPGRADE_TO_NEWER_DEPENDENCIES}" != "false" ]]; then
echo
- echo Force re-installing airflow and providers from local files with eager upgrade
+ echo "${COLOR_BLUE}Force re-installing airflow and providers from local files with eager upgrade${COLOR_RESET}"
echo
# force reinstall all airflow + provider package local files with eager upgrade
pip install "${pip_flags[@]}" --upgrade --upgrade-strategy eager \
@@ -73,7 +75,7 @@ function install_airflow_and_providers_from_docker_context_files(){
${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS}
else
echo
- echo Force re-installing airflow and providers from local files with constraints and upgrade if needed
+ echo "${COLOR_BLUE}Force re-installing airflow and providers from local files with constraints and upgrade if needed${COLOR_RESET}"
echo
if [[ ${AIRFLOW_CONSTRAINTS_LOCATION} == "/"* ]]; then
grep -ve '^apache-airflow' <"${AIRFLOW_CONSTRAINTS_LOCATION}" > /tmp/constraints.txt
@@ -104,8 +106,9 @@ function install_airflow_and_providers_from_docker_context_files(){
# method on air-gaped system where you do not want to download any dependencies from remote hosts
# which is a requirement for serious installations
function install_all_other_packages_from_docker_context_files() {
+
echo
- echo Force re-installing all other package from local files without dependencies
+ echo "${COLOR_BLUE}Force re-installing all other package from local files without dependencies${COLOR_RESET}"
echo
local reinstalling_other_packages
# shellcheck disable=SC2010
@@ -118,6 +121,7 @@ function install_all_other_packages_from_docker_context_files() {
fi
}
+common::get_colors
common::get_airflow_version_specification
common::override_pip_version_if_needed
common::get_constraints_location
diff --git a/scripts/docker/install_mssql.sh b/scripts/docker/install_mssql.sh
index 14eb0fe..8aab499 100755
--- a/scripts/docker/install_mssql.sh
+++ b/scripts/docker/install_mssql.sh
@@ -16,9 +16,16 @@
# specific language governing permissions and limitations
# under the License.
set -euo pipefail
+
+: "${INSTALL_MSSQL_CLIENT:?Should be true or false}"
+
+COLOR_BLUE=$'\e[34m'
+readonly COLOR_BLUE
+COLOR_RESET=$'\e[0m'
+readonly COLOR_RESET
function install_mssql_client() {
echo
- echo Installing mssql client
+ echo "${COLOR_BLUE}Installing mssql client${COLOR_RESET}"
echo
curl --silent https://packages.microsoft.com/keys/microsoft.asc | apt-key add - >/dev/null 2>&1
curl --silent https://packages.microsoft.com/config/debian/10/prod.list > /etc/apt/sources.list.d/mssql-release.list
diff --git a/scripts/docker/install_mysql.sh b/scripts/docker/install_mysql.sh
index abd4108..52aca49 100755
--- a/scripts/docker/install_mysql.sh
+++ b/scripts/docker/install_mysql.sh
@@ -21,9 +21,16 @@ declare -a packages
MYSQL_VERSION="8.0"
readonly MYSQL_VERSION
+COLOR_BLUE=$'\e[34m'
+readonly COLOR_BLUE
+COLOR_RESET=$'\e[0m'
+readonly COLOR_RESET
+
+: "${INSTALL_MYSQL_CLIENT:?Should be true or false}"
+
install_mysql_client() {
echo
- echo Installing mysql client
+ echo "${COLOR_BLUE}Installing mysql client version ${MYSQL_VERSION}${COLOR_RESET}"
echo
if [[ "${1}" == "dev" ]]; then
diff --git a/scripts/docker/install_pip_version.sh b/scripts/docker/install_pip_version.sh
index 312a954..68b138f 100755
--- a/scripts/docker/install_pip_version.sh
+++ b/scripts/docker/install_pip_version.sh
@@ -16,27 +16,23 @@
# specific language governing permissions and limitations
# under the License.
-# Install airflow using regular 'pip install' command. This install airflow depending on the arguments:
-# AIRFLOW_INSTALLATION_METHOD - determines where to install airflow form:
-# "." - installs airflow from local sources
-# "apache-airflow" - installs airflow from PyPI 'apache-airflow' package
-# AIRFLOW_VERSION_SPECIFICATION - optional specification for Airflow version to install (
-# might be ==2.0.2 for example or <3.0.0
-# UPGRADE_TO_NEWER_DEPENDENCIES - determines whether eager-upgrade should be performed with the
-# dependencies (with EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS added)
-#
# shellcheck disable=SC2086
# shellcheck source=scripts/docker/common.sh
. "$( dirname "${BASH_SOURCE[0]}" )/common.sh"
+: "${AIRFLOW_PIP_VERSION:?Should be set}"
+
function install_pip_version() {
+ echo
+ echo "${COLOR_BLUE}Installing pip version ${AIRFLOW_PIP_VERSION}${COLOR_RESET}"
+ echo
pip install --disable-pip-version-check --no-cache-dir --upgrade "pip==${AIRFLOW_PIP_VERSION}" &&
mkdir -p ${HOME}/.local/bin
}
+common::get_colors
common::get_airflow_version_specification
common::override_pip_version_if_needed
-common::get_constraints_location
common::show_pip_version_and_location
install_pip_version