You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2022/01/22 15:49:24 UTC
[airflow] 11/33: Cleaner output of docker image building scripts (#20679)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v2-2-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit ea900a0f307d2c308750dc2fc71f6c09b7342383
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Thu Jan 6 04:46:16 2022 +0100
Cleaner output of docker image building scripts (#20679)
(cherry picked from commit fb8780013227a20462878b7f9286d083630c0bc2)
---
Dockerfile | 5 +----
Dockerfile.ci | 3 +--
IMAGES.rst | 7 +++----
dev/README_RELEASE_PROVIDER_PACKAGES.md | 4 ++--
docs/docker-stack/build-arg-ref.rst | 2 +-
scripts/ci/kubernetes/ci_run_kubernetes_tests.sh | 2 +-
scripts/ci/libraries/_initialization.sh | 2 +-
scripts/docker/common.sh | 20 ++++++++++--------
scripts/docker/compile_www_assets.sh | 24 +++++++++++++++++++---
scripts/docker/install_additional_dependencies.sh | 19 ++++++++---------
scripts/docker/install_airflow.sh | 15 +++++++-------
...install_airflow_dependencies_from_branch_tip.sh | 7 ++++---
.../docker/install_from_docker_context_files.sh | 16 ++++++++-------
scripts/docker/install_mssql.sh | 6 +++---
scripts/docker/install_mysql.sh | 5 ++---
scripts/docker/install_pip_version.sh | 4 +++-
scripts/in_container/_in_container_utils.sh | 4 ++--
scripts/in_container/prod/entrypoint_prod.sh | 2 +-
.../in_container/run_prepare_airflow_packages.sh | 2 +-
19 files changed, 84 insertions(+), 65 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index a28de84..bb24aa7 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -43,7 +43,7 @@ ARG AIRFLOW_UID="50000"
ARG PYTHON_BASE_IMAGE="python:3.6-slim-buster"
-ARG AIRFLOW_PIP_VERSION=21.2.4
+ARG AIRFLOW_PIP_VERSION=21.3.1
ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow"
# By default PIP has progress bar but you can disable it.
@@ -186,9 +186,6 @@ ENV INSTALL_MYSQL_CLIENT=${INSTALL_MYSQL_CLIENT} \
PATH=${PATH}:/root/.local/bin \
AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \
PIP_PROGRESS_BAR=${PIP_PROGRESS_BAR} \
- # Install Airflow with "--user" flag, so that we can copy the whole .local folder to the final image
- # from the build image and always in non-editable mode
- AIRFLOW_INSTALL_USER_FLAG="--user" \
AIRFLOW_INSTALL_EDITABLE_FLAG="" \
UPGRADE_TO_NEWER_DEPENDENCIES=${UPGRADE_TO_NEWER_DEPENDENCIES}
diff --git a/Dockerfile.ci b/Dockerfile.ci
index 48962e6..0d956c2 100644
--- a/Dockerfile.ci
+++ b/Dockerfile.ci
@@ -223,7 +223,7 @@ ARG AIRFLOW_PRE_CACHED_PIP_PACKAGES="true"
# By default in the image, we are installing all providers when installing from sources
ARG INSTALL_PROVIDERS_FROM_SOURCES="true"
ARG INSTALL_FROM_PYPI="true"
-ARG AIRFLOW_PIP_VERSION=21.2.4
+ARG AIRFLOW_PIP_VERSION=21.3.1
# Setup PIP
# By default PIP install run without cache to make image smaller
ARG PIP_NO_CACHE_DIR="true"
@@ -258,7 +258,6 @@ ENV AIRFLOW_REPO=${AIRFLOW_REPO}\
INSTALL_MYSQL_CLIENT="true" \
INSTALL_MSSQL_CLIENT="true" \
AIRFLOW_INSTALLATION_METHOD="." \
- AIRFLOW_INSTALL_USER_FLAG="" \
AIRFLOW_INSTALL_EDITABLE_FLAG="--editable" \
AIRFLOW_VERSION_SPECIFICATION="" \
PIP_NO_CACHE_DIR=${PIP_NO_CACHE_DIR} \
diff --git a/IMAGES.rst b/IMAGES.rst
index 559ecdb..12f0b91 100644
--- a/IMAGES.rst
+++ b/IMAGES.rst
@@ -41,14 +41,13 @@ to run Kubernetes tests. See below for the list of arguments that should be prov
production image from the local sources.
The image is primarily optimised for size of the final image, but also for speed of rebuilds - the
-'airflow-build-image' segment uses the same technique as the CI jobs for pre-installing PIP dependencies.
+'airflow-build-image' segment uses the same technique as the CI jobs for pre-installing dependencies.
It first pre-installs them from the right GitHub branch and only after that final airflow installation is
-done from either local sources or remote location (PIP or GitHub repository).
+done from either local sources or remote location (PyPI or GitHub repository).
You can read more details about building, extending and customizing the PROD image in the
`Latest documentation <https://airflow.apache.org/docs/docker-stack/index.html>`_
-
CI image
--------
@@ -492,7 +491,7 @@ The following build arguments (``--build-arg`` in docker build command) can be u
| ``ADDITIONAL_RUNTIME_APT_ENV`` | | Additional env variables defined |
| | | when installing runtime deps |
+------------------------------------------+------------------------------------------+------------------------------------------+
-| ``AIRFLOW_PIP_VERSION`` | ``21.2.4`` | PIP version used. |
+| ``AIRFLOW_PIP_VERSION`` | ``21.3.1`` | PIP version used. |
+------------------------------------------+------------------------------------------+------------------------------------------+
| ``PIP_PROGRESS_BAR`` | ``on`` | Progress bar for PIP installation |
+------------------------------------------+------------------------------------------+------------------------------------------+
diff --git a/dev/README_RELEASE_PROVIDER_PACKAGES.md b/dev/README_RELEASE_PROVIDER_PACKAGES.md
index e0b55db..3d928b0 100644
--- a/dev/README_RELEASE_PROVIDER_PACKAGES.md
+++ b/dev/README_RELEASE_PROVIDER_PACKAGES.md
@@ -611,9 +611,9 @@ provider packages. This is especially helpful when you want to test integrations
additional tools. Below is an example Dockerfile, which installs providers for Google/
```dockerfile
-FROM apache/airflow:2.0.0
+FROM apache/airflow:2.2.3
-RUN pip install --upgrade --user apache-airflow-providers-google==2.0.0.rc1
+RUN pip install --user apache-airflow-providers-google==2.2.2.rc1
USER ${AIRFLOW_UID}
```
diff --git a/docs/docker-stack/build-arg-ref.rst b/docs/docker-stack/build-arg-ref.rst
index f142b37..ab75d4a 100644
--- a/docs/docker-stack/build-arg-ref.rst
+++ b/docs/docker-stack/build-arg-ref.rst
@@ -45,7 +45,7 @@ Those are the most common arguments that you use when you want to build a custom
+------------------------------------------+------------------------------------------+---------------------------------------------+
| ``AIRFLOW_USER_HOME_DIR`` | ``/home/airflow`` | Home directory of the Airflow user. |
+------------------------------------------+------------------------------------------+---------------------------------------------+
-| ``AIRFLOW_PIP_VERSION`` | ``21.2.4`` | PIP version used. |
+| ``AIRFLOW_PIP_VERSION`` | ``21.3.1`` | PIP version used. |
+------------------------------------------+------------------------------------------+---------------------------------------------+
| ``PIP_PROGRESS_BAR`` | ``on`` | Progress bar for PIP installation |
+------------------------------------------+------------------------------------------+---------------------------------------------+
diff --git a/scripts/ci/kubernetes/ci_run_kubernetes_tests.sh b/scripts/ci/kubernetes/ci_run_kubernetes_tests.sh
index ef920b4..6cf054f 100755
--- a/scripts/ci/kubernetes/ci_run_kubernetes_tests.sh
+++ b/scripts/ci/kubernetes/ci_run_kubernetes_tests.sh
@@ -82,7 +82,7 @@ function create_virtualenv() {
. "${virtualenv_path}/bin/activate"
- pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}" "wheel==${WHEEL_VERSION}"
+ pip install "pip==${AIRFLOW_PIP_VERSION}" "wheel==${WHEEL_VERSION}"
local constraints=(
--constraint
diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh
index a95ce19..a0e9e30 100644
--- a/scripts/ci/libraries/_initialization.sh
+++ b/scripts/ci/libraries/_initialization.sh
@@ -421,7 +421,7 @@ function initialization::initialize_image_build_variables() {
export INSTALLED_EXTRAS="async,amazon,celery,cncf.kubernetes,docker,dask,elasticsearch,ftp,grpc,hashicorp,http,imap,ldap,google,microsoft.azure,mysql,postgres,redis,sendgrid,sftp,slack,ssh,statsd,virtualenv"
- AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION:="21.2.4"}
+ AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION:="21.3.1"}
export AIRFLOW_PIP_VERSION
# We also pin version of wheel used to get consistent builds
diff --git a/scripts/docker/common.sh b/scripts/docker/common.sh
index d11715e..c3c33f7 100755
--- a/scripts/docker/common.sh
+++ b/scripts/docker/common.sh
@@ -17,14 +17,11 @@
# under the License.
set -euo pipefail
-test -v INSTALL_MYSQL_CLIENT
-test -v INSTALL_MSSQL_CLIENT
-test -v AIRFLOW_INSTALL_USER_FLAG
-test -v AIRFLOW_REPO
-test -v AIRFLOW_BRANCH
-test -v AIRFLOW_PIP_VERSION
-
-set -x
+: "${INSTALL_MYSQL_CLIENT:?Should be true or false}"
+: "${INSTALL_MSSQL_CLIENT:?Should be true or false}"
+: "${AIRFLOW_REPO:?Should be set}"
+: "${AIRFLOW_BRANCH:?Should be set}"
+: "${AIRFLOW_PIP_VERSION:?Should be set}"
function common::get_airflow_version_specification() {
if [[ -z ${AIRFLOW_VERSION_SPECIFICATION}
@@ -42,7 +39,6 @@ function common::override_pip_version_if_needed() {
fi
}
-
function common::get_constraints_location() {
# auto-detect Airflow-constraint reference and location
if [[ -z "${AIRFLOW_CONSTRAINTS_REFERENCE}" ]]; then
@@ -60,3 +56,9 @@ function common::get_constraints_location() {
AIRFLOW_CONSTRAINTS_LOCATION="${constraints_base}/${AIRFLOW_CONSTRAINTS}-${python_version}.txt"
fi
}
+
+function common::show_pip_version_and_location() {
+ echo "PATH=${PATH}"
+ echo "pip on path: $(which pip)"
+ echo "Using pip: $(pip --version)"
+}
diff --git a/scripts/docker/compile_www_assets.sh b/scripts/docker/compile_www_assets.sh
index 59a7017..5813f1b 100755
--- a/scripts/docker/compile_www_assets.sh
+++ b/scripts/docker/compile_www_assets.sh
@@ -17,7 +17,6 @@
# under the License.
# shellcheck disable=SC2086
set -euo pipefail
-set -x
# Installs additional dependencies passed as Argument to the Docker build command
function compile_www_assets() {
@@ -35,8 +34,27 @@ function compile_www_assets() {
www_dir="$(python -m site --user-site)/airflow/www"
fi
pushd ${www_dir} || exit 1
- yarn install --frozen-lockfile --no-cache
- yarn run prod
+ set +e
+ yarn install --frozen-lockfile --no-cache 2>/tmp/out-yarn-install.txt
+ local res=$?
+ if [[ ${res} != 0 ]]; then
+ >&2 echo
+ >&2 echo "Error when running yarn install:"
+ >&2 echo
+ >&2 cat /tmp/out-yarn-install.txt && rm -f /tmp/out-yarn-install.txt
+ exit 1
+ fi
+ yarn run prod 2>/tmp/out-yarn-run.txt
+ res=$?
+ if [[ ${res} != 0 ]]; then
+ >&2 echo
+ >&2 echo "Error when running yarn install:"
+ >&2 echo
+ >&2 cat /tmp/out-yarn-run.txt && rm -f /tmp/out-yarn-run.txt
+ exit 1
+ fi
+ rm -f /tmp/out-yarn-run.txt
+ set -e
find package.json yarn.lock static/css static/js -type f | sort | xargs md5sum > "${md5sum_file}"
rm -rf "${www_dir}/node_modules"
rm -vf "${www_dir}"/{package.json,yarn.lock,.eslintignore,.eslintrc,.stylelintignore,.stylelintrc,compile_assets.sh,webpack.config.js}
diff --git a/scripts/docker/install_additional_dependencies.sh b/scripts/docker/install_additional_dependencies.sh
index 4f9c05f..3ab4ac7 100755
--- a/scripts/docker/install_additional_dependencies.sh
+++ b/scripts/docker/install_additional_dependencies.sh
@@ -18,11 +18,10 @@
# shellcheck disable=SC2086
set -euo pipefail
-test -v UPGRADE_TO_NEWER_DEPENDENCIES
-test -v ADDITIONAL_PYTHON_DEPS
-test -v EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS
-test -v AIRFLOW_INSTALL_USER_FLAG
-test -v AIRFLOW_PIP_VERSION
+: "${UPGRADE_TO_NEWER_DEPENDENCIES:?Should be true or false}"
+: "${ADDITIONAL_PYTHON_DEPS:?Should be set}"
+: "${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS:?Should be set}"
+: "${AIRFLOW_PIP_VERSION:?Should be set}"
# shellcheck source=scripts/docker/common.sh
. "$( dirname "${BASH_SOURCE[0]}" )/common.sh"
@@ -36,20 +35,19 @@ function install_additional_dependencies() {
echo
echo Installing additional dependencies while upgrading to newer dependencies
echo
- pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade --upgrade-strategy eager \
+ pip install --upgrade --upgrade-strategy eager \
${ADDITIONAL_PYTHON_DEPS} ${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS}
# make sure correct PIP version is used
- pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}"
+ pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}"
pip check
else
echo
echo Installing additional dependencies upgrading only if needed
echo
- pip install ${AIRFLOW_INSTALL_USER_FLAG} \
- --upgrade --upgrade-strategy only-if-needed \
+ pip install --upgrade --upgrade-strategy only-if-needed \
${ADDITIONAL_PYTHON_DEPS}
# make sure correct PIP version is used
- pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}"
+ pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}"
pip check
fi
}
@@ -57,5 +55,6 @@ function install_additional_dependencies() {
common::get_airflow_version_specification
common::override_pip_version_if_needed
common::get_constraints_location
+common::show_pip_version_and_location
install_additional_dependencies
diff --git a/scripts/docker/install_airflow.sh b/scripts/docker/install_airflow.sh
index 61a30c4..e89e09b 100755
--- a/scripts/docker/install_airflow.sh
+++ b/scripts/docker/install_airflow.sh
@@ -47,7 +47,7 @@ function install_airflow() {
echo Installing all packages with eager upgrade
echo
# eager upgrade
- pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade --upgrade-strategy eager \
+ pip install --upgrade --upgrade-strategy eager \
"${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \
${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS}
if [[ -n "${AIRFLOW_INSTALL_EDITABLE_FLAG}" ]]; then
@@ -59,23 +59,23 @@ function install_airflow() {
fi
# make sure correct PIP version is used
- pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}"
+ pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}"
pip check
else \
echo
echo Installing all packages with constraints and upgrade if needed
echo
- pip install ${AIRFLOW_INSTALL_USER_FLAG} ${AIRFLOW_INSTALL_EDITABLE_FLAG} \
+ pip install ${AIRFLOW_INSTALL_EDITABLE_FLAG} \
"${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \
--constraint "${AIRFLOW_CONSTRAINTS_LOCATION}"
# make sure correct PIP version is used
- pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}"
+ pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}"
# then upgrade if needed without using constraints to account for new limits in setup.py
- pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade --upgrade-strategy only-if-needed \
+ pip install --upgrade --upgrade-strategy only-if-needed \
${AIRFLOW_INSTALL_EDITABLE_FLAG} \
- "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}" \
+ "${AIRFLOW_INSTALLATION_METHOD}[${AIRFLOW_EXTRAS}]${AIRFLOW_VERSION_SPECIFICATION}"
# make sure correct PIP version is used
- pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}"
+ pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}"
pip check
fi
@@ -84,5 +84,6 @@ function install_airflow() {
common::get_airflow_version_specification
common::override_pip_version_if_needed
common::get_constraints_location
+common::show_pip_version_and_location
install_airflow
diff --git a/scripts/docker/install_airflow_dependencies_from_branch_tip.sh b/scripts/docker/install_airflow_dependencies_from_branch_tip.sh
index 61aaa13..8ebb619 100755
--- a/scripts/docker/install_airflow_dependencies_from_branch_tip.sh
+++ b/scripts/docker/install_airflow_dependencies_from_branch_tip.sh
@@ -39,12 +39,12 @@ function install_airflow_dependencies_from_branch_tip() {
fi
# Install latest set of dependencies using constraints. In case constraints were upgraded and there
# are conflicts, this might fail, but it should be fixed in the following installation steps
- pip install ${AIRFLOW_INSTALL_USER_FLAG} \
+ pip install \
"https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz#egg=apache-airflow[${AIRFLOW_EXTRAS}]" \
--constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" || true
# make sure correct PIP version is used
- pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}"
- pip freeze | grep apache-airflow-providers | xargs pip uninstall --yes || true
+ pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}"
+ pip freeze | grep apache-airflow-providers | xargs pip uninstall --yes 2>/dev/null || true
echo
echo Uninstalling just airflow. Dependencies remain.
echo
@@ -54,5 +54,6 @@ function install_airflow_dependencies_from_branch_tip() {
common::get_airflow_version_specification
common::override_pip_version_if_needed
common::get_constraints_location
+common::show_pip_version_and_location
install_airflow_dependencies_from_branch_tip
diff --git a/scripts/docker/install_from_docker_context_files.sh b/scripts/docker/install_from_docker_context_files.sh
index d8ed6bc..0afa9b6 100755
--- a/scripts/docker/install_from_docker_context_files.sh
+++ b/scripts/docker/install_from_docker_context_files.sh
@@ -34,7 +34,6 @@ function install_airflow_and_providers_from_docker_context_files(){
local pip_flags=(
# Don't quote this -- if it is empty we don't want it to create an
# empty array element
- ${AIRFLOW_INSTALL_USER_FLAG}
--find-links="file:///docker-context-files"
)
@@ -88,14 +87,14 @@ function install_airflow_and_providers_from_docker_context_files(){
--constraint /tmp/constraints.txt
rm /tmp/constraints.txt
# make sure correct PIP version is used \
- pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}"
+ pip install "pip==${AIRFLOW_PIP_VERSION}"
# then upgrade if needed without using constraints to account for new limits in setup.py
- pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade --upgrade-strategy only-if-needed \
+ pip install --upgrade --upgrade-strategy only-if-needed \
${reinstalling_apache_airflow_package} ${reinstalling_apache_airflow_providers_packages}
fi
# make sure correct PIP version is left installed
- pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}"
+ pip install "pip==${AIRFLOW_PIP_VERSION}"
pip check
}
@@ -104,7 +103,7 @@ function install_airflow_and_providers_from_docker_context_files(){
# without dependencies. This is extremely useful in case you want to install via pip-download
# method on air-gaped system where you do not want to download any dependencies from remote hosts
# which is a requirement for serious installations
-install_all_other_packages_from_docker_context_files() {
+function install_all_other_packages_from_docker_context_files() {
echo
echo Force re-installing all other package from local files without dependencies
echo
@@ -113,15 +112,18 @@ install_all_other_packages_from_docker_context_files() {
reinstalling_other_packages=$(ls /docker-context-files/*.{whl,tar.gz} 2>/dev/null | \
grep -v apache_airflow | grep -v apache-airflow || true)
if [[ -n "${reinstalling_other_packages}" ]]; then \
- pip install ${AIRFLOW_INSTALL_USER_FLAG} --force-reinstall --no-deps --no-index ${reinstalling_other_packages}
+ pip install --force-reinstall --no-deps --no-index ${reinstalling_other_packages}
# make sure correct PIP version is used
- pip install ${AIRFLOW_INSTALL_USER_FLAG} --upgrade "pip==${AIRFLOW_PIP_VERSION}"
+ pip install "pip==${AIRFLOW_PIP_VERSION}"
fi
}
common::get_airflow_version_specification
common::override_pip_version_if_needed
common::get_constraints_location
+common::show_pip_version_and_location
install_airflow_and_providers_from_docker_context_files
+
+common::show_pip_version_and_location
install_all_other_packages_from_docker_context_files
diff --git a/scripts/docker/install_mssql.sh b/scripts/docker/install_mssql.sh
index b5f8b51..14eb0fe 100755
--- a/scripts/docker/install_mssql.sh
+++ b/scripts/docker/install_mssql.sh
@@ -15,13 +15,13 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-set -exuo pipefail
+set -euo pipefail
function install_mssql_client() {
echo
echo Installing mssql client
echo
- curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add -
- curl https://packages.microsoft.com/config/debian/10/prod.list > /etc/apt/sources.list.d/mssql-release.list
+ curl --silent https://packages.microsoft.com/keys/microsoft.asc | apt-key add - >/dev/null 2>&1
+ curl --silent https://packages.microsoft.com/config/debian/10/prod.list > /etc/apt/sources.list.d/mssql-release.list
apt-get update -yqq
apt-get upgrade -yqq
ACCEPT_EULA=Y apt-get -yqq install -y --no-install-recommends msodbcsql17 mssql-tools
diff --git a/scripts/docker/install_mysql.sh b/scripts/docker/install_mysql.sh
index 7983eb0..abd4108 100755
--- a/scripts/docker/install_mysql.sh
+++ b/scripts/docker/install_mysql.sh
@@ -15,7 +15,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-set -exuo pipefail
+set -euo pipefail
declare -a packages
MYSQL_VERSION="8.0"
@@ -46,14 +46,13 @@ install_mysql_client() {
for keyserver in $(shuf -e ha.pool.sks-keyservers.net hkp://p80.pool.sks-keyservers.net:80 \
keyserver.ubuntu.com hkp://keyserver.ubuntu.com:80)
do
- gpg --keyserver "${keyserver}" --recv-keys "${key}" && break
+ gpg --keyserver "${keyserver}" --recv-keys "${key}" 2>&1 && break
done
set -e
gpg --export "${key}" > /etc/apt/trusted.gpg.d/mysql.gpg
gpgconf --kill all
rm -rf "${GNUPGHOME}"
unset GNUPGHOME
- apt-key list > /dev/null 2>&1
echo "deb http://repo.mysql.com/apt/debian/ buster mysql-${MYSQL_VERSION}" | tee -a /etc/apt/sources.list.d/mysql.list
apt-get update
apt-get install --no-install-recommends -y "${packages[@]}"
diff --git a/scripts/docker/install_pip_version.sh b/scripts/docker/install_pip_version.sh
index 6e0c3c1..312a954 100755
--- a/scripts/docker/install_pip_version.sh
+++ b/scripts/docker/install_pip_version.sh
@@ -30,11 +30,13 @@
. "$( dirname "${BASH_SOURCE[0]}" )/common.sh"
function install_pip_version() {
- pip install --no-cache-dir --upgrade "pip==${AIRFLOW_PIP_VERSION}" && mkdir -p /root/.local/bin
+ pip install --disable-pip-version-check --no-cache-dir --upgrade "pip==${AIRFLOW_PIP_VERSION}" &&
+ mkdir -p ${HOME}/.local/bin
}
common::get_airflow_version_specification
common::override_pip_version_if_needed
common::get_constraints_location
+common::show_pip_version_and_location
install_pip_version
diff --git a/scripts/in_container/_in_container_utils.sh b/scripts/in_container/_in_container_utils.sh
index 1cf30dd..783ea7d 100644
--- a/scripts/in_container/_in_container_utils.sh
+++ b/scripts/in_container/_in_container_utils.sh
@@ -242,7 +242,7 @@ function install_released_airflow_version() {
echo
rm -rf "${AIRFLOW_SOURCES}"/*.egg-info
- pip install --upgrade "apache-airflow==${version}"
+ pip install "apache-airflow==${version}"
}
function install_local_airflow_with_eager_upgrade() {
@@ -332,7 +332,7 @@ function setup_provider_packages() {
function install_supported_pip_version() {
group_start "Install supported PIP version ${AIRFLOW_PIP_VERSION}"
- pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}"
+ pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}"
group_end
}
diff --git a/scripts/in_container/prod/entrypoint_prod.sh b/scripts/in_container/prod/entrypoint_prod.sh
index 2e57773..6f74ce6 100755
--- a/scripts/in_container/prod/entrypoint_prod.sh
+++ b/scripts/in_container/prod/entrypoint_prod.sh
@@ -290,7 +290,7 @@ if [[ -n "${_PIP_ADDITIONAL_REQUIREMENTS=}" ]] ; then
>&2 echo " the container starts, so it is onlny useful for testing and trying out"
>&2 echo " of adding dependencies."
>&2 echo
- pip install --no-cache-dir --user ${_PIP_ADDITIONAL_REQUIREMENTS}
+ pip install --no-cache-dir ${_PIP_ADDITIONAL_REQUIREMENTS}
fi
diff --git a/scripts/in_container/run_prepare_airflow_packages.sh b/scripts/in_container/run_prepare_airflow_packages.sh
index 5148f64..eab50a6 100755
--- a/scripts/in_container/run_prepare_airflow_packages.sh
+++ b/scripts/in_container/run_prepare_airflow_packages.sh
@@ -34,7 +34,7 @@ function prepare_airflow_packages() {
rm -rf -- *egg-info*
rm -rf -- build
- pip install --upgrade "pip==${AIRFLOW_PIP_VERSION}" "wheel==${WHEEL_VERSION}"
+ pip install --disable-pip-version-check "pip==${AIRFLOW_PIP_VERSION}" "wheel==${WHEEL_VERSION}"
local packages=()