You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2020/11/29 23:59:56 UTC
[airflow] 09/18: Fixes unneeded docker-context-files added in CI
(#12534)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v1-10-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 08d9e0eeac1b6fd2af5db3d9b30b0b40f29cd1e8
Author: Jarek Potiuk <ja...@polidea.com>
AuthorDate: Sat Nov 21 19:21:43 2020 +0100
Fixes unneeded docker-context-files added in CI (#12534)
We do not need to add docker-context-files in CI before we run
first "cache" PIP installation. Adding it might cause the effect
that the cache will always be invalidated in case someone has
a file added there before building and pushing the image.
This PR fixes the problem by adding docker-context files later
in the Dockerfile and changing the constraints location
used in the "cache" step to always use the github constraints in
this case.
Closes #12509
(cherry picked from commit 37548f09acb91edd041565f52051f58610402cb3)
---
Dockerfile | 3 ++-
Dockerfile.ci | 11 ++++++-----
IMAGES.rst | 9 ++++++++-
scripts/ci/libraries/_build_images.sh | 35 +++++++++++++++--------------------
4 files changed, 31 insertions(+), 27 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index 8ad3db9..00442bc 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -176,7 +176,8 @@ RUN if [[ ${AIRFLOW_PRE_CACHED_PIP_PACKAGES} == "true" ]]; then \
fi; \
pip install --user \
"https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz#egg=apache-airflow[${AIRFLOW_EXTRAS}]" \
- --constraint "${AIRFLOW_CONSTRAINTS_LOCATION}" && pip uninstall --yes apache-airflow; \
+ --constraint "https://raw.githubusercontent.com/apache/airflow/${AIRFLOW_CONSTRAINTS_REFERENCE}/constraints-${PYTHON_MAJOR_MINOR_VERSION}.txt" \
+ && pip uninstall --yes apache-airflow; \
fi
ARG AIRFLOW_SOURCES_FROM="."
diff --git a/Dockerfile.ci b/Dockerfile.ci
index aa426ef..ac51a56 100644
--- a/Dockerfile.ci
+++ b/Dockerfile.ci
@@ -262,10 +262,6 @@ ENV AIRFLOW_LOCAL_PIP_WHEELS=${AIRFLOW_LOCAL_PIP_WHEELS}
ARG INSTALL_AIRFLOW_VIA_PIP="true"
ENV INSTALL_AIRFLOW_VIA_PIP=${INSTALL_AIRFLOW_VIA_PIP}
-# If wheel files are found in /docker-context-files during installation
-# they are also installed additionally to whatever is installed from Airflow.
-COPY docker-context-files /docker-context-files
-
# In case of CI builds we want to pre-install master version of airflow dependencies so that
# We do not have to always reinstall it from the scratch.
# This can be reinstalled from latest master by increasing PIP_DEPENDENCIES_EPOCH_NUMBER.
@@ -273,7 +269,8 @@ COPY docker-context-files /docker-context-files
RUN if [[ ${AIRFLOW_PRE_CACHED_PIP_PACKAGES} == "true" ]]; then \
pip install \
"https://github.com/${AIRFLOW_REPO}/archive/${AIRFLOW_BRANCH}.tar.gz#egg=apache-airflow[${AIRFLOW_EXTRAS}]" \
- --constraint "${AIRFLOW_CONSTRAINTS_URL}" && pip uninstall --yes apache-airflow; \
+ --constraint "https://raw.githubusercontent.com/apache/airflow/${AIRFLOW_CONSTRAINTS_REFERENCE}/constraints-${PYTHON_MAJOR_MINOR_VERSION}.txt" \
+ && pip uninstall --yes apache-airflow; \
fi
@@ -322,6 +319,10 @@ RUN if [[ ${INSTALL_AIRFLOW_VIA_PIP} == "true" ]]; then \
fi; \
fi
+# If wheel files are found in /docker-context-files during installation
+# they are also installed additionally to whatever is installed from Airflow.
+COPY docker-context-files/ /docker-context-files/
+
RUN if [[ ${AIRFLOW_LOCAL_PIP_WHEELS} != "true" ]]; then \
if ls /docker-context-files/*.whl 1> /dev/null 2>&1; then \
pip install --no-deps /docker-context-files/*.whl; \
diff --git a/IMAGES.rst b/IMAGES.rst
index 8c913db..13ce935 100644
--- a/IMAGES.rst
+++ b/IMAGES.rst
@@ -399,7 +399,14 @@ The following build arguments (``--build-arg`` in docker build command) can be u
| | | file has to be in docker context so |
| | | it's best to place such file in |
| | | one of the folders included in |
-| | | dockerignore |
+| | | dockerignore . for example in the |
+| | | 'docker-context-files'. Note that the |
+| | | location does not work for the first |
+| | | stage of installation when the |
+| | | stage of installation when the |
+| | | ``AIRFLOW_PRE_CACHED_PIP_PACKAGES`` is |
+| | | set to true. Default location from |
+| | | GitHub is used in this case. |
+------------------------------------------+------------------------------------------+------------------------------------------+
| ``AIRFLOW_LOCAL_PIP_WHEELS`` | ``false`` | If set to true, Airflow and it's |
| | | dependencies are installed from locally |
diff --git a/scripts/ci/libraries/_build_images.sh b/scripts/ci/libraries/_build_images.sh
index fb756ba..5bd2d06 100644
--- a/scripts/ci/libraries/_build_images.sh
+++ b/scripts/ci/libraries/_build_images.sh
@@ -97,6 +97,19 @@ function build_images::forget_last_answer() {
fi
}
+function build_images::confirm_via_terminal() {
+ echo > "${DETECTED_TERMINAL}"
+ echo > "${DETECTED_TERMINAL}"
+ echo "Make sure that you rebased to latest master before rebuilding!" > "${DETECTED_TERMINAL}"
+ echo > "${DETECTED_TERMINAL}"
+ # Make sure to use output of tty rather than stdin/stdout when available - this way confirm
+ # will works also in case of pre-commits (git does not pass stdin/stdout to pre-commit hooks)
+ # shellcheck disable=SC2094
+ "${AIRFLOW_SOURCES}/confirm" "${ACTION} image ${THE_IMAGE_TYPE}-python${PYTHON_MAJOR_MINOR_VERSION}" \
+ <"${DETECTED_TERMINAL}" >"${DETECTED_TERMINAL}"
+ RES=$?
+}
+
# Confirms if hte image should be rebuild and interactively checks it with the user.
# In case iit needs to be rebuild. It only ask the user if it determines that the rebuild
# is needed and that the rebuild is not already forced. It asks the user using available terminals
@@ -144,29 +157,11 @@ function build_images::confirm_image_rebuild() {
"${AIRFLOW_SOURCES}/confirm" "${ACTION} image ${THE_IMAGE_TYPE}-python${PYTHON_MAJOR_MINOR_VERSION}"
RES=$?
elif [[ ${DETECTED_TERMINAL:=$(tty)} != "not a tty" ]]; then
- echo > "${DETECTED_TERMINAL}"
- echo > "${DETECTED_TERMINAL}"
- echo "Make sure that you rebased to latest master before rebuilding!" > "${DETECTED_TERMINAL}"
- echo > "${DETECTED_TERMINAL}"
- # Make sure to use output of tty rather than stdin/stdout when available - this way confirm
- # will works also in case of pre-commits (git does not pass stdin/stdout to pre-commit hooks)
- # shellcheck disable=SC2094
- "${AIRFLOW_SOURCES}/confirm" "${ACTION} image ${THE_IMAGE_TYPE}-python${PYTHON_MAJOR_MINOR_VERSION}" \
- <"${DETECTED_TERMINAL}" >"${DETECTED_TERMINAL}"
- RES=$?
export DETECTED_TERMINAL
+ build_images::confirm_via_terminal
elif [[ -c /dev/tty ]]; then
export DETECTED_TERMINAL=/dev/tty
- # Make sure to use /dev/tty first rather than stdin/stdout when available - this way confirm
- # will works also in case of pre-commits (git does not pass stdin/stdout to pre-commit hooks)
- echo > "${DETECTED_TERMINAL}"
- echo > "${DETECTED_TERMINAL}"
- echo "Make sure that you rebased to latest master before rebuilding!" > "${DETECTED_TERMINAL}"
- echo > "${DETECTED_TERMINAL}"
- # shellcheck disable=SC2094
- "${AIRFLOW_SOURCES}/confirm" "${ACTION} image ${THE_IMAGE_TYPE}-python${PYTHON_MAJOR_MINOR_VERSION}" \
- <"${DETECTED_TERMINAL}" >"${DETECTED_TERMINAL}"
- RES=$?
+ build_images::confirm_via_terminal
else
verbosity::print_info
verbosity::print_info "No terminal, no stdin - quitting"