[airflow] 12/44: Builds prod images on DockerHub from packages (#12908)

[ka...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

kaxilnaik pushed a commit to branch v1-10-test
in repository https://gitbox.apache.org/repos/asf/airflow.git

commit 6e3c049337275c097b1221c4e61d5ddac7832798
Author: Jarek Potiuk <ja...@polidea.com>
AuthorDate: Tue Dec 8 12:45:03 2020 +0100

    Builds prod images on DockerHub from packages (#12908)
    
    This build combines building both CI and PROD image in one
    script execution on DockerHub per python version.
    
    First the CI image is build and secondly, the image is used
    to build all the packages from sources and then those
    packages are used to build the PROD image.
    
    Resulting image will be package image built from latest sources.
    
    Closes: #12261
    (cherry picked from commit f9e9ad2b096ff9d8ee78224333f799ca3968b6bd)
---
 scripts/ci/images/ci_build_dockerhub.sh          | 64 ++++++++++++++----------
 scripts/ci/images/ci_prepare_prod_image_on_ci.sh | 16 +++---
 scripts/ci/libraries/_build_images.sh            | 33 ++++++++++++
 scripts/ci/libraries/_initialization.sh          |  9 ++++
 4 files changed, 90 insertions(+), 32 deletions(-)

diff --git a/scripts/ci/images/ci_build_dockerhub.sh b/scripts/ci/images/ci_build_dockerhub.sh
index f176448..a0ad0e6 100755
--- a/scripts/ci/images/ci_build_dockerhub.sh
+++ b/scripts/ci/images/ci_build_dockerhub.sh
@@ -16,11 +16,14 @@
 # specific language governing permissions and limitations
 # under the License.
 
+# shellcheck disable=SC2030,SC2031
+
 # This is hook build used by DockerHub. We are also using it
 # on CI to potentially rebuild (and refresh layers that
 # are not cached) Docker images that are used to run CI jobs
 export FORCE_ANSWER_TO_QUESTIONS="yes"
 export VERBOSE_COMMANDS="true"
+export VERBOSE="true"
 
 : "${DOCKER_REPO:?"ERROR: Please specify DOCKER_REPO variable following the pattern HOST/DOCKERHUB_USER/DOCKERHUB_REPO"}"
 
@@ -35,48 +38,57 @@ echo "DOCKERHUB_USER=${DOCKERHUB_USER}"
 echo "DOCKERHUB_REPO=${DOCKERHUB_REPO}"
 echo
 
-: "${DOCKER_TAG:?"ERROR: Please specify DOCKER_TAG variable following the pattern BRANCH-pythonX.Y[-ci]"}"
+: "${DOCKER_TAG:?"ERROR: Please specify DOCKER_TAG variable following the pattern BRANCH-pythonX.Y"}"
 
 echo "DOCKER_TAG=${DOCKER_TAG}"
 
-[[ ${DOCKER_TAG:=} =~ ${DEFAULT_BRANCH}-python([0-9.]*)(.*) ]] && export PYTHON_MAJOR_MINOR_VERSION=${BASH_REMATCH[1]}
+[[ ${DOCKER_TAG:=} =~ ${DEFAULT_BRANCH}-python([0-9.]*) ]] && export PYTHON_MAJOR_MINOR_VERSION=${BASH_REMATCH[1]}
 
-: "${PYTHON_MAJOR_MINOR_VERSION:?"The tag '${DOCKER_TAG}' should follow the pattern .*-pythonX.Y[-ci]"}"
+: "${PYTHON_MAJOR_MINOR_VERSION:?"The tag '${DOCKER_TAG}' should follow the pattern .*-pythonX.Y"}"
 
 echo "Detected PYTHON_MAJOR_MINOR_VERSION=${PYTHON_MAJOR_MINOR_VERSION}"
 echo
 
-FORCE_AIRFLOW_PROD_BASE_TAG="${DOCKER_TAG}"
-export FORCE_AIRFLOW_PROD_BASE_TAG
-
-readonly FORCE_AIRFLOW_PROD_BASE_TAG
-
-if [[ "${FORCE_AIRFLOW_PROD_BASE_TAG}" =~ [0-9].* ]]; then
-    # Disable cache if we are building a tagged version
-    export DOCKER_CACHE="disabled"
-fi
-
-# shellcheck source=scripts/ci/libraries/_script_init.sh
-. "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
+(
+    export INSTALL_FROM_PYPI="true"
+    export INSTALL_FROM_DOCKER_CONTEXT_FILES="false"
+    export INSTALL_PROVIDERS_FROM_SOURCES="true"
+    export AIRFLOW_PRE_CACHED_PIP_PACKAGES="true"
+    export DOCKER_CACHE="pulled"
+    # shellcheck source=scripts/ci/libraries/_script_init.sh
+    . "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
 
-if [[ ${DOCKER_TAG} == *python*-ci ]]; then
     echo
-    echo "Building CI image"
+    echo "Building and pushing CI image for ${PYTHON_MAJOR_MINOR_VERSION} in a sub-process"
     echo
     rm -rf "${BUILD_CACHE_DIR}"
     build_images::prepare_ci_build
     build_images::rebuild_ci_image_if_needed
-    push_pull_remove_images::push_ci_images
-elif [[ ${DOCKER_TAG} == *python* ]]; then
+    if [[ ! "${DOCKER_TAG}" =~ ^[0-9].* ]]; then
+        # Do not push if we are building a tagged version
+        push_pull_remove_images::push_ci_images
+    fi
+)
+
+(
+    export INSTALL_FROM_PYPI="false"
+    export INSTALL_FROM_DOCKER_CONTEXT_FILES="true"
+    export INSTALL_PROVIDERS_FROM_SOURCES="false"
+    export AIRFLOW_PRE_CACHED_PIP_PACKAGES="false"
+    export DOCKER_CACHE="pulled"
+
+    if [[ "${DOCKER_TAG}" =~ ^[0-9].* ]]; then
+        # Disable cache and set name of the tag as image name if we are building a tagged version
+        export DOCKER_CACHE="disabled"
+        export FORCE_AIRFLOW_PROD_BASE_TAG="${DOCKER_TAG}"
+    fi
+    # shellcheck source=scripts/ci/libraries/_script_init.sh
+    . "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
     echo
-    echo "Building prod image"
+    echo "Building and pushing PROD image for ${PYTHON_MAJOR_MINOR_VERSION} in a sub-process"
     echo
     rm -rf "${BUILD_CACHE_DIR}"
     build_images::prepare_prod_build
-    build_images::build_prod_images
+    build_images::build_prod_images_from_packages
     push_pull_remove_images::push_prod_images
-else
-    echo
-    echo "Skipping the build in Dockerhub. The tag is not good: ${DOCKER_TAG}"
-    echo
-fi
+)
diff --git a/scripts/ci/images/ci_prepare_prod_image_on_ci.sh b/scripts/ci/images/ci_prepare_prod_image_on_ci.sh
index d38182b..700487c 100755
--- a/scripts/ci/images/ci_prepare_prod_image_on_ci.sh
+++ b/scripts/ci/images/ci_prepare_prod_image_on_ci.sh
@@ -15,6 +15,15 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
+
+export INSTALL_FROM_PYPI="false"
+export INSTALL_FROM_DOCKER_CONTEXT_FILES="true"
+export INSTALL_PROVIDERS_FROM_SOURCES="false"
+export AIRFLOW_PRE_CACHED_PIP_PACKAGES="false"
+export DOCKER_CACHE="local"
+export VERBOSE="true"
+
+
 # shellcheck source=scripts/ci/libraries/_script_init.sh
 . "$( dirname "${BASH_SOURCE[0]}" )/../libraries/_script_init.sh"
 
@@ -23,9 +32,6 @@
 function build_prod_images_on_ci() {
     build_images::prepare_prod_build
 
-    rm -rf "${BUILD_CACHE_DIR}"
-    mkdir -pv "${BUILD_CACHE_DIR}"
-
     if [[ ${USE_GITHUB_REGISTRY} == "true" && ${GITHUB_REGISTRY_WAIT_FOR_IMAGE} == "true" ]]; then
 
         # Tries to wait for the image indefinitely
@@ -34,10 +40,8 @@ function build_prod_images_on_ci() {
         build_images::wait_for_image_tag "${GITHUB_REGISTRY_AIRFLOW_PROD_IMAGE}" \
             ":${GITHUB_REGISTRY_PULL_IMAGE_TAG}" "${AIRFLOW_PROD_IMAGE}"
 
-        build_images::wait_for_image_tag "${GITHUB_REGISTRY_AIRFLOW_PROD_BUILD_IMAGE}" \
-            ":${GITHUB_REGISTRY_PULL_IMAGE_TAG}" "${AIRFLOW_PROD_BUILD_IMAGE}"
     else
-        build_images::build_prod_images
+        build_images::build_prod_images_from_packages
     fi
 
 
diff --git a/scripts/ci/libraries/_build_images.sh b/scripts/ci/libraries/_build_images.sh
index d054f15..8f48c16 100644
--- a/scripts/ci/libraries/_build_images.sh
+++ b/scripts/ci/libraries/_build_images.sh
@@ -856,6 +856,39 @@ function build_images::determine_docker_cache_strategy() {
 }
 
 
+function build_images::build_prod_images_from_packages() {
+    # Cleanup dist and docker-context-files folders
+    mkdir -pv "${AIRFLOW_SOURCES}/dist"
+    mkdir -pv "${AIRFLOW_SOURCES}/docker-context-files"
+    rm -f "${AIRFLOW_SOURCES}/dist/"*.{whl,tar.gz}
+    rm -f "${AIRFLOW_SOURCES}/docker-context-files/"*.{whl,tar.gz}
+
+    pip_download_command="pip download -d /dist '.[${INSTALLED_EXTRAS}]' --constraint 'https://raw.githubusercontent.com/apache/airflow/${DEFAULT_CONSTRAINTS_BRANCH}/constraints-${PYTHON_MAJOR_MINOR_VERSION}.txt'"
+
+    # Download all dependencies needed
+    docker run --rm --entrypoint /bin/bash \
+        "${EXTRA_DOCKER_FLAGS[@]}" \
+        "${AIRFLOW_CI_IMAGE}" -c "${pip_download_command}"
+
+    # Remove all downloaded apache airflow packages
+    rm -f "${AIRFLOW_SOURCES}/dist/"apache_airflow*.whl
+    rm -f "${AIRFLOW_SOURCES}/dist/"apache-airflow*.tar.gz
+
+    # Remove all downloaded apache airflow packages
+    mv -f "${AIRFLOW_SOURCES}/dist/"* "${AIRFLOW_SOURCES}/docker-context-files/"
+
+    # Build apache airflow packages
+    build_airflow_packages::build_airflow_packages
+
+    # Remove generated tar.gz packages
+    rm -f "${AIRFLOW_SOURCES}/dist/"apache-airflow*.tar.gz
+
+    # move the packages to docker-context-files folder
+    mkdir -pv "${AIRFLOW_SOURCES}/docker-context-files"
+    mv "${AIRFLOW_SOURCES}/dist/"* "${AIRFLOW_SOURCES}/docker-context-files/"
+    build_images::build_prod_images
+}
+
 # Useful information for people who stumble upon a pip check failure
 function build_images::inform_about_pip_check() {
         echo """
diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh
index 9c3ab67..5f2ec6d 100644
--- a/scripts/ci/libraries/_initialization.sh
+++ b/scripts/ci/libraries/_initialization.sh
@@ -144,6 +144,15 @@ function initialization::initialize_base_variables() {
     # If no Airflow Home defined - fallback to ${HOME}/airflow
     AIRFLOW_HOME_DIR=${AIRFLOW_HOME:=${HOME}/airflow}
     export AIRFLOW_HOME_DIR
+
+    export INSTALLED_EXTRAS="async,amazon,celery,kubernetes,docker,dask,elasticsearch,ftp,grpc,hashicorp,http,imap,google,azure,mysql,postgres,redis,sendgrid,sftp,slack,ssh,statsd,virtualenv"
+    readonly INSTALLED_EXTRAS
+
+    PIP_VERSION="20.2.4"
+    export PIP_VERSION
+
+    WHEEL_VERSION="0.35.1"
+    export WHEEL_VERSION
 }
 
 # Determine current branch