You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by zh...@apache.org on 2021/07/15 22:02:43 UTC
[incubator-mxnet] branch v1.x updated: [v1.x] Fix docker cache
build pipeline (#20385)
This is an automated email from the ASF dual-hosted git repository.
zha0q1 pushed a commit to branch v1.x
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
The following commit(s) were added to refs/heads/v1.x by this push:
new 0675fa3 [v1.x] Fix docker cache build pipeline (#20385)
0675fa3 is described below
commit 0675fa366895072f5533955e239d311385e45562
Author: Joe Evans <jo...@gmail.com>
AuthorDate: Thu Jul 15 15:00:31 2021 -0700
[v1.x] Fix docker cache build pipeline (#20385)
* libturbojpeg0-dev is only for ubuntu 18.04, so don't install on other ubuntu versions.
* Change logic to not install on 16.04 (works on 18.04 and 20.04.)
* Fix docker registry variable - was inadvertently changed to DOCKER_CACHE_REGISTRY from DOCKER_ECR_REGISTRY.
* Use our local ECR registry for docker-compose images as well.
* Revert changing DOCKER_CACHE_REGISTRY variable.
* Override DOCKER_CACHE_REGISTRY for docker-compose in docker cache pipeline.
* Login to ECR if using ECR for docker registry.
* Import docker_cache when needed.
---
ci/Jenkinsfile_docker_cache | 2 +-
ci/Jenkinsfile_utils.groovy | 2 +-
ci/build.py | 6 ++++++
ci/docker/docker-compose.yml | 2 +-
ci/docker/install/ubuntu_core.sh | 6 +++++-
ci/docker_cache.py | 2 ++
6 files changed, 16 insertions(+), 4 deletions(-)
diff --git a/ci/Jenkinsfile_docker_cache b/ci/Jenkinsfile_docker_cache
index 1be54a3..19003c7 100644
--- a/ci/Jenkinsfile_docker_cache
+++ b/ci/Jenkinsfile_docker_cache
@@ -38,7 +38,7 @@ core_logic: {
timeout(time: total_timeout, unit: 'MINUTES') {
utils.init_git()
sh "ci/docker_cache.py --docker-registry ${env.DOCKER_ECR_REGISTRY}"
- sh "cd ci && python3 ./docker_login.py --secret-name ${env.DOCKERHUB_SECRET_NAME} && docker-compose -f docker/docker-compose.yml build --parallel && docker-compose -f docker/docker-compose.yml push && docker logout"
+ sh "cd ci && $(aws ecr get-login --region ${env.DOCKER_ECR_REGION} --no-include-email) && DOCKER_CACHE_REGISTRY=${env.DOCKER_ECR_REGISTRY} docker-compose -f docker/docker-compose.yml build --parallel && DOCKER_CACHE_REGISTRY=${env.DOCKER_ECR_REGISTRY} docker-compose -f docker/docker-compose.yml push"
}
}
}
diff --git a/ci/Jenkinsfile_utils.groovy b/ci/Jenkinsfile_utils.groovy
index a758c72..a8e5faf 100644
--- a/ci/Jenkinsfile_utils.groovy
+++ b/ci/Jenkinsfile_utils.groovy
@@ -148,7 +148,7 @@ def collect_test_results_windows(original_file_name, new_file_name) {
def docker_run(platform, function_name, use_nvidia, shared_mem = '500m', env_vars = "", build_args = "") {
- def command = "ci/build.py %ENV_VARS% %BUILD_ARGS% --docker-registry ${env.DOCKER_CACHE_REGISTRY} %USE_NVIDIA% --platform %PLATFORM% --docker-build-retries 3 --shm-size %SHARED_MEM% /work/runtime_functions.sh %FUNCTION_NAME%"
+ def command = "ci/build.py %ENV_VARS% %BUILD_ARGS% --docker-registry ${env.DOCKER_ECR_REGISTRY} %USE_NVIDIA% --platform %PLATFORM% --docker-build-retries 3 --shm-size %SHARED_MEM% /work/runtime_functions.sh %FUNCTION_NAME%"
command = command.replaceAll('%ENV_VARS%', env_vars.length() > 0 ? "-e ${env_vars}" : '')
command = command.replaceAll('%BUILD_ARGS%', env_vars.length() > 0 ? "${build_args}" : '')
command = command.replaceAll('%USE_NVIDIA%', use_nvidia ? '--nvidiadocker' : '')
diff --git a/ci/build.py b/ci/build.py
index a316225..110d4d3 100755
--- a/ci/build.py
+++ b/ci/build.py
@@ -296,6 +296,12 @@ def load_docker_cache(platform, tag, docker_registry) -> None:
if platform in DOCKER_COMPOSE_WHITELIST:
env = os.environ.copy()
env["DOCKER_CACHE_REGISTRY"] = docker_registry
+ if "dkr.ecr" in docker_registry:
+ try:
+ import docker_cache
+ docker_cache._ecr_login(docker_registry)
+ except Exception:
+ logging.exception('Unable to login to ECR...')
cmd = ['docker-compose', '-f', 'docker/docker-compose.yml', 'pull', platform]
logging.info("Running command: 'DOCKER_CACHE_REGISTRY=%s %s'", docker_registry, ' '.join(cmd))
check_call(cmd, env=env)
diff --git a/ci/docker/docker-compose.yml b/ci/docker/docker-compose.yml
index 0784987..983dc45 100644
--- a/ci/docker/docker-compose.yml
+++ b/ci/docker/docker-compose.yml
@@ -91,4 +91,4 @@ services:
args:
BASE_IMAGE: nvidia/cuda:11.2.1-cudnn8-devel-centos7
cache_from:
- - ${DOCKER_CACHE_REGISTRY}/build.centos7_gpu_cu112:latest
\ No newline at end of file
+ - ${DOCKER_CACHE_REGISTRY}/build.centos7_gpu_cu112:latest
diff --git a/ci/docker/install/ubuntu_core.sh b/ci/docker/install/ubuntu_core.sh
index 53b5d4a..db50ffe 100755
--- a/ci/docker/install/ubuntu_core.sh
+++ b/ci/docker/install/ubuntu_core.sh
@@ -45,7 +45,6 @@ apt-get install -y \
libopencv-dev \
libjpeg-turbo8-dev \
libjpeg8-dev \
- libturbojpeg0-dev \
libzmq3-dev \
libtinfo-dev \
zlib1g-dev \
@@ -62,6 +61,11 @@ apt-get install -y \
# Use libturbojpeg package as it is correctly compiled with -fPIC flag
# https://github.com/HaxeFoundation/hashlink/issues/147
+# libturbojpeg0-dev is not available on 16.04
+source /etc/os-release
+if [[ "$VERSION_ID" != "16.04" ]]; then
+ apt-get install -y libturbojpeg0-dev
+fi
# CMake 3.13.2+ is required
diff --git a/ci/docker_cache.py b/ci/docker_cache.py
index 0ac96e7..600a15e 100755
--- a/ci/docker_cache.py
+++ b/ci/docker_cache.py
@@ -77,6 +77,8 @@ def _build_save_container(platform, registry, load_cache) -> Optional[str]:
"""
# docker-compose
if platform in build_util.DOCKER_COMPOSE_WHITELIST:
+ if "dkr.ecr" in registry:
+ _ecr_login(registry)
build_util.build_docker(platform=platform, registry=registry, num_retries=10, no_cache=False)
push_cmd = ['docker-compose', 'push', platform]
subprocess.check_call(push_cmd)