You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by je...@apache.org on 2021/08/13 23:58:45 UTC
[incubator-mxnet] branch v1.x updated: [v1.x] Docker cache
enhancements (#20525)
This is an automated email from the ASF dual-hosted git repository.
jevans pushed a commit to branch v1.x
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
The following commit(s) were added to refs/heads/v1.x by this push:
new 2e754d1 [v1.x] Docker cache enhancements (#20525)
2e754d1 is described below
commit 2e754d18d736ee743efdd4ccddfe258cd73caf79
Author: Joe Evans <jo...@gmail.com>
AuthorDate: Fri Aug 13 16:57:26 2021 -0700
[v1.x] Docker cache enhancements (#20525)
* Properly initialize set so that docker cache builds exclude the centos7 dockerfile.
* Change the number of parallel builds to 1 for easier debugging and bandwidth control, also parameterize the number of build retries and reduce from 10 to 1 for easier debugging.
* Remove old dockerfiles.
* Update docker-compose config to use new ECR-based naming scheme.
* Refactor build.py and docker_cache.py to support building and pushing containers for the docker-compose based platforms. Add options for selecting architecture (defaults to x86) since aarch64 container builds will fail on x86 systems.
* Remove old commands for building docker-compose based containers since they are not built in docker_cache.py.
* Fix running of docker-compose based containers via build.py.
---
ci/Jenkinsfile_docker_cache | 1 -
ci/build.py | 71 +++++++++++++++++---------
ci/docker/Dockerfile.build.ubuntu_gpu_cu80 | 75 ---------------------------
ci/docker/Dockerfile.build.ubuntu_gpu_cu90 | 81 ------------------------------
ci/docker/Dockerfile.build.ubuntu_gpu_cu92 | 80 -----------------------------
ci/docker/docker-compose.yml | 26 +++++-----
ci/docker_cache.py | 20 +++++---
7 files changed, 74 insertions(+), 280 deletions(-)
diff --git a/ci/Jenkinsfile_docker_cache b/ci/Jenkinsfile_docker_cache
index c6ec4f4..f90bf04 100644
--- a/ci/Jenkinsfile_docker_cache
+++ b/ci/Jenkinsfile_docker_cache
@@ -38,7 +38,6 @@ core_logic: {
timeout(time: total_timeout, unit: 'MINUTES') {
utils.init_git()
sh "ci/docker_cache.py --docker-registry ${env.DOCKER_ECR_REGISTRY}"
- sh "cd ci && DOCKER_CACHE_REGISTRY=${env.DOCKER_ECR_REGISTRY} docker-compose -f docker/docker-compose.yml build --parallel && DOCKER_CACHE_REGISTRY=${env.DOCKER_ECR_REGISTRY} docker-compose -f docker/docker-compose.yml push"
}
}
}
diff --git a/ci/build.py b/ci/build.py
index 110d4d3..06a7475 100755
--- a/ci/build.py
+++ b/ci/build.py
@@ -42,24 +42,36 @@ import yaml
from util import *
-DOCKER_COMPOSE_WHITELIST = ('centos7_cpu', 'centos7_gpu_cu92', 'centos7_gpu_cu100',
- 'centos7_gpu_cu101', 'centos7_gpu_cu102', 'centos7_gpu_cu110',
- 'centos7_gpu_cu112')
-
# Files for docker compose
-DOCKER_COMPOSE_FILES = set(('docker/build.centos7'))
+DOCKER_COMPOSE_FILES = set(['docker/build.centos7'])
+
+# keywords to identify arm-based dockerfiles
+AARCH_FILE_KEYWORDS = ['armv', 'aarch64']
def get_dockerfiles_path():
return "docker"
+def get_docker_compose_platforms(path: str = get_dockerfiles_path()):
+ platforms = set()
+ with open(os.path.join(path, "docker-compose.yml"), "r") as f:
+ compose_config = yaml.load(f.read(), yaml.SafeLoader)
+ for platform in compose_config["services"]:
+ platforms.add(platform)
+ return platforms
-def get_platforms(path: str = get_dockerfiles_path(), legacy_only=False) -> List[str]:
- """Get a list of architectures given our dockerfiles"""
+
+def get_platforms(path: str = get_dockerfiles_path(), arch='x86') -> List[str]:
+ """Get a list of platforms given our dockerfiles"""
dockerfiles = glob.glob(os.path.join(path, "Dockerfile.*"))
dockerfiles = set(filter(lambda x: x[-1] != '~', dockerfiles))
files = set(map(lambda x: re.sub(r"Dockerfile.(.*)", r"\1", x), dockerfiles))
- if legacy_only:
- files = files - DOCKER_COMPOSE_FILES
+ files = files - DOCKER_COMPOSE_FILES
+ files.update(["build."+x for x in get_docker_compose_platforms()])
+ arm_files = set(filter(lambda x: any(y in x for y in AARCH_FILE_KEYWORDS), files))
+ if arch == 'x86':
+ files = files - arm_files
+ elif arch == 'aarch64':
+ files = arm_files
platforms = list(map(lambda x: os.path.split(x)[1], sorted(files)))
return platforms
@@ -87,14 +99,21 @@ def _hash_file(ctx, filename):
break
ctx.update(d)
+def is_docker_compose(platform: str) -> bool:
+ """:return: boolean whether specified platform container uses docker-compose"""
+ platlist = get_docker_compose_platforms()
+ platform = platform.split(".")[1] if any(x in platform for x in ['build.', 'publish.']) else platform
+ return platform in platlist
+
+
def get_docker_tag(platform: str, registry: str) -> str:
""":return: docker tag to be used for the container"""
- if platform in DOCKER_COMPOSE_WHITELIST:
+ platform = platform if any(x in platform for x in ['build.', 'publish.']) else 'build.{}'.format(platform)
+ if is_docker_compose(platform):
with open("docker/docker-compose.yml", "r") as f:
compose_config = yaml.load(f.read(), yaml.SafeLoader)
- return compose_config["services"][platform]["image"].replace('${DOCKER_CACHE_REGISTRY}', registry)
+ return compose_config["services"][platform.split(".")[1]]["image"].replace('${DOCKER_CACHE_REGISTRY}', registry)
- platform = platform if any(x in platform for x in ['build.', 'publish.']) else 'build.{}'.format(platform)
if not registry:
registry = "mxnet_local"
dockerfile = get_dockerfile(platform)
@@ -121,9 +140,9 @@ def build_docker(platform: str, registry: str, num_retries: int, no_cache: bool,
:return: Id of the top level image
"""
tag = get_docker_tag(platform=platform, registry=registry)
-
# docker-compose
- if platform in DOCKER_COMPOSE_WHITELIST:
+ if is_docker_compose(platform):
+ docker_compose_platform = platform.split(".")[1] if any(x in platform for x in ['build.', 'publish.']) else platform
logging.info('Building docker container tagged \'%s\' based on ci/docker/docker-compose.yml', tag)
# We add a user with the same group as the executing non-root user so files created in the
# container match permissions of the local user. Same for the group.
@@ -132,7 +151,7 @@ def build_docker(platform: str, registry: str, num_retries: int, no_cache: bool,
"--build-arg", "GROUP_ID={}".format(os.getgid())]
if cache_intermediate:
cmd.append('--no-rm')
- cmd.append(platform)
+ cmd.append(docker_compose_platform)
else:
logging.info("Building docker container tagged '%s'", tag)
#
@@ -286,14 +305,15 @@ def container_run(platform: str,
return 0
-def list_platforms() -> str:
- return "\nSupported platforms:\n{}".format('\n'.join(get_platforms()))
+def list_platforms(arch='x86') -> str:
+ return "\nSupported platforms:\n{}".format('\n'.join(get_platforms(arch=arch)))
def load_docker_cache(platform, tag, docker_registry) -> None:
"""Imports tagged container from the given docker registry"""
if docker_registry:
- if platform in DOCKER_COMPOSE_WHITELIST:
+ if is_docker_compose(platform):
+ docker_compose_platform = platform.split(".")[1] if any(x in platform for x in ['build.', 'publish.']) else platform
env = os.environ.copy()
env["DOCKER_CACHE_REGISTRY"] = docker_registry
if "dkr.ecr" in docker_registry:
@@ -302,7 +322,7 @@ def load_docker_cache(platform, tag, docker_registry) -> None:
docker_cache._ecr_login(docker_registry)
except Exception:
logging.exception('Unable to login to ECR...')
- cmd = ['docker-compose', '-f', 'docker/docker-compose.yml', 'pull', platform]
+ cmd = ['docker-compose', '-f', 'docker/docker-compose.yml', 'pull', docker_compose_platform]
logging.info("Running command: 'DOCKER_CACHE_REGISTRY=%s %s'", docker_registry, ' '.join(cmd))
check_call(cmd, env=env)
return
@@ -341,6 +361,11 @@ def main() -> int:
help="platform",
type=str)
+ parser.add_argument("-A", "--architecture",
+ help="Architecture of images to build (x86 or aarch64). Default is x86.",
+ default='x86',
+ dest='architecture')
+
parser.add_argument("-b", "--build-only",
help="Only build the container, don't build the project",
action='store_true')
@@ -407,7 +432,7 @@ def main() -> int:
for e in args.environment])
if args.list:
- print(list_platforms())
+ print(list_platforms(arch=args.architecture))
elif args.platform:
platform = args.platform
tag = get_docker_tag(platform=platform, registry=args.docker_registry)
@@ -451,9 +476,9 @@ def main() -> int:
return ret
elif args.all:
- platforms = get_platforms()
+ platforms = get_platforms(arch=args.architecture)
platforms = [platform for platform in platforms if 'build.' in platform]
- logging.info("Building for all architectures: %s", platforms)
+ logging.info("Building for all platforms: %s", platforms)
logging.info("Artifacts will be produced in the build/ directory.")
for platform in platforms:
tag = get_docker_tag(platform=platform, registry=args.docker_registry)
@@ -480,7 +505,7 @@ def main() -> int:
else:
parser.print_help()
- list_platforms()
+ list_platforms(arch=args.architecture)
print("""
Examples:
diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu_cu80 b/ci/docker/Dockerfile.build.ubuntu_gpu_cu80
deleted file mode 100644
index c203161..0000000
--- a/ci/docker/Dockerfile.build.ubuntu_gpu_cu80
+++ /dev/null
@@ -1,75 +0,0 @@
-# -*- mode: dockerfile -*-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-# Dockerfile to run MXNet on Ubuntu 16.04 for GPU
-
-FROM nvidia/cuda:8.0-cudnn7-devel-ubuntu16.04
-
-WORKDIR /work/deps
-
-COPY install/ubuntu_core.sh /work/
-RUN /work/ubuntu_core.sh
-
-COPY install/deb_ubuntu_ccache.sh /work/
-RUN /work/deb_ubuntu_ccache.sh
-
-COPY install/ubuntu_python.sh /work/
-COPY install/requirements /work/
-RUN /work/ubuntu_python.sh
-
-COPY install/ubuntu_scala.sh /work/
-COPY install/sbt.gpg /work/
-RUN /work/ubuntu_scala.sh
-
-COPY install/ubuntu_r.sh /work/
-RUN /work/ubuntu_r.sh
-
-COPY install/ubuntu_perl.sh /work/
-RUN /work/ubuntu_perl.sh
-
-COPY install/ubuntu_clang.sh /work/
-RUN /work/ubuntu_clang.sh
-
-COPY install/ubuntu_tvm.sh /work/
-RUN /work/ubuntu_tvm.sh
-
-COPY install/ubuntu_llvm.sh /work/
-RUN /work/ubuntu_llvm.sh
-
-COPY install/ubuntu_caffe.sh /work/
-RUN /work/ubuntu_caffe.sh
-
-COPY install/ubuntu_onnx.sh /work/
-RUN /work/ubuntu_onnx.sh
-
-COPY install/ubuntu_docs.sh /work/
-COPY install/requirements /work/
-RUN /work/ubuntu_docs.sh
-
-COPY install/ubuntu_tutorials.sh /work/
-RUN /work/ubuntu_tutorials.sh
-
-ARG USER_ID=0
-ARG GROUP_ID=0
-COPY install/ubuntu_adduser.sh /work/
-RUN /work/ubuntu_adduser.sh
-
-COPY runtime_functions.sh /work/
-
-WORKDIR /work/mxnet
-ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib
diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu_cu90 b/ci/docker/Dockerfile.build.ubuntu_gpu_cu90
deleted file mode 100644
index 0e180a3..0000000
--- a/ci/docker/Dockerfile.build.ubuntu_gpu_cu90
+++ /dev/null
@@ -1,81 +0,0 @@
-# -*- mode: dockerfile -*-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-# Dockerfile to run MXNet on Ubuntu 16.04 for GPU
-
-FROM nvidia/cuda:9.0-devel-ubuntu16.04
-
-WORKDIR /work/deps
-
-COPY install/ubuntu_core.sh /work/
-RUN /work/ubuntu_core.sh
-
-COPY install/deb_ubuntu_ccache.sh /work/
-RUN /work/deb_ubuntu_ccache.sh
-
-COPY install/ubuntu_python.sh /work/
-COPY install/requirements /work/
-RUN /work/ubuntu_python.sh
-
-COPY install/ubuntu_scala.sh /work/
-COPY install/sbt.gpg /work/
-RUN /work/ubuntu_scala.sh
-
-COPY install/ubuntu_r.sh /work/
-RUN /work/ubuntu_r.sh
-
-COPY install/ubuntu_perl.sh /work/
-RUN /work/ubuntu_perl.sh
-
-COPY install/ubuntu_clang.sh /work/
-RUN /work/ubuntu_clang.sh
-
-COPY install/ubuntu_tvm.sh /work/
-RUN /work/ubuntu_tvm.sh
-
-COPY install/ubuntu_llvm.sh /work/
-RUN /work/ubuntu_llvm.sh
-
-COPY install/ubuntu_caffe.sh /work/
-RUN /work/ubuntu_caffe.sh
-
-COPY install/ubuntu_onnx.sh /work/
-RUN /work/ubuntu_onnx.sh
-
-COPY install/ubuntu_docs.sh /work/
-COPY install/requirements /work/
-RUN /work/ubuntu_docs.sh
-
-COPY install/ubuntu_tutorials.sh /work/
-RUN /work/ubuntu_tutorials.sh
-
-ENV CUDA_VERSION=9.0.176
-ENV CUDNN_VERSION=7.6.5.32
-COPY install/ubuntu_cudnn.sh /work/
-RUN /work/ubuntu_cudnn.sh
-
-# Always last
-ARG USER_ID=0
-ARG GROUP_ID=0
-COPY install/ubuntu_adduser.sh /work/
-RUN /work/ubuntu_adduser.sh
-
-COPY runtime_functions.sh /work/
-
-WORKDIR /work/mxnet
-ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib
diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu_cu92 b/ci/docker/Dockerfile.build.ubuntu_gpu_cu92
deleted file mode 100644
index fe72b2e..0000000
--- a/ci/docker/Dockerfile.build.ubuntu_gpu_cu92
+++ /dev/null
@@ -1,80 +0,0 @@
-# -*- mode: dockerfile -*-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-# Dockerfile to run MXNet on Ubuntu 16.04 for GPU
-
-FROM nvidia/cuda:9.2-devel-ubuntu16.04
-
-WORKDIR /work/deps
-
-COPY install/ubuntu_core.sh /work/
-RUN /work/ubuntu_core.sh
-
-COPY install/deb_ubuntu_ccache.sh /work/
-RUN /work/deb_ubuntu_ccache.sh
-
-COPY install/ubuntu_python.sh /work/
-COPY install/requirements /work/
-RUN /work/ubuntu_python.sh
-
-COPY install/ubuntu_scala.sh /work/
-COPY install/sbt.gpg /work/
-RUN /work/ubuntu_scala.sh
-
-COPY install/ubuntu_r.sh /work/
-RUN /work/ubuntu_r.sh
-
-COPY install/ubuntu_perl.sh /work/
-RUN /work/ubuntu_perl.sh
-
-COPY install/ubuntu_clang.sh /work/
-RUN /work/ubuntu_clang.sh
-
-COPY install/ubuntu_tvm.sh /work/
-RUN /work/ubuntu_tvm.sh
-
-COPY install/ubuntu_llvm.sh /work/
-RUN /work/ubuntu_llvm.sh
-
-COPY install/ubuntu_caffe.sh /work/
-RUN /work/ubuntu_caffe.sh
-
-COPY install/ubuntu_onnx.sh /work/
-RUN /work/ubuntu_onnx.sh
-
-COPY install/ubuntu_docs.sh /work/
-COPY install/requirements /work/
-RUN /work/ubuntu_docs.sh
-
-COPY install/ubuntu_tutorials.sh /work/
-RUN /work/ubuntu_tutorials.sh
-
-ENV CUDA_VERSION=9.2.148
-ENV CUDNN_VERSION=7.6.5.32
-COPY install/ubuntu_cudnn.sh /work/
-RUN /work/ubuntu_cudnn.sh
-
-ARG USER_ID=0
-ARG GROUP_ID=0
-COPY install/ubuntu_adduser.sh /work/
-RUN /work/ubuntu_adduser.sh
-
-COPY runtime_functions.sh /work/
-
-WORKDIR /work/mxnet
-ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib
diff --git a/ci/docker/docker-compose.yml b/ci/docker/docker-compose.yml
index 983dc45..07d538f 100644
--- a/ci/docker/docker-compose.yml
+++ b/ci/docker/docker-compose.yml
@@ -28,10 +28,10 @@ services:
# versions
###################################################################################################
centos7_cpu:
- # The resulting image will be named build.centos7_cpu:latest and will be
+ # The resulting image will be named build.centos7_cpu and will be
# pushed to the dockerhub user specified in the environment variable
# ${DOCKER_CACHE_REGISTRY} (typicall "mxnetci") under this name
- image: ${DOCKER_CACHE_REGISTRY}/build.centos7_cpu:latest
+ image: ${DOCKER_CACHE_REGISTRY}:build.centos7_cpu
build:
context: .
dockerfile: Dockerfile.build.centos7
@@ -41,9 +41,9 @@ services:
# BASE_IMAGE is used to dynamically specify the FROM image in Dockerfile.build.centos7
BASE_IMAGE: centos:7
cache_from:
- - ${DOCKER_CACHE_REGISTRY}/build.centos7_cpu:latest
+ - ${DOCKER_CACHE_REGISTRY}:build.centos7_cpu
centos7_gpu_cu100:
- image: ${DOCKER_CACHE_REGISTRY}/build.centos7_gpu_cu100:latest
+ image: ${DOCKER_CACHE_REGISTRY}:build.centos7_gpu_cu100
build:
context: .
dockerfile: Dockerfile.build.centos7
@@ -51,9 +51,9 @@ services:
args:
BASE_IMAGE: nvidia/cuda:10.0-cudnn7-devel-centos7
cache_from:
- - ${DOCKER_CACHE_REGISTRY}/build.centos7_gpu_cu100:latest
+ - ${DOCKER_CACHE_REGISTRY}:build.centos7_gpu_cu100
centos7_gpu_cu101:
- image: ${DOCKER_CACHE_REGISTRY}/build.centos7_gpu_cu101:latest
+ image: ${DOCKER_CACHE_REGISTRY}:build.centos7_gpu_cu101
build:
context: .
dockerfile: Dockerfile.build.centos7
@@ -61,9 +61,9 @@ services:
args:
BASE_IMAGE: nvidia/cuda:10.1-cudnn7-devel-centos7
cache_from:
- - ${DOCKER_CACHE_REGISTRY}/build.centos7_gpu_cu101:latest
+ - ${DOCKER_CACHE_REGISTRY}:build.centos7_gpu_cu101
centos7_gpu_cu102:
- image: ${DOCKER_CACHE_REGISTRY}/build.centos7_gpu_cu102:latest
+ image: ${DOCKER_CACHE_REGISTRY}:build.centos7_gpu_cu102
build:
context: .
dockerfile: Dockerfile.build.centos7
@@ -71,9 +71,9 @@ services:
args:
BASE_IMAGE: nvidia/cuda:10.2-cudnn8-devel-centos7
cache_from:
- - ${DOCKER_CACHE_REGISTRY}/build.centos7_gpu_cu102:latest
+ - ${DOCKER_CACHE_REGISTRY}:build.centos7_gpu_cu102
centos7_gpu_cu110:
- image: ${DOCKER_CACHE_REGISTRY}/build.centos7_gpu_cu110:latest
+ image: ${DOCKER_CACHE_REGISTRY}:build.centos7_gpu_cu110
build:
context: .
dockerfile: Dockerfile.build.centos7
@@ -81,9 +81,9 @@ services:
args:
BASE_IMAGE: nvidia/cuda:11.0-cudnn8-devel-centos7
cache_from:
- - ${DOCKER_CACHE_REGISTRY}/build.centos7_gpu_cu110:latest
+ - ${DOCKER_CACHE_REGISTRY}:build.centos7_gpu_cu110
centos7_gpu_cu112:
- image: ${DOCKER_CACHE_REGISTRY}/build.centos7_gpu_cu112:latest
+ image: ${DOCKER_CACHE_REGISTRY}:build.centos7_gpu_cu112
build:
context: .
dockerfile: Dockerfile.build.centos7
@@ -91,4 +91,4 @@ services:
args:
BASE_IMAGE: nvidia/cuda:11.2.1-cudnn8-devel-centos7
cache_from:
- - ${DOCKER_CACHE_REGISTRY}/build.centos7_gpu_cu112:latest
+ - ${DOCKER_CACHE_REGISTRY}:build.centos7_gpu_cu112
diff --git a/ci/docker_cache.py b/ci/docker_cache.py
index dc24ba4..f0a72b5 100755
--- a/ci/docker_cache.py
+++ b/ci/docker_cache.py
@@ -38,8 +38,9 @@ from util import retry
DOCKER_CACHE_NUM_RETRIES = 3
DOCKER_CACHE_TIMEOUT_MINS = 45
-PARALLEL_BUILDS = 10
+PARALLEL_BUILDS = 5
DOCKER_CACHE_RETRY_SECONDS = 5
+DOCKER_BUILD_NUM_RETRIES = 2
def build_save_containers(platforms, registry, load_cache) -> int:
@@ -76,12 +77,16 @@ def _build_save_container(platform, registry, load_cache) -> Optional[str]:
:return: Platform if failed, None otherwise
"""
# docker-compose
- if platform in build_util.DOCKER_COMPOSE_WHITELIST:
+ if build_util.is_docker_compose(platform):
if "dkr.ecr" in registry:
_ecr_login(registry)
- build_util.build_docker(platform=platform, registry=registry, num_retries=10, no_cache=False)
- push_cmd = ['docker-compose', 'push', platform]
- subprocess.check_call(push_cmd)
+ build_util.build_docker(platform=platform, registry=registry,
+ num_retries=DOCKER_BUILD_NUM_RETRIES, no_cache=False)
+ docker_compose_service = platform.split(".")[1]
+ env = os.environ.copy()
+ env["DOCKER_CACHE_REGISTRY"] = registry
+ push_cmd = ['docker-compose', '-f', 'docker/docker-compose.yml', 'push', docker_compose_service]
+ subprocess.check_call(push_cmd, env=env)
return None
docker_tag = build_util.get_docker_tag(platform=platform, registry=registry)
@@ -94,7 +99,8 @@ def _build_save_container(platform, registry, load_cache) -> Optional[str]:
logging.debug('Building %s as %s', platform, docker_tag)
try:
# Increase the number of retries for building the cache.
- image_id = build_util.build_docker(platform=platform, registry=registry, num_retries=10, no_cache=False)
+ image_id = build_util.build_docker(platform=platform, registry=registry,
+ num_retries=DOCKER_BUILD_NUM_RETRIES, no_cache=False)
logging.info('Built %s as %s', docker_tag, image_id)
# Push cache to registry
@@ -219,7 +225,7 @@ def main() -> int:
args = parser.parse_args()
- platforms = build_util.get_platforms(legacy_only=True)
+ platforms = build_util.get_platforms()
if "dkr.ecr" in args.docker_registry:
_ecr_login(args.docker_registry)