You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by ma...@apache.org on 2018/11/13 12:17:39 UTC
[incubator-mxnet] branch master updated: Add --no-cache option to
build.py when building containers (#13182)
This is an automated email from the ASF dual-hosted git repository.
marcoabreu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
The following commit(s) were added to refs/heads/master by this push:
new 8b38528 Add --no-cache option to build.py when building containers (#13182)
8b38528 is described below
commit 8b385288eddf068bfd5689f834fcbaf23dfac864
Author: Pedro Larroy <92...@users.noreply.github.com>
AuthorDate: Tue Nov 13 13:17:13 2018 +0100
Add --no-cache option to build.py when building containers (#13182)
Add functionality to build.py to disable caching
---
ci/README.md | 2 ++
ci/build.py | 30 ++++++++++++++----------------
ci/docker_cache.py | 4 ++++
ci/requirements.txt | 1 +
4 files changed, 21 insertions(+), 16 deletions(-)
diff --git a/ci/README.md b/ci/README.md
index 6c8a23f..5b6cc66 100644
--- a/ci/README.md
+++ b/ci/README.md
@@ -5,6 +5,8 @@ Docker containers
You need docker and nvidia docker if you have a GPU.
+Also you need to run `pip3 install docker` as it uses the [docker python module](https://docker-py.readthedocs.io/en/stable/containers.html#)
+
If you are in ubuntu an easy way to install Docker CE is executing the
following script:
diff --git a/ci/build.py b/ci/build.py
index 8f3fe2d..628f49e 100755
--- a/ci/build.py
+++ b/ci/build.py
@@ -101,6 +101,8 @@ def get_platforms(path: str = get_dockerfiles_path()) -> List[str]:
def get_docker_tag(platform: str, registry: str) -> str:
""":return: docker tag to be used for the container"""
+ if not registry:
+ registry = "mxnet_local"
return "{0}/build.{1}".format(registry, platform)
@@ -112,14 +114,14 @@ def get_docker_binary(use_nvidia_docker: bool) -> str:
return "nvidia-docker" if use_nvidia_docker else "docker"
-def build_docker(platform: str, docker_binary: str, registry: str, num_retries: int, use_cache: bool) -> str:
+def build_docker(platform: str, docker_binary: str, registry: str, num_retries: int, no_cache: bool) -> str:
"""
Build a container for the given platform
:param platform: Platform
:param docker_binary: docker binary to use (docker/nvidia-docker)
:param registry: Dockerhub registry name
:param num_retries: Number of retries to build the docker image
- :param use_cache: will pass cache_from to docker to use the previously pulled tag
+ :param no_cache: pass no-cache to docker to rebuild the images
:return: Id of the top level image
"""
tag = get_docker_tag(platform=platform, registry=registry)
@@ -144,7 +146,9 @@ def build_docker(platform: str, docker_binary: str, registry: str, num_retries:
"-f", get_dockerfile(platform),
"--build-arg", "USER_ID={}".format(os.getuid()),
"--build-arg", "GROUP_ID={}".format(os.getgid())]
- if use_cache:
+ if no_cache:
+ cmd.append("--no-cache")
+ elif registry:
cmd.extend(["--cache-from", tag])
cmd.extend(["-t", tag, get_dockerfiles_path()])
@@ -422,7 +426,7 @@ def main() -> int:
action='store_true')
parser.add_argument("-d", "--docker-registry",
- help="Dockerhub registry name to retrieve cache from. Default is 'mxnetci'",
+ help="Dockerhub registry name to retrieve cache from.",
default='mxnetci',
type=str)
@@ -431,10 +435,8 @@ def main() -> int:
default=1,
type=int)
- parser.add_argument("-c", "--no-dockerhub-cache", action="store_true",
- help="Disables use of --cache-from option on docker build, allowing docker"
- " to use local layers for caching. If absent, we use the cache from dockerhub"
- " which is the default.")
+ parser.add_argument("--no-cache", action="store_true",
+ help="passes --no-cache to docker build")
parser.add_argument("command",
help="command to run in the container",
@@ -447,9 +449,6 @@ def main() -> int:
args = parser.parse_args()
- def use_cache():
- return not args.no_dockerhub_cache or under_ci()
-
command = list(chain(*args.command))
docker_binary = get_docker_binary(args.nvidiadocker)
@@ -472,10 +471,10 @@ def main() -> int:
elif args.platform:
platform = args.platform
tag = get_docker_tag(platform=platform, registry=args.docker_registry)
- if use_cache():
+ if args.docker_registry:
load_docker_cache(tag=tag, docker_registry=args.docker_registry)
build_docker(platform=platform, docker_binary=docker_binary, registry=args.docker_registry,
- num_retries=args.docker_build_retries, use_cache=use_cache())
+ num_retries=args.docker_build_retries, no_cache=args.no_cache)
if args.build_only:
logging.warning("Container was just built. Exiting due to build-only.")
return 0
@@ -512,10 +511,9 @@ def main() -> int:
logging.info("Artifacts will be produced in the build/ directory.")
for platform in platforms:
tag = get_docker_tag(platform=platform, registry=args.docker_registry)
- if use_cache():
- load_docker_cache(tag=tag, docker_registry=args.docker_registry)
+ load_docker_cache(tag=tag, docker_registry=args.docker_registry)
build_docker(platform, docker_binary=docker_binary, registry=args.docker_registry,
- num_retries=args.docker_build_retries, use_cache=use_cache())
+ num_retries=args.docker_build_retries, no_cache=args.no_cache)
if args.build_only:
continue
shutil.rmtree(buildir(), ignore_errors=True)
diff --git a/ci/docker_cache.py b/ci/docker_cache.py
index bebcb25..23c173c 100755
--- a/ci/docker_cache.py
+++ b/ci/docker_cache.py
@@ -125,6 +125,10 @@ def load_docker_cache(registry, docker_tag) -> None:
:return: None
"""
# We don't have to retag the image since it's already in the right format
+ if not registry:
+ return
+ assert docker_tag
+
logging.info('Loading Docker cache for %s from %s', docker_tag, registry)
pull_cmd = ['docker', 'pull', docker_tag]
subprocess.call(pull_cmd) # Don't throw an error if the image does not exist
diff --git a/ci/requirements.txt b/ci/requirements.txt
new file mode 100644
index 0000000..8f21ead
--- /dev/null
+++ b/ci/requirements.txt
@@ -0,0 +1 @@
+docker==3.5.0