You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by GitBox <gi...@apache.org> on 2018/11/13 12:17:15 UTC
[GitHub] marcoabreu closed pull request #13182: Add functionality to
build.py to disable caching
marcoabreu closed pull request #13182: Add functionality to build.py to disable caching
URL: https://github.com/apache/incubator-mxnet/pull/13182
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/ci/README.md b/ci/README.md
index 6c8a23f03cf..5b6cc668978 100644
--- a/ci/README.md
+++ b/ci/README.md
@@ -5,6 +5,8 @@ Docker containers
You need docker and nvidia docker if you have a GPU.
+Also you need to run `pip3 install docker` as it uses the [docker python module](https://docker-py.readthedocs.io/en/stable/containers.html#)
+
If you are in ubuntu an easy way to install Docker CE is executing the
following script:
diff --git a/ci/build.py b/ci/build.py
index 8f3fe2d1244..628f49e706c 100755
--- a/ci/build.py
+++ b/ci/build.py
@@ -101,6 +101,8 @@ def get_platforms(path: str = get_dockerfiles_path()) -> List[str]:
def get_docker_tag(platform: str, registry: str) -> str:
""":return: docker tag to be used for the container"""
+ if not registry:
+ registry = "mxnet_local"
return "{0}/build.{1}".format(registry, platform)
@@ -112,14 +114,14 @@ def get_docker_binary(use_nvidia_docker: bool) -> str:
return "nvidia-docker" if use_nvidia_docker else "docker"
-def build_docker(platform: str, docker_binary: str, registry: str, num_retries: int, use_cache: bool) -> str:
+def build_docker(platform: str, docker_binary: str, registry: str, num_retries: int, no_cache: bool) -> str:
"""
Build a container for the given platform
:param platform: Platform
:param docker_binary: docker binary to use (docker/nvidia-docker)
:param registry: Dockerhub registry name
:param num_retries: Number of retries to build the docker image
- :param use_cache: will pass cache_from to docker to use the previously pulled tag
+ :param no_cache: pass no-cache to docker to rebuild the images
:return: Id of the top level image
"""
tag = get_docker_tag(platform=platform, registry=registry)
@@ -144,7 +146,9 @@ def build_docker(platform: str, docker_binary: str, registry: str, num_retries:
"-f", get_dockerfile(platform),
"--build-arg", "USER_ID={}".format(os.getuid()),
"--build-arg", "GROUP_ID={}".format(os.getgid())]
- if use_cache:
+ if no_cache:
+ cmd.append("--no-cache")
+ elif registry:
cmd.extend(["--cache-from", tag])
cmd.extend(["-t", tag, get_dockerfiles_path()])
@@ -422,7 +426,7 @@ def main() -> int:
action='store_true')
parser.add_argument("-d", "--docker-registry",
- help="Dockerhub registry name to retrieve cache from. Default is 'mxnetci'",
+ help="Dockerhub registry name to retrieve cache from.",
default='mxnetci',
type=str)
@@ -431,10 +435,8 @@ def main() -> int:
default=1,
type=int)
- parser.add_argument("-c", "--no-dockerhub-cache", action="store_true",
- help="Disables use of --cache-from option on docker build, allowing docker"
- " to use local layers for caching. If absent, we use the cache from dockerhub"
- " which is the default.")
+ parser.add_argument("--no-cache", action="store_true",
+ help="passes --no-cache to docker build")
parser.add_argument("command",
help="command to run in the container",
@@ -447,9 +449,6 @@ def main() -> int:
args = parser.parse_args()
- def use_cache():
- return not args.no_dockerhub_cache or under_ci()
-
command = list(chain(*args.command))
docker_binary = get_docker_binary(args.nvidiadocker)
@@ -472,10 +471,10 @@ def signal_handler(signum, _):
elif args.platform:
platform = args.platform
tag = get_docker_tag(platform=platform, registry=args.docker_registry)
- if use_cache():
+ if args.docker_registry:
load_docker_cache(tag=tag, docker_registry=args.docker_registry)
build_docker(platform=platform, docker_binary=docker_binary, registry=args.docker_registry,
- num_retries=args.docker_build_retries, use_cache=use_cache())
+ num_retries=args.docker_build_retries, no_cache=args.no_cache)
if args.build_only:
logging.warning("Container was just built. Exiting due to build-only.")
return 0
@@ -512,10 +511,9 @@ def signal_handler(signum, _):
logging.info("Artifacts will be produced in the build/ directory.")
for platform in platforms:
tag = get_docker_tag(platform=platform, registry=args.docker_registry)
- if use_cache():
- load_docker_cache(tag=tag, docker_registry=args.docker_registry)
+ load_docker_cache(tag=tag, docker_registry=args.docker_registry)
build_docker(platform, docker_binary=docker_binary, registry=args.docker_registry,
- num_retries=args.docker_build_retries, use_cache=use_cache())
+ num_retries=args.docker_build_retries, no_cache=args.no_cache)
if args.build_only:
continue
shutil.rmtree(buildir(), ignore_errors=True)
diff --git a/ci/docker_cache.py b/ci/docker_cache.py
index bebcb25fb8f..23c173c39e0 100755
--- a/ci/docker_cache.py
+++ b/ci/docker_cache.py
@@ -125,6 +125,10 @@ def load_docker_cache(registry, docker_tag) -> None:
:return: None
"""
# We don't have to retag the image since it's already in the right format
+ if not registry:
+ return
+ assert docker_tag
+
logging.info('Loading Docker cache for %s from %s', docker_tag, registry)
pull_cmd = ['docker', 'pull', docker_tag]
subprocess.call(pull_cmd) # Don't throw an error if the image does not exist
diff --git a/ci/requirements.txt b/ci/requirements.txt
new file mode 100644
index 00000000000..8f21ead27f7
--- /dev/null
+++ b/ci/requirements.txt
@@ -0,0 +1 @@
+docker==3.5.0
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services