You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by GitBox <gi...@apache.org> on 2018/08/11 14:20:31 UTC
[GitHub] marcoabreu closed pull request #12125: CI scripts refinements.
Separate Py2 and Py3 installs cripts. Fix perms.
marcoabreu closed pull request #12125: CI scripts refinements. Separate Py2 and Py3 installs cripts. Fix perms.
URL: https://github.com/apache/incubator-mxnet/pull/12125
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/ci/build.py b/ci/build.py
index a9d6a63537f..0a1ad4cf575 100755
--- a/ci/build.py
+++ b/ci/build.py
@@ -43,6 +43,43 @@
CCACHE_MAXSIZE = '500G'
+
+
+def retry(ExceptionToCheck, tries=4, delay_s=1, backoff=2):
+ """Retry calling the decorated function using an exponential backoff.
+
+ http://www.saltycrane.com/blog/2009/11/trying-out-retry-decorator-python/
+ original from: http://wiki.python.org/moin/PythonDecoratorLibrary#Retry
+
+ :param ExceptionToCheck: the exception to check. may be a tuple of
+ exceptions to check
+ :type ExceptionToCheck: Exception or tuple
+ :param tries: number of times to try (not retry) before giving up
+ :type tries: int
+ :param delay_s: initial delay between retries in seconds
+ :type delay_s: int
+ :param backoff: backoff multiplier e.g. value of 2 will double the delay
+ each retry
+ :type backoff: int
+ """
+ import time
+ from functools import wraps
+ def decorated_retry(f):
+ @wraps(f)
+ def f_retry(*args, **kwargs):
+ mtries, mdelay = tries, delay_s
+ while mtries > 1:
+ try:
+ return f(*args, **kwargs)
+ except ExceptionToCheck as e:
+ logging.warning("Exception: %s, Retrying in %d seconds...", str(e), mdelay)
+ time.sleep(mdelay)
+ mtries -= 1
+ mdelay *= backoff
+ return f(*args, **kwargs)
+ return f_retry # true decorator
+ return decorated_retry
+
def under_ci() -> bool:
""":return: True if we run in Jenkins."""
return 'JOB_NAME' in os.environ
@@ -77,9 +114,8 @@ def build_docker(platform: str, docker_binary: str, registry: str, num_retries:
:param num_retries: Number of retries to build the docker image
:return: Id of the top level image
"""
-
tag = get_docker_tag(platform=platform, registry=registry)
- logging.info("Building container tagged '%s' with %s", tag, docker_binary)
+ logging.info("Building docker container tagged '%s' with %s", tag, docker_binary)
#
# We add a user with the same group as the executing non-root user so files created in the
# container match permissions of the local user. Same for the group.
@@ -91,40 +127,24 @@ def build_docker(platform: str, docker_binary: str, registry: str, num_retries:
# docker pull see: docker_cache.load_docker_cache
#
# This doesn't work with multi head docker files.
- #
-
- for i in range(num_retries):
- logging.info('%d out of %d tries to build the docker image.', i + 1, num_retries)
-
- cmd = [docker_binary, "build",
- "-f", get_dockerfile(platform),
- "--build-arg", "USER_ID={}".format(os.getuid()),
- "--build-arg", "GROUP_ID={}".format(os.getgid()),
- "--cache-from", tag,
- "-t", tag,
- "docker"]
+ #
+ cmd = [docker_binary, "build",
+ "-f", get_dockerfile(platform),
+ "--build-arg", "USER_ID={}".format(os.getuid()),
+ "--build-arg", "GROUP_ID={}".format(os.getgid()),
+ "--cache-from", tag,
+ "-t", tag,
+ "docker"]
+
+ @retry(subprocess.CalledProcessError, tries=num_retries)
+ def run_cmd():
logging.info("Running command: '%s'", ' '.join(cmd))
- try:
- check_call(cmd)
- # Docker build was successful. Call break to break out of the retry mechanism
- break
- except subprocess.CalledProcessError as e:
- saved_exception = e
- logging.error('Failed to build docker image')
- # Building the docker image failed. Call continue to trigger the retry mechanism
- continue
- else:
- # Num retries exceeded
- logging.exception('Exception during build of docker image', saved_exception)
- logging.fatal('Failed to build the docker image, aborting...')
- sys.exit(1)
+ check_call(cmd)
+ run_cmd()
# Get image id by reading the tag. It's guaranteed (except race condition) that the tag exists. Otherwise, the
# check_call would have failed
- image_id = _get_local_image_id(docker_binary=docker_binary, docker_tag=tag)
- if not image_id:
- raise FileNotFoundError('Unable to find docker image id matching with {}'.format(tag))
- return image_id
+ return _get_local_image_id(docker_binary=docker_binary, docker_tag=tag)
def _get_local_image_id(docker_binary, docker_tag):
@@ -136,6 +156,8 @@ def _get_local_image_id(docker_binary, docker_tag):
cmd = [docker_binary, "images", "-q", docker_tag]
image_id_b = subprocess.check_output(cmd)
image_id = image_id_b.decode('utf-8').strip()
+ if not image_id:
+ raise RuntimeError('Unable to find docker image id matching with tag {}'.format(tag))
return image_id
@@ -186,7 +208,7 @@ def container_run(platform: str,
'-e', "CCACHE_LOGFILE=/tmp/ccache.log", # a container-scoped log, useful for ccache verification.
tag]
runlist.extend(command)
- cmd = '\\\n\t'.join(runlist)
+ cmd = ' \\\n\t'.join(runlist)
ret = 0
if not dry_run and not interactive:
logging.info("Running %s in container %s", command, tag)
@@ -199,14 +221,14 @@ def container_run(platform: str,
# -ti can't be after the tag, as is interpreted as a command so hook it up after the -u argument
idx = into_cmd.index('-u') + 2
into_cmd[idx:idx] = ['-ti']
- cmd = '\\\n\t'.join(into_cmd)
+ cmd = ' \\\n\t'.join(into_cmd)
logging.info("Executing:\n%s\n", cmd)
docker_run_cmd = ' '.join(into_cmd)
ret = call(into_cmd)
if not dry_run and not interactive and ret != 0:
logging.error("Running of command in container failed (%s):\n%s\n", ret, cmd)
- logging.error("You can get into the container by adding the -i option")
+ logging.error("You can get into the container by adding the -i option to this script")
raise subprocess.CalledProcessError(ret, cmd)
return docker_run_cmd
@@ -303,7 +325,6 @@ def use_cache():
command = list(chain(*args.command))
docker_binary = get_docker_binary(args.nvidiadocker)
shared_memory_size = args.shared_memory_size
- num_docker_build_retires = args.docker_build_retries
if args.list:
list_platforms()
@@ -312,7 +333,7 @@ def use_cache():
tag = get_docker_tag(platform=platform, registry=args.docker_registry)
if use_cache():
load_docker_cache(tag=tag, docker_registry=args.docker_registry)
- build_docker(platform, docker_binary, registry=args.docker_registry, num_retries=num_docker_build_retires)
+ build_docker(platform, docker_binary, registry=args.docker_registry, num_retries=args.docker_build_retries)
if args.build_only:
logging.warning("Container was just built. Exiting due to build-only.")
return 0
@@ -346,7 +367,7 @@ def use_cache():
tag = get_docker_tag(platform=platform, registry=args.docker_registry)
if use_cache():
load_docker_cache(tag=tag, docker_registry=args.docker_registry)
- build_docker(platform, docker_binary, args.docker_registry, num_retries=num_docker_build_retires)
+ build_docker(platform, docker_binary, args.docker_registry, num_retries=args.docker_build_retries)
if args.build_only:
continue
build_platform = "build_{}".format(platform)
diff --git a/ci/docker/Dockerfile.build.android_armv7 b/ci/docker/Dockerfile.build.android_armv7
old mode 100755
new mode 100644
diff --git a/ci/docker/Dockerfile.build.android_armv8 b/ci/docker/Dockerfile.build.android_armv8
old mode 100755
new mode 100644
diff --git a/ci/docker/Dockerfile.build.armv6 b/ci/docker/Dockerfile.build.armv6
old mode 100755
new mode 100644
diff --git a/ci/docker/Dockerfile.build.armv7 b/ci/docker/Dockerfile.build.armv7
old mode 100755
new mode 100644
diff --git a/ci/docker/Dockerfile.build.armv8 b/ci/docker/Dockerfile.build.armv8
old mode 100755
new mode 100644
diff --git a/ci/docker/Dockerfile.build.centos7_cpu b/ci/docker/Dockerfile.build.centos7_cpu
old mode 100755
new mode 100644
diff --git a/ci/docker/Dockerfile.build.centos7_gpu b/ci/docker/Dockerfile.build.centos7_gpu
old mode 100755
new mode 100644
diff --git a/ci/docker/Dockerfile.build.jetson b/ci/docker/Dockerfile.build.jetson
old mode 100755
new mode 100644
diff --git a/ci/docker/Dockerfile.build.ubuntu_base_cpu b/ci/docker/Dockerfile.build.ubuntu_base_cpu
old mode 100755
new mode 100644
diff --git a/ci/docker/Dockerfile.build.ubuntu_base_gpu b/ci/docker/Dockerfile.build.ubuntu_base_gpu
old mode 100755
new mode 100644
diff --git a/ci/docker/Dockerfile.build.ubuntu_blc b/ci/docker/Dockerfile.build.ubuntu_blc
old mode 100755
new mode 100644
index 294740ce139..208cba2111f
--- a/ci/docker/Dockerfile.build.ubuntu_blc
+++ b/ci/docker/Dockerfile.build.ubuntu_blc
@@ -24,8 +24,13 @@ WORKDIR /work/deps
COPY install/ubuntu_core.sh /work/
RUN /work/ubuntu_core.sh
-COPY install/ubuntu_python.sh /work/
-RUN /work/ubuntu_python.sh
+
+COPY install/ubuntu_python2.sh /work/
+RUN /work/ubuntu_python2.sh
+
+COPY install/ubuntu_python3.sh /work/
+RUN /work/ubuntu_python3.sh
+
COPY install/ubuntu_npm_blc.sh /work/
RUN /work/ubuntu_npm_blc.sh
diff --git a/ci/docker/Dockerfile.build.ubuntu_build_cuda b/ci/docker/Dockerfile.build.ubuntu_build_cuda
old mode 100755
new mode 100644
index 9ed0cbbe3e5..19e9265f88d
--- a/ci/docker/Dockerfile.build.ubuntu_build_cuda
+++ b/ci/docker/Dockerfile.build.ubuntu_build_cuda
@@ -27,20 +27,30 @@ WORKDIR /work/deps
COPY install/ubuntu_core.sh /work/
RUN /work/ubuntu_core.sh
+
COPY install/deb_ubuntu_ccache.sh /work/
RUN /work/deb_ubuntu_ccache.sh
-COPY install/ubuntu_python.sh /work/
-RUN /work/ubuntu_python.sh
+
+COPY install/ubuntu_python2.sh /work/
+RUN /work/ubuntu_python2.sh
+
+COPY install/ubuntu_python3.sh /work/
+RUN /work/ubuntu_python3.sh
+
COPY install/ubuntu_scala.sh /work/
COPY install/sbt.gpg /work/
RUN /work/ubuntu_scala.sh
+
COPY install/ubuntu_r.sh /work/
COPY install/r.gpg /work/
RUN /work/ubuntu_r.sh
+
COPY install/ubuntu_perl.sh /work/
RUN /work/ubuntu_perl.sh
+
COPY install/ubuntu_clang.sh /work/
RUN /work/ubuntu_clang.sh
+
COPY install/ubuntu_mklml.sh /work/
RUN /work/ubuntu_mklml.sh
diff --git a/ci/docker/Dockerfile.build.ubuntu_cpu b/ci/docker/Dockerfile.build.ubuntu_cpu
old mode 100755
new mode 100644
index 58a8e9a50d7..76df3dfc1c1
--- a/ci/docker/Dockerfile.build.ubuntu_cpu
+++ b/ci/docker/Dockerfile.build.ubuntu_cpu
@@ -28,8 +28,11 @@ RUN /work/ubuntu_core.sh
COPY install/deb_ubuntu_ccache.sh /work/
RUN /work/deb_ubuntu_ccache.sh
-COPY install/ubuntu_python.sh /work/
-RUN /work/ubuntu_python.sh
+COPY install/ubuntu_python2.sh /work/
+RUN /work/ubuntu_python2.sh
+
+COPY install/ubuntu_python3.sh /work/
+RUN /work/ubuntu_python3.sh
COPY install/ubuntu_scala.sh /work/
COPY install/sbt.gpg /work/
diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu b/ci/docker/Dockerfile.build.ubuntu_gpu
old mode 100755
new mode 100644
index de38948e623..8b4cd1e5a3f
--- a/ci/docker/Dockerfile.build.ubuntu_gpu
+++ b/ci/docker/Dockerfile.build.ubuntu_gpu
@@ -28,8 +28,11 @@ RUN /work/ubuntu_core.sh
COPY install/deb_ubuntu_ccache.sh /work/
RUN /work/deb_ubuntu_ccache.sh
-COPY install/ubuntu_python.sh /work/
-RUN /work/ubuntu_python.sh
+COPY install/ubuntu_python2.sh /work/
+RUN /work/ubuntu_python2.sh
+
+COPY install/ubuntu_python3.sh /work/
+RUN /work/ubuntu_python3.sh
COPY install/ubuntu_scala.sh /work/
COPY install/sbt.gpg /work/
diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu_tensorrt b/ci/docker/Dockerfile.build.ubuntu_gpu_tensorrt
old mode 100755
new mode 100644
index 255da316041..3f0bbc666f2
--- a/ci/docker/Dockerfile.build.ubuntu_gpu_tensorrt
+++ b/ci/docker/Dockerfile.build.ubuntu_gpu_tensorrt
@@ -24,10 +24,16 @@ WORKDIR /work/deps
COPY install/ubuntu_core.sh /work/
RUN /work/ubuntu_core.sh
+
COPY install/deb_ubuntu_ccache.sh /work/
RUN /work/deb_ubuntu_ccache.sh
-COPY install/ubuntu_python.sh /work/
-RUN /work/ubuntu_python.sh
+
+COPY install/ubuntu_python2.sh /work/
+RUN /work/ubuntu_python2.sh
+
+COPY install/ubuntu_python3.sh /work/
+RUN /work/ubuntu_python3.sh
+
COPY install/tensorrt.sh /work
RUN /work/tensorrt.sh
diff --git a/ci/docker/Dockerfile.build.ubuntu_nightly_cpu b/ci/docker/Dockerfile.build.ubuntu_nightly_cpu
old mode 100755
new mode 100644
index 58ff33e3013..c803188263f
--- a/ci/docker/Dockerfile.build.ubuntu_nightly_cpu
+++ b/ci/docker/Dockerfile.build.ubuntu_nightly_cpu
@@ -28,8 +28,11 @@ RUN /work/ubuntu_core.sh
COPY install/deb_ubuntu_ccache.sh /work/
RUN /work/deb_ubuntu_ccache.sh
-COPY install/ubuntu_python.sh /work/
-RUN /work/ubuntu_python.sh
+COPY install/ubuntu_python2.sh /work/
+RUN /work/ubuntu_python2.sh
+
+COPY install/ubuntu_python3.sh /work/
+RUN /work/ubuntu_python3.sh
COPY install/ubuntu_scala.sh /work/
COPY install/sbt.gpg /work/
diff --git a/ci/docker/Dockerfile.build.ubuntu_nightly_gpu b/ci/docker/Dockerfile.build.ubuntu_nightly_gpu
old mode 100755
new mode 100644
index 017bade1775..c0e31e2c15e
--- a/ci/docker/Dockerfile.build.ubuntu_nightly_gpu
+++ b/ci/docker/Dockerfile.build.ubuntu_nightly_gpu
@@ -28,8 +28,11 @@ RUN /work/ubuntu_core.sh
COPY install/deb_ubuntu_ccache.sh /work/
RUN /work/deb_ubuntu_ccache.sh
-COPY install/ubuntu_python.sh /work/
-RUN /work/ubuntu_python.sh
+COPY install/ubuntu_python2.sh /work/
+RUN /work/ubuntu_python2.sh
+
+COPY install/ubuntu_python3.sh /work/
+RUN /work/ubuntu_python3.sh
COPY install/ubuntu_scala.sh /work/
COPY install/sbt.gpg /work/
diff --git a/ci/docker/Dockerfile.build.ubuntu_rat b/ci/docker/Dockerfile.build.ubuntu_rat
old mode 100755
new mode 100644
diff --git a/ci/docker/install/ubuntu_python.sh b/ci/docker/install/ubuntu_python2.sh
similarity index 84%
rename from ci/docker/install/ubuntu_python.sh
rename to ci/docker/install/ubuntu_python2.sh
index e71cac8a389..f0526e2d830 100755
--- a/ci/docker/install/ubuntu_python.sh
+++ b/ci/docker/install/ubuntu_python2.sh
@@ -22,12 +22,10 @@
set -ex
# install libraries for mxnet's python package on ubuntu
-apt-get install -y python-dev python3-dev virtualenv
+apt-get install -y python-dev virtualenv wget
# the version of the pip shipped with ubuntu may be too lower, install a recent version here
wget -nv https://bootstrap.pypa.io/get-pip.py
-python3 get-pip.py
python2 get-pip.py
-pip2 install nose cpplint==1.3.0 pylint==1.8.3 'numpy<1.15.0,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3
-pip3 install nose cpplint==1.3.0 pylint==1.8.3 'numpy<1.15.0,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3
+pip2 install nose cpplint==1.3.0 pylint==1.8.3 'numpy<1.15.0,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3 mock
diff --git a/ci/docker/install/ubuntu_python3.sh b/ci/docker/install/ubuntu_python3.sh
new file mode 100755
index 00000000000..1dad5a7aa20
--- /dev/null
+++ b/ci/docker/install/ubuntu_python3.sh
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
+
+set -ex
+# install libraries for mxnet's python package on ubuntu
+apt-get install -y python3-dev virtualenv wget
+
+# the version of the pip shipped with ubuntu may be too lower, install a recent version here
+wget -nv https://bootstrap.pypa.io/get-pip.py
+python3 get-pip.py
+
+pip3 install nose cpplint==1.3.0 pylint==1.8.3 'numpy<1.15.0,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3 mock
diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index 815eae9c55b..2a813c833e0 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -1005,6 +1005,7 @@ broken_link_checker() {
./tests/nightly/broken_link_checker_test/broken_link_checker.sh
}
+
##############################################################
# MAIN
#
diff --git a/ci/test_docker_cache.py b/ci/test_docker_cache.py
old mode 100644
new mode 100755
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services