You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by ma...@apache.org on 2018/08/11 14:20:38 UTC
[incubator-mxnet] branch master updated: CI scripts refinements.
Separate Py2 and Py3 installs cripts. Fix perms. (#12125)
This is an automated email from the ASF dual-hosted git repository.
marcoabreu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
The following commit(s) were added to refs/heads/master by this push:
new 1b60478 CI scripts refinements. Separate Py2 and Py3 installs cripts. Fix perms. (#12125)
1b60478 is described below
commit 1b60478db2c66ef82f5b3ed48b12832d6da72e61
Author: Pedro Larroy <92...@users.noreply.github.com>
AuthorDate: Sat Aug 11 16:20:29 2018 +0200
CI scripts refinements. Separate Py2 and Py3 installs cripts. Fix perms. (#12125)
---
ci/build.py | 97 +++++++++++++---------
ci/docker/Dockerfile.build.android_armv7 | 0
ci/docker/Dockerfile.build.android_armv8 | 0
ci/docker/Dockerfile.build.armv6 | 0
ci/docker/Dockerfile.build.armv7 | 0
ci/docker/Dockerfile.build.armv8 | 0
ci/docker/Dockerfile.build.centos7_cpu | 0
ci/docker/Dockerfile.build.centos7_gpu | 0
ci/docker/Dockerfile.build.jetson | 0
ci/docker/Dockerfile.build.ubuntu_base_cpu | 0
ci/docker/Dockerfile.build.ubuntu_base_gpu | 0
ci/docker/Dockerfile.build.ubuntu_blc | 9 +-
ci/docker/Dockerfile.build.ubuntu_build_cuda | 14 +++-
ci/docker/Dockerfile.build.ubuntu_cpu | 7 +-
ci/docker/Dockerfile.build.ubuntu_gpu | 7 +-
ci/docker/Dockerfile.build.ubuntu_gpu_tensorrt | 10 ++-
ci/docker/Dockerfile.build.ubuntu_nightly_cpu | 7 +-
ci/docker/Dockerfile.build.ubuntu_nightly_gpu | 7 +-
ci/docker/Dockerfile.build.ubuntu_rat | 0
.../{ubuntu_python.sh => ubuntu_python2.sh} | 6 +-
.../{ubuntu_python.sh => ubuntu_python3.sh} | 6 +-
ci/docker/runtime_functions.sh | 1 +
ci/test_docker_cache.py | 0
23 files changed, 111 insertions(+), 60 deletions(-)
diff --git a/ci/build.py b/ci/build.py
index a9d6a63..0a1ad4c 100755
--- a/ci/build.py
+++ b/ci/build.py
@@ -43,6 +43,43 @@ from util import *
CCACHE_MAXSIZE = '500G'
+
+
+def retry(ExceptionToCheck, tries=4, delay_s=1, backoff=2):
+ """Retry calling the decorated function using an exponential backoff.
+
+ http://www.saltycrane.com/blog/2009/11/trying-out-retry-decorator-python/
+ original from: http://wiki.python.org/moin/PythonDecoratorLibrary#Retry
+
+ :param ExceptionToCheck: the exception to check. may be a tuple of
+ exceptions to check
+ :type ExceptionToCheck: Exception or tuple
+ :param tries: number of times to try (not retry) before giving up
+ :type tries: int
+ :param delay_s: initial delay between retries in seconds
+ :type delay_s: int
+ :param backoff: backoff multiplier e.g. value of 2 will double the delay
+ each retry
+ :type backoff: int
+ """
+ import time
+ from functools import wraps
+ def decorated_retry(f):
+ @wraps(f)
+ def f_retry(*args, **kwargs):
+ mtries, mdelay = tries, delay_s
+ while mtries > 1:
+ try:
+ return f(*args, **kwargs)
+ except ExceptionToCheck as e:
+ logging.warning("Exception: %s, Retrying in %d seconds...", str(e), mdelay)
+ time.sleep(mdelay)
+ mtries -= 1
+ mdelay *= backoff
+ return f(*args, **kwargs)
+ return f_retry # true decorator
+ return decorated_retry
+
def under_ci() -> bool:
""":return: True if we run in Jenkins."""
return 'JOB_NAME' in os.environ
@@ -77,9 +114,8 @@ def build_docker(platform: str, docker_binary: str, registry: str, num_retries:
:param num_retries: Number of retries to build the docker image
:return: Id of the top level image
"""
-
tag = get_docker_tag(platform=platform, registry=registry)
- logging.info("Building container tagged '%s' with %s", tag, docker_binary)
+ logging.info("Building docker container tagged '%s' with %s", tag, docker_binary)
#
# We add a user with the same group as the executing non-root user so files created in the
# container match permissions of the local user. Same for the group.
@@ -91,40 +127,24 @@ def build_docker(platform: str, docker_binary: str, registry: str, num_retries:
# docker pull see: docker_cache.load_docker_cache
#
# This doesn't work with multi head docker files.
- #
-
- for i in range(num_retries):
- logging.info('%d out of %d tries to build the docker image.', i + 1, num_retries)
-
- cmd = [docker_binary, "build",
- "-f", get_dockerfile(platform),
- "--build-arg", "USER_ID={}".format(os.getuid()),
- "--build-arg", "GROUP_ID={}".format(os.getgid()),
- "--cache-from", tag,
- "-t", tag,
- "docker"]
+ #
+ cmd = [docker_binary, "build",
+ "-f", get_dockerfile(platform),
+ "--build-arg", "USER_ID={}".format(os.getuid()),
+ "--build-arg", "GROUP_ID={}".format(os.getgid()),
+ "--cache-from", tag,
+ "-t", tag,
+ "docker"]
+
+ @retry(subprocess.CalledProcessError, tries=num_retries)
+ def run_cmd():
logging.info("Running command: '%s'", ' '.join(cmd))
- try:
- check_call(cmd)
- # Docker build was successful. Call break to break out of the retry mechanism
- break
- except subprocess.CalledProcessError as e:
- saved_exception = e
- logging.error('Failed to build docker image')
- # Building the docker image failed. Call continue to trigger the retry mechanism
- continue
- else:
- # Num retries exceeded
- logging.exception('Exception during build of docker image', saved_exception)
- logging.fatal('Failed to build the docker image, aborting...')
- sys.exit(1)
+ check_call(cmd)
+ run_cmd()
# Get image id by reading the tag. It's guaranteed (except race condition) that the tag exists. Otherwise, the
# check_call would have failed
- image_id = _get_local_image_id(docker_binary=docker_binary, docker_tag=tag)
- if not image_id:
- raise FileNotFoundError('Unable to find docker image id matching with {}'.format(tag))
- return image_id
+ return _get_local_image_id(docker_binary=docker_binary, docker_tag=tag)
def _get_local_image_id(docker_binary, docker_tag):
@@ -136,6 +156,8 @@ def _get_local_image_id(docker_binary, docker_tag):
cmd = [docker_binary, "images", "-q", docker_tag]
image_id_b = subprocess.check_output(cmd)
image_id = image_id_b.decode('utf-8').strip()
+ if not image_id:
+ raise RuntimeError('Unable to find docker image id matching with tag {}'.format(tag))
return image_id
@@ -186,7 +208,7 @@ def container_run(platform: str,
'-e', "CCACHE_LOGFILE=/tmp/ccache.log", # a container-scoped log, useful for ccache verification.
tag]
runlist.extend(command)
- cmd = '\\\n\t'.join(runlist)
+ cmd = ' \\\n\t'.join(runlist)
ret = 0
if not dry_run and not interactive:
logging.info("Running %s in container %s", command, tag)
@@ -199,14 +221,14 @@ def container_run(platform: str,
# -ti can't be after the tag, as is interpreted as a command so hook it up after the -u argument
idx = into_cmd.index('-u') + 2
into_cmd[idx:idx] = ['-ti']
- cmd = '\\\n\t'.join(into_cmd)
+ cmd = ' \\\n\t'.join(into_cmd)
logging.info("Executing:\n%s\n", cmd)
docker_run_cmd = ' '.join(into_cmd)
ret = call(into_cmd)
if not dry_run and not interactive and ret != 0:
logging.error("Running of command in container failed (%s):\n%s\n", ret, cmd)
- logging.error("You can get into the container by adding the -i option")
+ logging.error("You can get into the container by adding the -i option to this script")
raise subprocess.CalledProcessError(ret, cmd)
return docker_run_cmd
@@ -303,7 +325,6 @@ def main() -> int:
command = list(chain(*args.command))
docker_binary = get_docker_binary(args.nvidiadocker)
shared_memory_size = args.shared_memory_size
- num_docker_build_retires = args.docker_build_retries
if args.list:
list_platforms()
@@ -312,7 +333,7 @@ def main() -> int:
tag = get_docker_tag(platform=platform, registry=args.docker_registry)
if use_cache():
load_docker_cache(tag=tag, docker_registry=args.docker_registry)
- build_docker(platform, docker_binary, registry=args.docker_registry, num_retries=num_docker_build_retires)
+ build_docker(platform, docker_binary, registry=args.docker_registry, num_retries=args.docker_build_retries)
if args.build_only:
logging.warning("Container was just built. Exiting due to build-only.")
return 0
@@ -346,7 +367,7 @@ def main() -> int:
tag = get_docker_tag(platform=platform, registry=args.docker_registry)
if use_cache():
load_docker_cache(tag=tag, docker_registry=args.docker_registry)
- build_docker(platform, docker_binary, args.docker_registry, num_retries=num_docker_build_retires)
+ build_docker(platform, docker_binary, args.docker_registry, num_retries=args.docker_build_retries)
if args.build_only:
continue
build_platform = "build_{}".format(platform)
diff --git a/ci/docker/Dockerfile.build.android_armv7 b/ci/docker/Dockerfile.build.android_armv7
old mode 100755
new mode 100644
diff --git a/ci/docker/Dockerfile.build.android_armv8 b/ci/docker/Dockerfile.build.android_armv8
old mode 100755
new mode 100644
diff --git a/ci/docker/Dockerfile.build.armv6 b/ci/docker/Dockerfile.build.armv6
old mode 100755
new mode 100644
diff --git a/ci/docker/Dockerfile.build.armv7 b/ci/docker/Dockerfile.build.armv7
old mode 100755
new mode 100644
diff --git a/ci/docker/Dockerfile.build.armv8 b/ci/docker/Dockerfile.build.armv8
old mode 100755
new mode 100644
diff --git a/ci/docker/Dockerfile.build.centos7_cpu b/ci/docker/Dockerfile.build.centos7_cpu
old mode 100755
new mode 100644
diff --git a/ci/docker/Dockerfile.build.centos7_gpu b/ci/docker/Dockerfile.build.centos7_gpu
old mode 100755
new mode 100644
diff --git a/ci/docker/Dockerfile.build.jetson b/ci/docker/Dockerfile.build.jetson
old mode 100755
new mode 100644
diff --git a/ci/docker/Dockerfile.build.ubuntu_base_cpu b/ci/docker/Dockerfile.build.ubuntu_base_cpu
old mode 100755
new mode 100644
diff --git a/ci/docker/Dockerfile.build.ubuntu_base_gpu b/ci/docker/Dockerfile.build.ubuntu_base_gpu
old mode 100755
new mode 100644
diff --git a/ci/docker/Dockerfile.build.ubuntu_blc b/ci/docker/Dockerfile.build.ubuntu_blc
old mode 100755
new mode 100644
index 294740c..208cba2
--- a/ci/docker/Dockerfile.build.ubuntu_blc
+++ b/ci/docker/Dockerfile.build.ubuntu_blc
@@ -24,8 +24,13 @@ WORKDIR /work/deps
COPY install/ubuntu_core.sh /work/
RUN /work/ubuntu_core.sh
-COPY install/ubuntu_python.sh /work/
-RUN /work/ubuntu_python.sh
+
+COPY install/ubuntu_python2.sh /work/
+RUN /work/ubuntu_python2.sh
+
+COPY install/ubuntu_python3.sh /work/
+RUN /work/ubuntu_python3.sh
+
COPY install/ubuntu_npm_blc.sh /work/
RUN /work/ubuntu_npm_blc.sh
diff --git a/ci/docker/Dockerfile.build.ubuntu_build_cuda b/ci/docker/Dockerfile.build.ubuntu_build_cuda
old mode 100755
new mode 100644
index 9ed0cbb..19e9265
--- a/ci/docker/Dockerfile.build.ubuntu_build_cuda
+++ b/ci/docker/Dockerfile.build.ubuntu_build_cuda
@@ -27,20 +27,30 @@ WORKDIR /work/deps
COPY install/ubuntu_core.sh /work/
RUN /work/ubuntu_core.sh
+
COPY install/deb_ubuntu_ccache.sh /work/
RUN /work/deb_ubuntu_ccache.sh
-COPY install/ubuntu_python.sh /work/
-RUN /work/ubuntu_python.sh
+
+COPY install/ubuntu_python2.sh /work/
+RUN /work/ubuntu_python2.sh
+
+COPY install/ubuntu_python3.sh /work/
+RUN /work/ubuntu_python3.sh
+
COPY install/ubuntu_scala.sh /work/
COPY install/sbt.gpg /work/
RUN /work/ubuntu_scala.sh
+
COPY install/ubuntu_r.sh /work/
COPY install/r.gpg /work/
RUN /work/ubuntu_r.sh
+
COPY install/ubuntu_perl.sh /work/
RUN /work/ubuntu_perl.sh
+
COPY install/ubuntu_clang.sh /work/
RUN /work/ubuntu_clang.sh
+
COPY install/ubuntu_mklml.sh /work/
RUN /work/ubuntu_mklml.sh
diff --git a/ci/docker/Dockerfile.build.ubuntu_cpu b/ci/docker/Dockerfile.build.ubuntu_cpu
old mode 100755
new mode 100644
index 58a8e9a..76df3df
--- a/ci/docker/Dockerfile.build.ubuntu_cpu
+++ b/ci/docker/Dockerfile.build.ubuntu_cpu
@@ -28,8 +28,11 @@ RUN /work/ubuntu_core.sh
COPY install/deb_ubuntu_ccache.sh /work/
RUN /work/deb_ubuntu_ccache.sh
-COPY install/ubuntu_python.sh /work/
-RUN /work/ubuntu_python.sh
+COPY install/ubuntu_python2.sh /work/
+RUN /work/ubuntu_python2.sh
+
+COPY install/ubuntu_python3.sh /work/
+RUN /work/ubuntu_python3.sh
COPY install/ubuntu_scala.sh /work/
COPY install/sbt.gpg /work/
diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu b/ci/docker/Dockerfile.build.ubuntu_gpu
old mode 100755
new mode 100644
index de38948..8b4cd1e
--- a/ci/docker/Dockerfile.build.ubuntu_gpu
+++ b/ci/docker/Dockerfile.build.ubuntu_gpu
@@ -28,8 +28,11 @@ RUN /work/ubuntu_core.sh
COPY install/deb_ubuntu_ccache.sh /work/
RUN /work/deb_ubuntu_ccache.sh
-COPY install/ubuntu_python.sh /work/
-RUN /work/ubuntu_python.sh
+COPY install/ubuntu_python2.sh /work/
+RUN /work/ubuntu_python2.sh
+
+COPY install/ubuntu_python3.sh /work/
+RUN /work/ubuntu_python3.sh
COPY install/ubuntu_scala.sh /work/
COPY install/sbt.gpg /work/
diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu_tensorrt b/ci/docker/Dockerfile.build.ubuntu_gpu_tensorrt
old mode 100755
new mode 100644
index 255da31..3f0bbc6
--- a/ci/docker/Dockerfile.build.ubuntu_gpu_tensorrt
+++ b/ci/docker/Dockerfile.build.ubuntu_gpu_tensorrt
@@ -24,10 +24,16 @@ WORKDIR /work/deps
COPY install/ubuntu_core.sh /work/
RUN /work/ubuntu_core.sh
+
COPY install/deb_ubuntu_ccache.sh /work/
RUN /work/deb_ubuntu_ccache.sh
-COPY install/ubuntu_python.sh /work/
-RUN /work/ubuntu_python.sh
+
+COPY install/ubuntu_python2.sh /work/
+RUN /work/ubuntu_python2.sh
+
+COPY install/ubuntu_python3.sh /work/
+RUN /work/ubuntu_python3.sh
+
COPY install/tensorrt.sh /work
RUN /work/tensorrt.sh
diff --git a/ci/docker/Dockerfile.build.ubuntu_nightly_cpu b/ci/docker/Dockerfile.build.ubuntu_nightly_cpu
old mode 100755
new mode 100644
index 58ff33e..c803188
--- a/ci/docker/Dockerfile.build.ubuntu_nightly_cpu
+++ b/ci/docker/Dockerfile.build.ubuntu_nightly_cpu
@@ -28,8 +28,11 @@ RUN /work/ubuntu_core.sh
COPY install/deb_ubuntu_ccache.sh /work/
RUN /work/deb_ubuntu_ccache.sh
-COPY install/ubuntu_python.sh /work/
-RUN /work/ubuntu_python.sh
+COPY install/ubuntu_python2.sh /work/
+RUN /work/ubuntu_python2.sh
+
+COPY install/ubuntu_python3.sh /work/
+RUN /work/ubuntu_python3.sh
COPY install/ubuntu_scala.sh /work/
COPY install/sbt.gpg /work/
diff --git a/ci/docker/Dockerfile.build.ubuntu_nightly_gpu b/ci/docker/Dockerfile.build.ubuntu_nightly_gpu
old mode 100755
new mode 100644
index 017bade..c0e31e2
--- a/ci/docker/Dockerfile.build.ubuntu_nightly_gpu
+++ b/ci/docker/Dockerfile.build.ubuntu_nightly_gpu
@@ -28,8 +28,11 @@ RUN /work/ubuntu_core.sh
COPY install/deb_ubuntu_ccache.sh /work/
RUN /work/deb_ubuntu_ccache.sh
-COPY install/ubuntu_python.sh /work/
-RUN /work/ubuntu_python.sh
+COPY install/ubuntu_python2.sh /work/
+RUN /work/ubuntu_python2.sh
+
+COPY install/ubuntu_python3.sh /work/
+RUN /work/ubuntu_python3.sh
COPY install/ubuntu_scala.sh /work/
COPY install/sbt.gpg /work/
diff --git a/ci/docker/Dockerfile.build.ubuntu_rat b/ci/docker/Dockerfile.build.ubuntu_rat
old mode 100755
new mode 100644
diff --git a/ci/docker/install/ubuntu_python.sh b/ci/docker/install/ubuntu_python2.sh
similarity index 84%
copy from ci/docker/install/ubuntu_python.sh
copy to ci/docker/install/ubuntu_python2.sh
index e71cac8..f0526e2 100755
--- a/ci/docker/install/ubuntu_python.sh
+++ b/ci/docker/install/ubuntu_python2.sh
@@ -22,12 +22,10 @@
set -ex
# install libraries for mxnet's python package on ubuntu
-apt-get install -y python-dev python3-dev virtualenv
+apt-get install -y python-dev virtualenv wget
# the version of the pip shipped with ubuntu may be too lower, install a recent version here
wget -nv https://bootstrap.pypa.io/get-pip.py
-python3 get-pip.py
python2 get-pip.py
-pip2 install nose cpplint==1.3.0 pylint==1.8.3 'numpy<1.15.0,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3
-pip3 install nose cpplint==1.3.0 pylint==1.8.3 'numpy<1.15.0,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3
+pip2 install nose cpplint==1.3.0 pylint==1.8.3 'numpy<1.15.0,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3 mock
diff --git a/ci/docker/install/ubuntu_python.sh b/ci/docker/install/ubuntu_python3.sh
similarity index 84%
rename from ci/docker/install/ubuntu_python.sh
rename to ci/docker/install/ubuntu_python3.sh
index e71cac8..1dad5a7 100755
--- a/ci/docker/install/ubuntu_python.sh
+++ b/ci/docker/install/ubuntu_python3.sh
@@ -22,12 +22,10 @@
set -ex
# install libraries for mxnet's python package on ubuntu
-apt-get install -y python-dev python3-dev virtualenv
+apt-get install -y python3-dev virtualenv wget
# the version of the pip shipped with ubuntu may be too lower, install a recent version here
wget -nv https://bootstrap.pypa.io/get-pip.py
python3 get-pip.py
-python2 get-pip.py
-pip2 install nose cpplint==1.3.0 pylint==1.8.3 'numpy<1.15.0,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3
-pip3 install nose cpplint==1.3.0 pylint==1.8.3 'numpy<1.15.0,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3
+pip3 install nose cpplint==1.3.0 pylint==1.8.3 'numpy<1.15.0,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3 mock
diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index 815eae9..2a813c8 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -1005,6 +1005,7 @@ broken_link_checker() {
./tests/nightly/broken_link_checker_test/broken_link_checker.sh
}
+
##############################################################
# MAIN
#
diff --git a/ci/test_docker_cache.py b/ci/test_docker_cache.py
old mode 100644
new mode 100755