You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by ma...@apache.org on 2018/08/17 18:44:46 UTC
[incubator-mxnet] branch master updated: Revert "CI scripts
refinements. Separate Py2 and Py3 installs cripts. Fix perms. (#12125)"
(#12223)
This is an automated email from the ASF dual-hosted git repository.
marcoabreu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
The following commit(s) were added to refs/heads/master by this push:
new 2ff1837 Revert "CI scripts refinements. Separate Py2 and Py3 installs cripts. Fix perms. (#12125)" (#12223)
2ff1837 is described below
commit 2ff1837c4ba5238c96055e502a26dfe1f69c5253
Author: Anton Chernov <me...@gmail.com>
AuthorDate: Fri Aug 17 20:44:32 2018 +0200
Revert "CI scripts refinements. Separate Py2 and Py3 installs cripts. Fix perms. (#12125)" (#12223)
This reverts commit 1b60478db2c66ef82f5b3ed48b12832d6da72e61.
---
ci/build.py | 97 +++++++++-------------
ci/docker/Dockerfile.build.android_armv7 | 0
ci/docker/Dockerfile.build.android_armv8 | 0
ci/docker/Dockerfile.build.armv6 | 0
ci/docker/Dockerfile.build.armv7 | 0
ci/docker/Dockerfile.build.armv8 | 0
ci/docker/Dockerfile.build.centos7_cpu | 0
ci/docker/Dockerfile.build.centos7_gpu | 0
ci/docker/Dockerfile.build.jetson | 0
ci/docker/Dockerfile.build.ubuntu_base_cpu | 0
ci/docker/Dockerfile.build.ubuntu_base_gpu | 0
ci/docker/Dockerfile.build.ubuntu_blc | 9 +-
ci/docker/Dockerfile.build.ubuntu_build_cuda | 14 +---
ci/docker/Dockerfile.build.ubuntu_cpu | 7 +-
ci/docker/Dockerfile.build.ubuntu_gpu | 7 +-
ci/docker/Dockerfile.build.ubuntu_gpu_tensorrt | 10 +--
ci/docker/Dockerfile.build.ubuntu_nightly_cpu | 7 +-
ci/docker/Dockerfile.build.ubuntu_nightly_gpu | 7 +-
ci/docker/Dockerfile.build.ubuntu_rat | 0
.../{ubuntu_python2.sh => ubuntu_python.sh} | 6 +-
ci/docker/install/ubuntu_python3.sh | 31 -------
ci/docker/runtime_functions.sh | 1 -
ci/test_docker_cache.py | 0
23 files changed, 56 insertions(+), 140 deletions(-)
diff --git a/ci/build.py b/ci/build.py
index 0a1ad4c..a9d6a63 100755
--- a/ci/build.py
+++ b/ci/build.py
@@ -43,43 +43,6 @@ from util import *
CCACHE_MAXSIZE = '500G'
-
-
-def retry(ExceptionToCheck, tries=4, delay_s=1, backoff=2):
- """Retry calling the decorated function using an exponential backoff.
-
- http://www.saltycrane.com/blog/2009/11/trying-out-retry-decorator-python/
- original from: http://wiki.python.org/moin/PythonDecoratorLibrary#Retry
-
- :param ExceptionToCheck: the exception to check. may be a tuple of
- exceptions to check
- :type ExceptionToCheck: Exception or tuple
- :param tries: number of times to try (not retry) before giving up
- :type tries: int
- :param delay_s: initial delay between retries in seconds
- :type delay_s: int
- :param backoff: backoff multiplier e.g. value of 2 will double the delay
- each retry
- :type backoff: int
- """
- import time
- from functools import wraps
- def decorated_retry(f):
- @wraps(f)
- def f_retry(*args, **kwargs):
- mtries, mdelay = tries, delay_s
- while mtries > 1:
- try:
- return f(*args, **kwargs)
- except ExceptionToCheck as e:
- logging.warning("Exception: %s, Retrying in %d seconds...", str(e), mdelay)
- time.sleep(mdelay)
- mtries -= 1
- mdelay *= backoff
- return f(*args, **kwargs)
- return f_retry # true decorator
- return decorated_retry
-
def under_ci() -> bool:
""":return: True if we run in Jenkins."""
return 'JOB_NAME' in os.environ
@@ -114,8 +77,9 @@ def build_docker(platform: str, docker_binary: str, registry: str, num_retries:
:param num_retries: Number of retries to build the docker image
:return: Id of the top level image
"""
+
tag = get_docker_tag(platform=platform, registry=registry)
- logging.info("Building docker container tagged '%s' with %s", tag, docker_binary)
+ logging.info("Building container tagged '%s' with %s", tag, docker_binary)
#
# We add a user with the same group as the executing non-root user so files created in the
# container match permissions of the local user. Same for the group.
@@ -127,24 +91,40 @@ def build_docker(platform: str, docker_binary: str, registry: str, num_retries:
# docker pull see: docker_cache.load_docker_cache
#
# This doesn't work with multi head docker files.
- #
- cmd = [docker_binary, "build",
- "-f", get_dockerfile(platform),
- "--build-arg", "USER_ID={}".format(os.getuid()),
- "--build-arg", "GROUP_ID={}".format(os.getgid()),
- "--cache-from", tag,
- "-t", tag,
- "docker"]
-
- @retry(subprocess.CalledProcessError, tries=num_retries)
- def run_cmd():
+ #
+
+ for i in range(num_retries):
+ logging.info('%d out of %d tries to build the docker image.', i + 1, num_retries)
+
+ cmd = [docker_binary, "build",
+ "-f", get_dockerfile(platform),
+ "--build-arg", "USER_ID={}".format(os.getuid()),
+ "--build-arg", "GROUP_ID={}".format(os.getgid()),
+ "--cache-from", tag,
+ "-t", tag,
+ "docker"]
logging.info("Running command: '%s'", ' '.join(cmd))
- check_call(cmd)
+ try:
+ check_call(cmd)
+ # Docker build was successful. Call break to break out of the retry mechanism
+ break
+ except subprocess.CalledProcessError as e:
+ saved_exception = e
+ logging.error('Failed to build docker image')
+ # Building the docker image failed. Call continue to trigger the retry mechanism
+ continue
+ else:
+ # Num retries exceeded
+ logging.exception('Exception during build of docker image', saved_exception)
+ logging.fatal('Failed to build the docker image, aborting...')
+ sys.exit(1)
- run_cmd()
# Get image id by reading the tag. It's guaranteed (except race condition) that the tag exists. Otherwise, the
# check_call would have failed
- return _get_local_image_id(docker_binary=docker_binary, docker_tag=tag)
+ image_id = _get_local_image_id(docker_binary=docker_binary, docker_tag=tag)
+ if not image_id:
+ raise FileNotFoundError('Unable to find docker image id matching with {}'.format(tag))
+ return image_id
def _get_local_image_id(docker_binary, docker_tag):
@@ -156,8 +136,6 @@ def _get_local_image_id(docker_binary, docker_tag):
cmd = [docker_binary, "images", "-q", docker_tag]
image_id_b = subprocess.check_output(cmd)
image_id = image_id_b.decode('utf-8').strip()
- if not image_id:
- raise RuntimeError('Unable to find docker image id matching with tag {}'.format(tag))
return image_id
@@ -208,7 +186,7 @@ def container_run(platform: str,
'-e', "CCACHE_LOGFILE=/tmp/ccache.log", # a container-scoped log, useful for ccache verification.
tag]
runlist.extend(command)
- cmd = ' \\\n\t'.join(runlist)
+ cmd = '\\\n\t'.join(runlist)
ret = 0
if not dry_run and not interactive:
logging.info("Running %s in container %s", command, tag)
@@ -221,14 +199,14 @@ def container_run(platform: str,
# -ti can't be after the tag, as is interpreted as a command so hook it up after the -u argument
idx = into_cmd.index('-u') + 2
into_cmd[idx:idx] = ['-ti']
- cmd = ' \\\n\t'.join(into_cmd)
+ cmd = '\\\n\t'.join(into_cmd)
logging.info("Executing:\n%s\n", cmd)
docker_run_cmd = ' '.join(into_cmd)
ret = call(into_cmd)
if not dry_run and not interactive and ret != 0:
logging.error("Running of command in container failed (%s):\n%s\n", ret, cmd)
- logging.error("You can get into the container by adding the -i option to this script")
+ logging.error("You can get into the container by adding the -i option")
raise subprocess.CalledProcessError(ret, cmd)
return docker_run_cmd
@@ -325,6 +303,7 @@ def main() -> int:
command = list(chain(*args.command))
docker_binary = get_docker_binary(args.nvidiadocker)
shared_memory_size = args.shared_memory_size
+ num_docker_build_retires = args.docker_build_retries
if args.list:
list_platforms()
@@ -333,7 +312,7 @@ def main() -> int:
tag = get_docker_tag(platform=platform, registry=args.docker_registry)
if use_cache():
load_docker_cache(tag=tag, docker_registry=args.docker_registry)
- build_docker(platform, docker_binary, registry=args.docker_registry, num_retries=args.docker_build_retries)
+ build_docker(platform, docker_binary, registry=args.docker_registry, num_retries=num_docker_build_retires)
if args.build_only:
logging.warning("Container was just built. Exiting due to build-only.")
return 0
@@ -367,7 +346,7 @@ def main() -> int:
tag = get_docker_tag(platform=platform, registry=args.docker_registry)
if use_cache():
load_docker_cache(tag=tag, docker_registry=args.docker_registry)
- build_docker(platform, docker_binary, args.docker_registry, num_retries=args.docker_build_retries)
+ build_docker(platform, docker_binary, args.docker_registry, num_retries=num_docker_build_retires)
if args.build_only:
continue
build_platform = "build_{}".format(platform)
diff --git a/ci/docker/Dockerfile.build.android_armv7 b/ci/docker/Dockerfile.build.android_armv7
old mode 100644
new mode 100755
diff --git a/ci/docker/Dockerfile.build.android_armv8 b/ci/docker/Dockerfile.build.android_armv8
old mode 100644
new mode 100755
diff --git a/ci/docker/Dockerfile.build.armv6 b/ci/docker/Dockerfile.build.armv6
old mode 100644
new mode 100755
diff --git a/ci/docker/Dockerfile.build.armv7 b/ci/docker/Dockerfile.build.armv7
old mode 100644
new mode 100755
diff --git a/ci/docker/Dockerfile.build.armv8 b/ci/docker/Dockerfile.build.armv8
old mode 100644
new mode 100755
diff --git a/ci/docker/Dockerfile.build.centos7_cpu b/ci/docker/Dockerfile.build.centos7_cpu
old mode 100644
new mode 100755
diff --git a/ci/docker/Dockerfile.build.centos7_gpu b/ci/docker/Dockerfile.build.centos7_gpu
old mode 100644
new mode 100755
diff --git a/ci/docker/Dockerfile.build.jetson b/ci/docker/Dockerfile.build.jetson
old mode 100644
new mode 100755
diff --git a/ci/docker/Dockerfile.build.ubuntu_base_cpu b/ci/docker/Dockerfile.build.ubuntu_base_cpu
old mode 100644
new mode 100755
diff --git a/ci/docker/Dockerfile.build.ubuntu_base_gpu b/ci/docker/Dockerfile.build.ubuntu_base_gpu
old mode 100644
new mode 100755
diff --git a/ci/docker/Dockerfile.build.ubuntu_blc b/ci/docker/Dockerfile.build.ubuntu_blc
old mode 100644
new mode 100755
index 208cba2..294740c
--- a/ci/docker/Dockerfile.build.ubuntu_blc
+++ b/ci/docker/Dockerfile.build.ubuntu_blc
@@ -24,13 +24,8 @@ WORKDIR /work/deps
COPY install/ubuntu_core.sh /work/
RUN /work/ubuntu_core.sh
-
-COPY install/ubuntu_python2.sh /work/
-RUN /work/ubuntu_python2.sh
-
-COPY install/ubuntu_python3.sh /work/
-RUN /work/ubuntu_python3.sh
-
+COPY install/ubuntu_python.sh /work/
+RUN /work/ubuntu_python.sh
COPY install/ubuntu_npm_blc.sh /work/
RUN /work/ubuntu_npm_blc.sh
diff --git a/ci/docker/Dockerfile.build.ubuntu_build_cuda b/ci/docker/Dockerfile.build.ubuntu_build_cuda
old mode 100644
new mode 100755
index 19e9265..9ed0cbb
--- a/ci/docker/Dockerfile.build.ubuntu_build_cuda
+++ b/ci/docker/Dockerfile.build.ubuntu_build_cuda
@@ -27,30 +27,20 @@ WORKDIR /work/deps
COPY install/ubuntu_core.sh /work/
RUN /work/ubuntu_core.sh
-
COPY install/deb_ubuntu_ccache.sh /work/
RUN /work/deb_ubuntu_ccache.sh
-
-COPY install/ubuntu_python2.sh /work/
-RUN /work/ubuntu_python2.sh
-
-COPY install/ubuntu_python3.sh /work/
-RUN /work/ubuntu_python3.sh
-
+COPY install/ubuntu_python.sh /work/
+RUN /work/ubuntu_python.sh
COPY install/ubuntu_scala.sh /work/
COPY install/sbt.gpg /work/
RUN /work/ubuntu_scala.sh
-
COPY install/ubuntu_r.sh /work/
COPY install/r.gpg /work/
RUN /work/ubuntu_r.sh
-
COPY install/ubuntu_perl.sh /work/
RUN /work/ubuntu_perl.sh
-
COPY install/ubuntu_clang.sh /work/
RUN /work/ubuntu_clang.sh
-
COPY install/ubuntu_mklml.sh /work/
RUN /work/ubuntu_mklml.sh
diff --git a/ci/docker/Dockerfile.build.ubuntu_cpu b/ci/docker/Dockerfile.build.ubuntu_cpu
old mode 100644
new mode 100755
index 08fb04d..6751465
--- a/ci/docker/Dockerfile.build.ubuntu_cpu
+++ b/ci/docker/Dockerfile.build.ubuntu_cpu
@@ -28,11 +28,8 @@ RUN /work/ubuntu_core.sh
COPY install/deb_ubuntu_ccache.sh /work/
RUN /work/deb_ubuntu_ccache.sh
-COPY install/ubuntu_python2.sh /work/
-RUN /work/ubuntu_python2.sh
-
-COPY install/ubuntu_python3.sh /work/
-RUN /work/ubuntu_python3.sh
+COPY install/ubuntu_python.sh /work/
+RUN /work/ubuntu_python.sh
COPY install/ubuntu_scala.sh /work/
COPY install/sbt.gpg /work/
diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu b/ci/docker/Dockerfile.build.ubuntu_gpu
old mode 100644
new mode 100755
index d99dafb..8fcbcbb
--- a/ci/docker/Dockerfile.build.ubuntu_gpu
+++ b/ci/docker/Dockerfile.build.ubuntu_gpu
@@ -28,11 +28,8 @@ RUN /work/ubuntu_core.sh
COPY install/deb_ubuntu_ccache.sh /work/
RUN /work/deb_ubuntu_ccache.sh
-COPY install/ubuntu_python2.sh /work/
-RUN /work/ubuntu_python2.sh
-
-COPY install/ubuntu_python3.sh /work/
-RUN /work/ubuntu_python3.sh
+COPY install/ubuntu_python.sh /work/
+RUN /work/ubuntu_python.sh
COPY install/ubuntu_scala.sh /work/
COPY install/sbt.gpg /work/
diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu_tensorrt b/ci/docker/Dockerfile.build.ubuntu_gpu_tensorrt
old mode 100644
new mode 100755
index 3f0bbc6..255da31
--- a/ci/docker/Dockerfile.build.ubuntu_gpu_tensorrt
+++ b/ci/docker/Dockerfile.build.ubuntu_gpu_tensorrt
@@ -24,16 +24,10 @@ WORKDIR /work/deps
COPY install/ubuntu_core.sh /work/
RUN /work/ubuntu_core.sh
-
COPY install/deb_ubuntu_ccache.sh /work/
RUN /work/deb_ubuntu_ccache.sh
-
-COPY install/ubuntu_python2.sh /work/
-RUN /work/ubuntu_python2.sh
-
-COPY install/ubuntu_python3.sh /work/
-RUN /work/ubuntu_python3.sh
-
+COPY install/ubuntu_python.sh /work/
+RUN /work/ubuntu_python.sh
COPY install/tensorrt.sh /work
RUN /work/tensorrt.sh
diff --git a/ci/docker/Dockerfile.build.ubuntu_nightly_cpu b/ci/docker/Dockerfile.build.ubuntu_nightly_cpu
old mode 100644
new mode 100755
index 834710c..1b126c1
--- a/ci/docker/Dockerfile.build.ubuntu_nightly_cpu
+++ b/ci/docker/Dockerfile.build.ubuntu_nightly_cpu
@@ -28,11 +28,8 @@ RUN /work/ubuntu_core.sh
COPY install/deb_ubuntu_ccache.sh /work/
RUN /work/deb_ubuntu_ccache.sh
-COPY install/ubuntu_python2.sh /work/
-RUN /work/ubuntu_python2.sh
-
-COPY install/ubuntu_python3.sh /work/
-RUN /work/ubuntu_python3.sh
+COPY install/ubuntu_python.sh /work/
+RUN /work/ubuntu_python.sh
COPY install/ubuntu_scala.sh /work/
COPY install/sbt.gpg /work/
diff --git a/ci/docker/Dockerfile.build.ubuntu_nightly_gpu b/ci/docker/Dockerfile.build.ubuntu_nightly_gpu
old mode 100644
new mode 100755
index fb34307..deeed8b
--- a/ci/docker/Dockerfile.build.ubuntu_nightly_gpu
+++ b/ci/docker/Dockerfile.build.ubuntu_nightly_gpu
@@ -28,11 +28,8 @@ RUN /work/ubuntu_core.sh
COPY install/deb_ubuntu_ccache.sh /work/
RUN /work/deb_ubuntu_ccache.sh
-COPY install/ubuntu_python2.sh /work/
-RUN /work/ubuntu_python2.sh
-
-COPY install/ubuntu_python3.sh /work/
-RUN /work/ubuntu_python3.sh
+COPY install/ubuntu_python.sh /work/
+RUN /work/ubuntu_python.sh
COPY install/ubuntu_scala.sh /work/
COPY install/sbt.gpg /work/
diff --git a/ci/docker/Dockerfile.build.ubuntu_rat b/ci/docker/Dockerfile.build.ubuntu_rat
old mode 100644
new mode 100755
diff --git a/ci/docker/install/ubuntu_python2.sh b/ci/docker/install/ubuntu_python.sh
similarity index 84%
rename from ci/docker/install/ubuntu_python2.sh
rename to ci/docker/install/ubuntu_python.sh
index f0526e2..e71cac8 100755
--- a/ci/docker/install/ubuntu_python2.sh
+++ b/ci/docker/install/ubuntu_python.sh
@@ -22,10 +22,12 @@
set -ex
# install libraries for mxnet's python package on ubuntu
-apt-get install -y python-dev virtualenv wget
+apt-get install -y python-dev python3-dev virtualenv
# the version of the pip shipped with ubuntu may be too lower, install a recent version here
wget -nv https://bootstrap.pypa.io/get-pip.py
+python3 get-pip.py
python2 get-pip.py
-pip2 install nose cpplint==1.3.0 pylint==1.8.3 'numpy<1.15.0,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3 mock
+pip2 install nose cpplint==1.3.0 pylint==1.8.3 'numpy<1.15.0,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3
+pip3 install nose cpplint==1.3.0 pylint==1.8.3 'numpy<1.15.0,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3
diff --git a/ci/docker/install/ubuntu_python3.sh b/ci/docker/install/ubuntu_python3.sh
deleted file mode 100755
index 1dad5a7..0000000
--- a/ci/docker/install/ubuntu_python3.sh
+++ /dev/null
@@ -1,31 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# build and install are separated so changes to build don't invalidate
-# the whole docker cache for the image
-
-set -ex
-# install libraries for mxnet's python package on ubuntu
-apt-get install -y python3-dev virtualenv wget
-
-# the version of the pip shipped with ubuntu may be too lower, install a recent version here
-wget -nv https://bootstrap.pypa.io/get-pip.py
-python3 get-pip.py
-
-pip3 install nose cpplint==1.3.0 pylint==1.8.3 'numpy<1.15.0,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3 mock
diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index e4aac8b..a0c4723 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -994,7 +994,6 @@ broken_link_checker() {
./tests/nightly/broken_link_checker_test/broken_link_checker.sh
}
-
##############################################################
# MAIN
#
diff --git a/ci/test_docker_cache.py b/ci/test_docker_cache.py
old mode 100755
new mode 100644