You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by GitBox <gi...@apache.org> on 2018/08/21 16:01:45 UTC
[GitHub] larroy closed pull request #12275: [Don't merge][Review] A solution
to prevent zombie containers locally and in CI
larroy closed pull request #12275: [Don't merge][Review] A solution to prevent zombie containers locally and in CI
URL: https://github.com/apache/incubator-mxnet/pull/12275
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/ci/README.md b/ci/README.md
index 548e9cb9b04..7338d65aecc 100644
--- a/ci/README.md
+++ b/ci/README.md
@@ -1,5 +1,6 @@
# Containerized build & test utilities
+
This folder contains scripts and dockerfiles used to build and test MXNet using
Docker containers
diff --git a/ci/build.py b/ci/build.py
index a9d6a63537f..6149d2ec7c7 100755
--- a/ci/build.py
+++ b/ci/build.py
@@ -23,7 +23,7 @@
"""
__author__ = 'Marco de Abreu, Kellen Sunderland, Anton Chernov, Pedro Larroy'
-__version__ = '0.1'
+__version__ = '0.8'
import argparse
import glob
@@ -37,17 +37,117 @@
import platform
from copy import deepcopy
from itertools import chain
-from subprocess import call, check_call
+from subprocess import call, check_call, check_output
from typing import *
from util import *
+import docker
+import docker.models
+import docker.errors
+import signal
+import atexit
+import pprint
+import requests
+
+
+class Cleanup:
+ def __init__(self):
+ self.containers = set()
+ self.docker_stop_timeout = 3
+
+ def add_container(self, container: docker.models.containers.Container):
+ assert isinstance(container, docker.models.containers.Container)
+ self.containers.add(container)
+
+ def remove_container(self, container: docker.models.containers.Container):
+ assert isinstance(container, docker.models.containers.Container)
+ self.containers.remove(container)
+
+ def _cleanup_containers(self):
+ if self.containers:
+ logging.warning("Cleaning up containers")
+ else:
+ return
+ docker_client = docker.from_env()
+ try:
+ stop_timeout = int(os.environ.get("DOCKER_STOP_TIMEOUT", self.docker_stop_timeout))
+ except Exception as e:
+ stop_timeout = 3
+ for container in self.containers:
+ try:
+ container.stop(timeout=stop_timeout)
+ logging.info("☠: stopped container %s", trim_container_id(container.id))
+ container.remove()
+ logging.info("🚽: removed container %s", trim_container_id(container.id))
+ except Exception as e:
+ logging.exception(e)
+ #pass
+ self.containers.clear()
+ logging.info("Cleaning up containers finished.")
+
+ def __call__(self):
+ """Perform cleanup"""
+ self._cleanup_containers()
+
+
+
+def retry(ExceptionToCheck, tries=4, delay_s=1, backoff=2):
+ """Retry calling the decorated function using an exponential backoff.
+
+ http://www.saltycrane.com/blog/2009/11/trying-out-retry-decorator-python/
+ original from: http://wiki.python.org/moin/PythonDecoratorLibrary#Retry
+
+ :param ExceptionToCheck: the exception to check. may be a tuple of
+ exceptions to check
+ :type ExceptionToCheck: Exception or tuple
+ :param tries: number of times to try (not retry) before giving up
+ :type tries: int
+ :param delay_s: initial delay between retries in seconds
+ :type delay_s: int
+ :param backoff: backoff multiplier e.g. value of 2 will double the delay
+ each retry
+ :type backoff: int
+ """
+ import time
+ from functools import wraps
+ def decorated_retry(f):
+ @wraps(f)
+ def f_retry(*args, **kwargs):
+ mtries, mdelay = tries, delay_s
+ while mtries > 1:
+ try:
+ return f(*args, **kwargs)
+ except ExceptionToCheck as e:
+ logging.warning("Exception: %s, Retrying in %d seconds...", str(e), mdelay)
+ time.sleep(mdelay)
+ mtries -= 1
+ mdelay *= backoff
+ return f(*args, **kwargs)
+
+ return f_retry # true decorator
+
+ return decorated_retry
-CCACHE_MAXSIZE = '500G'
def under_ci() -> bool:
""":return: True if we run in Jenkins."""
return 'JOB_NAME' in os.environ
-def get_platforms(path: Optional[str] = "docker"):
+
+def git_cleanup() -> None:
+ """Clean repo and subrepos, update subrepos"""
+ logging.info("cleaning up repository")
+ with remember_cwd():
+ os.chdir(get_mxnet_root())
+ check_call(['git', 'clean', '-ffdx'])
+ check_call(['git', 'submodule', 'foreach', '--recursive', 'git', 'clean', '-ffdx'])
+ check_call(['git', 'submodule', 'update', '--recursive', '--init'])
+
+
+def get_dockerfiles_path():
+ return "dockerfiles"
+
+
+def get_platforms(path: Optional[str] = get_dockerfiles_path()):
"""Get a list of architectures given our dockerfiles"""
dockerfiles = glob.glob(os.path.join(path, "Dockerfile.build.*"))
dockerfiles = list(filter(lambda x: x[-1] != '~', dockerfiles))
@@ -57,10 +157,11 @@ def get_platforms(path: Optional[str] = "docker"):
def get_docker_tag(platform: str, registry: str) -> str:
+ """:return: docker tag to be used for the container"""
return "{0}/build.{1}".format(registry, platform)
-def get_dockerfile(platform: str, path="docker") -> str:
+def get_dockerfile(platform: str, path=get_dockerfiles_path()) -> str:
return os.path.join(path, "Dockerfile.build.{0}".format(platform))
@@ -77,54 +178,37 @@ def build_docker(platform: str, docker_binary: str, registry: str, num_retries:
:param num_retries: Number of retries to build the docker image
:return: Id of the top level image
"""
-
tag = get_docker_tag(platform=platform, registry=registry)
- logging.info("Building container tagged '%s' with %s", tag, docker_binary)
+ logging.info("Building docker container tagged '%s' with %s", tag, docker_binary)
#
# We add a user with the same group as the executing non-root user so files created in the
# container match permissions of the local user. Same for the group.
#
# These variables are used in the docker files to create user and group with these ids.
- # see: docker/install/ubuntu_adduser.sh
+ # see: dockerfiles/install/ubuntu_adduser.sh
#
# cache-from is needed so we use the cached images tagged from the remote via
# docker pull see: docker_cache.load_docker_cache
#
# This doesn't work with multi head docker files.
- #
-
- for i in range(num_retries):
- logging.info('%d out of %d tries to build the docker image.', i + 1, num_retries)
-
- cmd = [docker_binary, "build",
- "-f", get_dockerfile(platform),
- "--build-arg", "USER_ID={}".format(os.getuid()),
- "--build-arg", "GROUP_ID={}".format(os.getgid()),
- "--cache-from", tag,
- "-t", tag,
- "docker"]
+ #
+ cmd = [docker_binary, "build",
+ "-f", get_dockerfile(platform),
+ "--build-arg", "USER_ID={}".format(os.getuid()),
+ "--build-arg", "GROUP_ID={}".format(os.getgid()),
+ "--cache-from", tag,
+ "-t", tag,
+ get_dockerfiles_path()]
+
+ @retry(subprocess.CalledProcessError, tries=num_retries)
+ def run_cmd():
logging.info("Running command: '%s'", ' '.join(cmd))
- try:
- check_call(cmd)
- # Docker build was successful. Call break to break out of the retry mechanism
- break
- except subprocess.CalledProcessError as e:
- saved_exception = e
- logging.error('Failed to build docker image')
- # Building the docker image failed. Call continue to trigger the retry mechanism
- continue
- else:
- # Num retries exceeded
- logging.exception('Exception during build of docker image', saved_exception)
- logging.fatal('Failed to build the docker image, aborting...')
- sys.exit(1)
+ check_call(cmd)
+ run_cmd()
# Get image id by reading the tag. It's guaranteed (except race condition) that the tag exists. Otherwise, the
# check_call would have failed
- image_id = _get_local_image_id(docker_binary=docker_binary, docker_tag=tag)
- if not image_id:
- raise FileNotFoundError('Unable to find docker image id matching with {}'.format(tag))
- return image_id
+ return _get_local_image_id(docker_binary=docker_binary, docker_tag=tag)
def _get_local_image_id(docker_binary, docker_tag):
@@ -136,12 +220,15 @@ def _get_local_image_id(docker_binary, docker_tag):
cmd = [docker_binary, "images", "-q", docker_tag]
image_id_b = subprocess.check_output(cmd)
image_id = image_id_b.decode('utf-8').strip()
+ if not image_id:
+ raise RuntimeError('Unable to find docker image id matching with tag {}'.format(tag))
return image_id
def buildir() -> str:
return os.path.join(get_mxnet_root(), "build")
+
def default_ccache_dir() -> str:
# Share ccache across containers
if 'CCACHE_DIR' in os.environ:
@@ -158,15 +245,36 @@ def default_ccache_dir() -> str:
return ccache_dir
return os.path.join(tempfile.gettempdir(), "ci_ccache")
+def trim_container_id(cid):
+ return cid[:12]
def container_run(platform: str,
- docker_binary: str,
+ nvidia_runtime: bool,
docker_registry: str,
shared_memory_size: str,
- local_ccache_dir: str,
command: List[str],
+ local_ccache_dir: str,
+ cleanup: Cleanup,
dry_run: bool = False,
- interactive: bool = False) -> str:
+ interactive: bool = False) -> int:
+ CONTAINER_WAIT_S = 600
+ #
+ # Environment setup
+ #
+ environment = {
+ 'CCACHE_MAXSIZE': '500G',
+ 'CCACHE_TEMPDIR': '/tmp/ccache', # temp dir should be local and not shared
+ 'CCACHE_DIR': '/work/ccache', # this path is inside the container as /work/ccache is mounted
+ 'CCACHE_LOGFILE': '/tmp/ccache.log', # a container-scoped log, useful for ccache verification.
+ }
+ # These variables are passed to the container to the process tree killer can find runaway process inside the container
+ # https://wiki.jenkins.io/display/JENKINS/ProcessTreeKiller
+ # https://github.com/jenkinsci/jenkins/blob/578d6bacb33a5e99f149de504c80275796f0b231/core/src/main/java/hudson/model/Run.java#L2393
+ #
+ JENKINS_ENV_VARS = ['BUILD_NUMBER', 'BUILD_ID', 'BUILD_TAG']
+ environment.update({k: os.environ[k] for k in JENKINS_ENV_VARS if k in os.environ})
+ environment.update({k: os.environ[k] for k in ['CCACHE_MAXSIZE'] if k in os.environ})
+
tag = get_docker_tag(platform=platform, registry=docker_registry)
mx_root = get_mxnet_root()
local_build_folder = buildir()
@@ -174,47 +282,106 @@ def container_run(platform: str,
os.makedirs(local_build_folder, exist_ok=True)
os.makedirs(local_ccache_dir, exist_ok=True)
logging.info("Using ccache directory: %s", local_ccache_dir)
- runlist = [docker_binary, 'run', '--rm', '-t',
+ docker_client = docker.from_env()
+ # Equivalent command
+ docker_cmd_list = [get_docker_binary(nvidia_runtime), 'run',
+ '--rm',
'--shm-size={}'.format(shared_memory_size),
'-v', "{}:/work/mxnet".format(mx_root), # mount mxnet root
'-v', "{}:/work/build".format(local_build_folder), # mount mxnet/build for storing build artifacts
'-v', "{}:/work/ccache".format(local_ccache_dir),
'-u', '{}:{}'.format(os.getuid(), os.getgid()),
- '-e', 'CCACHE_MAXSIZE={}'.format(CCACHE_MAXSIZE),
+ '-e', 'CCACHE_MAXSIZE=15G',
'-e', 'CCACHE_TEMPDIR=/tmp/ccache', # temp dir should be local and not shared
'-e', "CCACHE_DIR=/work/ccache", # this path is inside the container as /work/ccache is mounted
'-e', "CCACHE_LOGFILE=/tmp/ccache.log", # a container-scoped log, useful for ccache verification.
+ '-ti',
tag]
- runlist.extend(command)
- cmd = '\\\n\t'.join(runlist)
- ret = 0
- if not dry_run and not interactive:
- logging.info("Running %s in container %s", command, tag)
- logging.info("Executing:\n%s\n", cmd)
- ret = call(runlist)
-
- docker_run_cmd = ' '.join(runlist)
- if not dry_run and interactive:
- into_cmd = deepcopy(runlist)
- # -ti can't be after the tag, as is interpreted as a command so hook it up after the -u argument
- idx = into_cmd.index('-u') + 2
- into_cmd[idx:idx] = ['-ti']
- cmd = '\\\n\t'.join(into_cmd)
- logging.info("Executing:\n%s\n", cmd)
- docker_run_cmd = ' '.join(into_cmd)
- ret = call(into_cmd)
-
- if not dry_run and not interactive and ret != 0:
- logging.error("Running of command in container failed (%s):\n%s\n", ret, cmd)
- logging.error("You can get into the container by adding the -i option")
- raise subprocess.CalledProcessError(ret, cmd)
-
- return docker_run_cmd
+ docker_cmd_list.extend(command)
+ docker_cmd = ' \\\n\t'.join(docker_cmd_list)
+ logging.info("Running %s in container %s", command, tag)
+ logging.info("Executing the equivalent of:\n%s\n", docker_cmd)
+ ret = 0 # return code of the command inside docker
+ if not dry_run:
+
+
+ #############################
+ #
+ signal.pthread_sigmask(signal.SIG_BLOCK, {signal.SIGINT, signal.SIGTERM})
+ if nvidia_runtime:
+ runtime='nvidia'
+ else:
+ # runc is default (docker info | grep -i runtime)
+ runtime=None
+
+ container = docker_client.containers.run(
+ tag,
+ runtime=runtime,
+ detach=True,
+ command=command,
+ #auto_remove=True,
+ shm_size=shared_memory_size,
+ user='{}:{}'.format(os.getuid(), os.getgid()),
+ volumes={
+ mx_root:
+ {'bind': '/work/mxnet', 'mode': 'rw'},
+ local_build_folder:
+ {'bind': '/work/build', 'mode': 'rw'},
+ local_ccache_dir:
+ {'bind': '/work/ccache', 'mode': 'rw'},
+ },
+ environment=environment)
+ logging.info("Started container: %s", trim_container_id(container.id))
+ # Race condition:
+ # If the previous call is interrupted then it's possible that the container is not cleaned up
+ # We avoid by masking the signals temporarily
+ cleanup.add_container(container)
+ signal.pthread_sigmask(signal.SIG_UNBLOCK, {signal.SIGINT, signal.SIGTERM})
+ #
+ #############################
+
+ stream = container.logs(stream=True, stdout=True, stderr=True)
+ sys.stdout.flush()
+ for chunk in stream:
+ sys.stdout.buffer.write(chunk)
+ sys.stdout.buffer.flush()
+ sys.stdout.flush()
+ stream.close()
+ try:
+ logging.info("Waiting for status of container %s for %d s.", trim_container_id(container.id), CONTAINER_WAIT_S)
+ wait_result = container.wait(timeout=CONTAINER_WAIT_S)
+ logging.info("Container exit status: %s", wait_result)
+ ret = wait_result.get('StatusCode', 200)
+ except Exception as e:
+ logging.exception(e)
+ ret = 150
+
+ # Stop
+ try:
+ logging.info("Stopping container: %s", trim_container_id(container.id))
+ container.stop()
+ except Exception as e:
+ logging.exception(e)
+ ret = 151
+
+ # Remove
+ try:
+ logging.info("Removing container: %s", trim_container_id(container.id))
+ container.remove()
+ except Exception as e:
+ logging.exception(e)
+ ret = 152
+ cleanup.remove_container(container)
+ containers = docker_client.containers.list()
+ if containers:
+ logging.info("Other running containers: %s", [trim_container_id(x.id) for x in containers])
+ return ret
def list_platforms() -> str:
print("\nSupported platforms:\n{}".format('\n'.join(get_platforms())))
+
def load_docker_cache(tag, docker_registry) -> None:
if docker_registry:
try:
@@ -226,19 +393,48 @@ def load_docker_cache(tag, docker_registry) -> None:
else:
logging.info('Distributed docker cache disabled')
-def main() -> int:
+def ec2_instance_id_hostname() -> str:
+ if under_ci():
+ result = []
+ try:
+ r = requests.get("http://instance-data/latest/meta-data/instance-id")
+ if r.status_code == 200:
+ result.append(r.content.decode())
+ r = requests.get("http://instance-data/latest/meta-data/public-hostname")
+ if r.status_code == 200:
+ result.append(r.content.decode())
+ return ' '.join(result)
+ except ConnectionError as e:
+ pass
+ return '?'
+ else:
+ return ''
+
+def log_environment():
+ instance_id = ec2_instance_id_hostname()
+ if instance_id:
+ logging.info("Instance id: %s", instance_id)
+ pp = pprint.PrettyPrinter(indent=4)
+ logging.info("Build environment: %s", pp.pformat(dict(os.environ)))
+
+def chdir_to_script_directory():
# We need to be in the same directory than the script so the commands in the dockerfiles work as
# expected. But the script can be invoked from a different path
base = os.path.split(os.path.realpath(__file__))[0]
os.chdir(base)
- logging.getLogger().setLevel(logging.INFO)
+def script_name() -> str:
+ return os.path.split(sys.argv[0])[1]
- def script_name() -> str:
- return os.path.split(sys.argv[0])[1]
+def main() -> int:
+ logging.getLogger().setLevel(logging.INFO)
+ logging.getLogger("requests").setLevel(logging.WARNING)
logging.basicConfig(format='{}: %(asctime)-15s %(message)s'.format(script_name()))
+ log_environment()
+ chdir_to_script_directory()
+
parser = argparse.ArgumentParser(description="""Utility for building and testing MXNet on docker
containers""", epilog="")
parser.add_argument("-p", "--platform",
@@ -284,7 +480,7 @@ def script_name() -> str:
default=1,
type=int)
- parser.add_argument("-c", "--cache", action="store_true",
+ parser.add_argument("--no-cache", action="store_true",
help="Enable docker registry cache")
parser.add_argument("command",
@@ -297,13 +493,26 @@ def script_name() -> str:
type=str)
args = parser.parse_args()
+
def use_cache():
- return args.cache or under_ci()
+ return not args.no_cache or under_ci()
command = list(chain(*args.command))
docker_binary = get_docker_binary(args.nvidiadocker)
shared_memory_size = args.shared_memory_size
- num_docker_build_retires = args.docker_build_retries
+
+ # Cleanup on signals and exit
+ cleanup = Cleanup()
+ def signal_handler(signum, _):
+ signal.pthread_sigmask(signal.SIG_BLOCK, {signum})
+ logging.warning("Signal %d received, cleaning up...", signum)
+ cleanup()
+ logging.warning("done. Exiting with error.")
+ sys.exit(1)
+
+ atexit.register(cleanup)
+ signal.signal(signal.SIGTERM, signal_handler)
+ signal.signal(signal.SIGINT, signal_handler)
if args.list:
list_platforms()
@@ -312,31 +521,37 @@ def use_cache():
tag = get_docker_tag(platform=platform, registry=args.docker_registry)
if use_cache():
load_docker_cache(tag=tag, docker_registry=args.docker_registry)
- build_docker(platform, docker_binary, registry=args.docker_registry, num_retries=num_docker_build_retires)
+ build_docker(platform, docker_binary, registry=args.docker_registry, num_retries=args.docker_build_retries)
if args.build_only:
logging.warning("Container was just built. Exiting due to build-only.")
return 0
+ ret = 0
if command:
- container_run(platform=platform, docker_binary=docker_binary, shared_memory_size=shared_memory_size,
+ ret = container_run(platform=platform, nvidia_runtime=args.nvidiadocker, shared_memory_size=shared_memory_size,
command=command, docker_registry=args.docker_registry,
- local_ccache_dir=args.ccache_dir, interactive=args.interactive)
+ local_ccache_dir=args.ccache_dir, interactive=args.interactive, cleanup=cleanup)
elif args.print_docker_run:
- print(container_run(platform=platform, docker_binary=docker_binary, shared_memory_size=shared_memory_size,
- command=[], dry_run=True, docker_registry=args.docker_registry, local_ccache_dir=args.ccache_dir))
+ ret = container_run(platform=platform, nvidia_runtime=args.nvidiadocker, shared_memory_size=shared_memory_size,
+ command=[], dry_run=True, docker_registry=args.docker_registry,
+ local_ccache_dir=args.ccache_dir)
+ command=[]
elif args.interactive:
- container_run(platform=platform, docker_binary=docker_binary, shared_memory_size=shared_memory_size,
+ ret = container_run(platform=platform, nvidia_runtime=args.nvidiadocker, shared_memory_size=shared_memory_size,
command=command, docker_registry=args.docker_registry,
- local_ccache_dir=args.ccache_dir, interactive=args.interactive)
-
+ local_ccache_dir=args.ccache_dir, interactive=args.interactive, cleanup=cleanup)
else:
# With no commands, execute a build function for the target platform
assert not args.interactive, "when running with -i must provide a command"
- cmd = ["/work/mxnet/ci/docker/runtime_functions.sh", "build_{}".format(platform)]
- logging.info("No command specified, trying default build: %s", ' '.join(cmd))
- container_run(platform=platform, docker_binary=docker_binary, shared_memory_size=shared_memory_size,
- command=cmd, docker_registry=args.docker_registry,
- local_ccache_dir=args.ccache_dir)
+ command = ["/work/mxnet/ci/dockerfiles/runtime_functions.sh", "build_{}".format(platform)]
+ logging.info("No command specified, trying default build: %s", ' '.join(command))
+ ret = container_run(platform=platform, nvidia_runtime=args.nvidiadocker, shared_memory_size=shared_memory_size,
+ command=command, docker_registry=args.docker_registry,
+ local_ccache_dir=args.ccache_dir, cleanup=cleanup)
+
+ if ret != 0:
+ logging.critical("Execution of %s failed with status: %d", command, ret)
+ return(ret)
elif args.all:
platforms = get_platforms()
@@ -346,15 +561,21 @@ def use_cache():
tag = get_docker_tag(platform=platform, registry=args.docker_registry)
if use_cache():
load_docker_cache(tag=tag, docker_registry=args.docker_registry)
- build_docker(platform, docker_binary, args.docker_registry, num_retries=num_docker_build_retires)
+ build_docker(platform, docker_binary, args.docker_registry, num_retries=args.docker_build_retries)
if args.build_only:
continue
- build_platform = "build_{}".format(platform)
- cmd = ["/work/mxnet/ci/docker/runtime_functions.sh", build_platform]
+ git_cleanup()
shutil.rmtree(buildir(), ignore_errors=True)
- container_run(platform=platform, docker_binary=docker_binary, shared_memory_size=shared_memory_size,
- command=cmd, docker_registry=args.docker_registry, local_ccache_dir=args.ccache_dir)
- plat_buildir = os.path.join(get_mxnet_root(), build_platform)
+ build_platform = "build_{}".format(platform)
+ plat_buildir = os.path.abspath(os.path.join(get_mxnet_root(), '..',
+ "mxnet_{}".format(build_platform)))
+ if os.path.exists(plat_buildir):
+ logging.warning("{} already exists, skipping".format(plat_buildir))
+ continue
+ command = ["/work/mxnet/ci/dockerfiles/runtime_functions.sh", build_platform]
+ # BUILD in docker
+ container_run(platform=platform, nvidia_runtime=args.nvidiadocker, shared_memory_size=shared_memory_size,
+ command=command, docker_registry=args.docker_registry, local_ccache_dir=args.ccache_dir)
shutil.move(buildir(), plat_buildir)
logging.info("Built files left in: %s", plat_buildir)
@@ -367,7 +588,7 @@ def use_cache():
./build.py -p armv7
Will build a docker container with cross compilation tools and build MXNet for armv7 by
- running: ci/docker/runtime_functions.sh build_armv7 inside the container.
+ running: ci/dockerfiles/runtime_functions.sh build_armv7 inside the container.
./build.py -p armv7 ls
@@ -383,7 +604,8 @@ def use_cache():
./build.py -a
- Builds for all platforms and leaves artifacts in build_<platform>
+ Builds for all platforms and leaves artifacts in build_<platform>. **WARNING** it performs git
+ cleanup of the repo.
""")
diff --git a/ci/docker/Dockerfile.build.android_armv7 b/ci/dockerfiles/Dockerfile.build.android_armv7
old mode 100755
new mode 100644
similarity index 100%
rename from ci/docker/Dockerfile.build.android_armv7
rename to ci/dockerfiles/Dockerfile.build.android_armv7
diff --git a/ci/docker/Dockerfile.build.android_armv8 b/ci/dockerfiles/Dockerfile.build.android_armv8
old mode 100755
new mode 100644
similarity index 100%
rename from ci/docker/Dockerfile.build.android_armv8
rename to ci/dockerfiles/Dockerfile.build.android_armv8
diff --git a/ci/docker/Dockerfile.build.armv6 b/ci/dockerfiles/Dockerfile.build.armv6
old mode 100755
new mode 100644
similarity index 100%
rename from ci/docker/Dockerfile.build.armv6
rename to ci/dockerfiles/Dockerfile.build.armv6
diff --git a/ci/docker/Dockerfile.build.armv7 b/ci/dockerfiles/Dockerfile.build.armv7
old mode 100755
new mode 100644
similarity index 100%
rename from ci/docker/Dockerfile.build.armv7
rename to ci/dockerfiles/Dockerfile.build.armv7
diff --git a/ci/docker/Dockerfile.build.armv8 b/ci/dockerfiles/Dockerfile.build.armv8
old mode 100755
new mode 100644
similarity index 100%
rename from ci/docker/Dockerfile.build.armv8
rename to ci/dockerfiles/Dockerfile.build.armv8
diff --git a/ci/docker/Dockerfile.build.centos7_cpu b/ci/dockerfiles/Dockerfile.build.centos7_cpu
old mode 100755
new mode 100644
similarity index 100%
rename from ci/docker/Dockerfile.build.centos7_cpu
rename to ci/dockerfiles/Dockerfile.build.centos7_cpu
diff --git a/ci/docker/Dockerfile.build.centos7_gpu b/ci/dockerfiles/Dockerfile.build.centos7_gpu
old mode 100755
new mode 100644
similarity index 100%
rename from ci/docker/Dockerfile.build.centos7_gpu
rename to ci/dockerfiles/Dockerfile.build.centos7_gpu
diff --git a/ci/docker/Dockerfile.build.jetson b/ci/dockerfiles/Dockerfile.build.jetson
old mode 100755
new mode 100644
similarity index 100%
rename from ci/docker/Dockerfile.build.jetson
rename to ci/dockerfiles/Dockerfile.build.jetson
diff --git a/ci/docker/Dockerfile.build.ubuntu_base_cpu b/ci/dockerfiles/Dockerfile.build.ubuntu_base_cpu
old mode 100755
new mode 100644
similarity index 100%
rename from ci/docker/Dockerfile.build.ubuntu_base_cpu
rename to ci/dockerfiles/Dockerfile.build.ubuntu_base_cpu
diff --git a/ci/docker/Dockerfile.build.ubuntu_base_gpu b/ci/dockerfiles/Dockerfile.build.ubuntu_base_gpu
old mode 100755
new mode 100644
similarity index 100%
rename from ci/docker/Dockerfile.build.ubuntu_base_gpu
rename to ci/dockerfiles/Dockerfile.build.ubuntu_base_gpu
diff --git a/ci/docker/Dockerfile.build.ubuntu_blc b/ci/dockerfiles/Dockerfile.build.ubuntu_blc
old mode 100755
new mode 100644
similarity index 90%
rename from ci/docker/Dockerfile.build.ubuntu_blc
rename to ci/dockerfiles/Dockerfile.build.ubuntu_blc
index 294740ce139..208cba2111f
--- a/ci/docker/Dockerfile.build.ubuntu_blc
+++ b/ci/dockerfiles/Dockerfile.build.ubuntu_blc
@@ -24,8 +24,13 @@ WORKDIR /work/deps
COPY install/ubuntu_core.sh /work/
RUN /work/ubuntu_core.sh
-COPY install/ubuntu_python.sh /work/
-RUN /work/ubuntu_python.sh
+
+COPY install/ubuntu_python2.sh /work/
+RUN /work/ubuntu_python2.sh
+
+COPY install/ubuntu_python3.sh /work/
+RUN /work/ubuntu_python3.sh
+
COPY install/ubuntu_npm_blc.sh /work/
RUN /work/ubuntu_npm_blc.sh
diff --git a/ci/docker/Dockerfile.build.ubuntu_build_cuda b/ci/dockerfiles/Dockerfile.build.ubuntu_build_cuda
old mode 100755
new mode 100644
similarity index 93%
rename from ci/docker/Dockerfile.build.ubuntu_build_cuda
rename to ci/dockerfiles/Dockerfile.build.ubuntu_build_cuda
index 9ed0cbbe3e5..19e9265f88d
--- a/ci/docker/Dockerfile.build.ubuntu_build_cuda
+++ b/ci/dockerfiles/Dockerfile.build.ubuntu_build_cuda
@@ -27,20 +27,30 @@ WORKDIR /work/deps
COPY install/ubuntu_core.sh /work/
RUN /work/ubuntu_core.sh
+
COPY install/deb_ubuntu_ccache.sh /work/
RUN /work/deb_ubuntu_ccache.sh
-COPY install/ubuntu_python.sh /work/
-RUN /work/ubuntu_python.sh
+
+COPY install/ubuntu_python2.sh /work/
+RUN /work/ubuntu_python2.sh
+
+COPY install/ubuntu_python3.sh /work/
+RUN /work/ubuntu_python3.sh
+
COPY install/ubuntu_scala.sh /work/
COPY install/sbt.gpg /work/
RUN /work/ubuntu_scala.sh
+
COPY install/ubuntu_r.sh /work/
COPY install/r.gpg /work/
RUN /work/ubuntu_r.sh
+
COPY install/ubuntu_perl.sh /work/
RUN /work/ubuntu_perl.sh
+
COPY install/ubuntu_clang.sh /work/
RUN /work/ubuntu_clang.sh
+
COPY install/ubuntu_mklml.sh /work/
RUN /work/ubuntu_mklml.sh
diff --git a/ci/docker/Dockerfile.build.ubuntu_cpu b/ci/dockerfiles/Dockerfile.build.ubuntu_cpu
old mode 100755
new mode 100644
similarity index 91%
rename from ci/docker/Dockerfile.build.ubuntu_cpu
rename to ci/dockerfiles/Dockerfile.build.ubuntu_cpu
index 6751465758c..eef615af850
--- a/ci/docker/Dockerfile.build.ubuntu_cpu
+++ b/ci/dockerfiles/Dockerfile.build.ubuntu_cpu
@@ -28,8 +28,11 @@ RUN /work/ubuntu_core.sh
COPY install/deb_ubuntu_ccache.sh /work/
RUN /work/deb_ubuntu_ccache.sh
-COPY install/ubuntu_python.sh /work/
-RUN /work/ubuntu_python.sh
+COPY install/ubuntu_python2.sh /work/
+RUN /work/ubuntu_python2.sh
+
+COPY install/ubuntu_python3.sh /work/
+RUN /work/ubuntu_python3.sh
COPY install/ubuntu_scala.sh /work/
COPY install/sbt.gpg /work/
@@ -58,7 +61,7 @@ COPY install/ubuntu_onnx.sh /work/
RUN /work/ubuntu_onnx.sh
COPY install/ubuntu_docs.sh /work/
-COPY install/docs_requirements /work/
+COPY install/docs_requirements.txt /work/
RUN /work/ubuntu_docs.sh
ARG USER_ID=0
diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu b/ci/dockerfiles/Dockerfile.build.ubuntu_gpu
old mode 100755
new mode 100644
similarity index 91%
rename from ci/docker/Dockerfile.build.ubuntu_gpu
rename to ci/dockerfiles/Dockerfile.build.ubuntu_gpu
index 8fcbcbbb967..46eac512013
--- a/ci/docker/Dockerfile.build.ubuntu_gpu
+++ b/ci/dockerfiles/Dockerfile.build.ubuntu_gpu
@@ -28,8 +28,11 @@ RUN /work/ubuntu_core.sh
COPY install/deb_ubuntu_ccache.sh /work/
RUN /work/deb_ubuntu_ccache.sh
-COPY install/ubuntu_python.sh /work/
-RUN /work/ubuntu_python.sh
+COPY install/ubuntu_python2.sh /work/
+RUN /work/ubuntu_python2.sh
+
+COPY install/ubuntu_python3.sh /work/
+RUN /work/ubuntu_python3.sh
COPY install/ubuntu_scala.sh /work/
COPY install/sbt.gpg /work/
@@ -61,7 +64,7 @@ COPY install/ubuntu_onnx.sh /work/
RUN /work/ubuntu_onnx.sh
COPY install/ubuntu_docs.sh /work/
-COPY install/docs_requirements /work/
+COPY install/docs_requirements.txt /work/
RUN /work/ubuntu_docs.sh
COPY install/ubuntu_tutorials.sh /work/
diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu_tensorrt b/ci/dockerfiles/Dockerfile.build.ubuntu_gpu_tensorrt
old mode 100755
new mode 100644
similarity index 90%
rename from ci/docker/Dockerfile.build.ubuntu_gpu_tensorrt
rename to ci/dockerfiles/Dockerfile.build.ubuntu_gpu_tensorrt
index 255da316041..3f0bbc666f2
--- a/ci/docker/Dockerfile.build.ubuntu_gpu_tensorrt
+++ b/ci/dockerfiles/Dockerfile.build.ubuntu_gpu_tensorrt
@@ -24,10 +24,16 @@ WORKDIR /work/deps
COPY install/ubuntu_core.sh /work/
RUN /work/ubuntu_core.sh
+
COPY install/deb_ubuntu_ccache.sh /work/
RUN /work/deb_ubuntu_ccache.sh
-COPY install/ubuntu_python.sh /work/
-RUN /work/ubuntu_python.sh
+
+COPY install/ubuntu_python2.sh /work/
+RUN /work/ubuntu_python2.sh
+
+COPY install/ubuntu_python3.sh /work/
+RUN /work/ubuntu_python3.sh
+
COPY install/tensorrt.sh /work
RUN /work/tensorrt.sh
diff --git a/ci/docker/Dockerfile.build.ubuntu_nightly_cpu b/ci/dockerfiles/Dockerfile.build.ubuntu_nightly_cpu
old mode 100755
new mode 100644
similarity index 91%
rename from ci/docker/Dockerfile.build.ubuntu_nightly_cpu
rename to ci/dockerfiles/Dockerfile.build.ubuntu_nightly_cpu
index 1b126c18be4..7de78eb2c49
--- a/ci/docker/Dockerfile.build.ubuntu_nightly_cpu
+++ b/ci/dockerfiles/Dockerfile.build.ubuntu_nightly_cpu
@@ -28,8 +28,11 @@ RUN /work/ubuntu_core.sh
COPY install/deb_ubuntu_ccache.sh /work/
RUN /work/deb_ubuntu_ccache.sh
-COPY install/ubuntu_python.sh /work/
-RUN /work/ubuntu_python.sh
+COPY install/ubuntu_python2.sh /work/
+RUN /work/ubuntu_python2.sh
+
+COPY install/ubuntu_python3.sh /work/
+RUN /work/ubuntu_python3.sh
COPY install/ubuntu_scala.sh /work/
COPY install/sbt.gpg /work/
@@ -55,7 +58,7 @@ COPY install/ubuntu_onnx.sh /work/
RUN /work/ubuntu_onnx.sh
COPY install/ubuntu_docs.sh /work/
-COPY install/docs_requirements /work/
+COPY install/docs_requirements.txt /work/
RUN /work/ubuntu_docs.sh
COPY install/ubuntu_nightly_tests.sh /work/
diff --git a/ci/docker/Dockerfile.build.ubuntu_nightly_gpu b/ci/dockerfiles/Dockerfile.build.ubuntu_nightly_gpu
old mode 100755
new mode 100644
similarity index 92%
rename from ci/docker/Dockerfile.build.ubuntu_nightly_gpu
rename to ci/dockerfiles/Dockerfile.build.ubuntu_nightly_gpu
index deeed8b0d52..fa3727e20a0
--- a/ci/docker/Dockerfile.build.ubuntu_nightly_gpu
+++ b/ci/dockerfiles/Dockerfile.build.ubuntu_nightly_gpu
@@ -28,8 +28,11 @@ RUN /work/ubuntu_core.sh
COPY install/deb_ubuntu_ccache.sh /work/
RUN /work/deb_ubuntu_ccache.sh
-COPY install/ubuntu_python.sh /work/
-RUN /work/ubuntu_python.sh
+COPY install/ubuntu_python2.sh /work/
+RUN /work/ubuntu_python2.sh
+
+COPY install/ubuntu_python3.sh /work/
+RUN /work/ubuntu_python3.sh
COPY install/ubuntu_scala.sh /work/
COPY install/sbt.gpg /work/
@@ -61,7 +64,7 @@ COPY install/ubuntu_onnx.sh /work/
RUN /work/ubuntu_onnx.sh
COPY install/ubuntu_docs.sh /work/
-COPY install/docs_requirements /work/
+COPY install/docs_requirements.txt /work/
RUN /work/ubuntu_docs.sh
COPY install/ubuntu_tutorials.sh /work/
diff --git a/ci/docker/Dockerfile.build.ubuntu_rat b/ci/dockerfiles/Dockerfile.build.ubuntu_rat
old mode 100755
new mode 100644
similarity index 100%
rename from ci/docker/Dockerfile.build.ubuntu_rat
rename to ci/dockerfiles/Dockerfile.build.ubuntu_rat
diff --git a/ci/docker/install/android_arm64_openblas.sh b/ci/dockerfiles/install/android_arm64_openblas.sh
similarity index 100%
rename from ci/docker/install/android_arm64_openblas.sh
rename to ci/dockerfiles/install/android_arm64_openblas.sh
diff --git a/ci/docker/install/android_armv7_openblas.sh b/ci/dockerfiles/install/android_armv7_openblas.sh
similarity index 100%
rename from ci/docker/install/android_armv7_openblas.sh
rename to ci/dockerfiles/install/android_armv7_openblas.sh
diff --git a/ci/docker/install/android_ndk.sh b/ci/dockerfiles/install/android_ndk.sh
similarity index 100%
rename from ci/docker/install/android_ndk.sh
rename to ci/dockerfiles/install/android_ndk.sh
diff --git a/ci/docker/install/arm64_openblas.sh b/ci/dockerfiles/install/arm64_openblas.sh
similarity index 100%
rename from ci/docker/install/arm64_openblas.sh
rename to ci/dockerfiles/install/arm64_openblas.sh
diff --git a/ci/docker/install/arm_openblas.sh b/ci/dockerfiles/install/arm_openblas.sh
similarity index 100%
rename from ci/docker/install/arm_openblas.sh
rename to ci/dockerfiles/install/arm_openblas.sh
diff --git a/ci/docker/install/centos7_adduser.sh b/ci/dockerfiles/install/centos7_adduser.sh
similarity index 100%
rename from ci/docker/install/centos7_adduser.sh
rename to ci/dockerfiles/install/centos7_adduser.sh
diff --git a/ci/docker/install/centos7_ccache.sh b/ci/dockerfiles/install/centos7_ccache.sh
similarity index 100%
rename from ci/docker/install/centos7_ccache.sh
rename to ci/dockerfiles/install/centos7_ccache.sh
diff --git a/ci/docker/install/centos7_core.sh b/ci/dockerfiles/install/centos7_core.sh
similarity index 100%
rename from ci/docker/install/centos7_core.sh
rename to ci/dockerfiles/install/centos7_core.sh
diff --git a/ci/docker/install/centos7_python.sh b/ci/dockerfiles/install/centos7_python.sh
similarity index 100%
rename from ci/docker/install/centos7_python.sh
rename to ci/dockerfiles/install/centos7_python.sh
diff --git a/ci/docker/install/deb_ubuntu_ccache.sh b/ci/dockerfiles/install/deb_ubuntu_ccache.sh
similarity index 100%
rename from ci/docker/install/deb_ubuntu_ccache.sh
rename to ci/dockerfiles/install/deb_ubuntu_ccache.sh
diff --git a/ci/docker/install/docs_requirements b/ci/dockerfiles/install/docs_requirements.txt
similarity index 100%
rename from ci/docker/install/docs_requirements
rename to ci/dockerfiles/install/docs_requirements.txt
diff --git a/ci/docker/install/export_gpg_keys.sh b/ci/dockerfiles/install/export_gpg_keys.sh
similarity index 100%
rename from ci/docker/install/export_gpg_keys.sh
rename to ci/dockerfiles/install/export_gpg_keys.sh
diff --git a/ci/docker/install/r.gpg b/ci/dockerfiles/install/r.gpg
similarity index 100%
rename from ci/docker/install/r.gpg
rename to ci/dockerfiles/install/r.gpg
diff --git a/ci/docker/install/sbt.gpg b/ci/dockerfiles/install/sbt.gpg
similarity index 100%
rename from ci/docker/install/sbt.gpg
rename to ci/dockerfiles/install/sbt.gpg
diff --git a/ci/docker/install/tensorrt.sh b/ci/dockerfiles/install/tensorrt.sh
similarity index 100%
rename from ci/docker/install/tensorrt.sh
rename to ci/dockerfiles/install/tensorrt.sh
diff --git a/ci/docker/install/ubuntu_adduser.sh b/ci/dockerfiles/install/ubuntu_adduser.sh
similarity index 100%
rename from ci/docker/install/ubuntu_adduser.sh
rename to ci/dockerfiles/install/ubuntu_adduser.sh
diff --git a/ci/docker/install/ubuntu_arm.sh b/ci/dockerfiles/install/ubuntu_arm.sh
similarity index 100%
rename from ci/docker/install/ubuntu_arm.sh
rename to ci/dockerfiles/install/ubuntu_arm.sh
diff --git a/ci/docker/install/ubuntu_caffe.sh b/ci/dockerfiles/install/ubuntu_caffe.sh
similarity index 100%
rename from ci/docker/install/ubuntu_caffe.sh
rename to ci/dockerfiles/install/ubuntu_caffe.sh
diff --git a/ci/docker/install/ubuntu_clang.sh b/ci/dockerfiles/install/ubuntu_clang.sh
similarity index 100%
rename from ci/docker/install/ubuntu_clang.sh
rename to ci/dockerfiles/install/ubuntu_clang.sh
diff --git a/ci/docker/install/ubuntu_clojure.sh b/ci/dockerfiles/install/ubuntu_clojure.sh
similarity index 100%
rename from ci/docker/install/ubuntu_clojure.sh
rename to ci/dockerfiles/install/ubuntu_clojure.sh
diff --git a/ci/docker/install/ubuntu_core.sh b/ci/dockerfiles/install/ubuntu_core.sh
similarity index 100%
rename from ci/docker/install/ubuntu_core.sh
rename to ci/dockerfiles/install/ubuntu_core.sh
diff --git a/ci/docker/install/ubuntu_docs.sh b/ci/dockerfiles/install/ubuntu_docs.sh
similarity index 92%
rename from ci/docker/install/ubuntu_docs.sh
rename to ci/dockerfiles/install/ubuntu_docs.sh
index a709b3de784..1b11683e7a4 100755
--- a/ci/docker/install/ubuntu_docs.sh
+++ b/ci/dockerfiles/install/ubuntu_docs.sh
@@ -27,7 +27,7 @@ apt-get install -y \
doxygen \
pandoc
-pip3 install -r /work/docs_requirements
-pip2 install -r /work/docs_requirements
+pip3 install -r /work/docs_requirements.txt
+pip2 install -r /work/docs_requirements.txt
echo 'Dependency installation complete.'
diff --git a/ci/docker/install/ubuntu_emscripten.sh b/ci/dockerfiles/install/ubuntu_emscripten.sh
similarity index 100%
rename from ci/docker/install/ubuntu_emscripten.sh
rename to ci/dockerfiles/install/ubuntu_emscripten.sh
diff --git a/ci/docker/install/ubuntu_llvm.sh b/ci/dockerfiles/install/ubuntu_llvm.sh
similarity index 95%
rename from ci/docker/install/ubuntu_llvm.sh
rename to ci/dockerfiles/install/ubuntu_llvm.sh
index 09e13d3d1ed..db24cfd23f6 100755
--- a/ci/docker/install/ubuntu_llvm.sh
+++ b/ci/dockerfiles/install/ubuntu_llvm.sh
@@ -23,4 +23,4 @@ echo deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial-5.0 main\
>> /etc/apt/sources.list.d/llvm.list
wget -O - http://apt.llvm.org/llvm-snapshot.gpg.key|sudo apt-key add -
-apt-get update && apt-get install -y --force-yes llvm-5.0
\ No newline at end of file
+apt-get update && apt-get install -y --force-yes llvm-5.0
diff --git a/ci/docker/install/ubuntu_mklml.sh b/ci/dockerfiles/install/ubuntu_mklml.sh
similarity index 100%
rename from ci/docker/install/ubuntu_mklml.sh
rename to ci/dockerfiles/install/ubuntu_mklml.sh
diff --git a/ci/docker/install/ubuntu_nightly_tests.sh b/ci/dockerfiles/install/ubuntu_nightly_tests.sh
similarity index 100%
rename from ci/docker/install/ubuntu_nightly_tests.sh
rename to ci/dockerfiles/install/ubuntu_nightly_tests.sh
diff --git a/ci/docker/install/ubuntu_npm_blc.sh b/ci/dockerfiles/install/ubuntu_npm_blc.sh
similarity index 100%
rename from ci/docker/install/ubuntu_npm_blc.sh
rename to ci/dockerfiles/install/ubuntu_npm_blc.sh
diff --git a/ci/docker/install/ubuntu_nvidia.sh b/ci/dockerfiles/install/ubuntu_nvidia.sh
similarity index 100%
rename from ci/docker/install/ubuntu_nvidia.sh
rename to ci/dockerfiles/install/ubuntu_nvidia.sh
diff --git a/ci/docker/install/ubuntu_onnx.sh b/ci/dockerfiles/install/ubuntu_onnx.sh
similarity index 100%
rename from ci/docker/install/ubuntu_onnx.sh
rename to ci/dockerfiles/install/ubuntu_onnx.sh
diff --git a/ci/docker/install/ubuntu_perl.sh b/ci/dockerfiles/install/ubuntu_perl.sh
similarity index 100%
rename from ci/docker/install/ubuntu_perl.sh
rename to ci/dockerfiles/install/ubuntu_perl.sh
diff --git a/ci/docker/install/ubuntu_python.sh b/ci/dockerfiles/install/ubuntu_python2.sh
similarity index 84%
rename from ci/docker/install/ubuntu_python.sh
rename to ci/dockerfiles/install/ubuntu_python2.sh
index e71cac8a389..f0526e2d830 100755
--- a/ci/docker/install/ubuntu_python.sh
+++ b/ci/dockerfiles/install/ubuntu_python2.sh
@@ -22,12 +22,10 @@
set -ex
# install libraries for mxnet's python package on ubuntu
-apt-get install -y python-dev python3-dev virtualenv
+apt-get install -y python-dev virtualenv wget
# the version of the pip shipped with ubuntu may be too lower, install a recent version here
wget -nv https://bootstrap.pypa.io/get-pip.py
-python3 get-pip.py
python2 get-pip.py
-pip2 install nose cpplint==1.3.0 pylint==1.8.3 'numpy<1.15.0,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3
-pip3 install nose cpplint==1.3.0 pylint==1.8.3 'numpy<1.15.0,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3
+pip2 install nose cpplint==1.3.0 pylint==1.8.3 'numpy<1.15.0,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3 mock
diff --git a/ci/dockerfiles/install/ubuntu_python3.sh b/ci/dockerfiles/install/ubuntu_python3.sh
new file mode 100755
index 00000000000..1dad5a7aa20
--- /dev/null
+++ b/ci/dockerfiles/install/ubuntu_python3.sh
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
+
+set -ex
+# install libraries for mxnet's python package on ubuntu
+apt-get install -y python3-dev virtualenv wget
+
+# the version of the pip shipped with ubuntu may be too lower, install a recent version here
+wget -nv https://bootstrap.pypa.io/get-pip.py
+python3 get-pip.py
+
+pip3 install nose cpplint==1.3.0 pylint==1.8.3 'numpy<1.15.0,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3 mock
diff --git a/ci/docker/install/ubuntu_r.sh b/ci/dockerfiles/install/ubuntu_r.sh
similarity index 100%
rename from ci/docker/install/ubuntu_r.sh
rename to ci/dockerfiles/install/ubuntu_r.sh
diff --git a/ci/docker/install/ubuntu_rat.sh b/ci/dockerfiles/install/ubuntu_rat.sh
similarity index 100%
rename from ci/docker/install/ubuntu_rat.sh
rename to ci/dockerfiles/install/ubuntu_rat.sh
diff --git a/ci/docker/install/ubuntu_runas_sudo.sh b/ci/dockerfiles/install/ubuntu_runas_sudo.sh
similarity index 100%
rename from ci/docker/install/ubuntu_runas_sudo.sh
rename to ci/dockerfiles/install/ubuntu_runas_sudo.sh
diff --git a/ci/docker/install/ubuntu_scala.sh b/ci/dockerfiles/install/ubuntu_scala.sh
similarity index 100%
rename from ci/docker/install/ubuntu_scala.sh
rename to ci/dockerfiles/install/ubuntu_scala.sh
diff --git a/ci/docker/install/ubuntu_tutorials.sh b/ci/dockerfiles/install/ubuntu_tutorials.sh
similarity index 100%
rename from ci/docker/install/ubuntu_tutorials.sh
rename to ci/dockerfiles/install/ubuntu_tutorials.sh
diff --git a/ci/docker/install/ubuntu_tvm.sh b/ci/dockerfiles/install/ubuntu_tvm.sh
similarity index 100%
rename from ci/docker/install/ubuntu_tvm.sh
rename to ci/dockerfiles/install/ubuntu_tvm.sh
diff --git a/ci/docker/runtime_functions.sh b/ci/dockerfiles/runtime_functions.sh
similarity index 99%
rename from ci/docker/runtime_functions.sh
rename to ci/dockerfiles/runtime_functions.sh
index 2341674fa21..65054c88ab2 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/dockerfiles/runtime_functions.sh
@@ -163,7 +163,7 @@ build_armv7() {
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DUSE_CUDA=OFF \
-DUSE_OPENCV=OFF \
- -DUSE_OPENMP=ON \
+ -DUSE_OPENMP=OFF \
-DUSE_SIGNAL_HANDLER=ON \
-DCMAKE_BUILD_TYPE=Release \
-DUSE_MKL_IF_AVAILABLE=OFF \
@@ -305,6 +305,14 @@ build_ubuntu_cpu() {
build_ubuntu_cpu_openblas
}
+build_ubuntu_base_cpu() {
+ echo "skip"
+}
+
+build_ubuntu_base_gpu() {
+ echo "skip"
+}
+
build_ubuntu_cpu_openblas() {
set -ex
export CC="ccache gcc"
@@ -995,6 +1003,7 @@ broken_link_checker() {
./tests/nightly/broken_link_checker_test/broken_link_checker.sh
}
+
##############################################################
# MAIN
#
diff --git a/ci/requirements.txt b/ci/requirements.txt
new file mode 100644
index 00000000000..bdb9670965e
--- /dev/null
+++ b/ci/requirements.txt
@@ -0,0 +1 @@
+docker
diff --git a/ci/test_docker_cache.py b/ci/test/test_docker_cache.py
old mode 100644
new mode 100755
similarity index 98%
rename from ci/test_docker_cache.py
rename to ci/test/test_docker_cache.py
index 358d54985ac..1e76126a04b
--- a/ci/test_docker_cache.py
+++ b/ci/test/test_docker_cache.py
@@ -30,11 +30,14 @@
import sys
from unittest.mock import MagicMock
-sys.path.append(os.path.dirname(__file__))
+print('omg')
+parentdir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+print(parentdir)
+sys.path.append(parentdir)
import docker_cache
import build as build_util
-DOCKERFILE_DIR = 'docker'
+DOCKERFILE_DIR = 'dockerfiles'
DOCKER_REGISTRY_NAME = 'test_registry'
DOCKER_REGISTRY_PORT = 5000
DOCKER_REGISTRY_PATH = 'localhost:{}'.format(DOCKER_REGISTRY_PORT)
@@ -250,6 +253,3 @@ def _assert_docker_build(lambda_func, expected_cache_hit_count: int, expected_ca
format(expected_cache_hit_count, output.count('Using cache'), output)
-if __name__ == '__main__':
- import nose
- nose.main()
diff --git a/ci/util.py b/ci/util.py
index 22631f30435..16594bd9c69 100644
--- a/ci/util.py
+++ b/ci/util.py
@@ -37,7 +37,9 @@ def remember_cwd():
Restore current directory when exiting context
'''
curdir = os.getcwd()
- try: yield
- finally: os.chdir(curdir)
+ try:
+ yield
+ finally:
+ os.chdir(curdir)
diff --git a/docs/build_version_doc/setup_docs_ubuntu.sh b/docs/build_version_doc/setup_docs_ubuntu.sh
index 22b2fe2c2f7..ffc834bd62d 100755
--- a/docs/build_version_doc/setup_docs_ubuntu.sh
+++ b/docs/build_version_doc/setup_docs_ubuntu.sh
@@ -110,10 +110,10 @@ sudo apt-get update && sudo apt-get install -y \
wget -nv https://bootstrap.pypa.io/get-pip.py
echo "Installing for Python 3..."
sudo python3 get-pip.py
-pip3 install --user -r ../../ci/docker/install/docs_requirements
+pip3 install --user -r ../../ci/dockerfiles/install/docs_requirements.txt
echo "Installing for Python 2..."
sudo python2 get-pip.py
-pip2 install --user -r ../../ci/docker/install/docs_requirements
+pip2 install --user -r ../../ci/dockerfiles/install/docs_requirements.txt
cd ../../
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services