You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by GitBox <gi...@apache.org> on 2018/08/15 10:57:55 UTC
[GitHub] marcoabreu closed pull request #12161: [WIP] A solution to prevent
zombie containers locally and in CI
marcoabreu closed pull request #12161: [WIP] A solution to prevent zombie containers locally and in CI
URL: https://github.com/apache/incubator-mxnet/pull/12161
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/ci/README.md b/ci/README.md
index 548e9cb9b04..7338d65aecc 100644
--- a/ci/README.md
+++ b/ci/README.md
@@ -1,5 +1,6 @@
# Containerized build & test utilities
+
This folder contains scripts and dockerfiles used to build and test MXNet using
Docker containers
diff --git a/ci/build.py b/ci/build.py
index 0a1ad4cf575..0f849f45996 100755
--- a/ci/build.py
+++ b/ci/build.py
@@ -23,7 +23,7 @@
"""
__author__ = 'Marco de Abreu, Kellen Sunderland, Anton Chernov, Pedro Larroy'
-__version__ = '0.1'
+__version__ = '0.8'
import argparse
import glob
@@ -37,11 +37,54 @@
import platform
from copy import deepcopy
from itertools import chain
-from subprocess import call, check_call
+from subprocess import call, check_call, check_output
from typing import *
from util import *
-
-CCACHE_MAXSIZE = '500G'
+import docker
+import docker.models
+import docker.errors
+import signal
+import atexit
+import pprint as pp
+
+
+class Cleanup:
+ def __init__(self):
+ self.containers = set()
+ self.docker_stop_timeout = 3
+
+ def add_container(self, container: docker.models.containers.Container):
+ assert isinstance(container, docker.models.containers.Container)
+ self.containers.add(container)
+
+ def remove_container(self, container: docker.models.containers.Container):
+ assert isinstance(container, docker.models.containers.Container)
+ self.containers.remove(container)
+
+ def _cleanup_containers(self):
+ if self.containers:
+ logging.warning("Cleaning up containers")
+ else:
+ return
+ docker_client = docker.from_env()
+ try:
+ stop_timeout = int(os.environ.get("DOCKER_STOP_TIMEOUT", self.docker_stop_timeout))
+ except Exception as e:
+ stop_timeout = 3
+ for container in self.containers:
+ try:
+ container.stop(timeout=stop_timeout)
+ logging.info("☠: stopped container %s", trim_container_id(container.id))
+ container.remove()
+ logging.info("🚽: removed container %s", trim_container_id(container.id))
+ except Exception as e:
+ logging.exception(e)
+ #pass
+ self.containers.clear()
+
+ def __call__(self):
+ """Perform cleanup"""
+ self._cleanup_containers()
@@ -77,14 +120,30 @@ def f_retry(*args, **kwargs):
mtries -= 1
mdelay *= backoff
return f(*args, **kwargs)
+
return f_retry # true decorator
+
return decorated_retry
+
def under_ci() -> bool:
""":return: True if we run in Jenkins."""
return 'JOB_NAME' in os.environ
-def get_platforms(path: Optional[str] = "docker"):
+
+def git_cleanup() -> None:
+ """Clean repo and subrepos, update subrepos"""
+ logging.info("cleaning up repository")
+ with remember_cwd():
+ os.chdir(get_mxnet_root())
+ check_call(['git', 'clean', '-ffdx'])
+ check_call(['git', 'submodule', 'foreach', '--recursive', 'git', 'clean', '-ffdx'])
+ check_call(['git', 'submodule', 'update', '--recursive', '--init'])
+
+def get_dockerfiles_path():
+ return "dockerfiles"
+
+def get_platforms(path: Optional[str] = get_dockerfiles_path()):
"""Get a list of architectures given our dockerfiles"""
dockerfiles = glob.glob(os.path.join(path, "Dockerfile.build.*"))
dockerfiles = list(filter(lambda x: x[-1] != '~', dockerfiles))
@@ -94,10 +153,11 @@ def get_platforms(path: Optional[str] = "docker"):
def get_docker_tag(platform: str, registry: str) -> str:
+ """:return: docker tag to be used for the container"""
return "{0}/build.{1}".format(registry, platform)
-def get_dockerfile(platform: str, path="docker") -> str:
+def get_dockerfile(platform: str, path=get_dockerfiles_path()) -> str:
return os.path.join(path, "Dockerfile.build.{0}".format(platform))
@@ -121,7 +181,7 @@ def build_docker(platform: str, docker_binary: str, registry: str, num_retries:
# container match permissions of the local user. Same for the group.
#
# These variables are used in the docker files to create user and group with these ids.
- # see: docker/install/ubuntu_adduser.sh
+ # see: dockerfiles/install/ubuntu_adduser.sh
#
# cache-from is needed so we use the cached images tagged from the remote via
# docker pull see: docker_cache.load_docker_cache
@@ -134,7 +194,7 @@ def build_docker(platform: str, docker_binary: str, registry: str, num_retries:
"--build-arg", "GROUP_ID={}".format(os.getgid()),
"--cache-from", tag,
"-t", tag,
- "docker"]
+ get_dockerfiles_path()]
@retry(subprocess.CalledProcessError, tries=num_retries)
def run_cmd():
@@ -164,6 +224,7 @@ def _get_local_image_id(docker_binary, docker_tag):
def buildir() -> str:
return os.path.join(get_mxnet_root(), "build")
+
def default_ccache_dir() -> str:
# Share ccache across containers
if 'CCACHE_DIR' in os.environ:
@@ -180,63 +241,134 @@ def default_ccache_dir() -> str:
return ccache_dir
return os.path.join(tempfile.gettempdir(), "ci_ccache")
+def trim_container_id(cid):
+ return cid[:12]
def container_run(platform: str,
docker_binary: str,
docker_registry: str,
shared_memory_size: str,
- local_ccache_dir: str,
command: List[str],
+ local_ccache_dir: str,
+ cleanup: Cleanup,
dry_run: bool = False,
- interactive: bool = False) -> str:
+ interactive: bool = False) -> int:
+ CONTAINER_WAIT_S = 10
+ #
+ # Environment setup
+ #
+ environment = {
+ 'CCACHE_MAXSIZE': '20G',
+ 'CCACHE_TEMPDIR': '/tmp/ccache', # temp dir should be local and not shared
+ 'CCACHE_DIR': '/work/ccache', # this path is inside the container as /work/ccache is mounted
+ 'CCACHE_LOGFILE': '/tmp/ccache.log', # a container-scoped log, useful for ccache verification.
+ }
+ # These variables are passed to the container to the process tree killer can find runaway process inside the container
+ # https://wiki.jenkins.io/display/JENKINS/ProcessTreeKiller
+ # https://github.com/jenkinsci/jenkins/blob/578d6bacb33a5e99f149de504c80275796f0b231/core/src/main/java/hudson/model/Run.java#L2393
+ #
+ JENKINS_ENV_VARS = ['BUILD_NUMBER', 'BUILD_ID', 'BUILD_TAG']
+ environment.update({k: os.environ[k] for k in JENKINS_ENV_VARS if k in os.environ})
+ environment.update({k: os.environ[k] for k in ['CCACHE_MAXSIZE'] if k in os.environ})
+
tag = get_docker_tag(platform=platform, registry=docker_registry)
mx_root = get_mxnet_root()
local_build_folder = buildir()
# We need to create it first, otherwise it will be created by the docker daemon with root only permissions
os.makedirs(local_build_folder, exist_ok=True)
os.makedirs(local_ccache_dir, exist_ok=True)
- logging.info("Using ccache directory: %s", local_ccache_dir)
- runlist = [docker_binary, 'run', '--rm', '-t',
+ local_ccache_dir_sz = check_output(["du", "-hs", local_ccache_dir]).decode('utf-8').split('\t')[0]
+ logging.info("Using ccache directory: %s (size: %s)", local_ccache_dir, local_ccache_dir_sz)
+ docker_client = docker.from_env()
+ # Equivalent command
+ docker_cmd_list = [docker_binary, 'run',
+ '--rm',
'--shm-size={}'.format(shared_memory_size),
'-v', "{}:/work/mxnet".format(mx_root), # mount mxnet root
'-v', "{}:/work/build".format(local_build_folder), # mount mxnet/build for storing build artifacts
'-v', "{}:/work/ccache".format(local_ccache_dir),
'-u', '{}:{}'.format(os.getuid(), os.getgid()),
- '-e', 'CCACHE_MAXSIZE={}'.format(CCACHE_MAXSIZE),
+ '-e', 'CCACHE_MAXSIZE=15G',
'-e', 'CCACHE_TEMPDIR=/tmp/ccache', # temp dir should be local and not shared
'-e', "CCACHE_DIR=/work/ccache", # this path is inside the container as /work/ccache is mounted
'-e', "CCACHE_LOGFILE=/tmp/ccache.log", # a container-scoped log, useful for ccache verification.
+ '-ti',
tag]
- runlist.extend(command)
- cmd = ' \\\n\t'.join(runlist)
- ret = 0
- if not dry_run and not interactive:
- logging.info("Running %s in container %s", command, tag)
- logging.info("Executing:\n%s\n", cmd)
- ret = call(runlist)
-
- docker_run_cmd = ' '.join(runlist)
- if not dry_run and interactive:
- into_cmd = deepcopy(runlist)
- # -ti can't be after the tag, as is interpreted as a command so hook it up after the -u argument
- idx = into_cmd.index('-u') + 2
- into_cmd[idx:idx] = ['-ti']
- cmd = ' \\\n\t'.join(into_cmd)
- logging.info("Executing:\n%s\n", cmd)
- docker_run_cmd = ' '.join(into_cmd)
- ret = call(into_cmd)
-
- if not dry_run and not interactive and ret != 0:
- logging.error("Running of command in container failed (%s):\n%s\n", ret, cmd)
- logging.error("You can get into the container by adding the -i option to this script")
- raise subprocess.CalledProcessError(ret, cmd)
-
- return docker_run_cmd
+ docker_cmd_list.extend(command)
+ docker_cmd = ' \\\n\t'.join(docker_cmd_list)
+ logging.info("Running %s in container %s", command, tag)
+ logging.info("Executing the equivalent of:\n%s\n", docker_cmd)
+ ret = 0 # return code of the command inside docker
+ if not dry_run:
+
+
+ #############################
+ #
+ signal.pthread_sigmask(signal.SIG_BLOCK, {signal.SIGINT, signal.SIGTERM})
+ container = docker_client.containers.run(
+ tag,
+ detach=True,
+ command=command,
+ #auto_remove=True,
+ shm_size=shared_memory_size,
+ user='{}:{}'.format(os.getuid(), os.getgid()),
+ volumes={
+ mx_root:
+ {'bind': '/work/mxnet', 'mode': 'rw'},
+ local_build_folder:
+ {'bind': '/work/build', 'mode': 'rw'},
+ local_ccache_dir:
+ {'bind': '/work/ccache', 'mode': 'rw'},
+ },
+ environment=environment)
+ # Race condition:
+ # If the previous call is interrupted then it's possible that the container is not cleaned up
+ # We avoid by masking the signals temporarily
+ cleanup.add_container(container)
+ signal.pthread_sigmask(signal.SIG_UNBLOCK, {signal.SIGINT, signal.SIGTERM})
+ #
+ #############################
+
+ stream = container.logs(stream=True)
+ for chunk in stream:
+ sys.stdout.buffer.write(chunk)
+ sys.stdout.buffer.flush()
+ stream.close()
+ try:
+ logging.info("Waiting for status of container %s", trim_container_id(container.id))
+ wait_result = container.wait(timeout=CONTAINER_WAIT_S)
+ logging.info("Container exit status: %s", wait_result)
+ ret = wait_result.get('StatusCode', 200)
+ except Exception as e:
+ logging.exception(e)
+ ret = 150
+
+ # Stop
+ try:
+ logging.info("Stopping container: %s", trim_container_id(container.id))
+ container.stop()
+ except Exception as e:
+ logging.exception(e)
+ ret = 151
+
+ # Remove
+ try:
+ logging.info("removing container: %s", trim_container_id(container.id))
+ container.remove()
+ except Exception as e:
+ logging.exception(e)
+ ret = 152
+ cleanup.remove_container(container)
+ containers = docker_client.containers.list()
+ if containers:
+ logging.info("Other running containers: %s", [trim_container_id(x.id) for x in containers])
+ return ret
def list_platforms() -> str:
print("\nSupported platforms:\n{}".format('\n'.join(get_platforms())))
+
def load_docker_cache(tag, docker_registry) -> None:
if docker_registry:
try:
@@ -248,7 +380,10 @@ def load_docker_cache(tag, docker_registry) -> None:
else:
logging.info('Distributed docker cache disabled')
+
def main() -> int:
+ print("Environment: ")
+ pp.pprint(dict(os.environ))
# We need to be in the same directory than the script so the commands in the dockerfiles work as
# expected. But the script can be invoked from a different path
base = os.path.split(os.path.realpath(__file__))[0]
@@ -319,6 +454,7 @@ def script_name() -> str:
type=str)
args = parser.parse_args()
+
def use_cache():
return args.cache or under_ci()
@@ -326,6 +462,19 @@ def use_cache():
docker_binary = get_docker_binary(args.nvidiadocker)
shared_memory_size = args.shared_memory_size
+ # Cleanup on signals and exit
+ cleanup = Cleanup()
+ def signal_handler(signum, _):
+ signal.pthread_sigmask(signal.SIG_BLOCK, {signum})
+ logging.warning("Signal %d received, cleaning up...", signum)
+ cleanup()
+ logging.warning("done. Exiting with error.")
+ sys.exit(1)
+
+ atexit.register(cleanup)
+ signal.signal(signal.SIGTERM, signal_handler)
+ signal.signal(signal.SIGINT, signal_handler)
+
if args.list:
list_platforms()
elif args.platform:
@@ -338,26 +487,32 @@ def use_cache():
logging.warning("Container was just built. Exiting due to build-only.")
return 0
+ ret = 0
if command:
- container_run(platform=platform, docker_binary=docker_binary, shared_memory_size=shared_memory_size,
+ ret = container_run(platform=platform, docker_binary=docker_binary, shared_memory_size=shared_memory_size,
command=command, docker_registry=args.docker_registry,
- local_ccache_dir=args.ccache_dir, interactive=args.interactive)
+ local_ccache_dir=args.ccache_dir, interactive=args.interactive, cleanup=cleanup)
elif args.print_docker_run:
- print(container_run(platform=platform, docker_binary=docker_binary, shared_memory_size=shared_memory_size,
- command=[], dry_run=True, docker_registry=args.docker_registry, local_ccache_dir=args.ccache_dir))
+ ret = container_run(platform=platform, docker_binary=docker_binary, shared_memory_size=shared_memory_size,
+ command=[], dry_run=True, docker_registry=args.docker_registry,
+ local_ccache_dir=args.ccache_dir)
+ command=[]
elif args.interactive:
- container_run(platform=platform, docker_binary=docker_binary, shared_memory_size=shared_memory_size,
+ ret = container_run(platform=platform, docker_binary=docker_binary, shared_memory_size=shared_memory_size,
command=command, docker_registry=args.docker_registry,
- local_ccache_dir=args.ccache_dir, interactive=args.interactive)
-
+ local_ccache_dir=args.ccache_dir, interactive=args.interactive, cleanup=cleanup)
else:
# With no commands, execute a build function for the target platform
assert not args.interactive, "when running with -i must provide a command"
- cmd = ["/work/mxnet/ci/docker/runtime_functions.sh", "build_{}".format(platform)]
- logging.info("No command specified, trying default build: %s", ' '.join(cmd))
- container_run(platform=platform, docker_binary=docker_binary, shared_memory_size=shared_memory_size,
- command=cmd, docker_registry=args.docker_registry,
- local_ccache_dir=args.ccache_dir)
+ command = ["/work/mxnet/ci/dockerfiles/runtime_functions.sh", "build_{}".format(platform)]
+ logging.info("No command specified, trying default build: %s", ' '.join(command))
+ ret = container_run(platform=platform, docker_binary=docker_binary, shared_memory_size=shared_memory_size,
+ command=command, docker_registry=args.docker_registry,
+ local_ccache_dir=args.ccache_dir, cleanup=cleanup)
+
+ if ret != 0:
+ logging.critical("Execution of %s failed with status: %d", command, ret)
+ return(ret)
elif args.all:
platforms = get_platforms()
@@ -370,12 +525,18 @@ def use_cache():
build_docker(platform, docker_binary, args.docker_registry, num_retries=args.docker_build_retries)
if args.build_only:
continue
- build_platform = "build_{}".format(platform)
- cmd = ["/work/mxnet/ci/docker/runtime_functions.sh", build_platform]
+ git_cleanup()
shutil.rmtree(buildir(), ignore_errors=True)
+ build_platform = "build_{}".format(platform)
+ plat_buildir = os.path.abspath(os.path.join(get_mxnet_root(), '..',
+ "mxnet_{}".format(build_platform)))
+ if os.path.exists(plat_buildir):
+ logging.warning("{} already exists, skipping".format(plat_buildir))
+ continue
+ command = ["/work/mxnet/ci/dockerfiles/runtime_functions.sh", build_platform]
+ # BUILD in docker
container_run(platform=platform, docker_binary=docker_binary, shared_memory_size=shared_memory_size,
- command=cmd, docker_registry=args.docker_registry, local_ccache_dir=args.ccache_dir)
- plat_buildir = os.path.join(get_mxnet_root(), build_platform)
+ command=command, docker_registry=args.docker_registry, local_ccache_dir=args.ccache_dir)
shutil.move(buildir(), plat_buildir)
logging.info("Built files left in: %s", plat_buildir)
@@ -388,7 +549,7 @@ def use_cache():
./build.py -p armv7
Will build a docker container with cross compilation tools and build MXNet for armv7 by
- running: ci/docker/runtime_functions.sh build_armv7 inside the container.
+ running: ci/dockerfiles/runtime_functions.sh build_armv7 inside the container.
./build.py -p armv7 ls
@@ -404,7 +565,8 @@ def use_cache():
./build.py -a
- Builds for all platforms and leaves artifacts in build_<platform>
+ Builds for all platforms and leaves artifacts in build_<platform>. **WARNING** it performs git
+ cleanup of the repo.
""")
diff --git a/ci/docker/Dockerfile.build.android_armv7 b/ci/dockerfiles/Dockerfile.build.android_armv7
similarity index 100%
rename from ci/docker/Dockerfile.build.android_armv7
rename to ci/dockerfiles/Dockerfile.build.android_armv7
diff --git a/ci/docker/Dockerfile.build.android_armv8 b/ci/dockerfiles/Dockerfile.build.android_armv8
similarity index 100%
rename from ci/docker/Dockerfile.build.android_armv8
rename to ci/dockerfiles/Dockerfile.build.android_armv8
diff --git a/ci/docker/Dockerfile.build.armv6 b/ci/dockerfiles/Dockerfile.build.armv6
similarity index 100%
rename from ci/docker/Dockerfile.build.armv6
rename to ci/dockerfiles/Dockerfile.build.armv6
diff --git a/ci/docker/Dockerfile.build.armv7 b/ci/dockerfiles/Dockerfile.build.armv7
similarity index 100%
rename from ci/docker/Dockerfile.build.armv7
rename to ci/dockerfiles/Dockerfile.build.armv7
diff --git a/ci/docker/Dockerfile.build.armv8 b/ci/dockerfiles/Dockerfile.build.armv8
similarity index 100%
rename from ci/docker/Dockerfile.build.armv8
rename to ci/dockerfiles/Dockerfile.build.armv8
diff --git a/ci/docker/Dockerfile.build.centos7_cpu b/ci/dockerfiles/Dockerfile.build.centos7_cpu
similarity index 100%
rename from ci/docker/Dockerfile.build.centos7_cpu
rename to ci/dockerfiles/Dockerfile.build.centos7_cpu
diff --git a/ci/docker/Dockerfile.build.centos7_gpu b/ci/dockerfiles/Dockerfile.build.centos7_gpu
similarity index 100%
rename from ci/docker/Dockerfile.build.centos7_gpu
rename to ci/dockerfiles/Dockerfile.build.centos7_gpu
diff --git a/ci/docker/Dockerfile.build.jetson b/ci/dockerfiles/Dockerfile.build.jetson
similarity index 100%
rename from ci/docker/Dockerfile.build.jetson
rename to ci/dockerfiles/Dockerfile.build.jetson
diff --git a/ci/docker/Dockerfile.build.ubuntu_base_cpu b/ci/dockerfiles/Dockerfile.build.ubuntu_base_cpu
similarity index 100%
rename from ci/docker/Dockerfile.build.ubuntu_base_cpu
rename to ci/dockerfiles/Dockerfile.build.ubuntu_base_cpu
diff --git a/ci/docker/Dockerfile.build.ubuntu_base_gpu b/ci/dockerfiles/Dockerfile.build.ubuntu_base_gpu
similarity index 100%
rename from ci/docker/Dockerfile.build.ubuntu_base_gpu
rename to ci/dockerfiles/Dockerfile.build.ubuntu_base_gpu
diff --git a/ci/docker/Dockerfile.build.ubuntu_blc b/ci/dockerfiles/Dockerfile.build.ubuntu_blc
similarity index 100%
rename from ci/docker/Dockerfile.build.ubuntu_blc
rename to ci/dockerfiles/Dockerfile.build.ubuntu_blc
diff --git a/ci/docker/Dockerfile.build.ubuntu_build_cuda b/ci/dockerfiles/Dockerfile.build.ubuntu_build_cuda
similarity index 100%
rename from ci/docker/Dockerfile.build.ubuntu_build_cuda
rename to ci/dockerfiles/Dockerfile.build.ubuntu_build_cuda
diff --git a/ci/docker/Dockerfile.build.ubuntu_cpu b/ci/dockerfiles/Dockerfile.build.ubuntu_cpu
similarity index 100%
rename from ci/docker/Dockerfile.build.ubuntu_cpu
rename to ci/dockerfiles/Dockerfile.build.ubuntu_cpu
diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu b/ci/dockerfiles/Dockerfile.build.ubuntu_gpu
similarity index 100%
rename from ci/docker/Dockerfile.build.ubuntu_gpu
rename to ci/dockerfiles/Dockerfile.build.ubuntu_gpu
diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu_tensorrt b/ci/dockerfiles/Dockerfile.build.ubuntu_gpu_tensorrt
similarity index 100%
rename from ci/docker/Dockerfile.build.ubuntu_gpu_tensorrt
rename to ci/dockerfiles/Dockerfile.build.ubuntu_gpu_tensorrt
diff --git a/ci/docker/Dockerfile.build.ubuntu_nightly_cpu b/ci/dockerfiles/Dockerfile.build.ubuntu_nightly_cpu
similarity index 100%
rename from ci/docker/Dockerfile.build.ubuntu_nightly_cpu
rename to ci/dockerfiles/Dockerfile.build.ubuntu_nightly_cpu
diff --git a/ci/docker/Dockerfile.build.ubuntu_nightly_gpu b/ci/dockerfiles/Dockerfile.build.ubuntu_nightly_gpu
similarity index 100%
rename from ci/docker/Dockerfile.build.ubuntu_nightly_gpu
rename to ci/dockerfiles/Dockerfile.build.ubuntu_nightly_gpu
diff --git a/ci/docker/Dockerfile.build.ubuntu_rat b/ci/dockerfiles/Dockerfile.build.ubuntu_rat
similarity index 100%
rename from ci/docker/Dockerfile.build.ubuntu_rat
rename to ci/dockerfiles/Dockerfile.build.ubuntu_rat
diff --git a/ci/docker/install/android_arm64_openblas.sh b/ci/dockerfiles/install/android_arm64_openblas.sh
similarity index 100%
rename from ci/docker/install/android_arm64_openblas.sh
rename to ci/dockerfiles/install/android_arm64_openblas.sh
diff --git a/ci/docker/install/android_armv7_openblas.sh b/ci/dockerfiles/install/android_armv7_openblas.sh
similarity index 100%
rename from ci/docker/install/android_armv7_openblas.sh
rename to ci/dockerfiles/install/android_armv7_openblas.sh
diff --git a/ci/docker/install/android_ndk.sh b/ci/dockerfiles/install/android_ndk.sh
similarity index 100%
rename from ci/docker/install/android_ndk.sh
rename to ci/dockerfiles/install/android_ndk.sh
diff --git a/ci/docker/install/arm64_openblas.sh b/ci/dockerfiles/install/arm64_openblas.sh
similarity index 100%
rename from ci/docker/install/arm64_openblas.sh
rename to ci/dockerfiles/install/arm64_openblas.sh
diff --git a/ci/docker/install/arm_openblas.sh b/ci/dockerfiles/install/arm_openblas.sh
similarity index 100%
rename from ci/docker/install/arm_openblas.sh
rename to ci/dockerfiles/install/arm_openblas.sh
diff --git a/ci/docker/install/centos7_adduser.sh b/ci/dockerfiles/install/centos7_adduser.sh
similarity index 100%
rename from ci/docker/install/centos7_adduser.sh
rename to ci/dockerfiles/install/centos7_adduser.sh
diff --git a/ci/docker/install/centos7_ccache.sh b/ci/dockerfiles/install/centos7_ccache.sh
similarity index 100%
rename from ci/docker/install/centos7_ccache.sh
rename to ci/dockerfiles/install/centos7_ccache.sh
diff --git a/ci/docker/install/centos7_core.sh b/ci/dockerfiles/install/centos7_core.sh
similarity index 100%
rename from ci/docker/install/centos7_core.sh
rename to ci/dockerfiles/install/centos7_core.sh
diff --git a/ci/docker/install/centos7_python.sh b/ci/dockerfiles/install/centos7_python.sh
similarity index 100%
rename from ci/docker/install/centos7_python.sh
rename to ci/dockerfiles/install/centos7_python.sh
diff --git a/ci/docker/install/deb_ubuntu_ccache.sh b/ci/dockerfiles/install/deb_ubuntu_ccache.sh
similarity index 100%
rename from ci/docker/install/deb_ubuntu_ccache.sh
rename to ci/dockerfiles/install/deb_ubuntu_ccache.sh
diff --git a/ci/docker/install/export_gpg_keys.sh b/ci/dockerfiles/install/export_gpg_keys.sh
similarity index 100%
rename from ci/docker/install/export_gpg_keys.sh
rename to ci/dockerfiles/install/export_gpg_keys.sh
diff --git a/ci/docker/install/r.gpg b/ci/dockerfiles/install/r.gpg
similarity index 100%
rename from ci/docker/install/r.gpg
rename to ci/dockerfiles/install/r.gpg
diff --git a/ci/docker/install/sbt.gpg b/ci/dockerfiles/install/sbt.gpg
similarity index 100%
rename from ci/docker/install/sbt.gpg
rename to ci/dockerfiles/install/sbt.gpg
diff --git a/ci/docker/install/tensorrt.sh b/ci/dockerfiles/install/tensorrt.sh
similarity index 100%
rename from ci/docker/install/tensorrt.sh
rename to ci/dockerfiles/install/tensorrt.sh
diff --git a/ci/docker/install/ubuntu_adduser.sh b/ci/dockerfiles/install/ubuntu_adduser.sh
similarity index 100%
rename from ci/docker/install/ubuntu_adduser.sh
rename to ci/dockerfiles/install/ubuntu_adduser.sh
diff --git a/ci/docker/install/ubuntu_arm.sh b/ci/dockerfiles/install/ubuntu_arm.sh
similarity index 100%
rename from ci/docker/install/ubuntu_arm.sh
rename to ci/dockerfiles/install/ubuntu_arm.sh
diff --git a/ci/docker/install/ubuntu_caffe.sh b/ci/dockerfiles/install/ubuntu_caffe.sh
similarity index 100%
rename from ci/docker/install/ubuntu_caffe.sh
rename to ci/dockerfiles/install/ubuntu_caffe.sh
diff --git a/ci/docker/install/ubuntu_clang.sh b/ci/dockerfiles/install/ubuntu_clang.sh
similarity index 100%
rename from ci/docker/install/ubuntu_clang.sh
rename to ci/dockerfiles/install/ubuntu_clang.sh
diff --git a/ci/docker/install/ubuntu_clojure.sh b/ci/dockerfiles/install/ubuntu_clojure.sh
similarity index 100%
rename from ci/docker/install/ubuntu_clojure.sh
rename to ci/dockerfiles/install/ubuntu_clojure.sh
diff --git a/ci/docker/install/ubuntu_core.sh b/ci/dockerfiles/install/ubuntu_core.sh
similarity index 100%
rename from ci/docker/install/ubuntu_core.sh
rename to ci/dockerfiles/install/ubuntu_core.sh
diff --git a/ci/docker/install/ubuntu_docs.sh b/ci/dockerfiles/install/ubuntu_docs.sh
similarity index 100%
rename from ci/docker/install/ubuntu_docs.sh
rename to ci/dockerfiles/install/ubuntu_docs.sh
diff --git a/ci/docker/install/ubuntu_emscripten.sh b/ci/dockerfiles/install/ubuntu_emscripten.sh
similarity index 100%
rename from ci/docker/install/ubuntu_emscripten.sh
rename to ci/dockerfiles/install/ubuntu_emscripten.sh
diff --git a/ci/docker/install/ubuntu_llvm.sh b/ci/dockerfiles/install/ubuntu_llvm.sh
similarity index 100%
rename from ci/docker/install/ubuntu_llvm.sh
rename to ci/dockerfiles/install/ubuntu_llvm.sh
diff --git a/ci/docker/install/ubuntu_mklml.sh b/ci/dockerfiles/install/ubuntu_mklml.sh
similarity index 100%
rename from ci/docker/install/ubuntu_mklml.sh
rename to ci/dockerfiles/install/ubuntu_mklml.sh
diff --git a/ci/docker/install/ubuntu_nightly_tests.sh b/ci/dockerfiles/install/ubuntu_nightly_tests.sh
similarity index 100%
rename from ci/docker/install/ubuntu_nightly_tests.sh
rename to ci/dockerfiles/install/ubuntu_nightly_tests.sh
diff --git a/ci/docker/install/ubuntu_npm_blc.sh b/ci/dockerfiles/install/ubuntu_npm_blc.sh
similarity index 100%
rename from ci/docker/install/ubuntu_npm_blc.sh
rename to ci/dockerfiles/install/ubuntu_npm_blc.sh
diff --git a/ci/docker/install/ubuntu_nvidia.sh b/ci/dockerfiles/install/ubuntu_nvidia.sh
similarity index 100%
rename from ci/docker/install/ubuntu_nvidia.sh
rename to ci/dockerfiles/install/ubuntu_nvidia.sh
diff --git a/ci/docker/install/ubuntu_onnx.sh b/ci/dockerfiles/install/ubuntu_onnx.sh
similarity index 100%
rename from ci/docker/install/ubuntu_onnx.sh
rename to ci/dockerfiles/install/ubuntu_onnx.sh
diff --git a/ci/docker/install/ubuntu_perl.sh b/ci/dockerfiles/install/ubuntu_perl.sh
similarity index 100%
rename from ci/docker/install/ubuntu_perl.sh
rename to ci/dockerfiles/install/ubuntu_perl.sh
diff --git a/ci/docker/install/ubuntu_python2.sh b/ci/dockerfiles/install/ubuntu_python2.sh
similarity index 100%
rename from ci/docker/install/ubuntu_python2.sh
rename to ci/dockerfiles/install/ubuntu_python2.sh
diff --git a/ci/docker/install/ubuntu_python3.sh b/ci/dockerfiles/install/ubuntu_python3.sh
similarity index 100%
rename from ci/docker/install/ubuntu_python3.sh
rename to ci/dockerfiles/install/ubuntu_python3.sh
diff --git a/ci/docker/install/ubuntu_r.sh b/ci/dockerfiles/install/ubuntu_r.sh
similarity index 100%
rename from ci/docker/install/ubuntu_r.sh
rename to ci/dockerfiles/install/ubuntu_r.sh
diff --git a/ci/docker/install/ubuntu_rat.sh b/ci/dockerfiles/install/ubuntu_rat.sh
similarity index 100%
rename from ci/docker/install/ubuntu_rat.sh
rename to ci/dockerfiles/install/ubuntu_rat.sh
diff --git a/ci/docker/install/ubuntu_runas_sudo.sh b/ci/dockerfiles/install/ubuntu_runas_sudo.sh
similarity index 100%
rename from ci/docker/install/ubuntu_runas_sudo.sh
rename to ci/dockerfiles/install/ubuntu_runas_sudo.sh
diff --git a/ci/docker/install/ubuntu_scala.sh b/ci/dockerfiles/install/ubuntu_scala.sh
similarity index 100%
rename from ci/docker/install/ubuntu_scala.sh
rename to ci/dockerfiles/install/ubuntu_scala.sh
diff --git a/ci/docker/install/ubuntu_tutorials.sh b/ci/dockerfiles/install/ubuntu_tutorials.sh
similarity index 100%
rename from ci/docker/install/ubuntu_tutorials.sh
rename to ci/dockerfiles/install/ubuntu_tutorials.sh
diff --git a/ci/docker/install/ubuntu_tvm.sh b/ci/dockerfiles/install/ubuntu_tvm.sh
similarity index 100%
rename from ci/docker/install/ubuntu_tvm.sh
rename to ci/dockerfiles/install/ubuntu_tvm.sh
diff --git a/ci/docker/runtime_functions.sh b/ci/dockerfiles/runtime_functions.sh
similarity index 99%
rename from ci/docker/runtime_functions.sh
rename to ci/dockerfiles/runtime_functions.sh
index 24f844fdfc5..fc83fb16bd9 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/dockerfiles/runtime_functions.sh
@@ -305,6 +305,14 @@ build_ubuntu_cpu() {
build_ubuntu_cpu_openblas
}
+build_ubuntu_base_cpu() {
+ echo "skip"
+}
+
+build_ubuntu_base_gpu() {
+ echo "skip"
+}
+
build_ubuntu_cpu_openblas() {
set -ex
export CC="ccache gcc"
diff --git a/ci/requirements.txt b/ci/requirements.txt
new file mode 100644
index 00000000000..bdb9670965e
--- /dev/null
+++ b/ci/requirements.txt
@@ -0,0 +1 @@
+docker
diff --git a/ci/util.py b/ci/util.py
index 22631f30435..16594bd9c69 100644
--- a/ci/util.py
+++ b/ci/util.py
@@ -37,7 +37,9 @@ def remember_cwd():
Restore current directory when exiting context
'''
curdir = os.getcwd()
- try: yield
- finally: os.chdir(curdir)
+ try:
+ yield
+ finally:
+ os.chdir(curdir)
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services