You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by ma...@apache.org on 2018/05/16 12:31:28 UTC
[incubator-mxnet] branch master updated: [MXNET-416] Add docker
cache (#10917)
This is an automated email from the ASF dual-hosted git repository.
marcoabreu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
The following commit(s) were added to refs/heads/master by this push:
new 8ebc050 [MXNET-416] Add docker cache (#10917)
8ebc050 is described below
commit 8ebc050e270498844247af8196489d0d3cdfc300
Author: Marco de Abreu <ma...@users.noreply.github.com>
AuthorDate: Wed May 16 14:31:14 2018 +0200
[MXNET-416] Add docker cache (#10917)
* Initial work for distributed docker cache
* Fix Android ARMv7 dockerfile and improve docker cache script
* add docker image id
* Add s3 upload
* Add s3 upload
* Add cache download
* Extend script to build all dockerfiles
* Add loading of docker cache before build
* Switch to multiprocessing
* Add jenkins job
* Improve Jenkinsfile
* Reorganize Jenkinsfile
* Preserve entire image chain
* Fix bugs in cache generation
* Address review comments
* Address review comments
* Remove unused Dockerfiles (Amazon Linux)
* Address review comments
---
Jenkinsfile | 98 +++++----
ci/Jenkinsfile_docker_cache | 81 ++++++++
ci/__init__.py | 0
ci/build.py | 51 ++++-
ci/docker/Dockerfile.build.amzn_linux_cpu | 44 ----
ci/docker/Dockerfile.build.android_armv7 | 11 +-
ci/docker/Dockerfile.build.centos7_cpu | 4 +-
ci/docker/Dockerfile.build.centos7_gpu | 4 +-
ci/docker/Dockerfile.build.ubuntu_build_cuda | 3 +-
ci/docker/Dockerfile.build.ubuntu_cpu | 4 +-
ci/docker/Dockerfile.build.ubuntu_gpu | 4 +-
ci/docker/install/amzn_linux_core.sh | 45 -----
ci/docker/install/amzn_linux_julia.sh | 29 ---
ci/docker/install/amzn_linux_library.sh | 26 ---
ci/docker/install/amzn_linux_maven.sh | 27 ---
ci/docker/install/amzn_linux_openblas.sh | 29 ---
ci/docker/install/amzn_linux_opencv.sh | 33 ---
ci/docker/install/amzn_linux_python2.sh | 36 ----
ci/docker/install/amzn_linux_python3.sh | 44 ----
ci/docker/install/amzn_linux_testdeps.sh | 27 ---
ci/docker_cache.py | 292 +++++++++++++++++++++++++++
ci/docker_cache_requirements | 8 +
22 files changed, 493 insertions(+), 407 deletions(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index f18d7a9..4d8dfb7 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -92,24 +92,34 @@ echo ${libs} | sed -e 's/,/ /g' | xargs md5sum
"""
}
+def docker_run(platform, function_name, use_nvidia, shared_mem = '500m') {
+ def command = "ci/build.py --download-docker-cache --docker-cache-bucket ${env.DOCKER_CACHE_BUCKET} %USE_NVIDIA% --platform %PLATFORM% --shm-size %SHARED_MEM% /work/runtime_functions.sh %FUNCTION_NAME%"
+ command = command.replaceAll('%USE_NVIDIA%', use_nvidia ? '--nvidiadocker' : '')
+ command = command.replaceAll('%PLATFORM%', platform)
+ command = command.replaceAll('%FUNCTION_NAME%', function_name)
+ command = command.replaceAll('%SHARED_MEM%', shared_mem)
+
+ sh command
+}
+
// Python unittest for CPU
// Python 2
def python2_ut(docker_container_name) {
timeout(time: max_time, unit: 'MINUTES') {
- sh "ci/build.py --platform ${docker_container_name} /work/runtime_functions.sh unittest_ubuntu_python2_cpu"
+ docker_run(docker_container_name, 'unittest_ubuntu_python2_cpu', false)
}
}
// Python 3
def python3_ut(docker_container_name) {
timeout(time: max_time, unit: 'MINUTES') {
- sh "ci/build.py --platform ${docker_container_name} /work/runtime_functions.sh unittest_ubuntu_python3_cpu"
+ docker_run(docker_container_name, 'unittest_ubuntu_python3_cpu', false)
}
}
def python3_ut_mkldnn(docker_container_name) {
timeout(time: max_time, unit: 'MINUTES') {
- sh "ci/build.py --platform ${docker_container_name} /work/runtime_functions.sh unittest_ubuntu_python3_cpu_mkldnn"
+ docker_run(docker_container_name, 'unittest_ubuntu_python3_cpu_mkldnn', false)
}
}
@@ -118,14 +128,14 @@ def python3_ut_mkldnn(docker_container_name) {
// Python 2
def python2_gpu_ut(docker_container_name) {
timeout(time: max_time, unit: 'MINUTES') {
- sh "ci/build.py --nvidiadocker --platform ${docker_container_name} /work/runtime_functions.sh unittest_ubuntu_python2_gpu"
+ docker_run(docker_container_name, 'unittest_ubuntu_python2_gpu', true)
}
}
// Python 3
def python3_gpu_ut(docker_container_name) {
timeout(time: max_time, unit: 'MINUTES') {
- sh "ci/build.py --nvidiadocker --platform ${docker_container_name} /work/runtime_functions.sh unittest_ubuntu_python3_gpu"
+ docker_run(docker_container_name, 'unittest_ubuntu_python3_gpu', true)
}
}
@@ -134,7 +144,7 @@ try {
node('mxnetlinux-cpu') {
ws('workspace/sanity') {
init_git()
- sh "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh sanity_check"
+ docker_run('ubuntu_cpu', 'sanity_check', false)
}
}
}
@@ -145,7 +155,7 @@ try {
ws('workspace/build-centos7-cpu') {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
- sh "ci/build.py --platform centos7_cpu /work/runtime_functions.sh build_centos7_cpu"
+ docker_run('centos7_cpu', 'build_centos7_cpu', false)
pack_lib('centos7_cpu')
}
}
@@ -156,7 +166,7 @@ try {
ws('workspace/build-centos7-mkldnn') {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
- sh "ci/build.py --platform centos7_cpu /work/runtime_functions.sh build_centos7_mkldnn"
+ docker_run('centos7_cpu', 'build_centos7_mkldnn', false)
pack_lib('centos7_mkldnn')
}
}
@@ -167,7 +177,7 @@ try {
ws('workspace/build-centos7-gpu') {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
- sh "ci/build.py --platform centos7_gpu /work/runtime_functions.sh build_centos7_gpu"
+ docker_run('centos7_gpu', 'build_centos7_gpu', false)
pack_lib('centos7_gpu')
}
}
@@ -178,7 +188,7 @@ try {
ws('workspace/build-cpu-openblas') {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
- sh "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh build_ubuntu_cpu_openblas"
+ docker_run('ubuntu_cpu', 'build_ubuntu_cpu_openblas', false)
pack_lib('cpu', mx_dist_lib)
}
}
@@ -189,7 +199,7 @@ try {
ws('workspace/build-cpu-clang39') {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
- sh "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh build_ubuntu_cpu_clang39"
+ docker_run('ubuntu_cpu', 'build_ubuntu_cpu_clang39', false)
}
}
}
@@ -199,7 +209,7 @@ try {
ws('workspace/build-cpu-clang50') {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
- sh "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh build_ubuntu_cpu_clang50"
+ docker_run('ubuntu_cpu', 'build_ubuntu_cpu_clang50', false)
}
}
}
@@ -209,7 +219,7 @@ try {
ws('workspace/build-cpu-mkldnn-clang39') {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
- sh "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh build_ubuntu_cpu_clang39_mkldnn"
+ docker_run('ubuntu_cpu', 'build_ubuntu_cpu_clang39_mkldnn', false)
pack_lib('mkldnn_cpu_clang3', mx_mkldnn_lib)
}
}
@@ -220,7 +230,7 @@ try {
ws('workspace/build-cpu-mkldnn-clang50') {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
- sh "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh build_ubuntu_cpu_clang50_mkldnn"
+ docker_run('ubuntu_cpu', 'build_ubuntu_cpu_clang50_mkldnn', false)
pack_lib('mkldnn_cpu_clang5', mx_mkldnn_lib)
}
}
@@ -231,7 +241,7 @@ try {
ws('workspace/build-mkldnn-cpu') {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
- sh "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh build_ubuntu_cpu_mkldnn"
+ docker_run('ubuntu_cpu', 'build_ubuntu_cpu_mkldnn', false)
pack_lib('mkldnn_cpu', mx_mkldnn_lib)
}
}
@@ -242,7 +252,7 @@ try {
ws('workspace/build-mkldnn-gpu') {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
- sh "ci/build.py --platform ubuntu_build_cuda /work/runtime_functions.sh build_ubuntu_gpu_mkldnn"
+ docker_run('ubuntu_build_cuda', 'build_ubuntu_gpu_mkldnn', false)
pack_lib('mkldnn_gpu', mx_mkldnn_lib)
}
}
@@ -253,7 +263,7 @@ try {
ws('workspace/build-gpu') {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
- sh "ci/build.py --platform ubuntu_build_cuda /work/runtime_functions.sh build_ubuntu_gpu_cuda91_cudnn7"
+ docker_run('ubuntu_build_cuda', 'build_ubuntu_gpu_cuda91_cudnn7', false)
pack_lib('gpu', mx_dist_lib)
stash includes: 'build/cpp-package/example/lenet', name: 'cpp_lenet'
stash includes: 'build/cpp-package/example/alexnet', name: 'cpp_alexnet'
@@ -274,7 +284,7 @@ try {
ws('workspace/amalgamationmin') {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
- sh "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh build_ubuntu_amalgamation_min"
+ docker_run('ubuntu_cpu', 'build_ubuntu_amalgamation_min', false)
}
}
}
@@ -284,7 +294,7 @@ try {
ws('workspace/amalgamation') {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
- sh "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh build_ubuntu_amalgamation"
+ docker_run('ubuntu_cpu', 'build_ubuntu_amalgamation', false)
}
}
}
@@ -295,7 +305,7 @@ try {
ws('workspace/build-cmake-mkldnn-gpu') {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
- sh "ci/build.py --platform ubuntu_gpu /work/runtime_functions.sh build_ubuntu_gpu_cmake_mkldnn" //build_cuda
+ docker_run('ubuntu_gpu', 'build_ubuntu_gpu_cmake_mkldnn', false)
pack_lib('cmake_mkldnn_gpu', mx_cmake_mkldnn_lib)
}
}
@@ -306,7 +316,7 @@ try {
ws('workspace/build-cmake-gpu') {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
- sh "ci/build.py --platform ubuntu_gpu /work/runtime_functions.sh build_ubuntu_gpu_cmake" //build_cuda
+ docker_run('ubuntu_gpu', 'build_ubuntu_gpu_cmake', false)
pack_lib('cmake_gpu', mx_cmake_lib)
}
}
@@ -424,7 +434,7 @@ try {
ws('workspace/build-jetson-armv8') {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
- sh "ci/build.py --platform jetson /work/runtime_functions.sh build_jetson"
+ docker_run('jetson', 'build_jetson', false)
}
}
}
@@ -434,7 +444,7 @@ try {
ws('workspace/build-raspberry-armv7') {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
- sh "ci/build.py --platform armv7 /work/runtime_functions.sh build_armv7"
+ docker_run('armv7', 'build_armv7', false)
}
}
}
@@ -444,7 +454,7 @@ try {
ws('workspace/build-raspberry-armv6') {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
- sh "ci/build.py --platform armv6 /work/runtime_functions.sh build_armv6"
+ docker_run('armv6', 'build_armv6', false)
}
}
}
@@ -494,7 +504,7 @@ try {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
unpack_lib('gpu', mx_lib)
- sh "ci/build.py --nvidiadocker --platform ubuntu_gpu /work/runtime_functions.sh unittest_ubuntu_python2_quantization_gpu"
+ docker_run('ubuntu_gpu', 'unittest_ubuntu_python2_quantization_gpu', true)
}
}
}
@@ -505,7 +515,7 @@ try {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
unpack_lib('gpu', mx_lib)
- sh "ci/build.py --nvidiadocker --platform ubuntu_gpu /work/runtime_functions.sh unittest_ubuntu_python3_quantization_gpu"
+ docker_run('ubuntu_gpu', 'unittest_ubuntu_python3_quantization_gpu', true)
}
}
}
@@ -552,7 +562,7 @@ try {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
unpack_lib('centos7_cpu')
- sh "ci/build.py --platform centos7_cpu /work/runtime_functions.sh unittest_centos7_cpu"
+ docker_run('centos7_cpu', 'unittest_centos7_cpu', false)
}
}
}
@@ -563,7 +573,7 @@ try {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
unpack_lib('centos7_gpu')
- sh "ci/build.py --nvidiadocker --platform centos7_gpu /work/runtime_functions.sh unittest_centos7_gpu"
+ docker_run('centos7_gpu', 'unittest_centos7_gpu', true)
}
}
}
@@ -574,7 +584,7 @@ try {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
unpack_lib('cpu', mx_dist_lib)
- sh "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh unittest_ubuntu_cpu_scala"
+ docker_run('ubuntu_cpu', 'unittest_ubuntu_cpu_scala', false)
}
}
}
@@ -585,7 +595,7 @@ try {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
unpack_lib('gpu', mx_dist_lib)
- sh "ci/build.py --nvidiadocker --platform ubuntu_gpu /work/runtime_functions.sh unittest_ubuntu_gpu_scala"
+ docker_run('ubuntu_gpu', 'unittest_ubuntu_gpu_scala', true)
}
}
}
@@ -596,7 +606,7 @@ try {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
unpack_lib('cpu')
- sh "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh unittest_ubuntu_cpugpu_perl"
+ docker_run('ubuntu_cpu', 'unittest_ubuntu_cpugpu_perl', false)
}
}
}
@@ -607,7 +617,7 @@ try {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
unpack_lib('gpu')
- sh "ci/build.py --nvidiadocker --platform ubuntu_gpu /work/runtime_functions.sh unittest_ubuntu_cpugpu_perl"
+ docker_run('ubuntu_gpu', 'unittest_ubuntu_cpugpu_perl', true)
}
}
}
@@ -618,7 +628,7 @@ try {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
unpack_lib('cmake_gpu', mx_cmake_lib)
- sh "ci/build.py --nvidiadocker --platform ubuntu_gpu /work/runtime_functions.sh unittest_ubuntu_gpu_cpp"
+ docker_run('ubuntu_gpu', 'unittest_ubuntu_gpu_cpp', true)
}
}
}
@@ -629,7 +639,7 @@ try {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
unpack_lib('cmake_mkldnn_gpu', mx_cmake_mkldnn_lib)
- sh "ci/build.py --nvidiadocker --platform ubuntu_gpu /work/runtime_functions.sh unittest_ubuntu_gpu_cpp"
+ docker_run('ubuntu_gpu', 'unittest_ubuntu_gpu_cpp', true)
}
}
}
@@ -640,7 +650,7 @@ try {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
unpack_lib('cpu')
- sh "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh unittest_ubuntu_cpu_R"
+ docker_run('ubuntu_cpu', 'unittest_ubuntu_cpu_R', false)
}
}
}
@@ -651,7 +661,7 @@ try {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
unpack_lib('gpu')
- sh "ci/build.py --nvidiadocker --platform ubuntu_gpu /work/runtime_functions.sh unittest_ubuntu_gpu_R"
+ docker_run('ubuntu_gpu', 'unittest_ubuntu_gpu_R', true)
}
}
}
@@ -756,7 +766,7 @@ try {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
unpack_lib('cpu')
- sh "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh integrationtest_ubuntu_cpu_onnx"
+ docker_run('ubuntu_cpu', 'integrationtest_ubuntu_cpu_onnx', false)
}
}
}
@@ -767,7 +777,7 @@ try {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
unpack_lib('gpu')
- sh "ci/build.py --nvidiadocker --platform ubuntu_gpu /work/runtime_functions.sh integrationtest_ubuntu_gpu_python"
+ docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_python', true)
}
}
}
@@ -778,7 +788,7 @@ try {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
unpack_lib('gpu')
- sh "ci/build.py --nvidiadocker --platform ubuntu_gpu /work/runtime_functions.sh integrationtest_ubuntu_gpu_caffe"
+ docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_caffe', true)
}
}
}
@@ -799,7 +809,7 @@ try {
unstash 'cpp_mlp_gpu'
unstash 'cpp_test_score'
unstash 'cpp_test_optimizer'
- sh "ci/build.py --nvidiadocker --platform ubuntu_gpu /work/runtime_functions.sh integrationtest_ubuntu_gpu_cpp_package"
+ docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_cpp_package', true)
}
}
}
@@ -810,7 +820,7 @@ try {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
unpack_lib('gpu')
- sh "ci/build.py --nvidiadocker --platform ubuntu_gpu /work/runtime_functions.sh integrationtest_ubuntu_gpu_dist_kvstore"
+ docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_dist_kvstore', true)
}
}
}
@@ -821,7 +831,7 @@ try {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
unpack_lib('gpu')
- sh "ci/build.py --shm-size=3g --nvidiadocker --platform ubuntu_gpu /work/runtime_functions.sh tutorialtest_ubuntu_python2_gpu"
+ docker_run('ubuntu_gpu', 'tutorialtest_ubuntu_python2_gpu', true, '3g')
}
}
}
@@ -832,7 +842,7 @@ try {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
unpack_lib('gpu')
- sh "ci/build.py --shm-size=3g --nvidiadocker --platform ubuntu_gpu /work/runtime_functions.sh tutorialtest_ubuntu_python3_gpu"
+ docker_run('ubuntu_gpu', 'tutorialtest_ubuntu_python3_gpu', true, '3g')
}
}
}
@@ -844,7 +854,7 @@ try {
ws('workspace/docs') {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
- sh "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh deploy_docs"
+ docker_run('ubuntu_cpu', 'deploy_docs', false)
sh "tests/ci_build/deploy/ci_deploy_doc.sh ${env.BRANCH_NAME} ${env.BUILD_NUMBER}"
}
}
diff --git a/ci/Jenkinsfile_docker_cache b/ci/Jenkinsfile_docker_cache
new file mode 100644
index 0000000..8a0428b
--- /dev/null
+++ b/ci/Jenkinsfile_docker_cache
@@ -0,0 +1,81 @@
+// -*- mode: groovy -*-
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Jenkins pipeline to generate the centralized docker cache
+// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/
+
+// timeout in minutes
+total_timeout = 120
+git_timeout = 15
+// assign any caught errors here
+err = null
+
+// initialize source codes
+def init_git() {
+ deleteDir()
+ retry(5) {
+ try {
+ // Make sure wait long enough for api.github.com request quota. Important: Don't increase the amount of
+ // retries as this will increase the amount of requests and worsen the throttling
+ timeout(time: git_timeout, unit: 'MINUTES') {
+ checkout scm
+ sh 'git submodule update --init --recursive'
+ sh 'git clean -x -d -f'
+ }
+ } catch (exc) {
+ deleteDir()
+ error "Failed to fetch source codes with ${exc}"
+ sleep 2
+ }
+ }
+}
+
+
+try {
+ stage("Docker cache build & publish") {
+ node('mxnetlinux-cpu') {
+ ws('workspace/docker_cache') {
+ timeout(time: total_timeout, unit: 'MINUTES') {
+ init_git()
+ sh "ci/docker_cache.py --docker-cache-bucket ${env.DOCKER_CACHE_BUCKET}"
+ }
+ }
+ }
+ }
+
+ // set build status to success at the end
+ currentBuild.result = "SUCCESS"
+} catch (caughtError) {
+ node("mxnetlinux-cpu") {
+ sh "echo caught ${caughtError}"
+ err = caughtError
+ currentBuild.result = "FAILURE"
+ }
+} finally {
+ node("mxnetlinux-cpu") {
+ // Only send email if master failed
+ if (currentBuild.result == "FAILURE" && env.BRANCH_NAME == "master") {
+ emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}'
+ }
+ // Remember to rethrow so the build is marked as failing
+ if (err) {
+ throw err
+ }
+ }
+}
diff --git a/ci/__init__.py b/ci/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/ci/build.py b/ci/build.py
index 6d8d014..6b1d23e 100755
--- a/ci/build.py
+++ b/ci/build.py
@@ -33,6 +33,7 @@ import re
import shutil
import subprocess
import sys
+import docker_cache
from copy import deepcopy
from itertools import chain
from subprocess import call, check_call
@@ -61,17 +62,44 @@ def get_docker_binary(use_nvidia_docker: bool) -> str:
def build_docker(platform: str, docker_binary: str) -> None:
- """Build a container for the given platform"""
+ """
+ Build a container for the given platform
+ :param platform: Platform
+ :param docker_binary: docker binary to use (docker/nvidia-docker)
+ :return: Id of the top level image
+ """
+
tag = get_docker_tag(platform)
logging.info("Building container tagged '%s' with %s", tag, docker_binary)
cmd = [docker_binary, "build",
"-f", get_dockerfile(platform),
+ "--rm=false", # Keep intermediary layers to prime the build cache
"--build-arg", "USER_ID={}".format(os.getuid()),
+ "--cache-from", tag,
"-t", tag,
"docker"]
logging.info("Running command: '%s'", ' '.join(cmd))
check_call(cmd)
+ # Get image id by reading the tag. It's guaranteed (except race condition) that the tag exists. Otherwise, the
+ # check_call would have failed
+ image_id = _get_local_image_id(docker_binary=docker_binary, docker_tag=tag)
+ if not image_id:
+ raise FileNotFoundError('Unable to find docker image id matching with {}'.format(tag))
+ return image_id
+
+
+def _get_local_image_id(docker_binary, docker_tag):
+ """
+ Get the image id of the local docker layer with the passed tag
+ :param docker_tag: docker tag
+ :return: Image id as string or None if tag does not exist
+ """
+ cmd = [docker_binary, "images", "-q", docker_tag]
+ image_id_b = subprocess.check_output(cmd)
+ image_id = image_id_b.decode('utf-8').strip()
+ return image_id
+
def get_mxnet_root() -> str:
curpath = os.path.abspath(os.path.dirname(__file__))
@@ -123,6 +151,7 @@ def container_run(platform: str,
if not dry_run and ret != 0:
logging.error("Running of command in container failed (%s): %s", ret, cmd)
logging.error("You can try to get into the container by using the following command: %s", docker_run_cmd)
+
raise subprocess.CalledProcessError(ret, cmd)
return docker_run_cmd
@@ -131,7 +160,6 @@ def container_run(platform: str,
def list_platforms() -> str:
print("\nSupported platforms:\n{}".format('\n'.join(get_platforms())))
-
def main() -> int:
# We need to be in the same directory than the script so the commands in the dockerfiles work as
# expected. But the script can be invoked from a different path
@@ -180,6 +208,14 @@ def main() -> int:
help="go in a shell inside the container",
action='store_true')
+ parser.add_argument("--download-docker-cache",
+ help="Download the docker cache from our central repository instead of rebuilding locally",
+ action='store_true')
+
+ parser.add_argument("--docker-cache-bucket",
+ help="S3 docker cache bucket, e.g. mxnet-ci-docker-cache",
+ type=str)
+
parser.add_argument("command",
help="command to run in the container",
nargs='*', action='append', type=str)
@@ -194,12 +230,15 @@ def main() -> int:
list_platforms()
elif args.platform:
platform = args.platform
+ tag = get_docker_tag(platform)
+ if args.download_docker_cache:
+ logging.info('Docker cache download is enabled')
+ docker_cache.load_docker_cache(bucket_name=args.docker_cache_bucket, docker_tag=tag)
build_docker(platform, docker_binary)
if args.build_only:
- logging.warn("Container was just built. Exiting due to build-only.")
+ logging.warning("Container was just built. Exiting due to build-only.")
return 0
- tag = get_docker_tag(platform)
if command:
container_run(platform, docker_binary, shared_memory_size, command)
elif args.print_docker_run:
@@ -216,6 +255,10 @@ def main() -> int:
logging.info("Building for all architectures: {}".format(platforms))
logging.info("Artifacts will be produced in the build/ directory.")
for platform in platforms:
+ if args.download_docker_cache:
+ tag = get_docker_tag(platform)
+ logging.info('Docker cache download is enabled')
+ docker_cache.load_docker_cache(bucket_name=args.docker_cache_bucket, docker_tag=tag)
build_docker(platform, docker_binary)
if args.build_only:
continue
diff --git a/ci/docker/Dockerfile.build.amzn_linux_cpu b/ci/docker/Dockerfile.build.amzn_linux_cpu
deleted file mode 100755
index 7d6f223..0000000
--- a/ci/docker/Dockerfile.build.amzn_linux_cpu
+++ /dev/null
@@ -1,44 +0,0 @@
-# -*- mode: dockerfile -*-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-# Dockerfile to build and run MXNet for Amazon Linux on CPU
-
-FROM amazonlinux
-
-WORKDIR /work/deps
-COPY install/amzn_linux_core.sh /work/
-RUN /work/amzn_linux_core.sh
-COPY install/amzn_linux_opencv.sh /work/
-RUN /work/amzn_linux_opencv.sh
-COPY install/amzn_linux_openblas.sh /work/
-RUN /work/amzn_linux_openblas.sh
-COPY install/amzn_linux_python2.sh /work/
-RUN /work/amzn_linux_python2.sh
-COPY install/amzn_linux_python3.sh /work/
-RUN /work/amzn_linux_python3.sh
-COPY install/amzn_linux_testdeps.sh /work/
-RUN /work/amzn_linux_testdeps.sh
-COPY install/amzn_linux_julia.sh /work/
-RUN /work/amzn_linux_julia.sh
-COPY install/amzn_linux_maven.sh /work/
-RUN /work/amzn_linux_maven.sh
-COPY install/amzn_linux_library.sh /work/
-RUN /work/amzn_linux_library.sh
-WORKDIR /work/mxnet
-
-COPY runtime_functions.sh /work/
\ No newline at end of file
diff --git a/ci/docker/Dockerfile.build.android_armv7 b/ci/docker/Dockerfile.build.android_armv7
index 0074c1f..c22e000 100755
--- a/ci/docker/Dockerfile.build.android_armv7
+++ b/ci/docker/Dockerfile.build.android_armv7
@@ -84,13 +84,6 @@ ENV CC /usr/arm-linux-androideabi/bin/arm-linux-androideabi-clang
ENV CXX /usr/arm-linux-androideabi/bin/arm-linux-androideabi-clang++
ENV BUILD_OPTS "USE_BLAS=openblas USE_SSE=0 DMLC_LOG_STACK_TRACE=0 USE_OPENCV=0 USE_LAPACK=0"
-# Build MXNet
-ADD mxnet mxnet
-ADD arm.crosscompile.android.mk /work/mxnet/make/config.mk
-RUN cd mxnet && \
- make -j$(nproc) $BUILD_OPTS
+WORKDIR /work/mxnet
-WORKDIR /work/build/
-RUN cp /work/mxnet/lib/* .
-
-# TODO: Bring this into the new format
\ No newline at end of file
+COPY runtime_functions.sh /work/
diff --git a/ci/docker/Dockerfile.build.centos7_cpu b/ci/docker/Dockerfile.build.centos7_cpu
index a44d646..92314fa 100755
--- a/ci/docker/Dockerfile.build.centos7_cpu
+++ b/ci/docker/Dockerfile.build.centos7_cpu
@@ -20,8 +20,6 @@
FROM centos:7
-ARG USER_ID=0
-
WORKDIR /work/deps
COPY install/centos7_core.sh /work/
@@ -30,6 +28,8 @@ COPY install/centos7_python.sh /work/
RUN /work/centos7_python.sh
COPY install/ubuntu_mklml.sh /work/
RUN /work/ubuntu_mklml.sh
+
+ARG USER_ID=0
COPY install/centos7_adduser.sh /work/
RUN /work/centos7_adduser.sh
diff --git a/ci/docker/Dockerfile.build.centos7_gpu b/ci/docker/Dockerfile.build.centos7_gpu
index 4dcf5bf..2d28170 100755
--- a/ci/docker/Dockerfile.build.centos7_gpu
+++ b/ci/docker/Dockerfile.build.centos7_gpu
@@ -20,14 +20,14 @@
FROM nvidia/cuda:9.1-cudnn7-devel-centos7
-ARG USER_ID=0
-
WORKDIR /work/deps
COPY install/centos7_core.sh /work/
RUN /work/centos7_core.sh
COPY install/centos7_python.sh /work/
RUN /work/centos7_python.sh
+
+ARG USER_ID=0
COPY install/centos7_adduser.sh /work/
RUN /work/centos7_adduser.sh
diff --git a/ci/docker/Dockerfile.build.ubuntu_build_cuda b/ci/docker/Dockerfile.build.ubuntu_build_cuda
index 9156d6f..4d3c466 100755
--- a/ci/docker/Dockerfile.build.ubuntu_build_cuda
+++ b/ci/docker/Dockerfile.build.ubuntu_build_cuda
@@ -23,8 +23,6 @@
FROM nvidia/cuda:9.1-cudnn7-devel
-ARG USER_ID=0
-
WORKDIR /work/deps
COPY install/ubuntu_core.sh /work/
@@ -48,6 +46,7 @@ COPY install/ubuntu_nvidia.sh /work/
RUN /work/ubuntu_nvidia.sh
# Keep this at the end since this command is not cachable
+ARG USER_ID=0
COPY install/ubuntu_adduser.sh /work/
RUN /work/ubuntu_adduser.sh
diff --git a/ci/docker/Dockerfile.build.ubuntu_cpu b/ci/docker/Dockerfile.build.ubuntu_cpu
index f706f88..2dc7ef1 100755
--- a/ci/docker/Dockerfile.build.ubuntu_cpu
+++ b/ci/docker/Dockerfile.build.ubuntu_cpu
@@ -20,8 +20,6 @@
FROM ubuntu:16.04
-ARG USER_ID=0
-
WORKDIR /work/deps
COPY install/ubuntu_core.sh /work/
@@ -44,6 +42,8 @@ COPY install/ubuntu_onnx.sh /work/
RUN /work/ubuntu_onnx.sh
COPY install/ubuntu_docs.sh /work/
RUN /work/ubuntu_docs.sh
+
+ARG USER_ID=0
COPY install/ubuntu_adduser.sh /work/
RUN /work/ubuntu_adduser.sh
diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu b/ci/docker/Dockerfile.build.ubuntu_gpu
index 547f984..1097172 100755
--- a/ci/docker/Dockerfile.build.ubuntu_gpu
+++ b/ci/docker/Dockerfile.build.ubuntu_gpu
@@ -20,8 +20,6 @@
FROM nvidia/cuda:9.1-cudnn7-devel
-ARG USER_ID=0
-
WORKDIR /work/deps
COPY install/ubuntu_core.sh /work/
@@ -50,6 +48,8 @@ COPY install/ubuntu_docs.sh /work/
RUN /work/ubuntu_docs.sh
COPY install/ubuntu_tutorials.sh /work/
RUN /work/ubuntu_tutorials.sh
+
+ARG USER_ID=0
COPY install/ubuntu_adduser.sh /work/
RUN /work/ubuntu_adduser.sh
diff --git a/ci/docker/install/amzn_linux_core.sh b/ci/docker/install/amzn_linux_core.sh
deleted file mode 100755
index c13c969..0000000
--- a/ci/docker/install/amzn_linux_core.sh
+++ /dev/null
@@ -1,45 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# build and install are separated so changes to build don't invalidate
-# the whole docker cache for the image
-
-set -ex
-pushd .
-yum install -y git
-yum install -y wget
-yum install -y sudo
-yum install -y re2c
-yum groupinstall -y 'Development Tools'
-
-# Ninja
-git clone --recursive https://github.com/ninja-build/ninja.git
-cd ninja
-./configure.py --bootstrap
-cp ninja /usr/local/bin
-popd
-
-# CMake
-pushd .
-git clone --recursive https://github.com/Kitware/CMake.git --branch v3.10.2
-cd CMake
-./bootstrap
-make -j$(nproc)
-make install
-popd
\ No newline at end of file
diff --git a/ci/docker/install/amzn_linux_julia.sh b/ci/docker/install/amzn_linux_julia.sh
deleted file mode 100755
index bfaf3c4..0000000
--- a/ci/docker/install/amzn_linux_julia.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# build and install are separated so changes to build don't invalidate
-# the whole docker cache for the image
-
-set -ex
-wget -nv https://julialang.s3.amazonaws.com/bin/linux/x64/0.5/julia-0.5.0-linux-x86_64.tar.gz
-mv julia-0.5.0-linux-x86_64.tar.gz /tmp/
-tar xfvz /tmp/julia-0.5.0-linux-x86_64.tar.gz
-rm -f /tmp/julia-0.5.0-linux-x86_64.tar.gz
-# tar extracted in current directory
-ln -s -f ${PWD}/julia-3c9d75391c/bin/julia /usr/bin/julia
\ No newline at end of file
diff --git a/ci/docker/install/amzn_linux_library.sh b/ci/docker/install/amzn_linux_library.sh
deleted file mode 100755
index 0470895..0000000
--- a/ci/docker/install/amzn_linux_library.sh
+++ /dev/null
@@ -1,26 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# build and install are separated so changes to build don't invalidate
-# the whole docker cache for the image
-
-set -ex
-yum -y install graphviz
-pip install graphviz
-pip install opencv-python
\ No newline at end of file
diff --git a/ci/docker/install/amzn_linux_maven.sh b/ci/docker/install/amzn_linux_maven.sh
deleted file mode 100755
index 22875d0..0000000
--- a/ci/docker/install/amzn_linux_maven.sh
+++ /dev/null
@@ -1,27 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# build and install are separated so changes to build don't invalidate
-# the whole docker cache for the image
-
-set -ex
-wget -nv http://mirrors.ocf.berkeley.edu/apache/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz
-mv apache-maven-3.3.9-bin.tar.gz /tmp/
-tar xfvz /tmp/apache-maven-3.3.9-bin.tar.gz
-yum install -y java-1.8.0-openjdk-devel
\ No newline at end of file
diff --git a/ci/docker/install/amzn_linux_openblas.sh b/ci/docker/install/amzn_linux_openblas.sh
deleted file mode 100755
index 94088d6..0000000
--- a/ci/docker/install/amzn_linux_openblas.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# build and install are separated so changes to build don't invalidate
-# the whole docker cache for the image
-
-set -ex
-pushd .
-git clone https://github.com/xianyi/OpenBLAS
-cd OpenBLAS
-make FC=gfortran -j $(($(nproc) + 1))
-make PREFIX=/usr/local install
-popd
\ No newline at end of file
diff --git a/ci/docker/install/amzn_linux_opencv.sh b/ci/docker/install/amzn_linux_opencv.sh
deleted file mode 100755
index 956407e..0000000
--- a/ci/docker/install/amzn_linux_opencv.sh
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# build and install are separated so changes to build don't invalidate
-# the whole docker cache for the image
-
-set -ex
-pushd .
-yum install -y python27 python27-setuptools
-git clone https://github.com/opencv/opencv
-cd opencv
-mkdir -p build
-cd build
-cmake -DBUILD_opencv_gpu=OFF -DWITH_EIGEN=ON -DWITH_TBB=ON -DWITH_CUDA=OFF -DWITH_1394=OFF \
--DCMAKE_BUILD_TYPE=RELEASE -DCMAKE_INSTALL_PREFIX=/usr/local -GNinja ..
-ninja install
-popd
\ No newline at end of file
diff --git a/ci/docker/install/amzn_linux_python2.sh b/ci/docker/install/amzn_linux_python2.sh
deleted file mode 100755
index e099ad6..0000000
--- a/ci/docker/install/amzn_linux_python2.sh
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# build and install are separated so changes to build don't invalidate
-# the whole docker cache for the image
-
-set -ex
-yum groupinstall -y "Development Tools"
-yum install -y mlocate python27 python27-setuptools python27-tools python27-numpy python27-scipy python27-nose python27-matplotlib unzip
-ln -s -f /usr/bin/python2.7 /usr/bin/python2
-wget -nv https://bootstrap.pypa.io/get-pip.py
-python2 get-pip.py
-$(which easy_install-2.7) --upgrade pip
-if [ -f /usr/local/bin/pip ] && [ -f /usr/bin/pip ]; then
- mv /usr/bin/pip /usr/bin/pip.bak
- ln /usr/local/bin/pip /usr/bin/pip
-fi
-
-ln -s -f /usr/local/bin/pip /usr/bin/pip
-for i in ipython[all] jupyter pandas scikit-image h5py pandas sklearn sympy; do echo "${i}..."; pip install -U $i >/dev/null; done
diff --git a/ci/docker/install/amzn_linux_python3.sh b/ci/docker/install/amzn_linux_python3.sh
deleted file mode 100755
index 3f80d7d..0000000
--- a/ci/docker/install/amzn_linux_python3.sh
+++ /dev/null
@@ -1,44 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# build and install are separated so changes to build don't invalidate
-# the whole docker cache for the image
-
-set -ex
-pushd .
-wget -nv https://bootstrap.pypa.io/get-pip.py
-mkdir py3
-cd py3
-wget -nv https://www.python.org/ftp/python/3.5.2/Python-3.5.2.tgz
-tar -xvzf Python-3.5.2.tgz
-cd Python-3.5.2
-yum install -y zlib-devel openssl-devel sqlite-devel bzip2-devel gdbm-devel ncurses-devel xz-devel readline-devel
-./configure --prefix=/opt/ --with-zlib-dir=/usr/lib64
-make -j$(nproc)
-mkdir /opt/bin
-mkdir /opt/lib
-make install
-ln -s -f /opt/bin/python3 /usr/bin/python3
-cd ../..
-python3 get-pip.py
-ln -s -f /opt/bin/pip /usr/bin/pip3
-
-mkdir -p ~/.local/lib/python3.5/site-packages/
-pip3 install numpy
-popd
\ No newline at end of file
diff --git a/ci/docker/install/amzn_linux_testdeps.sh b/ci/docker/install/amzn_linux_testdeps.sh
deleted file mode 100755
index f5c49d9..0000000
--- a/ci/docker/install/amzn_linux_testdeps.sh
+++ /dev/null
@@ -1,27 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# build and install are separated so changes to build don't invalidate
-# the whole docker cache for the image
-
-set -ex
-pip install cpplint 'pylint==1.4.4' 'astroid==1.3.6'
-pip3 install nose
-ln -s -f /opt/bin/nosetests /usr/local/bin/nosetests3
-ln -s -f /opt/bin/nosetests-3.4 /usr/local/bin/nosetests-3.4
\ No newline at end of file
diff --git a/ci/docker_cache.py b/ci/docker_cache.py
new file mode 100755
index 0000000..7fdfbcf
--- /dev/null
+++ b/ci/docker_cache.py
@@ -0,0 +1,292 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Utility to handle distributed docker cache. This is done by keeping the entire image chain of a docker container
+on an S3 bucket. This utility allows cache creation and download. After execution, the cache will be in an identical
+state as if the container would have been built locally already.
+"""
+
+import os
+import logging
+import argparse
+import sys
+import boto3
+import tempfile
+import pprint
+import threading
+import build as build_util
+import botocore
+import subprocess
+from botocore.handlers import disable_signing
+from subprocess import call, check_call, CalledProcessError
+from joblib import Parallel, delayed
+
+S3_METADATA_IMAGE_ID_KEY = 'docker-image-id'
+LOG_PROGRESS_PERCENTAGE_THRESHOLD = 10
+
+cached_aws_session = None
+
+
+class ProgressPercentage(object):
+ def __init__(self, object_name, size):
+ self._object_name = object_name
+ self._size = size
+ self._seen_so_far = 0
+ self._last_percentage = 0
+ self._lock = threading.Lock()
+
+ def __call__(self, bytes_amount) -> None:
+ # To simplify we'll assume this is hooked up
+ # to a single filename.
+ with self._lock:
+ self._seen_so_far += bytes_amount
+ percentage = int((self._seen_so_far / self._size) * 100)
+ if (percentage - self._last_percentage) >= LOG_PROGRESS_PERCENTAGE_THRESHOLD:
+ self._last_percentage = percentage
+ logging.info('{}% of {}'.format(percentage, self._object_name))
+
+
+def build_save_containers(platforms, bucket) -> int:
+ """
+ Entry point to build and upload all built dockerimages in parallel
+ :param platforms: List of platforms
+ :param bucket: S3 bucket name
+ :return: 1 if error occurred, 0 otherwise
+ """
+ if len(platforms) == 0:
+ return 0
+
+ platform_results = Parallel(n_jobs=len(platforms), backend="multiprocessing")(
+ delayed(_build_save_container)(platform, bucket)
+ for platform in platforms)
+
+ is_error = False
+ for platform_result in platform_results:
+ if platform_result is not None:
+ logging.error('Failed to generate {}'.format(platform_result))
+ is_error = True
+
+ return 1 if is_error else 0
+
+
+def _build_save_container(platform, bucket) -> str:
+ """
+ Build image for passed platform and upload the cache to the specified S3 bucket
+ :param platform: Platform
+ :param bucket: Target s3 bucket
+ :return: Platform if failed, None otherwise
+ """
+ docker_tag = build_util.get_docker_tag(platform)
+
+ # Preload cache
+ # TODO: Allow to disable this in order to allow clean rebuilds
+ load_docker_cache(bucket_name=bucket, docker_tag=docker_tag)
+
+ # Start building
+ logging.debug('Building {} as {}'.format(platform, docker_tag))
+ try:
+ image_id = build_util.build_docker(docker_binary='docker', platform=platform)
+ logging.info('Built {} as {}'.format(docker_tag, image_id))
+
+ # Compile and upload tarfile
+ _compile_upload_cache_file(bucket_name=bucket, docker_tag=docker_tag, image_id=image_id)
+ return None
+ except Exception:
+ logging.exception('Unexpected exception during build of {}'.format(docker_tag))
+ return platform
+ # Error handling is done by returning the errorous platform name. This is necessary due to
+ # Parallel being unable to handle exceptions
+
+
+def _compile_upload_cache_file(bucket_name, docker_tag, image_id) -> None:
+ """
+ Upload the passed image by id, tag it with docker tag and upload to S3 bucket
+ :param bucket_name: S3 bucket name
+ :param docker_tag: Docker tag
+ :param image_id: Image id
+ :return: None
+ """
+ session = _get_aws_session()
+ s3_object = session.resource('s3').Object(bucket_name, docker_tag)
+
+ remote_image_id = _get_remote_image_id(s3_object)
+ if remote_image_id == image_id:
+ logging.info('{} ({}) for {} has not been updated - skipping'.format(docker_tag, image_id, docker_tag))
+ return
+ else:
+ logging.debug('Cached image {} differs from local {} for {}'.format(remote_image_id, image_id, docker_tag))
+
+ # Compile layers into tarfile
+ with tempfile.TemporaryDirectory() as temp_dir:
+ tar_file_path = _format_docker_cache_filepath(output_dir=temp_dir, docker_tag=docker_tag)
+ logging.debug('Writing layers of {} to {}'.format(docker_tag, tar_file_path))
+ history_cmd = ['docker', 'history', '-q', docker_tag]
+
+ image_ids_b = subprocess.check_output(history_cmd)
+ image_ids_str = image_ids_b.decode('utf-8').strip()
+ layer_ids = [id.strip() for id in image_ids_str.split('\n') if id != '<missing>']
+
+ # docker_tag is important to preserve the image name. Otherwise, the --cache-from feature will not be able to
+ # reference the loaded cache later on. The other layer ids are added to ensure all intermediary layers
+ # are preserved to allow resuming the cache at any point
+ cmd = ['docker', 'save', '-o', tar_file_path, docker_tag]
+ cmd.extend(layer_ids)
+ try:
+ check_call(cmd)
+ except CalledProcessError as e:
+ logging.error('Error during save of {} at {}. Command: {}'.
+ format(docker_tag, tar_file_path, pprint.pprint(cmd)))
+ return
+
+ # Upload file
+ logging.info('Uploading {} to S3'.format(docker_tag))
+ with open(tar_file_path, 'rb') as data:
+ s3_object.upload_fileobj(
+ Fileobj=data,
+ Callback=ProgressPercentage(object_name=docker_tag, size=os.path.getsize(tar_file_path)),
+ ExtraArgs={"Metadata": {S3_METADATA_IMAGE_ID_KEY: image_id}})
+ logging.info('Uploaded {} to S3'.format(docker_tag))
+
+
+def _get_remote_image_id(s3_object) -> str:
+ """
+ Get the image id of the docker cache which is represented by the S3 object
+ :param s3_object: S3 object
+ :return: Image id as string or None if object does not exist
+ """
+ try:
+ if S3_METADATA_IMAGE_ID_KEY in s3_object.metadata:
+ cached_image_id = s3_object.metadata[S3_METADATA_IMAGE_ID_KEY]
+ return cached_image_id
+ else:
+ logging.debug('No cached image available for {}'.format(s3_object.key))
+ except botocore.exceptions.ClientError as e:
+ if e.response['Error']['Code'] == "404":
+ logging.debug('{} does not exist in S3 yet'.format(s3_object.key))
+ else:
+ raise
+
+ return None
+
+
+def load_docker_cache(bucket_name, docker_tag) -> None:
+ """
+ Load the precompiled docker cache from the passed S3 bucket
+ :param bucket_name: S3 bucket name
+ :param docker_tag: Docker tag to load
+ :return: None
+ """
+ # Allow anonymous access
+ s3_resource = boto3.resource('s3')
+ s3_resource.meta.client.meta.events.register('choose-signer.s3.*', disable_signing)
+ s3_object = s3_resource.Object(bucket_name, docker_tag)
+
+ # Check if cache is still valid and exists
+ remote_image_id = _get_remote_image_id(s3_object)
+ if remote_image_id:
+ if _docker_layer_exists(remote_image_id):
+ logging.info('Local docker cache already present for {}'.format(docker_tag))
+ return
+ else:
+ logging.info('Local docker cache not present for {}'.format(docker_tag))
+
+ # Download using public S3 endpoint (without requiring credentials)
+ with tempfile.TemporaryDirectory() as temp_dir:
+ tar_file_path = os.path.join(temp_dir, 'layers.tar')
+ s3_object.download_file(
+ Filename=tar_file_path,
+ Callback=ProgressPercentage(object_name=docker_tag, size=s3_object.content_length))
+
+ # Load layers
+ cmd = ['docker', 'load', '-i', tar_file_path]
+ try:
+ check_call(cmd)
+ logging.info('Docker cache for {} loaded successfully'.format(docker_tag))
+ except CalledProcessError as e:
+ logging.error('Error during load of docker cache for {} at {}'.format(docker_tag, tar_file_path))
+ logging.exception(e)
+ return
+ else:
+ logging.info('No cached remote image of {} present'.format(docker_tag))
+
+
+def _docker_layer_exists(layer_id) -> bool:
+ """
+ Check if the docker cache contains the layer with the passed id
+ :param layer_id: layer id
+ :return: True if exists, False otherwise
+ """
+ cmd = ['docker', 'images', '-q']
+ image_ids_b = subprocess.check_output(cmd)
+ image_ids_str = image_ids_b.decode('utf-8').strip()
+ return layer_id in [id.strip() for id in image_ids_str.split('\n')]
+
+
+def _get_aws_session() -> boto3.Session: # pragma: no cover
+ """
+ Get the boto3 AWS session
+ :return: Session object
+ """
+ global cached_aws_session
+ if cached_aws_session:
+ return cached_aws_session
+
+ session = boto3.Session() # Uses IAM user credentials
+ cached_aws_session = session
+ return session
+
+
+def _format_docker_cache_filepath(output_dir, docker_tag) -> str:
+ return os.path.join(output_dir, docker_tag.replace('/', '_') + '.tar')
+
+
+def main() -> int:
+ # We need to be in the same directory than the script so the commands in the dockerfiles work as
+ # expected. But the script can be invoked from a different path
+ base = os.path.split(os.path.realpath(__file__))[0]
+ os.chdir(base)
+
+ logging.getLogger().setLevel(logging.DEBUG)
+ logging.getLogger('botocore').setLevel(logging.INFO)
+ logging.getLogger('boto3').setLevel(logging.INFO)
+ logging.getLogger('urllib3').setLevel(logging.INFO)
+ logging.getLogger('s3transfer').setLevel(logging.INFO)
+
+ def script_name() -> str:
+ return os.path.split(sys.argv[0])[1]
+
+ logging.basicConfig(format='{}: %(asctime)-15s %(message)s'.format(script_name()))
+
+ parser = argparse.ArgumentParser(description="Utility for preserving and loading Docker cache",epilog="")
+ parser.add_argument("--docker-cache-bucket",
+ help="S3 docker cache bucket, e.g. mxnet-ci-docker-cache",
+ type=str,
+ required=True)
+
+ args = parser.parse_args()
+
+ platforms = build_util.get_platforms()
+ _get_aws_session() # Init AWS credentials
+ return build_save_containers(platforms=platforms, bucket=args.docker_cache_bucket)
+
+
+if __name__ == '__main__':
+ sys.exit(main())
diff --git a/ci/docker_cache_requirements b/ci/docker_cache_requirements
new file mode 100644
index 0000000..47c16ff
--- /dev/null
+++ b/ci/docker_cache_requirements
@@ -0,0 +1,8 @@
+boto3==1.7.13
+botocore==1.10.13
+docutils==0.14
+jmespath==0.9.3
+joblib==0.11
+python-dateutil==2.7.2
+s3transfer==0.1.13
+six==1.11.0
--
To stop receiving notification emails like this one, please contact
marcoabreu@apache.org.