You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by zh...@apache.org on 2018/03/09 22:33:20 UTC
[incubator-mxnet] branch master updated: CI docker revamp;
Add Jetson, Raspberry and CentOS 7 build [MXNET-42][MXNET-43]
(#9995)
This is an automated email from the ASF dual-hosted git repository.
zhasheng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
The following commit(s) were added to refs/heads/master by this push:
new 85215b6 CI docker revamp; Add Jetson, Raspberry and CentOS 7 build [MXNET-42][MXNET-43] (#9995)
85215b6 is described below
commit 85215b6176ef3612d198f590268a6595b86565fb
Author: Marco de Abreu <ma...@users.noreply.github.com>
AuthorDate: Fri Mar 9 23:33:10 2018 +0100
CI docker revamp; Add Jetson, Raspberry and CentOS 7 build [MXNET-42][MXNET-43] (#9995)
* Start dockerfile revamp
* Fix lint
* Fix lint
* Fix lint
* Add python unit test
* Add new dependency
* Add user creation
* Fix file permission
* Determine USER_ID automatically
* REmove ENV command in dockerfile
* Remove python nose timer, improve useradd
* ENable nvidia docker
* Add remaining unittests
* Add CentOS 7 unittests
* Add integration tests
* Add TVM and LLVM to dockerfile
* Add ARMv7 and ARMv8 to build
* Fix amalgamation build
* Improvements and android_arm64 fixes, missing removing pthread
* Jetson fix (install unzip)
* Build jetson with make until issue with libomp.so is resolved #10011
* Fix Amalgamation builds
* Fix Jetson build by switching to cmake
* Assign CentOS gpu test to gpu instance
* Fix R builds
* Assign jobs to right docker containers
* Fix missing file permissions inside docker
* Create homedir on centos
* Enable lapack
* Fix Lapack on Cent OS 7
* Disable MXNET_MKLDNN_DEBUG
* Delete Dockerfiles
* Last general refinements before finish
* Remove docker_multiarch folder, superseeded by the new ci scripts
* Address review comments
* Fix Caffe Integrationstest
* Fix deploy stage
* Address review comments
* Address review comments
* Enable script to run on Mac
---
.mxnet_root | 0
CMakeLists.txt | 11 +-
Jenkinsfile | 326 ++++++--------
Makefile | 3 +-
ci/README.md | 50 +++
ci/build.py | 179 ++++++++
ci/docker/Dockerfile.build.amzn_linux_cpu | 44 ++
ci/docker/Dockerfile.build.android_arm64 | 66 +++
.../docker/Dockerfile.build.android_armv7 | 24 +-
.../docker/Dockerfile.build.arm64 | 26 +-
.../docker/Dockerfile.build.armv7 | 22 +-
.../docker/Dockerfile.build.centos7_cpu | 25 +-
.../docker/Dockerfile.build.centos7_gpu | 25 +-
.../docker}/Dockerfile.build.jetson | 74 ++--
ci/docker/Dockerfile.build.ubuntu_build_cuda | 57 +++
ci/docker/Dockerfile.build.ubuntu_cpu | 53 +++
ci/docker/Dockerfile.build.ubuntu_gpu | 55 +++
.../docker/install/amzn_linux_core.sh | 33 +-
.../docker/install/amzn_linux_julia.sh | 13 +-
.../docker/install/amzn_linux_library.sh | 11 +-
.../docker/install/amzn_linux_maven.sh | 13 +-
.../docker/install/amzn_linux_openblas.sh | 10 +-
.../docker/install/amzn_linux_opencv.sh | 19 +-
.../docker/install/amzn_linux_python2.sh | 15 +-
.../docker/install/amzn_linux_python3.sh | 18 +-
.../docker/install/amzn_linux_testdeps.sh | 11 +-
.../docker/install/android_arm64_ndk.sh | 24 +-
.../docker/install/android_arm64_openblas.sh | 19 +-
.../docker/install/arm64_openblas.sh | 22 +-
.../docker/install/centos7_adduser.sh | 25 +-
.../docker/install/centos7_core.sh | 28 +-
.../docker/install/centos7_python.sh | 22 +-
.../docker/install/ubuntu_adduser.sh | 25 +-
ci/docker/install/ubuntu_caffe.sh | 49 +++
.../docker/install/ubuntu_clang.sh | 19 +-
.../docker/install/ubuntu_core.sh | 27 +-
.../docker/install/ubuntu_docs.sh | 17 +-
.../docker/install/ubuntu_lint.sh | 12 +-
.../docker/install/ubuntu_llvm.sh | 4 +-
.../docker/install/ubuntu_mklml.sh | 13 +-
.../docker/install/ubuntu_nvidia.sh | 18 +-
.../docker/install/ubuntu_perl.sh | 10 +-
.../docker/install/ubuntu_python.sh | 16 +-
.../docker/install/ubuntu_r.sh | 12 +-
.../docker/install/ubuntu_scala.sh | 11 +-
.../docker/install/ubuntu_tvm.sh | 4 +-
ci/docker/runtime_functions.sh | 471 +++++++++++++++++++++
docker_multiarch/.gitignore | 2 -
docker_multiarch/Dockerfile.build.android.arm64 | 77 ----
docker_multiarch/Dockerfile.build.arm64 | 37 --
docker_multiarch/Dockerfile.build.armv6 | 38 --
docker_multiarch/Dockerfile.build.armv7 | 26 --
.../Dockerfile.build.cmake.ubuntu-17.04 | 37 --
.../Dockerfile.build.ubuntu-16.04-cuda_8.0_cudnn5 | 32 --
docker_multiarch/Dockerfile.build.ubuntu-17.04 | 30 --
.../Dockerfile.build.ubuntu-17.04.scala.docker | 17 -
docker_multiarch/Dockerfile.run.ubuntu-17.04.julia | 24 --
docker_multiarch/Dockerfile.run.ubuntu-17.04.perl | 23 -
.../Dockerfile.run.ubuntu-17.04.python | 17 -
docker_multiarch/Dockerfile.run.ubuntu-17.04.r | 36 --
docker_multiarch/Dockerfile.test.ubuntu-17.04 | 41 --
docker_multiarch/License.md | 20 -
docker_multiarch/README.md | 42 --
docker_multiarch/arm.crosscompile.android.mk | 161 -------
docker_multiarch/arm.crosscompile.mk | 161 -------
docker_multiarch/tool.py | 155 -------
python/mxnet/gluon/data/dataloader.py | 1 +
python/mxnet/image/detection.py | 4 +-
python/mxnet/libinfo.py | 12 +
tests/ci_build/Dockerfile.amzn_linux_cpu | 16 -
tests/ci_build/Dockerfile.build_cuda | 26 --
tests/ci_build/Dockerfile.caffe_gpu | 36 --
tests/ci_build/Dockerfile.cpu | 12 -
tests/ci_build/Dockerfile.cpu_clang | 21 -
tests/ci_build/Dockerfile.cpu_mklml | 18 -
tests/ci_build/Dockerfile.crosstool | 23 -
tests/ci_build/Dockerfile.doc | 15 -
tests/ci_build/Dockerfile.emscripten | 19 -
tests/ci_build/Dockerfile.gpu | 20 -
tests/ci_build/Dockerfile.gpu_mklml | 18 -
tests/ci_build/Dockerfile.lint | 5 -
tests/ci_build/Dockerfile.spell_checker | 6 -
tests/ci_build/Dockerfile.ubuntu1404_cuda75_cudnn5 | 40 --
tests/ci_build/README.md | 41 --
tests/ci_build/ci_build.sh | 188 --------
tests/ci_build/with_the_same_user | 47 --
tests/python/cpu/test_mkldnn.py | 58 ---
tools/caffe_converter/compare_layers.py | 2 +-
tools/caffe_converter/convert_mean.py | 2 +-
tools/caffe_converter/convert_model.py | 2 +-
90 files changed, 1570 insertions(+), 2069 deletions(-)
diff --git a/.mxnet_root b/.mxnet_root
new file mode 100644
index 0000000..e69de29
diff --git a/CMakeLists.txt b/CMakeLists.txt
index a076524..d596928 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -15,6 +15,7 @@ mxnet_option(USE_NCCL "Use NVidia NCCL with CUDA" OFF)
mxnet_option(USE_OPENCV "Build with OpenCV support" ON)
mxnet_option(USE_OPENMP "Build with Openmp support" ON)
mxnet_option(USE_CUDNN "Build with cudnn support" ON) # one could set CUDNN_ROOT for search path
+mxnet_option(USE_SSE "Build with x86 SSE instruction support" AUTO)
mxnet_option(USE_LAPACK "Build with lapack support" ON IF NOT MSVC)
mxnet_option(USE_MKL_IF_AVAILABLE "Use MKL if found" ON)
mxnet_option(USE_MKLML_MKL "Use MKLDNN variant of MKL (if MKL found)" ON IF USE_MKL_IF_AVAILABLE AND UNIX AND (NOT APPLE))
@@ -90,11 +91,17 @@ if(MSVC)
else(MSVC)
include(CheckCXXCompilerFlag)
if(USE_CXX14_IF_AVAILABLE)
- check_cxx_compiler_flag("-std=c++14" SUPPORT_CXX14)
+ check_cxx_compiler_flag("-std=c++14" SUPPORT_CXX14)
endif()
check_cxx_compiler_flag("-std=c++11" SUPPORT_CXX11)
check_cxx_compiler_flag("-std=c++0x" SUPPORT_CXX0X)
- check_cxx_compiler_flag("-msse2" SUPPORT_MSSE2)
+ # For cross compilation, we can't rely on the compiler which accepts the flag, but mshadow will
+ # add platform specific includes not available in other arches
+ if(USE_SSE STREQUAL "AUTO")
+ check_cxx_compiler_flag("-msse2" SUPPORT_MSSE2)
+ else()
+ set(SUPPORT_MSSE2 FALSE)
+ endif()
set(CMAKE_C_FLAGS "-Wall -Wno-unknown-pragmas -fPIC -Wno-sign-compare")
if ("${CMAKE_CXX_COMPILER_ID}" MATCHES ".*Clang$")
set(CMAKE_C_FLAGS "-Wno-braced-scalar-init")
diff --git a/Jenkinsfile b/Jenkinsfile
index d0295f0..2bac346 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -72,41 +72,6 @@ def init_git_win() {
}
}
-// Run make. First try to do an incremental build from a previous workspace in hope to
-// accelerate the compilation. If something wrong, clean the workspace and then
-// build from scratch.
-def make(docker_type, make_flag) {
- timeout(time: max_time, unit: 'MINUTES') {
- try {
- sh "${docker_run} ${docker_type} --dockerbinary docker make ${make_flag}"
- } catch (exc) {
- echo 'Incremental compilation failed with ${exc}. Fall back to build from scratch'
- sh "${docker_run} ${docker_type} --dockerbinary docker sudo make clean"
- sh "${docker_run} ${docker_type} --dockerbinary docker sudo make -C amalgamation/ clean"
- sh "${docker_run} ${docker_type} --dockerbinary docker make ${make_flag}"
- }
- }
-}
-
-// Run cmake. First try to do an incremental build from a previous workspace in hope to
-// accelerate the compilation. If something wrong, clean the workspace and then
-// build from scratch.
-def cmake(docker_type, cmake_defines, flags) {
- timeout(time: max_time, unit: 'MINUTES') {
- try {
- sh "${docker_run} ${docker_type} --dockerbinary docker mkdir -p build"
- sh "WORKDIR=/workspace/build ${docker_run} ${docker_type} --dockerbinary docker cmake ${cmake_defines} -G Ninja .."
- sh "WORKDIR=/workspace/build ${docker_run} ${docker_type} --dockerbinary docker ninja ${flags}"
- } catch (exc) {
- echo 'Incremental compilation failed with ${exc}. Fall back to build from scratch'
- sh "${docker_run} ${docker_type} --dockerbinary docker git clean -fdx"
- sh "${docker_run} ${docker_type} --dockerbinary docker mkdir -p build"
- sh "WORKDIR=/workspace/build ${docker_run} ${docker_type} --dockerbinary docker cmake ${cmake_defines} -G Ninja .."
- sh "WORKDIR=/workspace/build ${docker_run} ${docker_type} --dockerbinary docker ninja ${flags}"
- }
- }
-}
-
// pack libraries for later use
def pack_lib(name, libs=mx_lib) {
sh """
@@ -127,171 +92,95 @@ echo ${libs} | sed -e 's/,/ /g' | xargs md5sum
// Python unittest for CPU
// Python 2
-def python2_ut(docker_type) {
+def python2_ut(docker_container_name) {
timeout(time: max_time, unit: 'MINUTES') {
- sh "${docker_run} ${docker_type} --dockerbinary docker find . -name '*.pyc' -type f -delete"
- sh "${docker_run} ${docker_type} --dockerbinary docker PYTHONPATH=./python/ nosetests-2.7 --with-timer --verbose tests/python/unittest"
- sh "${docker_run} ${docker_type} --dockerbinary docker PYTHONPATH=./python/ nosetests-2.7 --with-timer --verbose tests/python/train"
+ sh "ci/build.py --build --platform ${docker_container_name} /work/runtime_functions.sh unittest_ubuntu_python2_cpu"
}
}
// Python 3
-def python3_ut(docker_type) {
+def python3_ut(docker_container_name) {
timeout(time: max_time, unit: 'MINUTES') {
- sh "${docker_run} ${docker_type} --dockerbinary docker find . -name '*.pyc' -type f -delete"
- sh "${docker_run} ${docker_type} --dockerbinary docker PYTHONPATH=./python/ nosetests-3.4 --with-timer --verbose tests/python/unittest"
+ sh "ci/build.py --build --platform ${docker_container_name} /work/runtime_functions.sh unittest_ubuntu_python3_cpu"
}
}
// GPU test has two parts. 1) run unittest on GPU, 2) compare the results on
// both CPU and GPU
// Python 2
-def python2_gpu_ut(docker_type) {
- timeout(time: max_time, unit: 'MINUTES') {
- sh "${docker_run} ${docker_type} find . -name '*.pyc' -type f -delete"
- sh "${docker_run} ${docker_type} PYTHONPATH=./python/ nosetests-2.7 --with-timer --verbose tests/python/gpu"
- }
-}
-
-// Python 3
-def python3_gpu_ut(docker_type) {
- timeout(time: max_time, unit: 'MINUTES') {
- sh "${docker_run} ${docker_type} find . -name '*.pyc' -type f -delete"
- sh "${docker_run} ${docker_type} PYTHONPATH=./python/ nosetests-3.4 --with-timer --verbose tests/python/gpu"
- }
-}
-
-// Python 2
-def python2_mkldnn_ut(docker_type) {
+def python2_gpu_ut(docker_container_name) {
timeout(time: max_time, unit: 'MINUTES') {
- sh "${docker_run} ${docker_type} find . -name '*.pyc' -type f -delete"
- sh "${docker_run} ${docker_type} PYTHONPATH=./python/ MXNET_MKLDNN_DEBUG=1 nosetests-2.7 --with-timer --verbose tests/python/cpu"
+ sh "ci/build.py --nvidiadocker --build --platform ${docker_container_name} /work/runtime_functions.sh unittest_ubuntu_python2_gpu"
}
}
// Python 3
-def python3_mkldnn_ut(docker_type) {
+def python3_gpu_ut(docker_container_name) {
timeout(time: max_time, unit: 'MINUTES') {
- sh "${docker_run} ${docker_type} find . -name '*.pyc' -type f -delete"
- sh "${docker_run} ${docker_type} PYTHONPATH=./python/ MXNET_MKLDNN_DEBUG=1 nosetests-3.4 --with-timer --verbose tests/python/cpu"
+ sh "ci/build.py --nvidiadocker --build --platform ${docker_container_name} /work/runtime_functions.sh unittest_ubuntu_python3_gpu"
}
}
try {
stage("Sanity Check") {
- timeout(time: max_time, unit: 'MINUTES') {
- node('mxnetlinux-cpu') {
- ws('workspace/sanity') {
- init_git()
- sh "tools/license_header.py check"
- make('lint', 'cpplint rcpplint jnilint')
- make('lint', 'pylint')
- }
+ node('mxnetlinux-cpu') {
+ ws('workspace/sanity') {
+ init_git()
+ sh "ci/build.py --build --platform ubuntu_cpu /work/runtime_functions.sh sanity_check"
}
}
}
stage('Build') {
- parallel 'CPU: Openblas': {
+ parallel 'CPU: CentOS 7': {
node('mxnetlinux-cpu') {
- ws('workspace/build-cpu') {
- init_git()
- def flag = """ \
- DEV=1 \
- USE_PROFILER=1 \
- USE_CPP_PACKAGE=1 \
- USE_BLAS=openblas \
- -j\$(nproc)
- """
- make("cpu", flag)
- pack_lib('cpu')
+ ws('workspace/build-centos7-cpu') {
+ init_git()
+ sh "ci/build.py --build --platform centos7_cpu /work/runtime_functions.sh build_centos7_cpu"
+ pack_lib('centos7_cpu')
}
}
},
- 'CPU: Clang 3.9': {
+ 'GPU: CentOS 7': {
node('mxnetlinux-cpu') {
- ws('workspace/build-cpu-clang') {
+ ws('workspace/build-centos7-gpu') {
init_git()
- def flag = """ \
- USE_PROFILER=1 \
- USE_CPP_PACKAGE=1 \
- USE_BLAS=openblas \
- USE_OPENMP=0 \
- CXX=clang++-3.9 \
- CC=clang-3.9 \
- -j\$(nproc)
- """
- make("cpu_clang", flag)
- pack_lib('cpu_clang')
+ sh "ci/build.py --build --platform centos7_gpu /work/runtime_functions.sh build_centos7_gpu"
+ pack_lib('centos7_gpu')
}
}
},
- 'CPU: Clang 5': {
+ 'CPU: Openblas': {
node('mxnetlinux-cpu') {
- ws('workspace/build-cpu-clang') {
+ ws('workspace/build-cpu-openblas') {
init_git()
- def flag = """ \
- USE_PROFILER=1 \
- USE_CPP_PACKAGE=1 \
- USE_BLAS=openblas \
- USE_OPENMP=1 \
- CXX=clang++-5.0 \
- CC=clang-5.0 \
- -j\$(nproc)
- """
- make("cpu_clang", flag)
- pack_lib('cpu_clang')
+ sh "ci/build.py --build --platform ubuntu_cpu /work/runtime_functions.sh build_ubuntu_cpu_openblas"
+ pack_lib('cpu')
}
}
},
- 'CPU: MKLDNN': {
+ 'CPU: Clang 3.9': {
node('mxnetlinux-cpu') {
- ws('workspace/build-mkldnn-cpu') {
+ ws('workspace/build-cpu-clang39') {
init_git()
- def flag = """ \
- DEV=1 \
- USE_PROFILER=1 \
- USE_CPP_PACKAGE=1 \
- USE_BLAS=openblas \
- USE_MKLDNN=1 \
- -j\$(nproc)
- """
- make("cpu_mklml", flag)
- pack_lib('mkldnn_cpu', mx_mkldnn_lib)
+ sh "ci/build.py --build --platform ubuntu_cpu /work/runtime_functions.sh build_ubuntu_cpu_clang39"
}
}
},
- 'GPU: CMake MKLDNN': {
+ 'CPU: Clang 5': {
node('mxnetlinux-cpu') {
- ws('workspace/build-cmake-mkldnn-gpu') {
+ ws('workspace/build-cpu-clang50') {
init_git()
- def defines = """ \
- -DUSE_CUDA=1 \
- -DUSE_CUDNN=1 \
- -DUSE_MKLML_MKL=1 \
- -DUSE_MKLDNN=1 \
- -DCMAKE_BUILD_TYPE=Release \
- """
- def flag = "-v"
- cmake("build_cuda", defines, flag)
- pack_lib('cmake_mkldnn_gpu', mx_cmake_mkldnn_lib)
+ sh "ci/build.py --build --platform ubuntu_cpu /work/runtime_functions.sh build_ubuntu_cpu_clang50"
}
}
},
- 'GPU: CMake': {
+ 'CPU: MKLDNN': {
node('mxnetlinux-cpu') {
- ws('workspace/build-cmake-gpu') {
+ ws('workspace/build-mkldnn-cpu') {
init_git()
- def defines = """ \
- -DUSE_CUDA=1 \
- -DUSE_CUDNN=1 \
- -DUSE_MKLML_MKL=0 \
- -DUSE_MKLDNN=0 \
- -DCMAKE_BUILD_TYPE=Release \
- """
- def flag = "-v"
- cmake("build_cuda", defines, flag)
- pack_lib('cmake_gpu', mx_cmake_lib)
+ sh "ci/build.py --build --platform ubuntu_cpu /work/runtime_functions.sh build_ubuntu_cpu_mkldnn"
+ pack_lib('mkldnn_cpu', mx_mkldnn_lib)
}
}
},
@@ -299,18 +188,7 @@ try {
node('mxnetlinux-cpu') {
ws('workspace/build-mkldnn-gpu') {
init_git()
- def flag = """ \
- DEV=1 \
- USE_PROFILER=1 \
- USE_CPP_PACKAGE=1 \
- USE_BLAS=openblas \
- USE_MKLDNN=1 \
- USE_CUDA=1 \
- USE_CUDA_PATH=/usr/local/cuda \
- USE_CUDNN=1 \
- -j\$(nproc)
- """
- make("build_cuda", flag)
+ sh "ci/build.py --build --platform ubuntu_build_cuda /work/runtime_functions.sh build_ubuntu_gpu_mkldnn"
pack_lib('mkldnn_gpu', mx_mkldnn_lib)
}
}
@@ -319,17 +197,7 @@ try {
node('mxnetlinux-cpu') {
ws('workspace/build-gpu') {
init_git()
- def flag = """ \
- DEV=1 \
- USE_PROFILER=1 \
- USE_BLAS=openblas \
- USE_CUDA=1 \
- USE_CUDA_PATH=/usr/local/cuda \
- USE_CUDNN=1 \
- USE_CPP_PACKAGE=1 \
- -j\$(nproc)
- """
- make('build_cuda', flag)
+ sh "ci/build.py --build --platform ubuntu_build_cuda /work/runtime_functions.sh build_ubuntu_gpu_cuda8_cudnn5"
pack_lib('gpu')
stash includes: 'build/cpp-package/example/test_score', name: 'cpp_test_score'
stash includes: 'build/cpp-package/example/test_optimizer', name: 'cpp_test_optimizer'
@@ -340,8 +208,7 @@ try {
node('mxnetlinux-cpu') {
ws('workspace/amalgamationmin') {
init_git()
- make('cpu', '-C amalgamation/ clean')
- make('cpu', '-C amalgamation/ USE_BLAS=openblas MIN=1')
+ sh "ci/build.py --build --platform ubuntu_cpu /work/runtime_functions.sh build_ubuntu_amalgamation_min"
}
}
},
@@ -349,8 +216,26 @@ try {
node('mxnetlinux-cpu') {
ws('workspace/amalgamation') {
init_git()
- make('cpu', '-C amalgamation/ clean')
- make('cpu', '-C amalgamation/ USE_BLAS=openblas')
+ sh "ci/build.py --build --platform ubuntu_cpu /work/runtime_functions.sh build_ubuntu_amalgamation"
+ }
+ }
+ },
+
+ 'GPU: CMake MKLDNN': {
+ node('mxnetlinux-cpu') {
+ ws('workspace/build-cmake-mkldnn-gpu') {
+ init_git()
+ sh "ci/build.py --build --platform ubuntu_gpu /work/runtime_functions.sh build_ubuntu_gpu_cmake_mkldnn" //build_cuda
+ pack_lib('cmake_mkldnn_gpu', mx_cmake_mkldnn_lib)
+ }
+ }
+ },
+ 'GPU: CMake': {
+ node('mxnetlinux-cpu') {
+ ws('workspace/build-cmake-gpu') {
+ init_git()
+ sh "ci/build.py --build --platform ubuntu_gpu /work/runtime_functions.sh build_ubuntu_gpu_cmake" //build_cuda
+ pack_lib('cmake_gpu', mx_cmake_lib)
}
}
},
@@ -419,6 +304,22 @@ try {
}
}
}
+ },
+ 'NVidia Jetson / ARMv8':{
+ node('mxnetlinux-cpu') {
+ ws('workspace/build-jetson-armv8') {
+ init_git()
+ sh "ci/build.py --build --platform jetson /work/runtime_functions.sh build_jetson"
+ }
+ }
+ },
+ 'Raspberry / ARMv7':{
+ node('mxnetlinux-cpu') {
+ ws('workspace/build-raspberry-armv7') {
+ init_git()
+ sh "ci/build.py --build --platform armv7 /work/runtime_functions.sh build_armv7"
+ }
+ }
}
} // End of stage('Build')
@@ -428,7 +329,7 @@ try {
ws('workspace/ut-python2-cpu') {
init_git()
unpack_lib('cpu')
- python2_ut('cpu')
+ python2_ut('ubuntu_cpu')
}
}
},
@@ -437,7 +338,7 @@ try {
ws('workspace/ut-python3-cpu') {
init_git()
unpack_lib('cpu')
- python3_ut('cpu')
+ python3_ut('ubuntu_cpu')
}
}
},
@@ -446,7 +347,7 @@ try {
ws('workspace/ut-python2-gpu') {
init_git()
unpack_lib('gpu', mx_lib)
- python2_gpu_ut('gpu')
+ python2_gpu_ut('ubuntu_gpu')
}
}
},
@@ -455,7 +356,7 @@ try {
ws('workspace/ut-python3-gpu') {
init_git()
unpack_lib('gpu', mx_lib)
- python3_gpu_ut('gpu')
+ python3_gpu_ut('ubuntu_gpu')
}
}
},
@@ -464,8 +365,7 @@ try {
ws('workspace/ut-python2-mkldnn-cpu') {
init_git()
unpack_lib('mkldnn_cpu', mx_mkldnn_lib)
- python2_ut('cpu_mklml')
- python2_mkldnn_ut('cpu_mklml')
+ python2_ut('ubuntu_cpu')
}
}
},
@@ -474,8 +374,7 @@ try {
ws('workspace/ut-python2-mkldnn-gpu') {
init_git()
unpack_lib('mkldnn_gpu', mx_mkldnn_lib)
- python2_gpu_ut('gpu_mklml')
- python2_mkldnn_ut('gpu_mklml')
+ python2_gpu_ut('ubuntu_gpu')
}
}
},
@@ -484,8 +383,7 @@ try {
ws('workspace/ut-python3-mkldnn-cpu') {
init_git()
unpack_lib('mkldnn_cpu', mx_mkldnn_lib)
- python3_ut('cpu_mklml')
- python3_mkldnn_ut('cpu_mklml')
+ python3_ut('ubuntu_cpu')
}
}
},
@@ -494,8 +392,29 @@ try {
ws('workspace/ut-python3-mkldnn-gpu') {
init_git()
unpack_lib('mkldnn_gpu', mx_mkldnn_lib)
- python3_gpu_ut('gpu_mklml')
- python3_mkldnn_ut('gpu_mklml')
+ python3_gpu_ut('ubuntu_gpu')
+ }
+ }
+ },
+ 'Python3: CentOS 7 CPU': {
+ node('mxnetlinux-cpu') {
+ ws('workspace/build-centos7-cpu') {
+ init_git()
+ unpack_lib('centos7_cpu')
+ timeout(time: max_time, unit: 'MINUTES') {
+ sh "ci/build.py --build --platform centos7_cpu /work/runtime_functions.sh unittest_centos7_cpu"
+ }
+ }
+ }
+ },
+ 'Python3: CentOS 7 GPU': {
+ node('mxnetlinux-gpu') {
+ ws('workspace/build-centos7-gpu') {
+ init_git()
+ unpack_lib('centos7_gpu')
+ timeout(time: max_time, unit: 'MINUTES') {
+ sh "ci/build.py --nvidiadocker --build --platform centos7_gpu /work/runtime_functions.sh unittest_centos7_gpu"
+ }
}
}
},
@@ -505,8 +424,7 @@ try {
init_git()
unpack_lib('cpu')
timeout(time: max_time, unit: 'MINUTES') {
- sh "${docker_run} cpu make scalapkg USE_BLAS=openblas"
- sh "${docker_run} cpu make scalatest USE_BLAS=openblas"
+ sh "ci/build.py --build --platform ubuntu_cpu /work/runtime_functions.sh unittest_ubuntu_cpu_scala"
}
}
}
@@ -517,7 +435,7 @@ try {
init_git()
unpack_lib('cpu')
timeout(time: max_time, unit: 'MINUTES') {
- sh "${docker_run} cpu ./perl-package/test.sh"
+ sh "ci/build.py --build --platform ubuntu_cpu /work/runtime_functions.sh unittest_ubuntu_cpugpu_perl"
}
}
}
@@ -528,7 +446,7 @@ try {
init_git()
unpack_lib('gpu')
timeout(time: max_time, unit: 'MINUTES') {
- sh "${docker_run} gpu ./perl-package/test.sh"
+ sh "ci/build.py --nvidiadocker --build --platform ubuntu_gpu /work/runtime_functions.sh unittest_ubuntu_cpugpu_perl"
}
}
}
@@ -539,7 +457,7 @@ try {
init_git()
unpack_lib('cmake_gpu', mx_cmake_lib)
timeout(time: max_time, unit: 'MINUTES') {
- sh "${docker_run} gpu_mklml build/tests/mxnet_unit_tests"
+ sh "ci/build.py --nvidiadocker --build --platform ubuntu_gpu /work/runtime_functions.sh unittest_ubuntu_gpu_cpp"
}
}
}
@@ -550,11 +468,7 @@ try {
init_git()
unpack_lib('cpu')
timeout(time: max_time, unit: 'MINUTES') {
- sh "${docker_run} cpu rm -rf .Renviron"
- sh "${docker_run} cpu mkdir -p /workspace/ut-r-cpu/site-library"
- sh "${docker_run} cpu make rpkg USE_BLAS=openblas R_LIBS=/workspace/ut-r-cpu/site-library"
- sh "${docker_run} cpu R CMD INSTALL --library=/workspace/ut-r-cpu/site-library R-package"
- sh "${docker_run} cpu make rpkgtest R_LIBS=/workspace/ut-r-cpu/site-library"
+ sh "ci/build.py --build --platform ubuntu_cpu /work/runtime_functions.sh unittest_ubuntu_cpu_R"
}
}
}
@@ -565,15 +479,12 @@ try {
init_git()
unpack_lib('gpu')
timeout(time: max_time, unit: 'MINUTES') {
- sh "${docker_run} gpu rm -rf .Renviron"
- sh "${docker_run} gpu mkdir -p /workspace/ut-r-gpu/site-library"
- sh "${docker_run} gpu make rpkg USE_BLAS=openblas R_LIBS=/workspace/ut-r-gpu/site-library"
- sh "${docker_run} gpu R CMD INSTALL --library=/workspace/ut-r-gpu/site-library R-package"
- sh "${docker_run} gpu make rpkgtest R_LIBS=/workspace/ut-r-gpu/site-library R_GPU_ENABLE=1"
+ sh "ci/build.py --nvidiadocker --build --platform ubuntu_gpu /work/runtime_functions.sh unittest_ubuntu_gpu_R"
}
}
}
},
+
'Python 2: CPU Win':{
node('mxnetwindows-cpu') {
timeout(time: max_time, unit: 'MINUTES') {
@@ -590,7 +501,7 @@ try {
C:\\mxnet\\test_cpu.bat"""
}
}
- }
+ }
},
'Python 3: CPU Win': {
node('mxnetwindows-cpu') {
@@ -655,7 +566,7 @@ try {
init_git()
unpack_lib('gpu')
timeout(time: max_time, unit: 'MINUTES') {
- sh "${docker_run} gpu --dockerbinary nvidia-docker PYTHONPATH=./python/ python example/image-classification/test_score.py"
+ sh "ci/build.py --nvidiadocker --build --platform ubuntu_gpu /work/runtime_functions.sh integrationtest_ubuntu_gpu_python"
}
}
}
@@ -666,7 +577,7 @@ try {
init_git()
unpack_lib('gpu')
timeout(time: max_time, unit: 'MINUTES') {
- sh "${docker_run} caffe_gpu --dockerbinary nvidia-docker PYTHONPATH=/caffe/python:./python python tools/caffe_converter/test_converter.py"
+ sh "ci/build.py --nvidiadocker --build --platform ubuntu_gpu /work/runtime_functions.sh integrationtest_ubuntu_gpu_caffe"
}
}
}
@@ -679,7 +590,7 @@ try {
unstash 'cpp_test_score'
unstash 'cpp_test_optimizer'
timeout(time: max_time, unit: 'MINUTES') {
- sh "${docker_run} gpu --dockerbinary nvidia-docker cpp-package/tests/ci_test.sh"
+ sh "ci/build.py --nvidiadocker --build --platform ubuntu_gpu /work/runtime_functions.sh integrationtest_ubuntu_gpu_cpp_package"
}
}
}
@@ -690,9 +601,10 @@ try {
node('mxnetlinux-cpu') {
ws('workspace/docs') {
init_git()
- sh "make clean"
- sh "make docs"
- sh "tests/ci_build/deploy/ci_deploy_doc.sh ${env.BRANCH_NAME} ${env.BUILD_NUMBER}"
+ timeout(time: max_time, unit: 'MINUTES') {
+ sh "ci/build.py --build --platform ubuntu_cpu /work/runtime_functions.sh deploy_docs"
+ sh "tests/ci_build/deploy/ci_deploy_doc.sh ${env.BRANCH_NAME} ${env.BUILD_NUMBER}"
+ }
}
}
}
diff --git a/Makefile b/Makefile
index 5d81c7f..50b125b 100644
--- a/Makefile
+++ b/Makefile
@@ -167,6 +167,7 @@ ifeq (,$(wildcard /usr/lib/liblapack.a))
ifeq (,$(wildcard /usr/lib64/liblapack.a))
ifeq (,$(wildcard $(USE_LAPACK_PATH)/liblapack.a))
USE_LAPACK = 0
+ $(warning "USE_LAPACK disabled because libraries were not found")
endif
endif
endif
@@ -473,7 +474,7 @@ pylint:
doc: docs
docs:
- tests/ci_build/ci_build.sh doc make -C docs html
+ make -C docs html
clean_docs:
make -C docs clean
diff --git a/ci/README.md b/ci/README.md
new file mode 100644
index 0000000..98c74e3
--- /dev/null
+++ b/ci/README.md
@@ -0,0 +1,50 @@
+# Containerized build & test utilities
+
+This folder contains scripts and dockerfiles used to build and test MXNet using Docker containers
+
+You need docker and nvidia docker if you have a GPU.
+
+If you are in ubuntu an easy way to install Docker CE is executing the following script:
+
+
+```
+#!/bin/bash
+set -e
+set -x
+export DEBIAN_FRONTEND=noninteractive
+apt-get -y install curl
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+add-apt-repository \
+ "deb [arch=amd64] https://download.docker.com/linux/ubuntu \
+ $(lsb_release -cs) \
+ stable"
+apt-get update
+apt-get -y install docker-ce
+service docker restart
+usermod -a -G docker $SUDO_USER
+```
+
+For detailed instructions go to the docker documentation.
+
+
+## build.py
+
+The main utility to build is build.py which will run docker and mount the mxnet folder as a volume
+to do in-place builds.
+
+The build.py script does two functions, build the docker image, and it can be also used to run
+commands inside this image with the propper mounts and paraphernalia required to build mxnet inside
+docker from the sources on the parent folder.
+
+A set of helper shell functions are in `functions.sh`. `build.py --help` will display usage
+information about the tool.
+
+To build for armv7 for example:
+
+```
+./build.py -p armv7 /work/functions.sh build_armv7
+```
+
+## Warning
+Due to current limitations of the CMake build system creating artifacts in the source 3rdparty
+folder of the parent mxnet sources concurrent builds of different platforms is NOT SUPPORTED.
diff --git a/ci/build.py b/ci/build.py
new file mode 100755
index 0000000..8caf52b
--- /dev/null
+++ b/ci/build.py
@@ -0,0 +1,179 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Multi arch dockerized build tool.
+
+"""
+
+__author__ = 'Marco de Abreu, Kellen Sunderland, Anton Chernov, Pedro Larroy'
+__version__ = '0.1'
+
+import os
+import sys
+import subprocess
+import logging
+import argparse
+from subprocess import check_call, call
+import glob
+import re
+from typing import *
+from itertools import chain
+from copy import deepcopy
+
+
+def get_platforms(path: Optional[str]="docker"):
+ """Get a list of architectures given our dockerfiles"""
+ dockerfiles = glob.glob(os.path.join(path, "Dockerfile.build.*"))
+ dockerfiles = list(filter(lambda x: x[-1] != '~', dockerfiles))
+ files = list(map(lambda x: re.sub(r"Dockerfile.build.(.*)", r"\1", x), dockerfiles))
+ files.sort()
+ platforms = list(map(lambda x: os.path.split(x)[1], files))
+ return platforms
+
+
+def get_docker_tag(platform: str) -> None:
+ return "mxnet/build.{0}".format(platform)
+
+
+def get_dockerfile(platform: str, path="docker"):
+ return os.path.join(path, "Dockerfile.build.{0}".format(platform))
+
+def get_docker_binary(use_nvidia_docker: bool):
+ if use_nvidia_docker:
+ return "nvidia-docker"
+ else:
+ return "docker"
+
+def build_docker(platform: str, docker_binary: str) -> None:
+ """Build a container for the given platform"""
+ tag = get_docker_tag(platform)
+ logging.info("Building container tagged '%s' with %s", tag, docker_binary)
+ cmd = [docker_binary, "build",
+ "-f", get_dockerfile(platform),
+ "--build-arg", "USER_ID={}".format(os.getuid()),
+ "-t", tag,
+ "docker"]
+ logging.info("Running command: '%s'", ' '.join(cmd))
+ check_call(cmd)
+
+def get_mxnet_root() -> str:
+ curpath = os.path.abspath(os.path.dirname(__file__))
+ def is_mxnet_root(path: str) -> bool:
+ return os.path.exists(os.path.join(path, ".mxnet_root"))
+ while not is_mxnet_root(curpath):
+ parent = os.path.abspath(os.path.join(curpath, os.pardir))
+ if parent == curpath:
+ raise RuntimeError("Got to the root and couldn't find a parent folder with .mxnet_root")
+ curpath = parent
+ return curpath
+
+
+def container_run(platform: str, docker_binary: str, command: List[str]) -> None:
+ tag = get_docker_tag(platform)
+ mx_root = get_mxnet_root()
+ local_build_folder = '{}/build'.format(mx_root)
+ # We need to create it first, otherwise it will be created by the docker daemon with root only permissions
+ os.makedirs(local_build_folder, exist_ok=True)
+ logging.info("Running %s in container %s", command, tag)
+ runlist = [docker_binary, 'run', '--rm',
+ '-v', "{}:/work/mxnet".format(mx_root), # mount mxnet root
+ '-v', "{}:/work/build".format(local_build_folder), # mount mxnet/build for storing build artifacts
+ '-u', '{}:{}'.format(os.getuid(), os.getgid()),
+ tag]
+ runlist.extend(command)
+ cmd = ' '.join(runlist)
+ logging.info("Executing: %s", cmd)
+ ret = call(runlist)
+ if ret != 0:
+ logging.error("Running of command in container failed: %s", cmd)
+ into_cmd = deepcopy(runlist)
+ idx = into_cmd.index('-u') + 2
+ into_cmd[idx:idx] = ['-ti', '--entrypoint', 'bash']
+ logging.error("You can try to get into the container by using the following command: %s", ' '.join(into_cmd))
+ raise subprocess.CalledProcessError(ret, cmd)
+
+def main() -> int:
+ # We need to be in the same directory than the script so the commands in the dockerfiles work as
+ # expected. But the script can be invoked from a different path
+ base = os.path.split(os.path.realpath(__file__))[0]
+ os.chdir(base)
+
+ logging.getLogger().setLevel(logging.INFO)
+ def script_name() -> str:
+ return os.path.split(sys.argv[0])[1]
+
+ logging.basicConfig(format='{}: %(asctime)-15s %(message)s'.format(script_name()))
+
+ parser = argparse.ArgumentParser()
+ parser.add_argument("-p", "--platform",
+ help="platform",
+ type=str)
+
+ parser.add_argument("-b", "--build",
+ help="Build the container",
+ action='store_true')
+
+ parser.add_argument("-n", "--nvidiadocker",
+ help="Use nvidia docker",
+ action='store_true')
+
+ parser.add_argument("-l", "--list",
+ help="List platforms",
+ action='store_true')
+
+ parser.add_argument("command",
+ help="command to run in the container",
+ nargs='*', action='append', type=str)
+
+ args = parser.parse_args()
+ command = list(chain(*args.command))
+ docker_binary = get_docker_binary(args.nvidiadocker)
+
+ if args.list:
+ platforms = get_platforms()
+ print(platforms)
+
+ elif args.platform:
+ platform = args.platform
+ if args.build:
+ build_docker(platform, docker_binary)
+ tag = get_docker_tag(platform)
+ if command:
+ container_run(platform, docker_binary, command)
+ else:
+ cmd = ["/work/mxnet/ci/docker/runtime_functions.sh", "build_{}".format(platform)]
+ logging.info("No command specified, trying default build: %s", ' '.join(cmd))
+ container_run(platform, docker_binary, cmd)
+
+ else:
+ platforms = get_platforms()
+ logging.info("Building for all architectures: {}".format(platforms))
+ logging.info("Artifacts will be produced in the build/ directory.")
+ for platform in platforms:
+ build_docker(platform, docker_binary)
+ cmd = ["/work/mxnet/ci/docker/runtime_functions.sh", "build_{}".format(platform)]
+ logging.info("No command specified, trying default build: %s", ' '.join(cmd))
+ container_run(platform, docker_binary, cmd)
+
+ return 0
+
+
+if __name__ == '__main__':
+ sys.exit(main())
diff --git a/ci/docker/Dockerfile.build.amzn_linux_cpu b/ci/docker/Dockerfile.build.amzn_linux_cpu
new file mode 100755
index 0000000..7d6f223
--- /dev/null
+++ b/ci/docker/Dockerfile.build.amzn_linux_cpu
@@ -0,0 +1,44 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to build and run MXNet for Amazon Linux on CPU
+
+FROM amazonlinux
+
+WORKDIR /work/deps
+COPY install/amzn_linux_core.sh /work/
+RUN /work/amzn_linux_core.sh
+COPY install/amzn_linux_opencv.sh /work/
+RUN /work/amzn_linux_opencv.sh
+COPY install/amzn_linux_openblas.sh /work/
+RUN /work/amzn_linux_openblas.sh
+COPY install/amzn_linux_python2.sh /work/
+RUN /work/amzn_linux_python2.sh
+COPY install/amzn_linux_python3.sh /work/
+RUN /work/amzn_linux_python3.sh
+COPY install/amzn_linux_testdeps.sh /work/
+RUN /work/amzn_linux_testdeps.sh
+COPY install/amzn_linux_julia.sh /work/
+RUN /work/amzn_linux_julia.sh
+COPY install/amzn_linux_maven.sh /work/
+RUN /work/amzn_linux_maven.sh
+COPY install/amzn_linux_library.sh /work/
+RUN /work/amzn_linux_library.sh
+WORKDIR /work/mxnet
+
+COPY runtime_functions.sh /work/
\ No newline at end of file
diff --git a/ci/docker/Dockerfile.build.android_arm64 b/ci/docker/Dockerfile.build.android_arm64
new file mode 100755
index 0000000..d768751
--- /dev/null
+++ b/ci/docker/Dockerfile.build.android_arm64
@@ -0,0 +1,66 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to build MXNet for Android ARM64/ARMv8
+
+FROM dockcross/base:latest
+MAINTAINER Pedro Larroy "pllarroy@amazon.com"
+
+# The cross-compiling emulator
+RUN apt-get update && apt-get install -y \
+ qemu-user \
+ qemu-user-static \
+ unzip
+
+ENV CROSS_TRIPLE=aarch64-linux-android
+ENV CROSS_ROOT=/usr/${CROSS_TRIPLE}
+ENV AS=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-as \
+ AR=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-ar \
+ CC=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-gcc \
+ CPP=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-cpp \
+ CXX=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-g++ \
+ LD=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-ld
+
+
+ENV DEFAULT_DOCKCROSS_IMAGE dockcross/android-arm
+
+# Build-time metadata as defined at http://label-schema.org
+ARG BUILD_DATE
+ARG IMAGE
+ARG VCS_REF
+ARG VCS_URL
+LABEL org.label-schema.build-date=$BUILD_DATE \
+ org.label-schema.name=$IMAGE \
+ org.label-schema.vcs-ref=$VCS_REF \
+ org.label-schema.vcs-url=$VCS_URL \
+ org.label-schema.schema-version="1.0"
+
+ENV ARCH aarch64
+
+ENV CC=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-clang
+ENV CXX=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-clang++
+
+WORKDIR /work/deps
+COPY install/android_arm64_ndk.sh /work/
+RUN /work/android_arm64_ndk.sh
+COPY install/android_arm64_openblas.sh /work/
+RUN /work/android_arm64_openblas.sh
+ENV CPLUS_INCLUDE_PATH /work/deps/OpenBLAS
+WORKDIR /work/build
+
+COPY runtime_functions.sh /work/
diff --git a/docker_multiarch/Dockerfile.build.android.armv7 b/ci/docker/Dockerfile.build.android_armv7
old mode 100644
new mode 100755
similarity index 74%
rename from docker_multiarch/Dockerfile.build.android.armv7
rename to ci/docker/Dockerfile.build.android_armv7
index 12d53a4..0074c1f
--- a/docker_multiarch/Dockerfile.build.android.armv7
+++ b/ci/docker/Dockerfile.build.android_armv7
@@ -1,4 +1,23 @@
# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to build MXNet for Android ARMv7
+
FROM dockcross/base:latest
MAINTAINER Pedro Larroy "pllarroy@amazon.com"
@@ -36,9 +55,6 @@ RUN mkdir -p /build && \
ENV DEFAULT_DOCKCROSS_IMAGE dockcross/android-arm
-# COPY Toolchain.cmake ${CROSS_ROOT}/
-# ENV CMAKE_TOOLCHAIN_FILE ${CROSS_ROOT}/Toolchain.cmake
-
# Build-time metadata as defined at http://label-schema.org
ARG BUILD_DATE
ARG IMAGE
@@ -76,3 +92,5 @@ RUN cd mxnet && \
WORKDIR /work/build/
RUN cp /work/mxnet/lib/* .
+
+# TODO: Bring this into the new format
\ No newline at end of file
diff --git a/tests/ci_build/install/ubuntu_install_llvm.sh b/ci/docker/Dockerfile.build.arm64
similarity index 62%
copy from tests/ci_build/install/ubuntu_install_llvm.sh
copy to ci/docker/Dockerfile.build.arm64
index d3282e7..eb68a81 100755
--- a/tests/ci_build/install/ubuntu_install_llvm.sh
+++ b/ci/docker/Dockerfile.build.arm64
@@ -1,5 +1,4 @@
-#!/usr/bin/env bash
-
+# -*- mode: dockerfile -*-
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -16,13 +15,24 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+#
+# Dockerfile to build MXNet for ARM64/ARMv8
+
+FROM dockcross/linux-arm64
+
+ENV ARCH aarch64
+ENV CC /usr/bin/aarch64-linux-gnu-gcc
+ENV CXX /usr/bin/aarch64-linux-gnu-g++
+ENV FC /usr/bin/aarch64-linux-gnu-gfortran-4.9
+ENV HOSTCC gcc
+WORKDIR /work
+COPY install/arm64_openblas.sh /work/
+RUN /work/arm64_openblas.sh
-echo deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-5.0 main\
- >> /etc/apt/sources.list.d/llvm.list
-echo deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial-5.0 main\
- >> /etc/apt/sources.list.d/llvm.list
+ENV LD_LIBRARY_PATH /opt/OpenBLAS/lib
+ENV CPLUS_INCLUDE_PATH /opt/OpenBLAS/include
+WORKDIR /work/mxnet
-wget -O - http://apt.llvm.org/llvm-snapshot.gpg.key|sudo apt-key add -
-apt-get update && apt-get install -y --force-yes llvm-5.0
+COPY runtime_functions.sh /work/
\ No newline at end of file
diff --git a/tests/ci_build/install/ubuntu_install_r.sh b/ci/docker/Dockerfile.build.armv7
similarity index 68%
copy from tests/ci_build/install/ubuntu_install_r.sh
copy to ci/docker/Dockerfile.build.armv7
index 6855b33..c249306 100755
--- a/tests/ci_build/install/ubuntu_install_r.sh
+++ b/ci/docker/Dockerfile.build.armv7
@@ -1,5 +1,4 @@
-#!/usr/bin/env bash
-
+# -*- mode: dockerfile -*-
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -16,15 +15,18 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-set -e
-set -x
+#
+# Dockerfile to build MXNet for Android ARMv7
-# install libraries for mxnet's r package on ubuntu
+FROM dockcross/linux-armv7
-echo "deb http://cran.rstudio.com/bin/linux/ubuntu trusty/" >> /etc/apt/sources.list
-gpg --keyserver keyserver.ubuntu.com --recv-key E084DAB9
-gpg -a --export E084DAB9 | apt-key add -
+ENV ARCH armv71
+ENV CC /usr/bin/arm-linux-gnueabihf-gcc
+ENV CXX /usr/bin/arm-linux-gnueabihf-g++
-apt-get update
-apt-get install -y r-base r-base-dev libxml2-dev libssl-dev libxt-dev
+RUN apt-get update && \
+ apt-get install -y libopenblas-dev:armhf && \
+ rm -rf /var/lib/apt/lists/*
+COPY runtime_functions.sh /work/
+WORKDIR /work/build
diff --git a/tests/ci_build/install/ubuntu_install_llvm.sh b/ci/docker/Dockerfile.build.centos7_cpu
similarity index 65%
copy from tests/ci_build/install/ubuntu_install_llvm.sh
copy to ci/docker/Dockerfile.build.centos7_cpu
index d3282e7..665f7dd 100755
--- a/tests/ci_build/install/ubuntu_install_llvm.sh
+++ b/ci/docker/Dockerfile.build.centos7_cpu
@@ -1,5 +1,4 @@
-#!/usr/bin/env bash
-
+# -*- mode: dockerfile -*-
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -16,13 +15,23 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+#
+# Dockerfile to build and run MXNet on CentOS 7 for CPU
+
+FROM centos:7
+
+ARG USER_ID=0
+WORKDIR /work/deps
+COPY install/centos7_core.sh /work/
+RUN /work/centos7_core.sh
+COPY install/centos7_python.sh /work/
+RUN /work/centos7_python.sh
+COPY install/centos7_adduser.sh /work/
+RUN /work/centos7_adduser.sh
-echo deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-5.0 main\
- >> /etc/apt/sources.list.d/llvm.list
-echo deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial-5.0 main\
- >> /etc/apt/sources.list.d/llvm.list
+ENV PYTHONPATH=./python/
+WORKDIR /work/mxnet
-wget -O - http://apt.llvm.org/llvm-snapshot.gpg.key|sudo apt-key add -
-apt-get update && apt-get install -y --force-yes llvm-5.0
+COPY runtime_functions.sh /work/
diff --git a/tests/ci_build/install/ubuntu_install_llvm.sh b/ci/docker/Dockerfile.build.centos7_gpu
similarity index 64%
copy from tests/ci_build/install/ubuntu_install_llvm.sh
copy to ci/docker/Dockerfile.build.centos7_gpu
index d3282e7..3d74821 100755
--- a/tests/ci_build/install/ubuntu_install_llvm.sh
+++ b/ci/docker/Dockerfile.build.centos7_gpu
@@ -1,5 +1,4 @@
-#!/usr/bin/env bash
-
+# -*- mode: dockerfile -*-
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -16,13 +15,23 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+#
+# Dockerfile to build and run MXNet on CentOS 7 for GPU
+
+FROM nvidia/cuda:8.0-cudnn5-devel-centos7
+
+ARG USER_ID=0
+WORKDIR /work/deps
+COPY install/centos7_core.sh /work/
+RUN /work/centos7_core.sh
+COPY install/centos7_python.sh /work/
+RUN /work/centos7_python.sh
+COPY install/centos7_adduser.sh /work/
+RUN /work/centos7_adduser.sh
-echo deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-5.0 main\
- >> /etc/apt/sources.list.d/llvm.list
-echo deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial-5.0 main\
- >> /etc/apt/sources.list.d/llvm.list
+ENV PYTHONPATH=./python/
+WORKDIR /work/mxnet
-wget -O - http://apt.llvm.org/llvm-snapshot.gpg.key|sudo apt-key add -
-apt-get update && apt-get install -y --force-yes llvm-5.0
+COPY runtime_functions.sh /work/
diff --git a/docker_multiarch/Dockerfile.build.jetson b/ci/docker/Dockerfile.build.jetson
old mode 100644
new mode 100755
similarity index 52%
rename from docker_multiarch/Dockerfile.build.jetson
rename to ci/docker/Dockerfile.build.jetson
index 8a60fae..e49b48e
--- a/docker_multiarch/Dockerfile.build.jetson
+++ b/ci/docker/Dockerfile.build.jetson
@@ -1,5 +1,24 @@
# -*- mode: dockerfile -*-
-# dockerfile to build libmxnet.so, and a python wheel for the Jetson TX1/TX2
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to build libmxnet.so, and a python wheel for the Jetson TX1/TX2
+# This script assumes /work/mxnet exists and contains the mxnet code you wish to compile and
+# that /work/build exists and is the target for your output.
FROM nvidia/cuda:8.0-cudnn6-devel as cudabuilder
@@ -7,8 +26,6 @@ FROM dockcross/linux-arm64
ENV ARCH aarch64
ENV NVCCFLAGS "-m64"
-ENV CUDA_ARCH "-gencode arch=compute_53,code=sm_53 -gencode arch=compute_62,code=sm_62"
-ENV BUILD_OPTS "USE_OPENCV=0 USE_BLAS=openblas USE_SSE=0 USE_CUDA=1 USE_CUDNN=1 ENABLE_CUDA_RTC=0 USE_NCCL=0 USE_CUDA_PATH=/usr/local/cuda/"
ENV CC /usr/bin/aarch64-linux-gnu-gcc
ENV CXX /usr/bin/aarch64-linux-gnu-g++
ENV FC /usr/bin/aarch64-linux-gnu-gfortran-4.9
@@ -34,59 +51,22 @@ RUN JETPACK_DOWNLOAD_PREFIX=http://developer.download.nvidia.com/devzone/devcent
ARM_CUDA_INSTALLER_PACKAGE=cuda-repo-l4t-8-0-local_8.0.84-1_arm64.deb && \
ARM_CUDNN_INSTALLER_PACKAGE=libcudnn6_6.0.21-1+cuda8.0_arm64.deb && \
ARM_CUDNN_DEV_INSTALLER_PACKAGE=libcudnn6-dev_6.0.21-1+cuda8.0_arm64.deb && \
- wget $JETPACK_DOWNLOAD_PREFIX/$ARM_CUDA_INSTALLER_PACKAGE && \
- wget $JETPACK_DOWNLOAD_PREFIX/$ARM_CUDNN_INSTALLER_PACKAGE && \
- wget $JETPACK_DOWNLOAD_PREFIX/$ARM_CUDNN_DEV_INSTALLER_PACKAGE && \
+ wget -nv $JETPACK_DOWNLOAD_PREFIX/$ARM_CUDA_INSTALLER_PACKAGE && \
+ wget -nv $JETPACK_DOWNLOAD_PREFIX/$ARM_CUDNN_INSTALLER_PACKAGE && \
+ wget -nv $JETPACK_DOWNLOAD_PREFIX/$ARM_CUDNN_DEV_INSTALLER_PACKAGE && \
dpkg -i $ARM_CUDA_INSTALLER_PACKAGE && \
dpkg -i $ARM_CUDNN_INSTALLER_PACKAGE && \
dpkg -i $ARM_CUDNN_DEV_INSTALLER_PACKAGE && \
- apt update -y && \
- apt install cuda-cudart-cross-aarch64-8-0 cuda-cublas-cross-aarch64-8-0 \
+ apt update -y && \
+ apt install -y unzip cuda-cudart-cross-aarch64-8-0 cuda-cublas-cross-aarch64-8-0 \
cuda-nvml-cross-aarch64-8-0 cuda-nvrtc-cross-aarch64-8-0 cuda-cufft-cross-aarch64-8-0 \
cuda-curand-cross-aarch64-8-0 cuda-cusolver-cross-aarch64-8-0 cuda-cusparse-cross-aarch64-8-0 \
- cuda-misc-headers-cross-aarch64-8-0 cuda-npp-cross-aarch64-8-0 libcudnn6 -y && \
+ cuda-misc-headers-cross-aarch64-8-0 cuda-npp-cross-aarch64-8-0 libcudnn6 && \
cp /usr/local/cuda-8.0/targets/aarch64-linux/lib/*.so /usr/local/cuda/lib64/ && \
cp /usr/local/cuda-8.0/targets/aarch64-linux/lib/stubs/*.so /usr/local/cuda/lib64/stubs/ && \
cp -r /usr/local/cuda-8.0/targets/aarch64-linux/include/ /usr/local/cuda/include/ && \
rm $ARM_CUDA_INSTALLER_PACKAGE $ARM_CUDNN_INSTALLER_PACKAGE $ARM_CUDNN_DEV_INSTALLER_PACKAGE
-# Build MXNet
-ADD mxnet mxnet
-
WORKDIR /work/mxnet
-# Add ARM specific settings
-ADD arm.crosscompile.mk make/config.mk
-
-# Build and link
-RUN make -j$(nproc) $BUILD_OPTS
-
-# Create a binary wheel for easy installation.
-# When using tool.py output will be in the jetson folder.
-# Scp the .whl file to your target device, and install via
-# pip install
-WORKDIR /work/mxnet/python
-RUN python setup.py bdist_wheel --universal
-
-# Copy build artifacts to output folder for tool.py script
-RUN mkdir -p /work/build & cp dist/*.whl /work/build && cp ../lib/* /work/build
-
-# Fix pathing issues in the wheel. We need to move libmxnet.so from the data folder to the root
-# of the wheel, then repackage the wheel.
-# Create a temp dir to do the work.
-WORKDIR /work/build
-RUN apt-get install -y unzip && \
- mkdir temp && \
- cp *.whl temp
-
-# Extract the wheel, move the libmxnet.so file, repackage the wheel.
-WORKDIR /work/build/temp
-RUN unzip *.whl && \
- rm *.whl && \
- mv *.data/data/mxnet/libmxnet.so mxnet && \
- zip -r ../temp.zip *
-
-# Replace the existing wheel with our fixed version.
-WORKDIR /work/build
-RUN rm -rf temp && \
- for f in *.whl; do rm "$f" && mv temp.zip "$f"; done
+COPY runtime_functions.sh /work/
diff --git a/ci/docker/Dockerfile.build.ubuntu_build_cuda b/ci/docker/Dockerfile.build.ubuntu_build_cuda
new file mode 100755
index 0000000..18c8af7
--- /dev/null
+++ b/ci/docker/Dockerfile.build.ubuntu_build_cuda
@@ -0,0 +1,57 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to build MXNet on Ubuntu 16.04 for GPU but on
+# a CPU-only instance. This restriction is caused by the CPP-
+# package generation, requiring the actual CUDA library to be
+# present
+
+FROM nvidia/cuda:8.0-cudnn5-devel
+
+ARG USER_ID=0
+
+WORKDIR /work/deps
+
+COPY install/ubuntu_core.sh /work/
+RUN /work/ubuntu_core.sh
+COPY install/ubuntu_python.sh /work/
+RUN /work/ubuntu_python.sh
+COPY install/ubuntu_scala.sh /work/
+RUN /work/ubuntu_scala.sh
+COPY install/ubuntu_r.sh /work/
+RUN /work/ubuntu_r.sh
+COPY install/ubuntu_perl.sh /work/
+RUN /work/ubuntu_perl.sh
+COPY install/ubuntu_lint.sh /work/
+RUN /work/ubuntu_lint.sh
+COPY install/ubuntu_clang.sh /work/
+RUN /work/ubuntu_clang.sh
+COPY install/ubuntu_mklml.sh /work/
+RUN /work/ubuntu_mklml.sh
+COPY install/ubuntu_adduser.sh /work/
+RUN /work/ubuntu_adduser.sh
+
+# Special case because the CPP-Package requires the CUDA runtime libs
+# and not only stubs..
+COPY install/ubuntu_nvidia.sh /work/
+RUN /work/ubuntu_nvidia.sh
+
+COPY runtime_functions.sh /work/
+
+WORKDIR /work/mxnet
+ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib
\ No newline at end of file
diff --git a/ci/docker/Dockerfile.build.ubuntu_cpu b/ci/docker/Dockerfile.build.ubuntu_cpu
new file mode 100755
index 0000000..f86c2f2
--- /dev/null
+++ b/ci/docker/Dockerfile.build.ubuntu_cpu
@@ -0,0 +1,53 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to build and run MXNet on Ubuntu 16.04 for CPU
+
+FROM ubuntu:16.04
+
+ARG USER_ID=0
+
+WORKDIR /work/deps
+
+COPY install/ubuntu_core.sh /work/
+RUN /work/ubuntu_core.sh
+COPY install/ubuntu_python.sh /work/
+RUN /work/ubuntu_python.sh
+COPY install/ubuntu_scala.sh /work/
+RUN /work/ubuntu_scala.sh
+COPY install/ubuntu_r.sh /work/
+RUN /work/ubuntu_r.sh
+COPY install/ubuntu_perl.sh /work/
+RUN /work/ubuntu_perl.sh
+COPY install/ubuntu_lint.sh /work/
+RUN /work/ubuntu_lint.sh
+COPY install/ubuntu_clang.sh /work/
+RUN /work/ubuntu_clang.sh
+COPY install/ubuntu_mklml.sh /work/
+RUN /work/ubuntu_mklml.sh
+COPY install/ubuntu_caffe.sh /work/
+RUN /work/ubuntu_caffe.sh
+COPY install/ubuntu_docs.sh /work/
+RUN /work/ubuntu_docs.sh
+COPY install/ubuntu_adduser.sh /work/
+RUN /work/ubuntu_adduser.sh
+
+COPY runtime_functions.sh /work/
+
+WORKDIR /work/mxnet
+ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib
diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu b/ci/docker/Dockerfile.build.ubuntu_gpu
new file mode 100755
index 0000000..826836c
--- /dev/null
+++ b/ci/docker/Dockerfile.build.ubuntu_gpu
@@ -0,0 +1,55 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to run MXNet on Ubuntu 16.04 for CPU
+
+FROM nvidia/cuda:8.0-cudnn5-devel
+
+ARG USER_ID=0
+
+WORKDIR /work/deps
+
+COPY install/ubuntu_core.sh /work/
+RUN /work/ubuntu_core.sh
+COPY install/ubuntu_python.sh /work/
+RUN /work/ubuntu_python.sh
+COPY install/ubuntu_scala.sh /work/
+RUN /work/ubuntu_scala.sh
+COPY install/ubuntu_r.sh /work/
+RUN /work/ubuntu_r.sh
+COPY install/ubuntu_perl.sh /work/
+RUN /work/ubuntu_perl.sh
+COPY install/ubuntu_lint.sh /work/
+RUN /work/ubuntu_lint.sh
+COPY install/ubuntu_clang.sh /work/
+RUN /work/ubuntu_clang.sh
+COPY install/ubuntu_mklml.sh /work/
+RUN /work/ubuntu_mklml.sh
+COPY install/ubuntu_tvm.sh /work/
+RUN /work/ubuntu_tvm.sh
+COPY install/ubuntu_llvm.sh /work/
+RUN /work/ubuntu_llvm.sh
+COPY install/ubuntu_caffe.sh /work/
+RUN /work/ubuntu_caffe.sh
+COPY install/ubuntu_adduser.sh /work/
+RUN /work/ubuntu_adduser.sh
+
+COPY runtime_functions.sh /work/
+
+WORKDIR /work/mxnet
+ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib
diff --git a/tests/ci_build/install/ubuntu_install_r.sh b/ci/docker/install/amzn_linux_core.sh
similarity index 59%
copy from tests/ci_build/install/ubuntu_install_r.sh
copy to ci/docker/install/amzn_linux_core.sh
index 6855b33..c13c969 100755
--- a/tests/ci_build/install/ubuntu_install_r.sh
+++ b/ci/docker/install/amzn_linux_core.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -16,15 +16,30 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-set -e
-set -x
-# install libraries for mxnet's r package on ubuntu
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
-echo "deb http://cran.rstudio.com/bin/linux/ubuntu trusty/" >> /etc/apt/sources.list
-gpg --keyserver keyserver.ubuntu.com --recv-key E084DAB9
-gpg -a --export E084DAB9 | apt-key add -
+set -ex
+pushd .
+yum install -y git
+yum install -y wget
+yum install -y sudo
+yum install -y re2c
+yum groupinstall -y 'Development Tools'
-apt-get update
-apt-get install -y r-base r-base-dev libxml2-dev libssl-dev libxt-dev
+# Ninja
+git clone --recursive https://github.com/ninja-build/ninja.git
+cd ninja
+./configure.py --bootstrap
+cp ninja /usr/local/bin
+popd
+# CMake
+pushd .
+git clone --recursive https://github.com/Kitware/CMake.git --branch v3.10.2
+cd CMake
+./bootstrap
+make -j$(nproc)
+make install
+popd
\ No newline at end of file
diff --git a/tests/ci_build/install/install_julia.sh b/ci/docker/install/amzn_linux_julia.sh
old mode 100644
new mode 100755
similarity index 77%
rename from tests/ci_build/install/install_julia.sh
rename to ci/docker/install/amzn_linux_julia.sh
index e36dfb5..bfaf3c4
--- a/tests/ci_build/install/install_julia.sh
+++ b/ci/docker/install/amzn_linux_julia.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -16,13 +16,14 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-set -e
-set -x
-wget https://julialang.s3.amazonaws.com/bin/linux/x64/0.5/julia-0.5.0-linux-x86_64.tar.gz
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
+
+set -ex
+wget -nv https://julialang.s3.amazonaws.com/bin/linux/x64/0.5/julia-0.5.0-linux-x86_64.tar.gz
mv julia-0.5.0-linux-x86_64.tar.gz /tmp/
tar xfvz /tmp/julia-0.5.0-linux-x86_64.tar.gz
rm -f /tmp/julia-0.5.0-linux-x86_64.tar.gz
-
# tar extracted in current directory
-ln -s -f ${PWD}/julia-3c9d75391c/bin/julia /usr/bin/julia
+ln -s -f ${PWD}/julia-3c9d75391c/bin/julia /usr/bin/julia
\ No newline at end of file
diff --git a/tests/ci_build/install/install_library.sh b/ci/docker/install/amzn_linux_library.sh
old mode 100644
new mode 100755
similarity index 81%
copy from tests/ci_build/install/install_library.sh
copy to ci/docker/install/amzn_linux_library.sh
index f3d7868..0470895
--- a/tests/ci_build/install/install_library.sh
+++ b/ci/docker/install/amzn_linux_library.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -16,10 +16,11 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-set -e
-set -x
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
-yum install graphviz
+set -ex
+yum -y install graphviz
pip install graphviz
-pip install opencv-python
+pip install opencv-python
\ No newline at end of file
diff --git a/tests/ci_build/install/install_maven.sh b/ci/docker/install/amzn_linux_maven.sh
old mode 100644
new mode 100755
similarity index 75%
copy from tests/ci_build/install/install_maven.sh
copy to ci/docker/install/amzn_linux_maven.sh
index f124186..22875d0
--- a/tests/ci_build/install/install_maven.sh
+++ b/ci/docker/install/amzn_linux_maven.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -16,11 +16,12 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-set -e
-set -x
-wget http://mirrors.ocf.berkeley.edu/apache/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
+
+set -ex
+wget -nv http://mirrors.ocf.berkeley.edu/apache/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz
mv apache-maven-3.3.9-bin.tar.gz /tmp/
tar xfvz /tmp/apache-maven-3.3.9-bin.tar.gz
-
-yum install -y java-1.8.0-openjdk-devel
+yum install -y java-1.8.0-openjdk-devel
\ No newline at end of file
diff --git a/tests/ci_build/install/install_openblas.sh b/ci/docker/install/amzn_linux_openblas.sh
old mode 100644
new mode 100755
similarity index 86%
copy from tests/ci_build/install/install_openblas.sh
copy to ci/docker/install/amzn_linux_openblas.sh
index 5c92823..94088d6
--- a/tests/ci_build/install/install_openblas.sh
+++ b/ci/docker/install/amzn_linux_openblas.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -16,10 +16,14 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-set -e
-set -x
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
+
+set -ex
+pushd .
git clone https://github.com/xianyi/OpenBLAS
cd OpenBLAS
make FC=gfortran -j $(($(nproc) + 1))
make PREFIX=/usr/local install
+popd
\ No newline at end of file
diff --git a/tests/ci_build/install/install_opencv.sh b/ci/docker/install/amzn_linux_opencv.sh
old mode 100644
new mode 100755
similarity index 70%
rename from tests/ci_build/install/install_opencv.sh
rename to ci/docker/install/amzn_linux_opencv.sh
index 9364a56..956407e
--- a/tests/ci_build/install/install_opencv.sh
+++ b/ci/docker/install/amzn_linux_opencv.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -16,15 +16,18 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-set -e
-set -x
-yum groupinstall -y "Development Tools"
-yum install -y cmake python27 python27-setuptools
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
+
+set -ex
+pushd .
+yum install -y python27 python27-setuptools
git clone https://github.com/opencv/opencv
cd opencv
mkdir -p build
cd build
-cmake -D BUILD_opencv_gpu=OFF -D WITH_EIGEN=ON -D WITH_TBB=ON -D WITH_CUDA=OFF -D WITH_1394=OFF -D CMAKE_BUILD_TYPE=RELEASE -D CMAKE_INSTALL_PREFIX=/usr/local ..
-make PREFIX=/usr/local install
-
+cmake -DBUILD_opencv_gpu=OFF -DWITH_EIGEN=ON -DWITH_TBB=ON -DWITH_CUDA=OFF -DWITH_1394=OFF \
+-DCMAKE_BUILD_TYPE=RELEASE -DCMAKE_INSTALL_PREFIX=/usr/local -GNinja ..
+ninja install
+popd
\ No newline at end of file
diff --git a/tests/ci_build/install/install_python2.sh b/ci/docker/install/amzn_linux_python2.sh
old mode 100644
new mode 100755
similarity index 83%
rename from tests/ci_build/install/install_python2.sh
rename to ci/docker/install/amzn_linux_python2.sh
index 5c7c453..e099ad6
--- a/tests/ci_build/install/install_python2.sh
+++ b/ci/docker/install/amzn_linux_python2.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -16,20 +16,21 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-set -e
-set -x
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
+
+set -ex
yum groupinstall -y "Development Tools"
yum install -y mlocate python27 python27-setuptools python27-tools python27-numpy python27-scipy python27-nose python27-matplotlib unzip
ln -s -f /usr/bin/python2.7 /usr/bin/python2
-wget https://bootstrap.pypa.io/get-pip.py
+wget -nv https://bootstrap.pypa.io/get-pip.py
python2 get-pip.py
$(which easy_install-2.7) --upgrade pip
if [ -f /usr/local/bin/pip ] && [ -f /usr/bin/pip ]; then
- mv /usr/bin/pip /usr/bin/pip.bak
- ln /usr/local/bin/pip /usr/bin/pip
+ mv /usr/bin/pip /usr/bin/pip.bak
+ ln /usr/local/bin/pip /usr/bin/pip
fi
ln -s -f /usr/local/bin/pip /usr/bin/pip
for i in ipython[all] jupyter pandas scikit-image h5py pandas sklearn sympy; do echo "${i}..."; pip install -U $i >/dev/null; done
-
diff --git a/tests/ci_build/install/install_python3.sh b/ci/docker/install/amzn_linux_python3.sh
old mode 100644
new mode 100755
similarity index 78%
rename from tests/ci_build/install/install_python3.sh
rename to ci/docker/install/amzn_linux_python3.sh
index 0df12eb..3f80d7d
--- a/tests/ci_build/install/install_python3.sh
+++ b/ci/docker/install/amzn_linux_python3.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -16,18 +16,21 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-set -e
-set -x
-wget https://bootstrap.pypa.io/get-pip.py || exit 1
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
+
+set -ex
+pushd .
+wget -nv https://bootstrap.pypa.io/get-pip.py
mkdir py3
cd py3
-wget https://www.python.org/ftp/python/3.5.2/Python-3.5.2.tgz || exit 1
+wget -nv https://www.python.org/ftp/python/3.5.2/Python-3.5.2.tgz
tar -xvzf Python-3.5.2.tgz
cd Python-3.5.2
yum install -y zlib-devel openssl-devel sqlite-devel bzip2-devel gdbm-devel ncurses-devel xz-devel readline-devel
./configure --prefix=/opt/ --with-zlib-dir=/usr/lib64
-make || exit 1
+make -j$(nproc)
mkdir /opt/bin
mkdir /opt/lib
make install
@@ -36,5 +39,6 @@ cd ../..
python3 get-pip.py
ln -s -f /opt/bin/pip /usr/bin/pip3
-mkdir -p /home/jenkins/.local/lib/python3.5/site-packages/
+mkdir -p ~/.local/lib/python3.5/site-packages/
pip3 install numpy
+popd
\ No newline at end of file
diff --git a/tests/ci_build/install/install_testdeps.sh b/ci/docker/install/amzn_linux_testdeps.sh
old mode 100644
new mode 100755
similarity index 82%
copy from tests/ci_build/install/install_testdeps.sh
copy to ci/docker/install/amzn_linux_testdeps.sh
index 26da186..f5c49d9
--- a/tests/ci_build/install/install_testdeps.sh
+++ b/ci/docker/install/amzn_linux_testdeps.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -16,11 +16,12 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-set -e
-set -x
-pip install cpplint 'pylint==1.4.4' 'astroid==1.3.6'
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
+set -ex
+pip install cpplint 'pylint==1.4.4' 'astroid==1.3.6'
pip3 install nose
ln -s -f /opt/bin/nosetests /usr/local/bin/nosetests3
-ln -s -f /opt/bin/nosetests-3.4 /usr/local/bin/nosetests-3.4
+ln -s -f /opt/bin/nosetests-3.4 /usr/local/bin/nosetests-3.4
\ No newline at end of file
diff --git a/tests/ci_build/install/ubuntu_install_core.sh b/ci/docker/install/android_arm64_ndk.sh
similarity index 59%
copy from tests/ci_build/install/ubuntu_install_core.sh
copy to ci/docker/install/android_arm64_ndk.sh
index 74fb05a..23d9ea2 100755
--- a/tests/ci_build/install/ubuntu_install_core.sh
+++ b/ci/docker/install/android_arm64_ndk.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -16,14 +16,20 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-set -e
-set -x
-# install libraries for building mxnet c++ core on ubuntu
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
-apt-get update && apt-get install -y \
- build-essential git libopenblas-dev liblapack-dev libopencv-dev \
- libcurl4-openssl-dev cmake wget unzip sudo ninja-build
+set -ex
+pushd .
+export ANDROID_NDK_REVISION=15c
+curl -O https://dl.google.com/android/repository/android-ndk-r${ANDROID_NDK_REVISION}-linux-x86_64.zip && \
+unzip ./android-ndk-r${ANDROID_NDK_REVISION}-linux-x86_64.zip && \
+cd android-ndk-r${ANDROID_NDK_REVISION} && \
+./build/tools/make_standalone_toolchain.py \
+ --stl=libc++ \
+ --arch arm64 \
+ --api 21 \
+ --install-dir=${CROSS_ROOT} && \
-# Link Openblas to Cblas as this link does not exist on ubuntu16.04
-ln -s /usr/lib/libopenblas.so /usr/lib/libcblas.so
+popd
\ No newline at end of file
diff --git a/tests/ci_build/install/install_openblas.sh b/ci/docker/install/android_arm64_openblas.sh
old mode 100644
new mode 100755
similarity index 66%
copy from tests/ci_build/install/install_openblas.sh
copy to ci/docker/install/android_arm64_openblas.sh
index 5c92823..87e0bf9
--- a/tests/ci_build/install/install_openblas.sh
+++ b/ci/docker/install/android_arm64_openblas.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -16,10 +16,17 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-set -e
-set -x
-git clone https://github.com/xianyi/OpenBLAS
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
+
+set -ex
+pushd .
+git clone https://github.com/xianyi/OpenBLAS.git
cd OpenBLAS
-make FC=gfortran -j $(($(nproc) + 1))
-make PREFIX=/usr/local install
+make -j$(nproc) TARGET=ARMV8 ARM_SOFTFP_ABI=1 HOSTCC=gcc NOFORTRAN=1 libs
+# Can't be run (utility not compiled for the target platform)
+#make install
+cp *.h /usr/include
+cp libopenblas.a /usr/local/lib
+popd
diff --git a/tests/ci_build/install/install_openblas.sh b/ci/docker/install/arm64_openblas.sh
old mode 100644
new mode 100755
similarity index 58%
rename from tests/ci_build/install/install_openblas.sh
rename to ci/docker/install/arm64_openblas.sh
index 5c92823..3151a4b
--- a/tests/ci_build/install/install_openblas.sh
+++ b/ci/docker/install/arm64_openblas.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -16,10 +16,20 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-set -e
-set -x
-git clone https://github.com/xianyi/OpenBLAS
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
+
+set -ex
+pushd .
+wget -nv https://api.github.com/repos/xianyi/OpenBLAS/git/refs/heads/master -O openblas_version.json
+echo "Using openblas:"
+cat openblas_version.json
+git clone https://github.com/xianyi/OpenBLAS.git
cd OpenBLAS
-make FC=gfortran -j $(($(nproc) + 1))
-make PREFIX=/usr/local install
+make -j$(nproc) TARGET=ARMV8
+make install
+ln -s /opt/OpenBLAS/lib/libopenblas.so /usr/lib/libopenblas.so
+ln -s /opt/OpenBLAS/lib/libopenblas.a /usr/lib/libopenblas.a
+ln -s /opt/OpenBLAS/lib/libopenblas.a /usr/lib/liblapack.a
+popd
\ No newline at end of file
diff --git a/tests/ci_build/install/ubuntu_install_r.sh b/ci/docker/install/centos7_adduser.sh
similarity index 56%
copy from tests/ci_build/install/ubuntu_install_r.sh
copy to ci/docker/install/centos7_adduser.sh
index 6855b33..7ed64c2 100755
--- a/tests/ci_build/install/ubuntu_install_r.sh
+++ b/ci/docker/install/centos7_adduser.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -16,15 +16,22 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-set -e
-set -x
-# install libraries for mxnet's r package on ubuntu
+# Add user in order to make sure the assumed user the container is running under
+# actually exists inside the container to avoid problems like missing home dir
-echo "deb http://cran.rstudio.com/bin/linux/ubuntu trusty/" >> /etc/apt/sources.list
-gpg --keyserver keyserver.ubuntu.com --recv-key E084DAB9
-gpg -a --export E084DAB9 | apt-key add -
-apt-get update
-apt-get install -y r-base r-base-dev libxml2-dev libssl-dev libxt-dev
+set -ex
+# $USER_ID is coming from build.py:build_docker passed as --build-arg
+if [[ "$USER_ID" -gt 0 ]]
+then
+ # -no-log-init required due to https://github.com/moby/moby/issues/5419
+ useradd -m --no-log-init --uid $USER_ID --system jenkins_slave
+ usermod -aG wheel jenkins_slave
+
+ # By default, docker creates all WORK_DIRs with root owner
+ mkdir /work/mxnet
+ mkdir /work/build
+ chown -R jenkins_slave /work/
+fi
diff --git a/tests/ci_build/install/ubuntu_install_core.sh b/ci/docker/install/centos7_core.sh
similarity index 59%
copy from tests/ci_build/install/ubuntu_install_core.sh
copy to ci/docker/install/centos7_core.sh
index 74fb05a..1688b81 100755
--- a/tests/ci_build/install/ubuntu_install_core.sh
+++ b/ci/docker/install/centos7_core.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -16,14 +16,24 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-set -e
-set -x
-# install libraries for building mxnet c++ core on ubuntu
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
-apt-get update && apt-get install -y \
- build-essential git libopenblas-dev liblapack-dev libopencv-dev \
- libcurl4-openssl-dev cmake wget unzip sudo ninja-build
+set -ex
-# Link Openblas to Cblas as this link does not exist on ubuntu16.04
-ln -s /usr/lib/libopenblas.so /usr/lib/libcblas.so
+# Multipackage installation does not fail in yum
+yum -y install epel-release
+yum -y install git
+yum -y install wget
+yum -y install atlas-devel # Provide clbas headerfiles
+yum -y install openblas-devel
+yum -y install lapack-devel
+yum -y install opencv-devel
+yum -y install openssl-devel
+yum -y install gcc-c++
+yum -y install make
+yum -y install cmake
+yum -y install wget
+yum -y install unzip
+yum -y install ninja-build
\ No newline at end of file
diff --git a/tests/ci_build/install/ubuntu_install_python.sh b/ci/docker/install/centos7_python.sh
similarity index 65%
copy from tests/ci_build/install/ubuntu_install_python.sh
copy to ci/docker/install/centos7_python.sh
index 0ba9800..9e076b6 100755
--- a/tests/ci_build/install/ubuntu_install_python.sh
+++ b/ci/docker/install/centos7_python.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -16,14 +16,20 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-set -e
-set -x
-# install libraries for mxnet's python package on ubuntu
-apt-get update && apt-get install -y python-dev python3-dev
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
-# the version of the pip shipped with ubuntu may be too lower, install a recent version here
-cd /tmp && wget https://bootstrap.pypa.io/get-pip.py && python3 get-pip.py && python2 get-pip.py
+set -ex
+
+ # Python 2.7 is installed by default, install 3.6 on top
+yum -y install https://centos7.iuscommunity.org/ius-release.rpm
+yum -y install python36u
+
+# Install PIP
+curl "https://bootstrap.pypa.io/get-pip.py" -o "get-pip.py"
+python2.7 get-pip.py
+python3.6 get-pip.py
pip2 install nose pylint numpy nose-timer requests h5py scipy
-pip3 install nose pylint numpy nose-timer requests h5py scipy
+pip3 install nose pylint numpy nose-timer requests h5py scipy
\ No newline at end of file
diff --git a/tests/ci_build/install/ubuntu_install_r.sh b/ci/docker/install/ubuntu_adduser.sh
similarity index 56%
copy from tests/ci_build/install/ubuntu_install_r.sh
copy to ci/docker/install/ubuntu_adduser.sh
index 6855b33..fd9a3f8 100755
--- a/tests/ci_build/install/ubuntu_install_r.sh
+++ b/ci/docker/install/ubuntu_adduser.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -16,15 +16,22 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-set -e
-set -x
-# install libraries for mxnet's r package on ubuntu
+# Add user in order to make sure the assumed user the container is running under
+# actually exists inside the container to avoid problems like missing home dir
-echo "deb http://cran.rstudio.com/bin/linux/ubuntu trusty/" >> /etc/apt/sources.list
-gpg --keyserver keyserver.ubuntu.com --recv-key E084DAB9
-gpg -a --export E084DAB9 | apt-key add -
-apt-get update
-apt-get install -y r-base r-base-dev libxml2-dev libssl-dev libxt-dev
+set -ex
+# $USER_ID is coming from build.py:build_docker passed as --build-arg
+if [[ "$USER_ID" -gt 0 ]]
+then
+ # -no-log-init required due to https://github.com/moby/moby/issues/5419
+ useradd -m --no-log-init --uid $USER_ID --system jenkins_slave
+ usermod -aG sudo jenkins_slave
+
+ # By default, docker creates all WORK_DIRs with root owner
+ mkdir /work/mxnet
+ mkdir /work/build
+ chown -R jenkins_slave /work/
+fi
diff --git a/ci/docker/install/ubuntu_caffe.sh b/ci/docker/install/ubuntu_caffe.sh
new file mode 100755
index 0000000..50e257b
--- /dev/null
+++ b/ci/docker/install/ubuntu_caffe.sh
@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+set -ex
+
+apt-get install -y libprotobuf-dev libleveldb-dev \
+ libsnappy-dev libopencv-dev libhdf5-serial-dev protobuf-compiler \
+ libatlas-base-dev python-dev libgflags-dev libgoogle-glog-dev liblmdb-dev \
+ python-numpy python-opencv
+
+apt-get install -y --no-install-recommends libboost-all-dev
+
+cd /work/deps
+git clone http://github.com/BVLC/caffe.git
+
+cd caffe
+cp Makefile.config.example Makefile.config
+
+echo "CPU_ONLY := 1" >> Makefile.config
+
+# Fixes https://github.com/BVLC/caffe/issues/5658 See https://github.com/intel/caffe/wiki/Ubuntu-16.04-or-15.10-Installation-Guide
+echo "INCLUDE_DIRS += /usr/lib /usr/lib/x86_64-linux-gnu /usr/include/hdf5/serial/ " >> Makefile.config
+echo "LIBRARY_DIRS += /usr/lib /usr/lib/x86_64-linux-gnu /usr/lib/x86_64-linux-gnu/hdf5/serial " >> Makefile.config
+
+# Fixes https://github.com/BVLC/caffe/issues/4333 See https://github.com/intel/caffe/wiki/Ubuntu-16.04-or-15.10-Installation-Guide
+# Note: This is only valid on Ubuntu16.04 - the version numbers are bound to the distribution
+ln -s /usr/lib/x86_64-linux-gnu/libhdf5_serial.so.10.0.2 /usr/lib/x86_64-linux-gnu/libhdf5.so
+ln -s /usr/lib/x86_64-linux-gnu/libhdf5_serial_hl.so.10.0.2 /usr/lib/x86_64-linux-gnu/libhdf5_hl.so
+
+make all pycaffe -j$(nproc)
+
+cd python
+for req in $(cat requirements.txt); do pip2 install $req; done
+
diff --git a/tests/ci_build/install/install_maven.sh b/ci/docker/install/ubuntu_clang.sh
old mode 100644
new mode 100755
similarity index 56%
rename from tests/ci_build/install/install_maven.sh
rename to ci/docker/install/ubuntu_clang.sh
index f124186..08b95bc
--- a/tests/ci_build/install/install_maven.sh
+++ b/ci/docker/install/ubuntu_clang.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -16,11 +16,16 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-set -e
-set -x
-wget http://mirrors.ocf.berkeley.edu/apache/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz
-mv apache-maven-3.3.9-bin.tar.gz /tmp/
-tar xfvz /tmp/apache-maven-3.3.9-bin.tar.gz
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
-yum install -y java-1.8.0-openjdk-devel
+set -ex
+# Install clang 3.9 (the same version as in XCode 8.*) and 5.0 (latest major release)
+wget -O - http://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
+ apt-add-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-3.9 main" && \
+ apt-add-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-5.0 main" && \
+ apt-get update && \
+ apt-get install -y clang-3.9 clang-5.0 && \
+ clang-3.9 --version && \
+ clang-5.0 --version
\ No newline at end of file
diff --git a/tests/ci_build/install/ubuntu_install_core.sh b/ci/docker/install/ubuntu_core.sh
similarity index 67%
copy from tests/ci_build/install/ubuntu_install_core.sh
copy to ci/docker/install/ubuntu_core.sh
index 74fb05a..e78f29a 100755
--- a/tests/ci_build/install/ubuntu_install_core.sh
+++ b/ci/docker/install/ubuntu_core.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -16,14 +16,27 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-set -e
-set -x
-# install libraries for building mxnet c++ core on ubuntu
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
-apt-get update && apt-get install -y \
- build-essential git libopenblas-dev liblapack-dev libopencv-dev \
- libcurl4-openssl-dev cmake wget unzip sudo ninja-build
+set -ex
+apt-get update
+apt-get install -y \
+ build-essential \
+ git \
+ libopenblas-dev \
+ liblapack-dev \
+ libopencv-dev \
+ libcurl4-openssl-dev \
+ cmake \
+ wget \
+ unzip \
+ sudo \
+ software-properties-common \
+ ninja-build \
+ python-pip
# Link Openblas to Cblas as this link does not exist on ubuntu16.04
ln -s /usr/lib/libopenblas.so /usr/lib/libcblas.so
+pip install cpplint==1.3.0 pylint==1.8.2
\ No newline at end of file
diff --git a/tests/ci_build/install/ubuntu_install_core.sh b/ci/docker/install/ubuntu_docs.sh
similarity index 65%
rename from tests/ci_build/install/ubuntu_install_core.sh
rename to ci/docker/install/ubuntu_docs.sh
index 74fb05a..bb8026f 100755
--- a/tests/ci_build/install/ubuntu_install_core.sh
+++ b/ci/docker/install/ubuntu_docs.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -16,14 +16,13 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-set -e
-set -x
-# install libraries for building mxnet c++ core on ubuntu
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
-apt-get update && apt-get install -y \
- build-essential git libopenblas-dev liblapack-dev libopencv-dev \
- libcurl4-openssl-dev cmake wget unzip sudo ninja-build
+set -ex
+wget http://downloads.lightbend.com/scala/2.11.8/scala-2.11.8.deb && \
+ dpkg -i scala-2.11.8.deb && rm scala-2.11.8.deb
-# Link Openblas to Cblas as this link does not exist on ubuntu16.04
-ln -s /usr/lib/libopenblas.so /usr/lib/libcblas.so
+apt-get install -y doxygen libatlas-base-dev graphviz pandoc
+pip install sphinx==1.3.5 CommonMark==0.5.4 breathe mock recommonmark pypandoc beautifulsoup4
diff --git a/tests/ci_build/install/install_library.sh b/ci/docker/install/ubuntu_lint.sh
old mode 100644
new mode 100755
similarity index 81%
rename from tests/ci_build/install/install_library.sh
rename to ci/docker/install/ubuntu_lint.sh
index f3d7868..f3a1d6b
--- a/tests/ci_build/install/install_library.sh
+++ b/ci/docker/install/ubuntu_lint.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -16,10 +16,10 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-set -e
-set -x
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
-yum install graphviz
-pip install graphviz
-pip install opencv-python
+set -ex
+apt-get update
+apt-get install -y python-pip sudo
\ No newline at end of file
diff --git a/tests/ci_build/install/ubuntu_install_llvm.sh b/ci/docker/install/ubuntu_llvm.sh
similarity index 94%
rename from tests/ci_build/install/ubuntu_install_llvm.sh
rename to ci/docker/install/ubuntu_llvm.sh
index d3282e7..09e13d3 100755
--- a/tests/ci_build/install/ubuntu_install_llvm.sh
+++ b/ci/docker/install/ubuntu_llvm.sh
@@ -17,12 +17,10 @@
# specific language governing permissions and limitations
# under the License.
-
-
echo deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-5.0 main\
>> /etc/apt/sources.list.d/llvm.list
echo deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial-5.0 main\
>> /etc/apt/sources.list.d/llvm.list
wget -O - http://apt.llvm.org/llvm-snapshot.gpg.key|sudo apt-key add -
-apt-get update && apt-get install -y --force-yes llvm-5.0
+apt-get update && apt-get install -y --force-yes llvm-5.0
\ No newline at end of file
diff --git a/tests/ci_build/install/install_testdeps.sh b/ci/docker/install/ubuntu_mklml.sh
old mode 100644
new mode 100755
similarity index 69%
rename from tests/ci_build/install/install_testdeps.sh
rename to ci/docker/install/ubuntu_mklml.sh
index 26da186..4f68fa7
--- a/tests/ci_build/install/install_testdeps.sh
+++ b/ci/docker/install/ubuntu_mklml.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -16,11 +16,10 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-set -e
-set -x
-pip install cpplint 'pylint==1.4.4' 'astroid==1.3.6'
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
-pip3 install nose
-ln -s -f /opt/bin/nosetests /usr/local/bin/nosetests3
-ln -s -f /opt/bin/nosetests-3.4 /usr/local/bin/nosetests-3.4
+set -ex
+wget --no-check-certificate -O /tmp/mklml.tgz https://github.com/01org/mkl-dnn/releases/download/v0.12/mklml_lnx_2018.0.1.20171227.tgz
+tar -zxvf /tmp/mklml.tgz && cp -rf mklml_*/* /usr/local/ && rm -rf mklml_*
\ No newline at end of file
diff --git a/tests/ci_build/install/ubuntu_install_nvidia.sh b/ci/docker/install/ubuntu_nvidia.sh
similarity index 74%
rename from tests/ci_build/install/ubuntu_install_nvidia.sh
rename to ci/docker/install/ubuntu_nvidia.sh
index d349809..bb1c73e 100755
--- a/tests/ci_build/install/ubuntu_install_nvidia.sh
+++ b/ci/docker/install/ubuntu_nvidia.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -17,17 +17,13 @@
# specific language governing permissions and limitations
# under the License.
-# install nvidia libraries to compile and run CUDA without
-# the necessity of nvidia-docker and a GPU
-set -e
-set -x
-
-# Needed to run add-apt-repository
-apt update && apt install -y software-properties-common
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
+set -ex
+apt install -y software-properties-common
add-apt-repository -y ppa:graphics-drivers
-
# Retrieve ppa:graphics-drivers and install nvidia-drivers.
# Note: DEBIAN_FRONTEND required to skip the interactive setup steps
-apt update && \
- DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends cuda-8-0
+apt update
+DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends cuda-8-0
\ No newline at end of file
diff --git a/tests/ci_build/install/ubuntu_install_perl.sh b/ci/docker/install/ubuntu_perl.sh
similarity index 81%
rename from tests/ci_build/install/ubuntu_install_perl.sh
rename to ci/docker/install/ubuntu_perl.sh
index bb0fa9d..52bd010 100755
--- a/tests/ci_build/install/ubuntu_install_perl.sh
+++ b/ci/docker/install/ubuntu_perl.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -16,9 +16,11 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-set -e
-set -x
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
+
+set -ex
# install libraries for mxnet's perl package on ubuntu
-apt-get update && apt-get install -y libmouse-perl pdl cpanminus swig libgraphviz-perl
+apt-get install -y libmouse-perl pdl cpanminus swig libgraphviz-perl
cpanm -q Function::Parameters Hash::Ordered PDL::CCS
diff --git a/tests/ci_build/install/ubuntu_install_python.sh b/ci/docker/install/ubuntu_python.sh
similarity index 75%
rename from tests/ci_build/install/ubuntu_install_python.sh
rename to ci/docker/install/ubuntu_python.sh
index 0ba9800..b19448e 100755
--- a/tests/ci_build/install/ubuntu_install_python.sh
+++ b/ci/docker/install/ubuntu_python.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -16,14 +16,18 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-set -e
-set -x
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
+
+set -ex
# install libraries for mxnet's python package on ubuntu
-apt-get update && apt-get install -y python-dev python3-dev
+apt-get install -y python-dev python3-dev virtualenv
# the version of the pip shipped with ubuntu may be too lower, install a recent version here
-cd /tmp && wget https://bootstrap.pypa.io/get-pip.py && python3 get-pip.py && python2 get-pip.py
+wget -nv https://bootstrap.pypa.io/get-pip.py
+python3 get-pip.py
+python2 get-pip.py
pip2 install nose pylint numpy nose-timer requests h5py scipy
-pip3 install nose pylint numpy nose-timer requests h5py scipy
+pip3 install nose pylint numpy nose-timer requests h5py scipy
\ No newline at end of file
diff --git a/tests/ci_build/install/ubuntu_install_r.sh b/ci/docker/install/ubuntu_r.sh
similarity index 89%
rename from tests/ci_build/install/ubuntu_install_r.sh
rename to ci/docker/install/ubuntu_r.sh
index 6855b33..e04e94d 100755
--- a/tests/ci_build/install/ubuntu_install_r.sh
+++ b/ci/docker/install/ubuntu_r.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -16,15 +16,15 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-set -e
-set -x
-# install libraries for mxnet's r package on ubuntu
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
+set -ex
+# install libraries for mxnet's r package on ubuntu
echo "deb http://cran.rstudio.com/bin/linux/ubuntu trusty/" >> /etc/apt/sources.list
gpg --keyserver keyserver.ubuntu.com --recv-key E084DAB9
gpg -a --export E084DAB9 | apt-key add -
apt-get update
-apt-get install -y r-base r-base-dev libxml2-dev libssl-dev libxt-dev
-
+apt-get install -y r-base r-base-dev libxml2-dev libssl-dev libxt-dev
\ No newline at end of file
diff --git a/tests/ci_build/install/ubuntu_install_scala.sh b/ci/docker/install/ubuntu_scala.sh
similarity index 86%
rename from tests/ci_build/install/ubuntu_install_scala.sh
rename to ci/docker/install/ubuntu_scala.sh
index 7dd343a..ee5554d 100755
--- a/tests/ci_build/install/ubuntu_install_scala.sh
+++ b/ci/docker/install/ubuntu_scala.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -16,15 +16,16 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-set -e
-set -x
-# install libraries for mxnet's scala package on ubuntu
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
+set -ex
+# install libraries for mxnet's scala package on ubuntu
apt-get install -y software-properties-common
add-apt-repository -y ppa:webupd8team/java
apt-get update
echo "oracle-java8-installer shared/accepted-oracle-license-v1-1 select true" | debconf-set-selections
apt-get install -y oracle-java8-installer
apt-get install -y oracle-java8-set-default
-apt-get update && apt-get install -y maven
+apt-get update && apt-get install -y maven
\ No newline at end of file
diff --git a/tests/ci_build/install/ubuntu_install_tvm.sh b/ci/docker/install/ubuntu_tvm.sh
similarity index 98%
rename from tests/ci_build/install/ubuntu_install_tvm.sh
rename to ci/docker/install/ubuntu_tvm.sh
index 2729c7f..9ab359b 100755
--- a/tests/ci_build/install/ubuntu_install_tvm.sh
+++ b/ci/docker/install/ubuntu_tvm.sh
@@ -33,7 +33,7 @@ echo LLVM_CONFIG=llvm-config-5.0 >> config.mk
echo USE_RPC=1 >> config.mk
echo USE_GRAPH_RUNTIME=1 >> config.mk
echo CUDA_PATH=/usr/local/cuda >> config.mk
-make -j`nproc`
+make -j$(nproc)
cd python
python setup.py install
@@ -41,4 +41,4 @@ cd -
cd topi/python
python setup.py install
-cd -
+cd -
\ No newline at end of file
diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
new file mode 100755
index 0000000..89ea44f
--- /dev/null
+++ b/ci/docker/runtime_functions.sh
@@ -0,0 +1,471 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
+
+set -ex
+
+clean_repo() {
+ set -ex
+ git clean -xfd
+ git submodule foreach --recursive git clean -xfd
+ git reset --hard
+ git submodule foreach --recursive git reset --hard
+ git submodule update --init --recursive
+}
+
+
+# Build commands: Every platform in docker/Dockerfile.build.<platform> should have a corresponding
+# function here with the same suffix:
+
+build_jetson() {
+ set -ex
+ pushd .
+
+ #cd /work/mxnet
+ #make -j$(nproc) USE_OPENCV=0 USE_BLAS=openblas USE_SSE=0 USE_CUDA=1 USE_CUDNN=1 ENABLE_CUDA_RTC=0 USE_NCCL=0 USE_CUDA_PATH=/usr/local/cuda/
+ cd /work/build
+ cmake\
+ -DUSE_CUDA=OFF\
+ -DUSE_OPENCV=OFF\
+ -DUSE_OPENMP=ON\
+ -DUSE_SIGNAL_HANDLER=ON\
+ -DUSE_MKL_IF_AVAILABLE=OFF\
+ -DUSE_LAPACK=OFF\
+ -DCMAKE_BUILD_TYPE=RelWithDebInfo\
+ -G Ninja /work/mxnet
+ ninja
+ export MXNET_LIBRARY_PATH=`pwd`/libmxnet.so
+ cd /work/mxnet/python
+ python setup.py bdist_wheel --universal
+
+
+ # Fix pathing issues in the wheel. We need to move libmxnet.so from the data folder to the root
+ # of the wheel, then repackage the wheel.
+ # Create a temp dir to do the work.
+ # TODO: move apt call to install
+ WHEEL=`readlink -f dist/*.whl`
+ TMPDIR=`mktemp -d`
+ unzip -d $TMPDIR $WHEEL
+ rm $WHEEL
+ cd $TMPDIR
+ mv *.data/data/mxnet/libmxnet.so mxnet
+ zip -r $WHEEL $TMPDIR
+ cp $WHEEL /work/build
+ rm -rf $TMPDIR
+ popd
+}
+
+build_armv7() {
+ set -ex
+ pushd .
+ cd /work/build
+ cmake\
+ -DUSE_CUDA=OFF\
+ -DUSE_OPENCV=OFF\
+ -DUSE_OPENMP=OFF\
+ -DUSE_SIGNAL_HANDLER=ON\
+ -DCMAKE_BUILD_TYPE=RelWithDebInfo\
+ -DUSE_MKL_IF_AVAILABLE=OFF\
+ -G Ninja /work/mxnet
+ ninja
+ export MXNET_LIBRARY_PATH=`pwd`/libmxnet.so
+ cd /work/mxnet/python
+ python setup.py bdist_wheel --universal
+ cp dist/*.whl /work/build
+ popd
+}
+
+build_amzn_linux_cpu() {
+ cd /work/build
+ cmake\
+ -DUSE_CUDA=OFF\
+ -DUSE_OPENCV=ON\
+ -DUSE_OPENMP=ON\
+ -DUSE_SIGNAL_HANDLER=ON\
+ -DCMAKE_BUILD_TYPE=RelWithDebInfo\
+ -DUSE_MKL_IF_AVAILABLE=OFF\
+ -DUSE_LAPACK=OFF\
+ -G Ninja /work/mxnet
+ ninja
+ export MXNET_LIBRARY_PATH=`pwd`/libmxnet.so
+}
+
+build_arm64() {
+ cmake\
+ -DUSE_CUDA=OFF\
+ -DUSE_OPENCV=OFF\
+ -DUSE_OPENMP=OFF\
+ -DUSE_SIGNAL_HANDLER=ON\
+ -DCMAKE_BUILD_TYPE=RelWithDebInfo\
+ -DUSE_MKL_IF_AVAILABLE=OFF\
+ -G Ninja /work/mxnet
+ ninja
+ export MXNET_LIBRARY_PATH=`pwd`/libmxnet.so
+ cd /work/mxnet/python
+ python setup.py bdist_wheel --universal
+ cp dist/*.whl /work/build
+}
+
+build_android_arm64() {
+ set -ex
+ cd /work/build
+ cmake\
+ -DUSE_CUDA=OFF\
+ -DUSE_SSE=OFF\
+ -DUSE_LAPACK=OFF\
+ -DUSE_OPENCV=OFF\
+ -DUSE_OPENMP=OFF\
+ -DUSE_SIGNAL_HANDLER=ON\
+ -DCMAKE_BUILD_TYPE=RelWithDebInfo\
+ -DUSE_MKL_IF_AVAILABLE=OFF\
+ -G Ninja /work/mxnet
+ ninja
+ export MXNET_LIBRARY_PATH=`pwd`/libmxnet.so
+ cd /work/mxnet/python
+ python setup.py bdist_wheel --universal
+ cp dist/*.whl /work/build
+}
+
+build_centos7_cpu() {
+ set -ex
+ cd /work/mxnet
+ make \
+ DEV=1 \
+ USE_LAPACK=1 \
+ USE_LAPACK_PATH=/usr/lib64/liblapack.so \
+ USE_PROFILER=1 \
+ USE_BLAS=openblas \
+ -j$(nproc)
+}
+
+build_centos7_gpu() {
+ set -ex
+ cd /work/mxnet
+ make \
+ DEV=1 \
+ USE_LAPACK=1 \
+ USE_LAPACK_PATH=/usr/lib64/liblapack.so \
+ USE_PROFILER=1 \
+ USE_BLAS=openblas \
+ USE_CUDA=1 \
+ USE_CUDA_PATH=/usr/local/cuda \
+ USE_CUDNN=1 \
+ -j$(nproc)
+}
+
+build_ubuntu_cpu_openblas() {
+ set -ex
+ make \
+ DEV=1 \
+ USE_PROFILER=1 \
+ USE_CPP_PACKAGE=1 \
+ USE_BLAS=openblas \
+ -j$(nproc)
+}
+
+build_ubuntu_cpu_clang39() {
+ set -ex
+ make \
+ USE_PROFILER=1 \
+ USE_CPP_PACKAGE=1 \
+ USE_BLAS=openblas \
+ USE_OPENMP=0 \
+ CXX=clang++-3.9 \
+ CC=clang-3.9 \
+ -j$(nproc)
+}
+
+build_ubuntu_cpu_clang50() {
+ set -ex
+ make \
+ USE_PROFILER=1 \
+ USE_CPP_PACKAGE=1 \
+ USE_BLAS=openblas \
+ USE_OPENMP=1 \
+ CXX=clang++-5.0 \
+ CC=clang-5.0 \
+ -j$(nproc)
+}
+
+build_ubuntu_cpu_mkldnn() {
+ set -ex
+ make \
+ DEV=1 \
+ USE_PROFILER=1 \
+ USE_CPP_PACKAGE=1 \
+ USE_BLAS=openblas \
+ USE_MKLDNN=1 \
+ -j$(nproc)
+}
+
+build_ubuntu_gpu_mkldnn() {
+ set -ex
+ make \
+ DEV=1 \
+ USE_PROFILER=1 \
+ USE_CPP_PACKAGE=1 \
+ USE_BLAS=openblas \
+ USE_MKLDNN=1 \
+ USE_CUDA=1 \
+ USE_CUDA_PATH=/usr/local/cuda \
+ USE_CUDNN=1 \
+ -j$(nproc)
+}
+
+build_ubuntu_gpu_cuda8_cudnn5() {
+ set -ex
+ make \
+ DEV=1 \
+ USE_PROFILER=1 \
+ USE_BLAS=openblas \
+ USE_CUDA=1 \
+ USE_CUDA_PATH=/usr/local/cuda \
+ USE_CUDNN=1 \
+ USE_CPP_PACKAGE=1 \
+ -j$(nproc)
+}
+
+build_ubuntu_amalgamation() {
+ set -ex
+ # Amalgamation can not be run with -j nproc
+ make -C amalgamation/ clean
+ make -C amalgamation/ USE_BLAS=openblas
+}
+
+build_ubuntu_amalgamation_min() {
+ set -ex
+ # Amalgamation can not be run with -j nproc
+ make -C amalgamation/ clean
+ make -C amalgamation/ USE_BLAS=openblas MIN=1
+}
+
+build_ubuntu_gpu_cmake_mkldnn() {
+ set -ex
+ cd /work/build
+ cmake \
+ -DUSE_CUDA=1 \
+ -DUSE_CUDNN=1 \
+ -DUSE_MKLML_MKL=1 \
+ -DUSE_MKLDNN=1 \
+ -DCMAKE_BUILD_TYPE=Release \
+ -G Ninja \
+ /work/mxnet
+
+ ninja -v
+}
+
+build_ubuntu_gpu_cmake() {
+ set -ex
+ cd /work/build
+ cmake \
+ -DUSE_CUDA=1 \
+ -DUSE_CUDNN=1 \
+ -DUSE_MKLML_MKL=0 \
+ -DUSE_MKLDNN=0 \
+ -DCMAKE_BUILD_TYPE=Release \
+ -G Ninja \
+ /work/mxnet
+
+ ninja -v
+}
+
+
+# Testing
+
+sanity_check() {
+ set -ex
+ tools/license_header.py check
+ make cpplint rcpplint jnilint
+ make pylint
+}
+
+
+unittest_ubuntu_python2_cpu() {
+ set -ex
+ export PYTHONPATH=./python/
+ # MXNET_MKLDNN_DEBUG is buggy and produces false positives
+ # https://github.com/apache/incubator-mxnet/issues/10026
+ #export MXNET_MKLDNN_DEBUG=1 # Ignored if not present
+ export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
+ nosetests-2.7 --verbose tests/python/unittest
+ nosetests-2.7 --verbose tests/python/train
+}
+
+unittest_ubuntu_python3_cpu() {
+ set -ex
+ export PYTHONPATH=./python/
+ # MXNET_MKLDNN_DEBUG is buggy and produces false positives
+ # https://github.com/apache/incubator-mxnet/issues/10026
+ #export MXNET_MKLDNN_DEBUG=1 # Ignored if not present
+ export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
+ nosetests-3.4 --verbose tests/python/unittest
+}
+
+unittest_ubuntu_python2_gpu() {
+ set -ex
+ export PYTHONPATH=./python/
+ # MXNET_MKLDNN_DEBUG is buggy and produces false positives
+ # https://github.com/apache/incubator-mxnet/issues/10026
+ #export MXNET_MKLDNN_DEBUG=1 # Ignored if not present
+ export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
+ nosetests-2.7 --verbose tests/python/gpu
+}
+
+unittest_ubuntu_python3_gpu() {
+ set -ex
+ export PYTHONPATH=./python/
+ # MXNET_MKLDNN_DEBUG is buggy and produces false positives
+ # https://github.com/apache/incubator-mxnet/issues/10026
+ #export MXNET_MKLDNN_DEBUG=1 # Ignored if not present
+ export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
+ nosetests-3.4 --verbose tests/python/gpu
+}
+
+unittest_ubuntu_cpu_scala() {
+ set -ex
+ make scalapkg USE_BLAS=openblas
+ make scalatest USE_BLAS=openblas
+}
+
+unittest_ubuntu_cpugpu_perl() {
+ set -ex
+ ./perl-package/test.sh
+}
+
+unittest_ubuntu_gpu_cpp() {
+ set -ex
+ build/tests/mxnet_unit_tests
+}
+
+unittest_ubuntu_cpu_R() {
+ set -ex
+ mkdir -p /tmp/r-site-library
+ # make -j not supported
+ make rpkg USE_BLAS=openblas R_LIBS=/tmp/r-site-library
+ R CMD INSTALL --library=/tmp/r-site-library R-package
+ make rpkgtest R_LIBS=/tmp/r-site-library
+}
+
+unittest_ubuntu_gpu_R() {
+ set -ex
+ mkdir -p /tmp/r-site-library
+ # make -j not supported
+ make rpkg USE_BLAS=openblas R_LIBS=/tmp/r-site-library
+ R CMD INSTALL --library=/tmp/r-site-library R-package
+ make rpkgtest R_LIBS=/tmp/r-site-library R_GPU_ENABLE=1
+}
+
+unittest_centos7_cpu() {
+ set -ex
+ cd /work/mxnet
+ python3.6 -m "nose" --with-timer --verbose tests/python/unittest
+ python3.6 -m "nose" --with-timer --verbose tests/python/train
+}
+
+unittest_centos7_gpu() {
+ set -ex
+ cd /work/mxnet
+ python3.6 -m "nose" --with-timer --verbose tests/python/gpu
+}
+
+integrationtest_ubuntu_gpu_python() {
+ set -ex
+ export PYTHONPATH=./python/
+ export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
+ python example/image-classification/test_score.py
+}
+
+integrationtest_ubuntu_gpu_caffe() {
+ set -ex
+ export PYTHONPATH=/work/deps/caffe/python:./python
+ python tools/caffe_converter/test_converter.py
+}
+
+integrationtest_ubuntu_gpu_cpp_package() {
+ set -ex
+ cpp-package/tests/ci_test.sh
+}
+
+
+test_ubuntu_cpu_python2() {
+ set -ex
+ pushd .
+ export MXNET_LIBRARY_PATH=/work/build/libmxnet.so
+
+ VENV=mxnet_py2_venv
+ virtualenv -p `which python2` $VENV
+ source $VENV/bin/activate
+ pip install nose nose-timer
+
+ cd /work/mxnet/python
+ pip install -e .
+ cd /work/mxnet
+ python -m "nose" --with-timer --verbose tests/python/unittest
+ popd
+}
+
+test_ubuntu_cpu_python3() {
+ set -ex
+ pushd .
+ export MXNET_LIBRARY_PATH=/work/build/libmxnet.so
+ VENV=mxnet_py3_venv
+ virtualenv -p `which python3` $VENV
+ source $VENV/bin/activate
+
+ cd /work/mxnet/python
+ pip3 install nose nose-timer
+ pip3 install -e .
+ cd /work/mxnet
+ python3 -m "nose" --with-timer --verbose tests/python/unittest
+
+ popd
+}
+
+# Deploy
+
+deploy_docs() {
+ set -ex
+ pushd .
+
+ make docs
+
+ popd
+}
+
+##############################################################
+# MAIN
+#
+# Run function passed as argument
+set +x
+if [ $# -gt 0 ]
+then
+ $@
+else
+ cat<<EOF
+
+$0: Execute a function by passing it as an argument to the script:
+
+Possible commands:
+
+EOF
+ declare -F | cut -d' ' -f3
+ echo
+fi
diff --git a/docker_multiarch/.gitignore b/docker_multiarch/.gitignore
deleted file mode 100644
index 2a07fbf..0000000
--- a/docker_multiarch/.gitignore
+++ /dev/null
@@ -1,2 +0,0 @@
-mxnet/
-build/
diff --git a/docker_multiarch/Dockerfile.build.android.arm64 b/docker_multiarch/Dockerfile.build.android.arm64
deleted file mode 100644
index 995e718..0000000
--- a/docker_multiarch/Dockerfile.build.android.arm64
+++ /dev/null
@@ -1,77 +0,0 @@
-# -*- mode: dockerfile -*-
-FROM dockcross/base:latest
-MAINTAINER Pedro Larroy "pllarroy@amazon.com"
-
-# The cross-compiling emulator
-RUN apt-get update && apt-get install -y \
- qemu-user \
- qemu-user-static \
- unzip
-
-ENV CROSS_TRIPLE=aarch64-linux-android
-ENV CROSS_ROOT=/usr/${CROSS_TRIPLE}
-ENV AS=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-as \
- AR=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-ar \
- CC=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-gcc \
- CPP=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-cpp \
- CXX=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-g++ \
- LD=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-ld
-
-ENV ANDROID_NDK_REVISION 15c
-RUN mkdir -p /build && \
- cd /build && \
- curl -O https://dl.google.com/android/repository/android-ndk-r${ANDROID_NDK_REVISION}-linux-x86_64.zip && \
- unzip ./android-ndk-r${ANDROID_NDK_REVISION}-linux-x86_64.zip && \
- cd android-ndk-r${ANDROID_NDK_REVISION} && \
- ./build/tools/make_standalone_toolchain.py \
- --stl=libc++ \
- --arch arm64 \
- --api 21 \
- --install-dir=${CROSS_ROOT} && \
- cd / && \
- rm -rf /build && \
- find ${CROSS_ROOT} -exec chmod a+r '{}' \; && \
- find ${CROSS_ROOT} -executable -exec chmod a+x '{}' \;
-
-
-ENV DEFAULT_DOCKCROSS_IMAGE dockcross/android-arm
-
-# COPY Toolchain.cmake ${CROSS_ROOT}/
-# ENV CMAKE_TOOLCHAIN_FILE ${CROSS_ROOT}/Toolchain.cmake
-
-# Build-time metadata as defined at http://label-schema.org
-ARG BUILD_DATE
-ARG IMAGE
-ARG VCS_REF
-ARG VCS_URL
-LABEL org.label-schema.build-date=$BUILD_DATE \
- org.label-schema.name=$IMAGE \
- org.label-schema.vcs-ref=$VCS_REF \
- org.label-schema.vcs-url=$VCS_URL \
- org.label-schema.schema-version="1.0"
-
-ENV ARCH aarch64
-
-# Build OpenBLAS
-# https://github.com/xianyi/OpenBLAS/wiki/How-to-build-OpenBLAS-for-Android
-RUN git clone https://github.com/xianyi/OpenBLAS.git && \
- cd OpenBLAS && \
- make -j$(nproc) TARGET=ARMV8 ARM_SOFTFP_ABI=1 HOSTCC=gcc NOFORTRAN=1 libs
-
-ENV OPENBLAS_ROOT /work/OpenBLAS
-ENV LIBRARY_PATH /work/OpenBLAS/lib/:/work/OpenBLAS/:$LIBRARY_PATH
-ENV CPLUS_INCLUDE_PATH /work/OpenBLAS/include/:/work/OpenBLAS/:$CPLUS_INCLUDE_PATH
-WORKDIR /work
-
-ENV CC=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-clang
-ENV CXX=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-clang++
-ENV BUILD_OPTS "USE_BLAS=openblas USE_SSE=0 DMLC_LOG_STACK_TRACE=0 USE_OPENCV=0 USE_LAPACK=0"
-
-# Build MXNet
-ADD mxnet mxnet
-ADD arm.crosscompile.android.mk /work/mxnet/make/config.mk
-RUN cd mxnet && \
- make -j$(nproc) $BUILD_OPTS
-
-WORKDIR /work/build/
-RUN cp /work/mxnet/lib/* .
diff --git a/docker_multiarch/Dockerfile.build.arm64 b/docker_multiarch/Dockerfile.build.arm64
deleted file mode 100644
index 33f7b3f..0000000
--- a/docker_multiarch/Dockerfile.build.arm64
+++ /dev/null
@@ -1,37 +0,0 @@
-# -*- mode: dockerfile -*-
-# dockerfile to build libmxnet.so for armv7
-FROM dockcross/linux-arm64
-
-ENV ARCH aarch64
-ENV BUILD_OPTS "USE_BLAS=openblas USE_SSE=0 USE_OPENCV=0"
-ENV CC /usr/bin/aarch64-linux-gnu-gcc
-ENV CXX /usr/bin/aarch64-linux-gnu-g++
-ENV FC /usr/bin/aarch64-linux-gnu-gfortran-4.9
-ENV HOSTCC gcc
-
-WORKDIR /work
-
-# Build OpenBLAS
-ADD https://api.github.com/repos/xianyi/OpenBLAS/git/refs/heads/master /tmp/openblas_version.json
-RUN git clone https://github.com/xianyi/OpenBLAS.git && \
- cd OpenBLAS && \
- make -j$(nproc) TARGET=ARMV8 && \
- make install && \
- ln -s /opt/OpenBLAS/lib/libopenblas.so /usr/lib/libopenblas.so && \
- ln -s /opt/OpenBLAS/lib/libopenblas.a /usr/lib/libopenblas.a && \
- ln -s /opt/OpenBLAS/lib/libopenblas.a /usr/lib/liblapack.a
-
-ENV LD_LIBRARY_PATH /opt/OpenBLAS/lib
-ENV CPLUS_INCLUDE_PATH /opt/OpenBLAS/include
-
-# Build MXNet
-#ADD https://api.github.com/repos/apache/incubator-mxnet/git/refs/heads/master mxnet_version.json
-#RUN git clone --recursive https://github.com/apache/incubator-mxnet.git mxnet
-ADD mxnet mxnet
-
-WORKDIR /work/mxnet
-ADD arm.crosscompile.mk make/config.mk
-RUN make -j$(nproc) $BUILD_OPTS
-
-WORKDIR /work/build/
-RUN cp /work/mxnet/lib/* .
diff --git a/docker_multiarch/Dockerfile.build.armv6 b/docker_multiarch/Dockerfile.build.armv6
deleted file mode 100644
index 9adfc5b..0000000
--- a/docker_multiarch/Dockerfile.build.armv6
+++ /dev/null
@@ -1,38 +0,0 @@
-# -*- mode: dockerfile -*-
-# dockerfile to build libmxnet.so for armv7
-FROM dockcross/linux-armv6
-
-ENV ARCH armv6l
-ENV BUILD_OPTS "USE_BLAS=openblas USE_SSE=0 USE_OPENCV=0"
-ENV CC /usr/bin/arm-linux-gnueabihf-gcc
-ENV CXX /usr/bin/arm-linux-gnueabihf-g++
-ENV FC /usr/bin/arm-linux-gnueabihf-gfortran
-ENV HOSTCC gcc
-
-WORKDIR /work
-
-# Build OpenBLAS
-ADD https://api.github.com/repos/xianyi/OpenBLAS/git/refs/heads/master openblas_version.json
-RUN git clone https://github.com/xianyi/OpenBLAS.git && \
- cd OpenBLAS && \
- make -j$(nproc) TARGET=ARMV6 && \
- make install && \
- ln -s /opt/OpenBLAS/lib/libopenblas.so /usr/lib/gcc/arm-linux-gnueabihf/libopenblas.so && \
- ln -s /opt/OpenBLAS/lib/libopenblas.a /usr/lib/gcc/arm-linux-gnueabihf/libopenblas.a && \
- ln -s /opt/OpenBLAS/lib/libopenblas.a /usr/lib/gcc/arm-linux-gnueabihf/liblapack.a && \
- ln -s /opt/OpenBLAS/lib/libopenblas.a /usr/lib/liblapack.a
-
-ENV LD_LIBRARY_PATH /opt/OpenBLAS/lib
-ENV CPLUS_INCLUDE_PATH /opt/OpenBLAS/include
-
-# Build MXNet
-#ADD https://api.github.com/repos/apache/incubator-mxnet/git/refs/heads/master mxnet_version.json
-#RUN git clone --recursive https://github.com/apache/incubator-mxnet.git mxnet
-ADD mxnet mxnet
-
-WORKDIR /work/mxnet
-ADD arm.crosscompile.mk make/config.mk
-RUN make -j$(nproc) $BUILD_OPTS
-
-WORKDIR /work/build/
-RUN cp /work/mxnet/lib/* .
diff --git a/docker_multiarch/Dockerfile.build.armv7 b/docker_multiarch/Dockerfile.build.armv7
deleted file mode 100644
index 740f2b2..0000000
--- a/docker_multiarch/Dockerfile.build.armv7
+++ /dev/null
@@ -1,26 +0,0 @@
-# -*- mode: dockerfile -*-
-# dockerfile to build libmxnet.so for armv7
-FROM dockcross/linux-armv7
-
-RUN apt-get update && \
- apt-get install -y libopenblas-dev:armhf && \
- rm -rf /var/lib/apt/lists/*
-
-ENV ARCH armv71
-ENV CC /usr/bin/arm-linux-gnueabihf-gcc
-ENV CXX /usr/bin/arm-linux-gnueabihf-g++
-ENV BUILD_OPTS "USE_OPENCV=0 USE_BLAS=openblas USE_SSE=0"
-
-# Build MXNet
-
-WORKDIR /work
-#ADD https://api.github.com/repos/apache/incubator-mxnet/git/refs/heads/master mxnet_version.json
-#RUN git clone --recursive https://github.com/apache/incubator-mxnet.git mxnet
-ADD mxnet mxnet
-
-WORKDIR /work/mxnet
-ADD arm.crosscompile.mk make/config.mk
-RUN make -j$(nproc) $BUILD_OPTS
-
-WORKDIR /work/build/
-RUN cp /work/mxnet/lib/* .
diff --git a/docker_multiarch/Dockerfile.build.cmake.ubuntu-17.04 b/docker_multiarch/Dockerfile.build.cmake.ubuntu-17.04
deleted file mode 100644
index cf0a981..0000000
--- a/docker_multiarch/Dockerfile.build.cmake.ubuntu-17.04
+++ /dev/null
@@ -1,37 +0,0 @@
-FROM ubuntu:17.04
-
-
-RUN apt-get update &&\
- apt-get install -y wget python3.5 gcc-4.9 gcc-5 g++-4.9 g++-5 cmake less python3-pip python3-dev\
- build-essential git pkgconf\
- libopenblas-dev liblapack-dev\
- maven default-jdk\
- ninja-build\
- libgtest-dev\
- &&\
- rm -rf /var/lib/apt/lists/*
-
-
-
-###########################
-# Build gtest
-WORKDIR /work/googletest
-RUN cmake /usr/src/googletest/googletest/ -GNinja
-RUN ninja
-RUN cp libgtest.a /usr/lib
-###########################
-
-
-
-WORKDIR /work
-#RUN git clone --recursive https://github.com/apache/incubator-mxnet.git mxnet
-ADD mxnet mxnet
-
-WORKDIR mxnet/build
-RUN cmake -DUSE_CUDA=OFF -DUSE_OPENCV=OFF -GNinja ..
-RUN ninja
-
-
-# Copy artifacts
-RUN mkdir -p /work/build
-RUN cp *.a *.so /work/build
diff --git a/docker_multiarch/Dockerfile.build.ubuntu-16.04-cuda_8.0_cudnn5 b/docker_multiarch/Dockerfile.build.ubuntu-16.04-cuda_8.0_cudnn5
deleted file mode 100644
index 071a351..0000000
--- a/docker_multiarch/Dockerfile.build.ubuntu-16.04-cuda_8.0_cudnn5
+++ /dev/null
@@ -1,32 +0,0 @@
-FROM nvidia/cuda:8.0-cudnn5-devel
-
-RUN apt-get update &&\
- apt-get install -y wget python3.5 gcc-4.9 gcc-5 g++-4.9 g++-5 cmake less python3-pip python3-dev\
- build-essential git pkgconf\
- libopenblas-dev liblapack-dev\
- maven default-jdk\
- &&\
- rm -rf /var/lib/apt/lists/*
-
-WORKDIR /work
-#RUN git clone --recursive https://github.com/apache/incubator-mxnet.git mxnet
-ADD mxnet mxnet
-
-# Compile MxNet
-ENV BUILD_OPTS "USE_OPENCV=0 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1"
-WORKDIR /work/mxnet
-RUN make -j$(nproc) $BUILD_OPTS
-
-WORKDIR /work/build/
-RUN cp /work/mxnet/lib/* .
-
-# Scala packag
-#WORKDIR /work
-#RUN wget --quiet http://downloads.lightbend.com/scala/2.11.8/scala-2.11.8.deb
-#RUN dpkg -i scala-2.11.8.deb && rm scala-2.11.8.deb
-
-#WORKDIR /work/mxnet
-#RUN make scalapkg $BUILD_OPTS
-
-#WORKDIR /work/build/scala_gpu
-#RUN cp /work/mxnet/scala-package/assembly/linux-x86_64-gpu/target/*.jar .
diff --git a/docker_multiarch/Dockerfile.build.ubuntu-17.04 b/docker_multiarch/Dockerfile.build.ubuntu-17.04
deleted file mode 100644
index 63b3c07..0000000
--- a/docker_multiarch/Dockerfile.build.ubuntu-17.04
+++ /dev/null
@@ -1,30 +0,0 @@
-#
-# Base image to build MXNet from source in ubuntu
-#
-# Other images depend on it, so build it like:
-#
-# docker build -f Dockerfile.build.ubuntu-17.04 -t mxnet.build.ubuntu-17.04 .
-#
-FROM ubuntu:17.04
-
-
-RUN apt-get update &&\
- apt-get install -y wget python3.5 gcc-4.9 gcc-5 g++-4.9 g++-5 cmake less python3-pip python3-dev\
- build-essential git pkgconf\
- libopenblas-dev liblapack-dev\
- maven default-jdk
-
-RUN rm -rf /var/lib/apt/lists/*
-
-WORKDIR /work
-#RUN git clone --recursive https://github.com/apache/incubator-mxnet.git mxnet
-ADD mxnet mxnet
-
-# Compile MxNet
-ENV BUILD_OPTS "USE_OPENCV=0 USE_BLAS=openblas"
-WORKDIR /work/mxnet
-RUN make -j$(nproc) $BUILD_OPTS
-
-# Copy artifacts
-WORKDIR /work/build/
-RUN cp /work/mxnet/lib/* .
diff --git a/docker_multiarch/Dockerfile.build.ubuntu-17.04.scala.docker b/docker_multiarch/Dockerfile.build.ubuntu-17.04.scala.docker
deleted file mode 100644
index a31ce89..0000000
--- a/docker_multiarch/Dockerfile.build.ubuntu-17.04.scala.docker
+++ /dev/null
@@ -1,17 +0,0 @@
-# Before building this image you would need to build MXNet by executing:
-# docker build -f Dockerfile.build.ubuntu-17.04 -t mxnet.build.ubuntu-17.04 .
-# if you haven't done it before.
-
-FROM mxnet.build.ubuntu-17.04
-
-# Scala package
-WORKDIR /work
-RUN wget --quiet http://downloads.lightbend.com/scala/2.11.8/scala-2.11.8.deb
-RUN dpkg -i scala-2.11.8.deb && rm scala-2.11.8.deb
-
-WORKDIR /work/mxnet
-RUN make scalapkg $BUILD_OPTS
-
-WORKDIR /work/build/scala
-RUN cp /work/mxnet/scala-package/core/target/*.jar .
-RUN cp /work/mxnet/scala-package/assembly/linux-x86_64-cpu/target/*.jar .
diff --git a/docker_multiarch/Dockerfile.run.ubuntu-17.04.julia b/docker_multiarch/Dockerfile.run.ubuntu-17.04.julia
deleted file mode 100644
index df3a036..0000000
--- a/docker_multiarch/Dockerfile.run.ubuntu-17.04.julia
+++ /dev/null
@@ -1,24 +0,0 @@
-# Before building this image you would need to build MXNet by executing:
-# docker build -f Dockerfile.build.ubuntu-17.04 -t mxnet.build.ubuntu-17.04 .
-# if you haven't done it before.
-
-FROM mxnet.build.ubuntu-17.04
-
-ENV DEBIAN_FRONTEND=noninteractive
-
-##################
-# Julia installation
-RUN wget -q https://julialang.s3.amazonaws.com/bin/linux/x64/0.5/julia-0.5.1-linux-x86_64.tar.gz\
- && tar -zxf julia-0.5.1-linux-x86_64.tar.gz\
- && rm julia-0.5.1-linux-x86_64.tar.gz\
- && ln -s $(pwd)/julia-6445c82d00/bin/julia /usr/bin/julia
-##################
-
-
-ENV MXNET_HOME /work/mxnet
-WORKDIR /work/mxnet
-RUN julia -e 'Pkg.add("MXNet")'
-
-
-
-
diff --git a/docker_multiarch/Dockerfile.run.ubuntu-17.04.perl b/docker_multiarch/Dockerfile.run.ubuntu-17.04.perl
deleted file mode 100644
index 17c5d8d..0000000
--- a/docker_multiarch/Dockerfile.run.ubuntu-17.04.perl
+++ /dev/null
@@ -1,23 +0,0 @@
-# Before building this image you would need to build MXNet by executing:
-# docker build -f Dockerfile.build.ubuntu-17.04 -t mxnet.build.ubuntu-17.04 .
-# if you haven't done it before.
-
-FROM mxnet.build.ubuntu-17.04
-
-ENV DEBIAN_FRONTEND=noninteractive
-
-RUN apt-get update
-RUN apt-get install -y\
- libmouse-perl pdl cpanminus swig libgraphviz-perl
-RUN rm -rf /var/lib/apt/lists/*
-
-RUN cpanm -q Function::Parameters Hash::Ordered PDL::CCS
-
-WORKDIR /work/mxnet/perl-package/AI-MXNetCAPI
-RUN perl Makefile.PL && make install
-
-WORKDIR /work/mxnet/perl-package/AI-NNVMCAPI/
-RUN perl Makefile.PL && make install
-
-WORKDIR /work/mxnet/perl-package/AI-MXNet/
-RUN perl Makefile.PL && make install
diff --git a/docker_multiarch/Dockerfile.run.ubuntu-17.04.python b/docker_multiarch/Dockerfile.run.ubuntu-17.04.python
deleted file mode 100644
index 8bd262b..0000000
--- a/docker_multiarch/Dockerfile.run.ubuntu-17.04.python
+++ /dev/null
@@ -1,17 +0,0 @@
-# Before building this image you would need to build MXNet by executing:
-# docker build -f Dockerfile.build.ubuntu-17.04 -t mxnet.build.ubuntu-17.04 .
-# if you haven't done it before.
-
-FROM mxnet.build.ubuntu-17.04
-
-ENV DEBIAN_FRONTEND=noninteractive
-
-
-RUN apt-get update
-RUN apt-get install -y python-pip
-RUN rm -rf /var/lib/apt/lists/*
-
-WORKDIR /work/mxnet/python
-RUN pip3 install -e .
-RUN pip install -e .
-
diff --git a/docker_multiarch/Dockerfile.run.ubuntu-17.04.r b/docker_multiarch/Dockerfile.run.ubuntu-17.04.r
deleted file mode 100644
index 493c7f0..0000000
--- a/docker_multiarch/Dockerfile.run.ubuntu-17.04.r
+++ /dev/null
@@ -1,36 +0,0 @@
-# Before building this image you would need to build MXNet by executing:
-# docker build -f Dockerfile.build.ubuntu-17.04 -t mxnet.build.ubuntu-17.04 .
-# if you haven't done it before.
-
-FROM mxnet.build.ubuntu-17.04
-
-ENV DEBIAN_FRONTEND=noninteractive
-#ENV BUILD_OPTS "USE_OPENCV=0 USE_BLAS=openblas GTEST_PATH=/usr/src/googletest/googletest"
-
-##################
-# R installation
-RUN apt-get update
-#RUN apt-get remove -y gnupg
-#RUN apt-get install -y --reinstall\
-# gnupg2 dirmngr
-
-RUN apt-get install -y dirmngr libopencv-dev
-RUN echo "deb http://cran.rstudio.com/bin/linux/ubuntu zesty/" >> /etc/apt/sources.list
-RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E084DAB9
-
-RUN apt-get install -y\
- r-base r-base-core r-recommended r-base-dev libxml2-dev libxt-dev libssl-dev libcurl4-openssl-dev
-
-
-WORKDIR /work/mxnet
-RUN cp R-package/DESCRIPTION .
-RUN Rscript -e "install.packages('devtools', repo = 'https://cran.rstudio.com')"
-RUN Rscript -e "library(devtools); library(methods); options(repos=c(CRAN='https://cran.rstudio.com')); install_deps(dependencies = TRUE)"
-
-
-##################
-# MXNet R package
-RUN make rpkg
-RUN R CMD INSTALL mxnet_current_r.tar.gz
-##################
-
diff --git a/docker_multiarch/Dockerfile.test.ubuntu-17.04 b/docker_multiarch/Dockerfile.test.ubuntu-17.04
deleted file mode 100644
index 1b0c145..0000000
--- a/docker_multiarch/Dockerfile.test.ubuntu-17.04
+++ /dev/null
@@ -1,41 +0,0 @@
-FROM ubuntu-17.04
-RUN apt-get update &&\
- apt-get install -y python3-nose python-nose python-pip libgtest-dev valgrind ninja-build\
- &&\
- rm -rf /var/lib/apt/lists/*
-
-###########################
-# Unit tests
-# Build google test
-WORKDIR /work/googletest
-RUN cmake /usr/src/googletest/googletest/ -GNinja
-RUN ninja
-# FIXME
-RUN mkdir -p /usr/src/googletest/googletest/lib/
-RUN cp libgtest.a /usr/src/googletest/googletest/lib/
-
-ENV BUILD_OPTS "USE_OPENCV=0 USE_BLAS=openblas GTEST_PATH=/usr/src/googletest/googletest"
-
-WORKDIR /work/mxnet
-RUN make -j$(nproc) test $BUILD_OPTS
-ENV MXNET_ENGINE_INFO=true
-RUN build/tests/cpp/mxnet_test
-RUN valgrind build/tests/cpp/mxnet_test
-############################
-
-############################
-# Python tests
-WORKDIR /work/mxnet/python
-RUN pip3 install -e .
-RUN pip install -e .
-
-WORKDIR /work/mxnet
-RUN nosetests3 --verbose tests/python/unittest
-RUN nosetests --verbose tests/python/unittest
-############################
-
-
-############################
-# Scala tests
-RUN make scalatest $BUILD_OPTS
-############################
diff --git a/docker_multiarch/License.md b/docker_multiarch/License.md
deleted file mode 100644
index 614c582..0000000
--- a/docker_multiarch/License.md
+++ /dev/null
@@ -1,20 +0,0 @@
-## Dockerfiles License
-
-### All Dockerfiles in this directory are provided under an Apache 2.0 license.
-
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
\ No newline at end of file
diff --git a/docker_multiarch/README.md b/docker_multiarch/README.md
deleted file mode 100644
index a463d15..0000000
--- a/docker_multiarch/README.md
+++ /dev/null
@@ -1,42 +0,0 @@
-# Dockerized multi-architecture build
-
-These docker files and utilities will build mxnet and run tests for different architectures using cross compilation and produce
-runtime binary artifacts.
-
-This utilities require that you have docker installed. [Docker CE](https://docs.docker.com/engine/installation/linux/docker-ce/ubuntu/#install-docker) is recommended.
-
-
-To compile for all the supported architectures you can run the script
-```
-$ ./tool.py
-```
-
-To build a single arch, you can invoke docker directly:
-
-```
-$ docker build -f Dockerfile.build.<arch> -t <tag> .
-```
-
-Or call the dockerfile directly:
-
-```
-docker build -f <dockerfile> -t <tag> .
-```
-
-Or pass the architecture id to the tool:
-```
-$ ./tool.py -a ubuntu-17.04
-```
-
-By convention all the Dockerfiles produce the build artifacts on /work/build so they can be copied
-after.
-
-
-The tool will leave the resulting artifacts on the build/ directory
-
-# TODO
-
-- Handle dependencies between docker files, for example having a yaml file with the dependency graph
- so they can be built in the right order. Right now the dependency is very simple so simple
- alphabetical sorting of the images does the trick.
-
diff --git a/docker_multiarch/arm.crosscompile.android.mk b/docker_multiarch/arm.crosscompile.android.mk
deleted file mode 100644
index 0302c5c..0000000
--- a/docker_multiarch/arm.crosscompile.android.mk
+++ /dev/null
@@ -1,161 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-#-------------------------------------------------------------------------------
-# Template configuration for compiling mxnet
-#
-# If you want to change the configuration, please use the following
-# steps. Assume you are on the root directory of mxnet. First copy the this
-# file so that any local changes will be ignored by git
-#
-# $ cp make/config.mk .
-#
-# Next modify the according entries, and then compile by
-#
-# $ make
-#
-# or build in parallel with 8 threads
-#
-# $ make -j8
-#-------------------------------------------------------------------------------
-
-#---------------------
-# We do not assign compilers here. Often when cross-compiling these will already
-# be set correctly.
-#--------------------
-
-export NVCC = nvcc
-
-# whether compile with options for MXNet developer
-DEV = 0
-
-# whether compile with debug
-DEBUG = 0
-
-# whether compiler with profiler
-USE_PROFILER =
-
-# the additional link flags you want to add
-# TODO: Move flags here
-ADD_LDFLAGS=-static-libstdc++ -L/work/OpenBLAS/
-
-# the additional compile flags you want to add
-ADD_CFLAGS =
-
-#---------------------------------------------
-# matrix computation libraries for CPU/GPU
-#---------------------------------------------
-
-# whether use CUDA during compile
-USE_CUDA = 0
-
-# add the path to CUDA library to link and compile flag
-# if you have already add them to environment variable, leave it as NONE
-# USE_CUDA_PATH = /usr/local/cuda
-USE_CUDA_PATH = NONE
-
-# whether use CuDNN R3 library
-USE_CUDNN = 0
-
-# whether use cuda runtime compiling for writing kernels in native language (i.e. Python)
-USE_NVRTC = 0
-
-# whether use opencv during compilation
-# you can disable it, however, you will not able to use
-# imbin iterator
-USE_OPENCV = 0
-
-# use openmp for parallelization
-USE_OPENMP = 1
-
-# whether use NNPACK library
-USE_NNPACK = 0
-
-# For arm builds we're using openblas
-USE_BLAS = openblas
-
-# whether use lapack during compilation
-# only effective when compiled with blas versions openblas/apple/atlas/mkl
-USE_LAPACK = 0
-
-# path to lapack library in case of a non-standard installation
-USE_LAPACK_PATH =
-
-# add path to intel library, you may need it for MKL, if you did not add the path
-# to environment variable
-USE_INTEL_PATH = NONE
-
-# If use MKL only for BLAS, choose static link automatically to allow python wrapper
-USE_STATIC_MKL = NONE
-ifeq ($(USE_BLAS), mkl)
-USE_STATIC_MKL = 1
-endif
-
-#----------------------------
-# distributed computing
-#----------------------------
-
-# whether or not to enable multi-machine supporting
-USE_DIST_KVSTORE = 0
-
-# whether or not allow to read and write HDFS directly. If yes, then hadoop is
-# required
-USE_HDFS = 0
-
-# path to libjvm.so. required if USE_HDFS=1
-LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server
-
-# whether or not allow to read and write AWS S3 directly. If yes, then
-# libcurl4-openssl-dev is required, it can be installed on Ubuntu by
-# sudo apt-get install -y libcurl4-openssl-dev
-USE_S3 = 0
-
-#----------------------------
-# additional operators
-#----------------------------
-
-# path to folders containing projects specific operators that you don't want to put in src/operators
-EXTRA_OPERATORS =
-
-#----------------------------
-# other features
-#----------------------------
-
-# Create C++ interface package
-USE_CPP_PACKAGE = 0
-
-#----------------------------
-# plugins
-#----------------------------
-
-# whether to use caffe integration. This requires installing caffe.
-# You also need to add CAFFE_PATH/build/lib to your LD_LIBRARY_PATH
-# CAFFE_PATH = $(HOME)/caffe
-# MXNET_PLUGINS += plugin/caffe/caffe.mk
-
-# whether to use torch integration. This requires installing torch.
-# You also need to add TORCH_PATH/install/lib to your LD_LIBRARY_PATH
-# TORCH_PATH = $(HOME)/torch
-# MXNET_PLUGINS += plugin/torch/torch.mk
-
-# WARPCTC_PATH = $(HOME)/warp-ctc
-# MXNET_PLUGINS += plugin/warpctc/warpctc.mk
-
-# whether to use sframe integration. This requires build sframe
-# git@github.com:dato-code/SFrame.git
-# SFRAME_PATH = $(HOME)/SFrame
-# MXNET_PLUGINS += plugin/sframe/plugin.mk
diff --git a/docker_multiarch/arm.crosscompile.mk b/docker_multiarch/arm.crosscompile.mk
deleted file mode 100644
index 2bca9e3..0000000
--- a/docker_multiarch/arm.crosscompile.mk
+++ /dev/null
@@ -1,161 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-#-------------------------------------------------------------------------------
-# Template configuration for compiling mxnet
-#
-# If you want to change the configuration, please use the following
-# steps. Assume you are on the root directory of mxnet. First copy the this
-# file so that any local changes will be ignored by git
-#
-# $ cp make/config.mk .
-#
-# Next modify the according entries, and then compile by
-#
-# $ make
-#
-# or build in parallel with 8 threads
-#
-# $ make -j8
-#-------------------------------------------------------------------------------
-
-#---------------------
-# We do not assign compilers here. Often when cross-compiling these will already
-# be set correctly.
-#--------------------
-
-export NVCC = nvcc
-
-# whether compile with options for MXNet developer
-DEV = 0
-
-# whether compile with debug
-DEBUG = 0
-
-# whether compiler with profiler
-USE_PROFILER =
-
-# the additional link flags you want to add
-# TODO: Move flags here
-ADD_LDFLAGS=-static-libstdc++
-
-# the additional compile flags you want to add
-ADD_CFLAGS =
-
-#---------------------------------------------
-# matrix computation libraries for CPU/GPU
-#---------------------------------------------
-
-# whether use CUDA during compile
-USE_CUDA = 0
-
-# add the path to CUDA library to link and compile flag
-# if you have already add them to environment variable, leave it as NONE
-# USE_CUDA_PATH = /usr/local/cuda
-USE_CUDA_PATH = NONE
-
-# whether use CuDNN R3 library
-USE_CUDNN = 0
-
-# whether use cuda runtime compiling for writing kernels in native language (i.e. Python)
-USE_NVRTC = 0
-
-# whether use opencv during compilation
-# you can disable it, however, you will not able to use
-# imbin iterator
-USE_OPENCV = 0
-
-# use openmp for parallelization
-USE_OPENMP = 1
-
-# whether use NNPACK library
-USE_NNPACK = 0
-
-# For arm builds we're using openblas
-USE_BLAS = openblas
-
-# whether use lapack during compilation
-# only effective when compiled with blas versions openblas/apple/atlas/mkl
-USE_LAPACK = 1
-
-# path to lapack library in case of a non-standard installation
-USE_LAPACK_PATH =
-
-# add path to intel library, you may need it for MKL, if you did not add the path
-# to environment variable
-USE_INTEL_PATH = NONE
-
-# If use MKL only for BLAS, choose static link automatically to allow python wrapper
-USE_STATIC_MKL = NONE
-ifeq ($(USE_BLAS), mkl)
-USE_STATIC_MKL = 1
-endif
-
-#----------------------------
-# distributed computing
-#----------------------------
-
-# whether or not to enable multi-machine supporting
-USE_DIST_KVSTORE = 0
-
-# whether or not allow to read and write HDFS directly. If yes, then hadoop is
-# required
-USE_HDFS = 0
-
-# path to libjvm.so. required if USE_HDFS=1
-LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server
-
-# whether or not allow to read and write AWS S3 directly. If yes, then
-# libcurl4-openssl-dev is required, it can be installed on Ubuntu by
-# sudo apt-get install -y libcurl4-openssl-dev
-USE_S3 = 0
-
-#----------------------------
-# additional operators
-#----------------------------
-
-# path to folders containing projects specific operators that you don't want to put in src/operators
-EXTRA_OPERATORS =
-
-#----------------------------
-# other features
-#----------------------------
-
-# Create C++ interface package
-USE_CPP_PACKAGE = 0
-
-#----------------------------
-# plugins
-#----------------------------
-
-# whether to use caffe integration. This requires installing caffe.
-# You also need to add CAFFE_PATH/build/lib to your LD_LIBRARY_PATH
-# CAFFE_PATH = $(HOME)/caffe
-# MXNET_PLUGINS += plugin/caffe/caffe.mk
-
-# whether to use torch integration. This requires installing torch.
-# You also need to add TORCH_PATH/install/lib to your LD_LIBRARY_PATH
-# TORCH_PATH = $(HOME)/torch
-# MXNET_PLUGINS += plugin/torch/torch.mk
-
-# WARPCTC_PATH = $(HOME)/warp-ctc
-# MXNET_PLUGINS += plugin/warpctc/warpctc.mk
-
-# whether to use sframe integration. This requires build sframe
-# git@github.com:dato-code/SFrame.git
-# SFRAME_PATH = $(HOME)/SFrame
-# MXNET_PLUGINS += plugin/sframe/plugin.mk
diff --git a/docker_multiarch/tool.py b/docker_multiarch/tool.py
deleted file mode 100755
index d0003ec05..0000000
--- a/docker_multiarch/tool.py
+++ /dev/null
@@ -1,155 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-"""Multi arch dockerized build tool.
-
-"""
-
-__author__ = 'Pedro Larroy'
-__version__ = '0.1'
-
-import os
-import sys
-import subprocess
-import logging
-import argparse
-from subprocess import check_call
-import glob
-import re
-
-class CmdResult(object):
- def __init__(self, std_out, std_err, status_code):
- self.std_out = std_out
- self.std_err = std_err
- self.status_code = status_code if status_code is not None else 0
-
- def __str__(self):
- return "%s, %s, %s" % (self.std_out, self.std_err, self.status_code)
-
-def run(cmd, fail_on_error=True):
- logging.debug("executing shell command:\n" + cmd)
- proc = subprocess.Popen(
- cmd,
- shell=True,
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE,
- )
- std_out, std_err = proc.communicate()
- if fail_on_error:
- if proc.returncode != 0:
- logging.warn('Error running command: {}'.format(cmd))
- assert proc.returncode == 0, std_err
- res = CmdResult(std_out.decode('utf-8'), std_err.decode('utf-8'), proc.returncode)
- return res
-
-
-def mkdir_p(d):
- rev_path_list = list()
- head = d
- while len(head) and head != os.sep:
- rev_path_list.append(head)
- (head, tail) = os.path.split(head)
-
- rev_path_list.reverse()
- for p in rev_path_list:
- try:
- os.mkdir(p)
- except OSError as e:
- if e.errno != 17:
- raise
-
-def get_arches():
- """Get a list of architectures given our dockerfiles"""
- dockerfiles = glob.glob("Dockerfile.build.*")
- dockerfiles = list(filter(lambda x: x[-1] != '~', dockerfiles))
- arches = list(map(lambda x: re.sub(r"Dockerfile.build.(.*)", r"\1", x), dockerfiles))
- arches.sort()
- return arches
-
-def sync_source():
- logging.info("Copying sources")
- check_call(["rsync","-a","--delete","--exclude=\".git/\"",'--exclude=/docker_multiarch/',"../","mxnet"])
-
-def get_docker_tag(arch):
- return "mxnet.build.{0}".format(arch)
-
-def get_dockerfile(arch):
- return "Dockerfile.build.{0}".format(arch)
-
-def build(arch):
- """Build the given architecture in the container"""
- assert arch in get_arches(), "No such architecture {0}, Dockerfile.build.{0} not found".format(arch)
- logging.info("Building for target platform {0}".format(arch))
- check_call(["docker", "build",
- "-f", get_dockerfile(arch),
- "-t", get_docker_tag(arch),
- "."])
-
-def collect_artifacts(arch):
- """Collects the artifacts built inside the docker container to the local fs"""
- def artifact_path(arch):
- return "{}/build/{}".format(os.getcwd(), arch)
- logging.info("Collect artifacts from build in {0}".format(artifact_path(arch)))
- mkdir_p("build/{}".format(arch))
-
- # Mount artifact_path on /$arch inside the container and copy the build output so we can access
- # locally from the host fs
- check_call(["docker","run",
- "-v", "{}:/{}".format(artifact_path(arch), arch),
- get_docker_tag(arch),
- "bash", "-c", "cp -r /work/build/* /{}".format(arch)])
-
-def main():
- logging.getLogger().setLevel(logging.INFO)
- logging.basicConfig(format='%(asctime)-15s %(message)s')
-
- parser = argparse.ArgumentParser()
- parser.add_argument("-a", "--arch",
- help="Architecture",
- type=str)
-
- parser.add_argument("-l", "--list_arch",
- help="List architectures",
- action='store_true')
- args = parser.parse_args()
-
- if args.list_arch:
- arches = get_arches()
- print(arches)
-
- elif args.arch:
- sync_source()
- build(args.arch)
- collect_artifacts(args.arch)
-
- else:
- arches = get_arches()
- logging.info("Building for all architectures: {}".format(arches))
- logging.info("Artifacts will be produced in the build/ directory.")
- sync_source()
- for arch in arches:
- build(arch)
- collect_artifacts(arch)
-
- return 0
-
-if __name__ == '__main__':
- sys.exit(main())
-
diff --git a/python/mxnet/gluon/data/dataloader.py b/python/mxnet/gluon/data/dataloader.py
index 02aa5c0..6efaf35 100644
--- a/python/mxnet/gluon/data/dataloader.py
+++ b/python/mxnet/gluon/data/dataloader.py
@@ -214,6 +214,7 @@ class DataLoader(object):
worker.start()
workers.append(worker)
+ idx = 0
for idx, batch in enumerate(self._batch_sampler):
key_queue.put((idx, batch))
num_batches = idx + 1
diff --git a/python/mxnet/image/detection.py b/python/mxnet/image/detection.py
index 191a2be..63a44ab 100644
--- a/python/mxnet/image/detection.py
+++ b/python/mxnet/image/detection.py
@@ -842,7 +842,7 @@ class ImageDetIter(ImageIter):
import cv2
except ImportError as e:
warnings.warn('Unable to import cv2, skip drawing: %s', str(e))
- raise StopIteration
+ return
count = 0
try:
while True:
@@ -896,7 +896,7 @@ class ImageDetIter(ImageIter):
yield image
except StopIteration:
if not count:
- raise StopIteration
+ return
def sync_label_shape(self, it, verbose=False):
"""Synchronize label shape with the input iterator. This is useful when
diff --git a/python/mxnet/libinfo.py b/python/mxnet/libinfo.py
index 9a82b79..3220b5a 100644
--- a/python/mxnet/libinfo.py
+++ b/python/mxnet/libinfo.py
@@ -20,6 +20,7 @@
from __future__ import absolute_import
import os
import platform
+import logging
def find_lib_path():
"""Find MXNet dynamic library files.
@@ -29,6 +30,17 @@ def find_lib_path():
lib_path : list(string)
List of all found path to the libraries.
"""
+ lib_from_env = os.environ.get('MXNET_LIBRARY_PATH')
+ if lib_from_env:
+ if os.path.isfile(lib_from_env):
+ if not os.path.isabs(lib_from_env):
+ logging.warning("MXNET_LIBRARY_PATH should be an absolute path, instead of: %s",
+ lib_from_env)
+ else:
+ return [lib_from_env]
+ else:
+ logging.warning("MXNET_LIBRARY_PATH '%s' doesn't exist", lib_from_env)
+
curr_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__)))
api_path = os.path.join(curr_path, '../../lib/')
cmake_build_path = os.path.join(curr_path, '../../build/')
diff --git a/tests/ci_build/Dockerfile.amzn_linux_cpu b/tests/ci_build/Dockerfile.amzn_linux_cpu
deleted file mode 100644
index d1fb4fc..0000000
--- a/tests/ci_build/Dockerfile.amzn_linux_cpu
+++ /dev/null
@@ -1,16 +0,0 @@
-FROM amazonlinux
-MAINTAINER Ly Nguyen <ly...@amazon.com>
-
-COPY install/* /install/
-
-RUN yum install -y git wget sudo
-
-RUN chmod -R 755 install
-RUN /install/install_opencv.sh
-RUN /install/install_openblas.sh
-RUN /install/install_python2.sh
-RUN /install/install_python3.sh
-RUN /install/install_testdeps.sh
-RUN /install/install_julia.sh
-RUN /install/install_maven.sh
-RUN /install/install_library.sh
diff --git a/tests/ci_build/Dockerfile.build_cuda b/tests/ci_build/Dockerfile.build_cuda
deleted file mode 100644
index d659a1a..0000000
--- a/tests/ci_build/Dockerfile.build_cuda
+++ /dev/null
@@ -1,26 +0,0 @@
-FROM nvidia/cuda:8.0-cudnn5-devel
-# cuda8.0 has to be used because this is the first ubuntu16.04 container
-# which is required due to OpenBLAS being incompatible with ubuntu14.04
-# the reason we used a gpu base container because we are going to test MKLDNN
-# operator implementation against GPU implementation
-
-COPY install/ubuntu_install_core.sh /install/
-RUN /install/ubuntu_install_core.sh
-COPY install/ubuntu_install_python.sh /install/
-RUN /install/ubuntu_install_python.sh
-COPY install/ubuntu_install_scala.sh /install/
-RUN /install/ubuntu_install_scala.sh
-COPY install/ubuntu_install_r.sh /install/
-RUN /install/ubuntu_install_r.sh
-COPY install/ubuntu_install_perl.sh /install/
-RUN /install/ubuntu_install_perl.sh
-
-# Allows to run tasks on a CPU without nvidia-docker and GPU
-COPY install/ubuntu_install_nvidia.sh /install/
-RUN /install/ubuntu_install_nvidia.sh
-
-# Add MKLML libraries
-RUN wget --no-check-certificate -O /tmp/mklml.tgz https://github.com/01org/mkl-dnn/releases/download/v0.12/mklml_lnx_2018.0.1.20171227.tgz
-RUN tar -zxvf /tmp/mklml.tgz && cp -rf mklml_*/* /usr/local/ && rm -rf mklml_*
-
-ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib
diff --git a/tests/ci_build/Dockerfile.caffe_gpu b/tests/ci_build/Dockerfile.caffe_gpu
deleted file mode 100644
index 34c4625..0000000
--- a/tests/ci_build/Dockerfile.caffe_gpu
+++ /dev/null
@@ -1,36 +0,0 @@
-FROM nvidia/cuda:8.0-cudnn5-devel
-# cuda8.0 has to be used because this is the first ubuntu16.04 container
-# which is required due to OpenBLAS being incompatible with ubuntu14.04
-
-COPY install/ubuntu_install_core.sh /install/
-RUN /install/ubuntu_install_core.sh
-
-COPY install/ubuntu_install_python.sh /install/
-RUN /install/ubuntu_install_python.sh
-
-RUN apt-get install -y libprotobuf-dev libleveldb-dev \
- libsnappy-dev libopencv-dev libhdf5-serial-dev protobuf-compiler \
- libatlas-base-dev python-dev libgflags-dev libgoogle-glog-dev liblmdb-dev \
- python-numpy python-opencv
-
-RUN apt-get install -y --no-install-recommends libboost-all-dev
-
-RUN cd /; git clone http://github.com/BVLC/caffe.git; cd caffe; \
- cp Makefile.config.example Makefile.config
-
-RUN echo "CPU_ONLY := 1" >> /caffe/Makefile.config
-
-# Fixes https://github.com/BVLC/caffe/issues/5658 See https://github.com/intel/caffe/wiki/Ubuntu-16.04-or-15.10-Installation-Guide
-RUN echo "INCLUDE_DIRS += /usr/lib /usr/lib/x86_64-linux-gnu /usr/include/hdf5/serial/ " >> /caffe/Makefile.config
-RUN echo "LIBRARY_DIRS += /usr/lib /usr/lib/x86_64-linux-gnu /usr/lib/x86_64-linux-gnu/hdf5/serial " >> /caffe/Makefile.config
-
-# Fixes https://github.com/BVLC/caffe/issues/4333 See https://github.com/intel/caffe/wiki/Ubuntu-16.04-or-15.10-Installation-Guide
-# Note: This is only valid on Ubuntu16.04 - the version numbers are bound to the distribution
-RUN ln -s /usr/lib/x86_64-linux-gnu/libhdf5_serial.so.10.0.2 /usr/lib/x86_64-linux-gnu/libhdf5.so
-RUN ln -s /usr/lib/x86_64-linux-gnu/libhdf5_serial_hl.so.10.0.2 /usr/lib/x86_64-linux-gnu/libhdf5_hl.so
-
-RUN cd caffe; make all pycaffe -j$(nproc)
-
-RUN cd caffe/python; for req in $(cat requirements.txt); do pip2 install $req; done
-
-ENV PYTHONPATH=${PYTHONPATH}:/caffe/python
diff --git a/tests/ci_build/Dockerfile.cpu b/tests/ci_build/Dockerfile.cpu
deleted file mode 100644
index 226054a..0000000
--- a/tests/ci_build/Dockerfile.cpu
+++ /dev/null
@@ -1,12 +0,0 @@
-FROM ubuntu:16.04
-
-COPY install/ubuntu_install_core.sh /install/
-RUN /install/ubuntu_install_core.sh
-COPY install/ubuntu_install_python.sh /install/
-RUN /install/ubuntu_install_python.sh
-COPY install/ubuntu_install_scala.sh /install/
-RUN /install/ubuntu_install_scala.sh
-COPY install/ubuntu_install_r.sh /install/
-RUN /install/ubuntu_install_r.sh
-COPY install/ubuntu_install_perl.sh /install/
-RUN /install/ubuntu_install_perl.sh
diff --git a/tests/ci_build/Dockerfile.cpu_clang b/tests/ci_build/Dockerfile.cpu_clang
deleted file mode 100644
index 5bcdee5..0000000
--- a/tests/ci_build/Dockerfile.cpu_clang
+++ /dev/null
@@ -1,21 +0,0 @@
-FROM ubuntu:16.04
-
-COPY install/ubuntu_install_core.sh /install/
-RUN /install/ubuntu_install_core.sh
-COPY install/ubuntu_install_python.sh /install/
-RUN /install/ubuntu_install_python.sh
-COPY install/ubuntu_install_scala.sh /install/
-RUN /install/ubuntu_install_scala.sh
-COPY install/ubuntu_install_r.sh /install/
-RUN /install/ubuntu_install_r.sh
-COPY install/ubuntu_install_perl.sh /install/
-RUN /install/ubuntu_install_perl.sh
-
-# Install clang 3.9 (the same version as in XCode 8.*) and 5.0 (latest major release)
-RUN wget -O - http://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
- apt-add-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-3.9 main" && \
- apt-add-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-5.0 main" && \
- apt-get update && \
- apt-get install -y clang-3.9 clang-5.0 && \
- clang-3.9 --version && \
- clang-5.0 --version
diff --git a/tests/ci_build/Dockerfile.cpu_mklml b/tests/ci_build/Dockerfile.cpu_mklml
deleted file mode 100644
index 9f235b4..0000000
--- a/tests/ci_build/Dockerfile.cpu_mklml
+++ /dev/null
@@ -1,18 +0,0 @@
-FROM ubuntu:16.04
-
-COPY install/ubuntu_install_core.sh /install/
-RUN /install/ubuntu_install_core.sh
-COPY install/ubuntu_install_python.sh /install/
-RUN /install/ubuntu_install_python.sh
-COPY install/ubuntu_install_scala.sh /install/
-RUN /install/ubuntu_install_scala.sh
-COPY install/ubuntu_install_r.sh /install/
-RUN /install/ubuntu_install_r.sh
-COPY install/ubuntu_install_perl.sh /install/
-RUN /install/ubuntu_install_perl.sh
-
-# Add MKLML library, compatiable with Ubuntu16.04
-RUN wget --no-check-certificate -O /tmp/mklml.tgz https://github.com/01org/mkl-dnn/releases/download/v0.12/mklml_lnx_2018.0.1.20171227.tgz
-RUN tar -zxvf /tmp/mklml.tgz && cp -rf mklml_*/* /usr/local/ && rm -rf mklml_*
-
-ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib
diff --git a/tests/ci_build/Dockerfile.crosstool b/tests/ci_build/Dockerfile.crosstool
deleted file mode 100644
index 371cdc7..0000000
--- a/tests/ci_build/Dockerfile.crosstool
+++ /dev/null
@@ -1,23 +0,0 @@
-FROM ubuntu
-MAINTAINER Aran Khanna <ar...@amazon.com>
-
-# UPDATE BOX
-RUN apt-get update && apt-get -y upgrade
-
-# TOOLCHAIN DEPS
-RUN apt-get install -y python python-setuptools python-pip python-dev unzip gfortran
-RUN apt-get install -y git bison cvs flex gperf texinfo automake libtool help2man make libtool-bin libncurses5-dev g++ cmake wget gawk
-RUN pip install numpy nose
-
-# BUILD TOOLCHAIN
-RUN git clone https://github.com/arank/crosstool-NG
-RUN cd crosstool-NG && ./bootstrap && ./configure && make && make install
-
-RUN useradd -ms /bin/bash aran
-RUN cd && cp -R .profile .bashrc /home/aran
-ADD . /home/aran/build
-
-RUN chown -R aran:aran /home/aran
-
-RUN cd /home/aran/build && su -m aran -c "export HOME=/home/aran;ct-ng arm-unknown-linux-gnueabi;ct-ng build"
-
diff --git a/tests/ci_build/Dockerfile.doc b/tests/ci_build/Dockerfile.doc
deleted file mode 100644
index 43d1fa9..0000000
--- a/tests/ci_build/Dockerfile.doc
+++ /dev/null
@@ -1,15 +0,0 @@
-FROM ubuntu:14.04
-
-
-COPY install/ubuntu_install_core.sh /install/
-RUN /install/ubuntu_install_core.sh
-COPY install/ubuntu_install_python.sh /install/
-RUN /install/ubuntu_install_python.sh
-COPY install/ubuntu_install_scala.sh /install/
-RUN /install/ubuntu_install_scala.sh
-
-RUN wget http://downloads.lightbend.com/scala/2.11.8/scala-2.11.8.deb && \
- dpkg -i scala-2.11.8.deb && rm scala-2.11.8.deb
-
-RUN apt-get install -y doxygen libatlas-base-dev graphviz pandoc
-RUN pip install sphinx==1.3.5 CommonMark==0.5.4 breathe mock recommonmark pypandoc beautifulsoup4
diff --git a/tests/ci_build/Dockerfile.emscripten b/tests/ci_build/Dockerfile.emscripten
deleted file mode 100644
index e359e6c..0000000
--- a/tests/ci_build/Dockerfile.emscripten
+++ /dev/null
@@ -1,19 +0,0 @@
-FROM ubuntu
-LABEL maintainer "arankhan@amazon.com"
-
-# UPDATE BOX
-RUN apt-get update && apt-get -y upgrade
-
-# TOOLCHAIN DEPS
-RUN apt-get install -y python python-setuptools python-pip python-dev unzip gfortran
-RUN apt-get install -y git nodejs build-essential cmake
-
-# BUILD EMSCRIPTEN
-RUN git clone https://github.com/kripken/emscripten.git
-RUN git clone https://github.com/kripken/emscripten-fastcomp
-RUN cd emscripten-fastcomp && \
-git clone https://github.com/kripken/emscripten-fastcomp-clang tools/clang && \
-mkdir build && cd build && \
-cmake .. -DCMAKE_BUILD_TYPE=Release -DLLVM_TARGETS_TO_BUILD="X86;JSBackend" \
--DLLVM_INCLUDE_EXAMPLES=OFF -DLLVM_INCLUDE_TESTS=OFF -DCLANG_INCLUDE_EXAMPLES=OFF \
--DCLANG_INCLUDE_TESTS=OFF && make
diff --git a/tests/ci_build/Dockerfile.gpu b/tests/ci_build/Dockerfile.gpu
deleted file mode 100644
index bd1a0083..0000000
--- a/tests/ci_build/Dockerfile.gpu
+++ /dev/null
@@ -1,20 +0,0 @@
-FROM nvidia/cuda:8.0-cudnn5-devel
-# cuda8.0 has to be used because this is the first ubuntu16.04 container
-# which is required due to OpenBLAS being incompatible with ubuntu14.04
-
-COPY install/ubuntu_install_core.sh /install/
-RUN /install/ubuntu_install_core.sh
-COPY install/ubuntu_install_python.sh /install/
-RUN /install/ubuntu_install_python.sh
-COPY install/ubuntu_install_scala.sh /install/
-RUN /install/ubuntu_install_scala.sh
-COPY install/ubuntu_install_r.sh /install/
-RUN /install/ubuntu_install_r.sh
-COPY install/ubuntu_install_perl.sh /install/
-RUN /install/ubuntu_install_perl.sh
-
-COPY install/ubuntu_install_llvm.sh /install/
-RUN /install/ubuntu_install_llvm.sh
-
-COPY install/ubuntu_install_tvm.sh /install/
-RUN /install/ubuntu_install_tvm.sh
diff --git a/tests/ci_build/Dockerfile.gpu_mklml b/tests/ci_build/Dockerfile.gpu_mklml
deleted file mode 100644
index f594b1c..0000000
--- a/tests/ci_build/Dockerfile.gpu_mklml
+++ /dev/null
@@ -1,18 +0,0 @@
-FROM nvidia/cuda:8.0-cudnn5-devel
-# cuda8.0 has to be used because this is the first ubuntu16.04 container
-# # which is required due to OpenBLAS being incompatible with ubuntu14.04
-# the reason we used a gpu base container because we are going to test MKLDNN
-# operator implementation against GPU implementation
-
-COPY install/ubuntu_install_core.sh /install/
-RUN /install/ubuntu_install_core.sh
-COPY install/ubuntu_install_python.sh /install/
-RUN /install/ubuntu_install_python.sh
-COPY install/ubuntu_install_scala.sh /install/
-RUN /install/ubuntu_install_scala.sh
-
-# Add MKLML library, compatible with Ubuntu16.04
-RUN wget --no-check-certificate -O /tmp/mklml.tgz https://github.com/01org/mkl-dnn/releases/download/v0.12/mklml_lnx_2018.0.1.20171227.tgz
-RUN tar -zxvf /tmp/mklml.tgz && cp -rf mklml_*/* /usr/local/ && rm -rf mklml_*
-
-ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib/:/usr/lib/gcc/x86_64-linux-gnu/5/
diff --git a/tests/ci_build/Dockerfile.lint b/tests/ci_build/Dockerfile.lint
deleted file mode 100644
index 7fde1f5..0000000
--- a/tests/ci_build/Dockerfile.lint
+++ /dev/null
@@ -1,5 +0,0 @@
-# For lint test
-FROM ubuntu:16.04
-
-RUN apt-get update && apt-get install -y python-pip sudo
-RUN pip install cpplint==1.3.0 pylint==1.8.2
diff --git a/tests/ci_build/Dockerfile.spell_checker b/tests/ci_build/Dockerfile.spell_checker
deleted file mode 100644
index c487959..0000000
--- a/tests/ci_build/Dockerfile.spell_checker
+++ /dev/null
@@ -1,6 +0,0 @@
-# For spell checker
-FROM mxnet/aml:latest
-
-RUN yum install -y enchant
-RUN pip install pyenchant grammar-check html2text
-RUN pip install sphinx==1.5.1 CommonMark==0.5.4 breathe mock==1.0.1 recommonmark
diff --git a/tests/ci_build/Dockerfile.ubuntu1404_cuda75_cudnn5 b/tests/ci_build/Dockerfile.ubuntu1404_cuda75_cudnn5
deleted file mode 100644
index 88fd7ce..0000000
--- a/tests/ci_build/Dockerfile.ubuntu1404_cuda75_cudnn5
+++ /dev/null
@@ -1,40 +0,0 @@
-FROM nvidia/cuda:7.5-cudnn5-devel
-MAINTAINER Ly Nguyen <ly...@amazon.com>
-
-# OPENCV
-RUN apt-mark hold libcudnn5 libcudnn5-dev
-RUN apt-get update && apt-get -y upgrade
-RUN apt-get install -y libopencv-dev
-
-# BLAS
-RUN apt-get install -y libatlas-base-dev
-
-# PYTHON2
-RUN apt-get install -y python-setuptools python-pip python-dev unzip gfortran
-RUN pip install numpy nose scipy
-
-# PYTHON3
-RUN apt-get install -y python3-setuptools python3-pip
-RUN pip3 install numpy nose scipy && ln -s -f /usr/local/bin/nosetests-3.4 /usr/local/bin/nosetests3
-
-# TESTDEPS
-RUN apt-get install -y libgtest-dev cmake wget
-RUN cd /usr/src/gtest && cmake CMakeLists.txt && make && cp *.a /usr/lib
-RUN pip install nose cpplint 'pylint==1.4.4' 'astroid==1.3.6'
-
-# MAVEN
-RUN apt-get install -y software-properties-common
-RUN add-apt-repository ppa:webupd8team/java -y
-RUN apt-get update
-RUN echo "oracle-java8-installer shared/accepted-oracle-license-v1-1 select true" | debconf-set-selections
-RUN apt-get install -y oracle-java8-installer
-RUN apt-get install -y oracle-java8-set-default
-RUN apt-get install -y maven
-
-# R
-RUN apt-get install -y software-properties-common r-base-core libcurl4-openssl-dev libssl-dev libxml2-dev
-RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E084DAB9
-RUN add-apt-repository -y ppa:marutter/rdev
-RUN apt-get update && apt-get -y upgrade
-RUN DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confnew" install r-base r-base-dev
-RUN Rscript -e "install.packages('devtools', repo = 'https://cran.rstudio.com')"
diff --git a/tests/ci_build/README.md b/tests/ci_build/README.md
deleted file mode 100644
index 4d647f2..0000000
--- a/tests/ci_build/README.md
+++ /dev/null
@@ -1,41 +0,0 @@
-# MXNET Builds
-
-This directory contains the files and setup instructions to run all tests. They
-are running on [ci.mxnet.io](http://ci.mxnet.io/blue/pipelines). But you can also
-run them locally easily.
-
-## Run locally
-
-To run locally, we need to first install
-[docker](https://docs.docker.com/engine/installation/) and
-[nvidia-docker](https://github.com/NVIDIA/nvidia-docker/wiki).
-
-We may use the AWS EC2 AMI `ami-d73bb4b7` available at US West (Oregon) which
-has both pre-installed.
-
-Then we can run the tasks defined in the [Jenkinsfile](../../Jenkinsfile) by
-using (`ci_build.sh`)[./ci_build.sh]. For example
-
-- lint the python codes
-
- ```bash
- ./ci_build.sh lint make pylint
- ```
-
-- build codes with CUDA supports
-
- ```bash
- ./ci_build.sh gpu make -j$(nproc) USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1
- ```
-
-- do the python unittest
-
- ```bash
- ./ci_build.sh gpu PYTHONPATH=./python/ nosetests --with-timer --verbose tests/python/unittest'
- ```
-
-- build the documents. The results will be available at `docs/_build/html`
-
- ```bash
- tests/ci_build/ci_build.sh doc make -C docs html
- ```
diff --git a/tests/ci_build/ci_build.sh b/tests/ci_build/ci_build.sh
deleted file mode 100755
index 768a2ed..0000000
--- a/tests/ci_build/ci_build.sh
+++ /dev/null
@@ -1,188 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-#
-# Execute command within a docker container
-#
-# Usage: ci_build.sh <CONTAINER_TYPE> [--dockerfile <DOCKERFILE_PATH>] [-it]
-# <COMMAND>
-#
-# CONTAINER_TYPE: Type of the docker container used the run the build: e.g.,
-# (cpu | gpu)
-#
-# DOCKERFILE_PATH: (Optional) Path to the Dockerfile used for docker build. If
-# this optional value is not supplied (via the --dockerfile
-# flag), will use Dockerfile.CONTAINER_TYPE in default
-#
-# COMMAND: Command to be executed in the docker container
-#
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-
-# Get the command line arguments.
-CONTAINER_TYPE=$( echo "$1" | tr '[:upper:]' '[:lower:]' )
-shift 1
-
-# Dockerfile to be used in docker build
-DOCKERFILE_PATH="${SCRIPT_DIR}/Dockerfile.${CONTAINER_TYPE}"
-DOCKER_CONTEXT_PATH="${SCRIPT_DIR}"
-
-if [[ "$1" == "--dockerfile" ]]; then
- DOCKERFILE_PATH="$2"
- DOCKER_CONTEXT_PATH=$(dirname "${DOCKERFILE_PATH}")
- echo "Using custom Dockerfile path: ${DOCKERFILE_PATH}"
- echo "Using custom docker build context path: ${DOCKER_CONTEXT_PATH}"
- shift 2
-fi
-
-if [[ "$1" == "-it" ]]; then
- CI_DOCKER_EXTRA_PARAMS+=('-it')
- shift 1
-fi
-
-if [[ "$1" == "--dockerbinary" ]]; then
- DOCKER_BINARY="$2"
- echo "Using custom Docker Engine: ${DOCKER_BINARY}"
- shift 2
-fi
-
-if [[ ! -f "${DOCKERFILE_PATH}" ]]; then
- echo "Invalid Dockerfile path: \"${DOCKERFILE_PATH}\""
- exit 1
-fi
-
-COMMAND=("$@")
-
-# Validate command line arguments.
-if [ "$#" -lt 1 ] || [ ! -e "${SCRIPT_DIR}/Dockerfile.${CONTAINER_TYPE}" ]; then
- supported_container_types=$( ls -1 ${SCRIPT_DIR}/Dockerfile.* | \
- sed -n 's/.*Dockerfile\.\([^\/]*\)/\1/p' | tr '\n' ' ' )
- echo "Usage: $(basename $0) CONTAINER_TYPE COMMAND"
- echo " CONTAINER_TYPE can be one of [${supported_container_types}]"
- echo " COMMAND is a command (with arguments) to run inside"
- echo " the container."
- exit 1
-fi
-
-# Only set docker binary automatically if it has not been specified
-if [[ -z "${DOCKER_BINARY}" ]]; then
- # Use nvidia-docker if the container is GPU.
- if [[ "${CONTAINER_TYPE}" == *"gpu"* ]]; then
- DOCKER_BINARY="nvidia-docker"
- else
- DOCKER_BINARY="docker"
- fi
- echo "Automatically assuming ${DOCKER_BINARY} as docker binary"
-fi
-
-# Helper function to traverse directories up until given file is found.
-function upsearch () {
- test / == "$PWD" && return || \
- test -e "$1" && echo "$PWD" && return || \
- cd .. && upsearch "$1"
-}
-
-# Set up WORKSPACE. Jenkins will set them for you or we pick
-# reasonable defaults if you run it outside of Jenkins.
-WORKSPACE="${WORKSPACE:-${SCRIPT_DIR}/../../}"
-
-# Set up a default WORKDIR, unless otherwise specified.
-if [[ -z "${WORKDIR}" ]]; then
- WORKDIR="/workspace"
-fi
-
-# Determine the docker image name
-DOCKER_IMG_NAME="mx-ci.${CONTAINER_TYPE}"
-
-# Under Jenkins matrix build, the build tag may contain characters such as
-# commas (,) and equal signs (=), which are not valid inside docker image names.
-DOCKER_IMG_NAME=$(echo "${DOCKER_IMG_NAME}" | sed -e 's/=/_/g' -e 's/,/-/g')
-
-# Convert to all lower-case, as per requirement of Docker image names
-DOCKER_IMG_NAME=$(echo "${DOCKER_IMG_NAME}" | tr '[:upper:]' '[:lower:]')
-
-# Skip with_the_same_user for non-linux
-uname=`uname`
-if [[ "$uname" == "Linux" ]]; then
- # By convention the root of our source dir is always mapped to the /workspace folder
- # inside the docker container. Our working directory when we start the container
- # is variable, so we should ensure we call commands such as with_the_same_user
- # with their absolute paths.
- PRE_COMMAND="/workspace/tests/ci_build/with_the_same_user"
-else
- PRE_COMMAND=""
-fi
-
-# Print arguments.
-echo "WORKSPACE: ${WORKSPACE}"
-echo "WORKDIR: ${WORKDIR}"
-echo "CI_DOCKER_EXTRA_PARAMS: ${CI_DOCKER_EXTRA_PARAMS[@]}"
-echo "COMMAND: ${COMMAND[@]}"
-echo "CONTAINER_TYPE: ${CONTAINER_TYPE}"
-echo "BUILD_TAG: ${BUILD_TAG}"
-echo "NODE_NAME: ${NODE_NAME}"
-echo "DOCKER CONTAINER NAME: ${DOCKER_IMG_NAME}"
-echo "PRE_COMMAND: ${PRE_COMMAND}"
-echo ""
-
-
-# Build the docker container.
-echo "Building container (${DOCKER_IMG_NAME})..."
-docker build -t ${DOCKER_IMG_NAME} \
- -f "${DOCKERFILE_PATH}" "${DOCKER_CONTEXT_PATH}"
-
-# Check docker build status
-if [[ $? != "0" ]]; then
- echo "ERROR: docker build failed."
- exit 1
-fi
-
-
-# Run the command inside the container.
-echo "Running '${COMMAND[@]}' inside ${DOCKER_IMG_NAME}..."
-
-# By default we cleanup - remove the container once it finish running (--rm)
-# and share the PID namespace (--pid=host) so the process inside does not have
-# pid 1 and SIGKILL is propagated to the process inside (jenkins can kill it).
-
-# Turning off MXNET_STORAGE_FALLBACK_LOG_WARNING temporarily per this issue:
-# https://github.com/apache/incubator-mxnet/issues/8980
-
-# By convention always map the root of the MXNet source directory into /workspace.
-# ${WORKSPACE} represents the path to the source folder on the host system.
-# ${WORKDIR} is the working directory we start the container in. By default this
-# is /workspace, but for example when running cmake it is sometimes /workspace/build.
-
-# Currently compiling for sm52 (g3s) and sm61(p3s). sm61 is required to test DP4A support on p3
-# instances. In the long term we should enable sm70 for p3 instances, but this will require
-# CUDA 9(.1) support with our base AMI drivers.
-
-${DOCKER_BINARY} run --rm --pid=host \
- -v ${WORKSPACE}:/workspace \
- -w ${WORKDIR} \
- -e "CI_BUILD_HOME=${WORKSPACE}" \
- -e "CI_BUILD_USER=$(id -u -n)" \
- -e "CI_BUILD_UID=$(id -u)" \
- -e "CI_BUILD_GROUP=$(id -g -n)" \
- -e "CI_BUILD_GID=$(id -g)" \
- -e "CUDA_ARCH=-gencode arch=compute_52,code=[sm_52,compute_52] -gencode arch=compute_61,code=sm_61 --fatbin-options -compress-all" \
- -e "MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0" \
- ${CI_DOCKER_EXTRA_PARAMS[@]} \
- ${DOCKER_IMG_NAME} \
- ${PRE_COMMAND} \
- ${COMMAND[@]}
diff --git a/tests/ci_build/with_the_same_user b/tests/ci_build/with_the_same_user
deleted file mode 100755
index 19f0de1..0000000
--- a/tests/ci_build/with_the_same_user
+++ /dev/null
@@ -1,47 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# This script is a wrapper creating the same user inside container as the one
-# running the ci_build.sh outside the container. It also set the home directory
-# for the user inside container to match the same absolute path as the workspace
-# outside of container. Do not run this manually. It does not make sense. It is
-# intended to be called by ci_build.sh only.
-
-set -e
-
-COMMAND=("$@")
-
-if ! touch /this_is_writable_file_system; then
- echo "You can't write to your filesystem!"
- echo "If you are in Docker you should check you do not have too many images" \
- "with too many files in them. Docker has some issue with it."
- exit 1
-else
- rm /this_is_writable_file_system
-fi
-
-getent group "${CI_BUILD_GID}" || addgroup --gid "${CI_BUILD_GID}" "${CI_BUILD_GROUP}"
-getent passwd "${CI_BUILD_UID}" || adduser --gid "${CI_BUILD_GID}" --uid "${CI_BUILD_UID}" \
- --gecos "${CI_BUILD_USER} (generated by with_the_same_user script)" \
- --disabled-password --home "${CI_BUILD_HOME}" --quiet "${CI_BUILD_USER}"
-usermod -a -G sudo "${CI_BUILD_USER}"
-echo "${CI_BUILD_USER} ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/90-nopasswd-sudo
-
-sudo -u "#${CI_BUILD_UID}" --preserve-env "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" \
-"HOME=${CI_BUILD_HOME}" ${COMMAND[@]}
diff --git a/tests/python/cpu/test_mkldnn.py b/tests/python/cpu/test_mkldnn.py
deleted file mode 100644
index 16b3003..0000000
--- a/tests/python/cpu/test_mkldnn.py
+++ /dev/null
@@ -1,58 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-"""
-MKL-DNN related test cases
-"""
-
-import logging
-import os
-from sys import platform
-
-def test_mkldnn_install():
- """
- This test will verify that MXNet is built/installed correctly when
- compiled with Intel MKL-DNN library. The method will try to import
- the mxnet module and see if the mkldnn library is mapped to this
- process's address space.
- """
- logging.basicConfig(level=logging.INFO)
-
- if not platform.startswith('linux'):
- logging.info("Bypass mkldnn install test for non-Linux OS")
- return
-
- try:
- #pylint: disable=unused-variable
- import mxnet as mx
- except (ImportError, OSError) as e:
- assert 0, "Import mxnet error: %s. Please double check your build/" \
- "install steps or environment variable settings" % str(e)
-
- pid = os.getpid()
- rc = os.system("cat /proc/" + str(pid) + \
- "/maps | grep libmkldnn > /dev/null")
-
- if rc == 0:
- logging.info("MXNet is built/installed correctly with MKL-DNN")
- else:
- assert 0, "MXNet is built/installed incorrectly with MKL-DNN, please " \
- "double check your build/install steps or environment " \
- "variable settings"
-
-if __name__ == '__main__':
- test_mkldnn_install()
diff --git a/tools/caffe_converter/compare_layers.py b/tools/caffe_converter/compare_layers.py
index 3f6883e..6cb82e8 100644
--- a/tools/caffe_converter/compare_layers.py
+++ b/tools/caffe_converter/compare_layers.py
@@ -22,9 +22,9 @@ import logging
import os
import warnings
+import numpy as np
import cv2
import mxnet as mx
-import numpy as np
logging.basicConfig(level=logging.INFO)
diff --git a/tools/caffe_converter/convert_mean.py b/tools/caffe_converter/convert_mean.py
index 3b6dc42..1a3df71 100644
--- a/tools/caffe_converter/convert_mean.py
+++ b/tools/caffe_converter/convert_mean.py
@@ -18,8 +18,8 @@
"""Convert caffe mean
"""
import argparse
-import mxnet as mx
import numpy as np
+import mxnet as mx
import caffe_parser
def convert_mean(binaryproto_fname, output=None):
diff --git a/tools/caffe_converter/convert_model.py b/tools/caffe_converter/convert_model.py
index b2ea57e..d5c069b 100644
--- a/tools/caffe_converter/convert_model.py
+++ b/tools/caffe_converter/convert_model.py
@@ -21,9 +21,9 @@ from __future__ import print_function
import argparse
import sys
import re
+import numpy as np
import caffe_parser
import mxnet as mx
-import numpy as np
from convert_symbol import convert_symbol
def prob_label(arg_names):
--
To stop receiving notification emails like this one, please contact
zhasheng@apache.org.