You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by GitBox <gi...@apache.org> on 2018/03/09 22:33:12 UTC
[GitHub] szha closed pull request #9995: CI docker revamp; Add Jetson, Raspberry and CentOS 7 build [MXNET-42][MXNET-43]
szha closed pull request #9995: CI docker revamp; Add Jetson, Raspberry and CentOS 7 build [MXNET-42][MXNET-43]
URL: https://github.com/apache/incubator-mxnet/pull/9995
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/.mxnet_root b/.mxnet_root
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/CMakeLists.txt b/CMakeLists.txt
index a076524992f..d5969288ced 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -15,6 +15,7 @@ mxnet_option(USE_NCCL "Use NVidia NCCL with CUDA" OFF)
mxnet_option(USE_OPENCV "Build with OpenCV support" ON)
mxnet_option(USE_OPENMP "Build with Openmp support" ON)
mxnet_option(USE_CUDNN "Build with cudnn support" ON) # one could set CUDNN_ROOT for search path
+mxnet_option(USE_SSE "Build with x86 SSE instruction support" AUTO)
mxnet_option(USE_LAPACK "Build with lapack support" ON IF NOT MSVC)
mxnet_option(USE_MKL_IF_AVAILABLE "Use MKL if found" ON)
mxnet_option(USE_MKLML_MKL "Use MKLDNN variant of MKL (if MKL found)" ON IF USE_MKL_IF_AVAILABLE AND UNIX AND (NOT APPLE))
@@ -90,11 +91,17 @@ if(MSVC)
else(MSVC)
include(CheckCXXCompilerFlag)
if(USE_CXX14_IF_AVAILABLE)
- check_cxx_compiler_flag("-std=c++14" SUPPORT_CXX14)
+ check_cxx_compiler_flag("-std=c++14" SUPPORT_CXX14)
endif()
check_cxx_compiler_flag("-std=c++11" SUPPORT_CXX11)
check_cxx_compiler_flag("-std=c++0x" SUPPORT_CXX0X)
- check_cxx_compiler_flag("-msse2" SUPPORT_MSSE2)
+ # For cross compilation, we can't rely on the compiler which accepts the flag, but mshadow will
+ # add platform specific includes not available in other arches
+ if(USE_SSE STREQUAL "AUTO")
+ check_cxx_compiler_flag("-msse2" SUPPORT_MSSE2)
+ else()
+ set(SUPPORT_MSSE2 FALSE)
+ endif()
set(CMAKE_C_FLAGS "-Wall -Wno-unknown-pragmas -fPIC -Wno-sign-compare")
if ("${CMAKE_CXX_COMPILER_ID}" MATCHES ".*Clang$")
set(CMAKE_C_FLAGS "-Wno-braced-scalar-init")
diff --git a/Jenkinsfile b/Jenkinsfile
index d0295f013eb..2bac346c765 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -72,41 +72,6 @@ def init_git_win() {
}
}
-// Run make. First try to do an incremental build from a previous workspace in hope to
-// accelerate the compilation. If something wrong, clean the workspace and then
-// build from scratch.
-def make(docker_type, make_flag) {
- timeout(time: max_time, unit: 'MINUTES') {
- try {
- sh "${docker_run} ${docker_type} --dockerbinary docker make ${make_flag}"
- } catch (exc) {
- echo 'Incremental compilation failed with ${exc}. Fall back to build from scratch'
- sh "${docker_run} ${docker_type} --dockerbinary docker sudo make clean"
- sh "${docker_run} ${docker_type} --dockerbinary docker sudo make -C amalgamation/ clean"
- sh "${docker_run} ${docker_type} --dockerbinary docker make ${make_flag}"
- }
- }
-}
-
-// Run cmake. First try to do an incremental build from a previous workspace in hope to
-// accelerate the compilation. If something wrong, clean the workspace and then
-// build from scratch.
-def cmake(docker_type, cmake_defines, flags) {
- timeout(time: max_time, unit: 'MINUTES') {
- try {
- sh "${docker_run} ${docker_type} --dockerbinary docker mkdir -p build"
- sh "WORKDIR=/workspace/build ${docker_run} ${docker_type} --dockerbinary docker cmake ${cmake_defines} -G Ninja .."
- sh "WORKDIR=/workspace/build ${docker_run} ${docker_type} --dockerbinary docker ninja ${flags}"
- } catch (exc) {
- echo 'Incremental compilation failed with ${exc}. Fall back to build from scratch'
- sh "${docker_run} ${docker_type} --dockerbinary docker git clean -fdx"
- sh "${docker_run} ${docker_type} --dockerbinary docker mkdir -p build"
- sh "WORKDIR=/workspace/build ${docker_run} ${docker_type} --dockerbinary docker cmake ${cmake_defines} -G Ninja .."
- sh "WORKDIR=/workspace/build ${docker_run} ${docker_type} --dockerbinary docker ninja ${flags}"
- }
- }
-}
-
// pack libraries for later use
def pack_lib(name, libs=mx_lib) {
sh """
@@ -127,171 +92,95 @@ echo ${libs} | sed -e 's/,/ /g' | xargs md5sum
// Python unittest for CPU
// Python 2
-def python2_ut(docker_type) {
+def python2_ut(docker_container_name) {
timeout(time: max_time, unit: 'MINUTES') {
- sh "${docker_run} ${docker_type} --dockerbinary docker find . -name '*.pyc' -type f -delete"
- sh "${docker_run} ${docker_type} --dockerbinary docker PYTHONPATH=./python/ nosetests-2.7 --with-timer --verbose tests/python/unittest"
- sh "${docker_run} ${docker_type} --dockerbinary docker PYTHONPATH=./python/ nosetests-2.7 --with-timer --verbose tests/python/train"
+ sh "ci/build.py --build --platform ${docker_container_name} /work/runtime_functions.sh unittest_ubuntu_python2_cpu"
}
}
// Python 3
-def python3_ut(docker_type) {
+def python3_ut(docker_container_name) {
timeout(time: max_time, unit: 'MINUTES') {
- sh "${docker_run} ${docker_type} --dockerbinary docker find . -name '*.pyc' -type f -delete"
- sh "${docker_run} ${docker_type} --dockerbinary docker PYTHONPATH=./python/ nosetests-3.4 --with-timer --verbose tests/python/unittest"
+ sh "ci/build.py --build --platform ${docker_container_name} /work/runtime_functions.sh unittest_ubuntu_python3_cpu"
}
}
// GPU test has two parts. 1) run unittest on GPU, 2) compare the results on
// both CPU and GPU
// Python 2
-def python2_gpu_ut(docker_type) {
- timeout(time: max_time, unit: 'MINUTES') {
- sh "${docker_run} ${docker_type} find . -name '*.pyc' -type f -delete"
- sh "${docker_run} ${docker_type} PYTHONPATH=./python/ nosetests-2.7 --with-timer --verbose tests/python/gpu"
- }
-}
-
-// Python 3
-def python3_gpu_ut(docker_type) {
- timeout(time: max_time, unit: 'MINUTES') {
- sh "${docker_run} ${docker_type} find . -name '*.pyc' -type f -delete"
- sh "${docker_run} ${docker_type} PYTHONPATH=./python/ nosetests-3.4 --with-timer --verbose tests/python/gpu"
- }
-}
-
-// Python 2
-def python2_mkldnn_ut(docker_type) {
+def python2_gpu_ut(docker_container_name) {
timeout(time: max_time, unit: 'MINUTES') {
- sh "${docker_run} ${docker_type} find . -name '*.pyc' -type f -delete"
- sh "${docker_run} ${docker_type} PYTHONPATH=./python/ MXNET_MKLDNN_DEBUG=1 nosetests-2.7 --with-timer --verbose tests/python/cpu"
+ sh "ci/build.py --nvidiadocker --build --platform ${docker_container_name} /work/runtime_functions.sh unittest_ubuntu_python2_gpu"
}
}
// Python 3
-def python3_mkldnn_ut(docker_type) {
+def python3_gpu_ut(docker_container_name) {
timeout(time: max_time, unit: 'MINUTES') {
- sh "${docker_run} ${docker_type} find . -name '*.pyc' -type f -delete"
- sh "${docker_run} ${docker_type} PYTHONPATH=./python/ MXNET_MKLDNN_DEBUG=1 nosetests-3.4 --with-timer --verbose tests/python/cpu"
+ sh "ci/build.py --nvidiadocker --build --platform ${docker_container_name} /work/runtime_functions.sh unittest_ubuntu_python3_gpu"
}
}
try {
stage("Sanity Check") {
- timeout(time: max_time, unit: 'MINUTES') {
- node('mxnetlinux-cpu') {
- ws('workspace/sanity') {
- init_git()
- sh "tools/license_header.py check"
- make('lint', 'cpplint rcpplint jnilint')
- make('lint', 'pylint')
- }
+ node('mxnetlinux-cpu') {
+ ws('workspace/sanity') {
+ init_git()
+ sh "ci/build.py --build --platform ubuntu_cpu /work/runtime_functions.sh sanity_check"
}
}
}
stage('Build') {
- parallel 'CPU: Openblas': {
+ parallel 'CPU: CentOS 7': {
node('mxnetlinux-cpu') {
- ws('workspace/build-cpu') {
- init_git()
- def flag = """ \
- DEV=1 \
- USE_PROFILER=1 \
- USE_CPP_PACKAGE=1 \
- USE_BLAS=openblas \
- -j\$(nproc)
- """
- make("cpu", flag)
- pack_lib('cpu')
+ ws('workspace/build-centos7-cpu') {
+ init_git()
+ sh "ci/build.py --build --platform centos7_cpu /work/runtime_functions.sh build_centos7_cpu"
+ pack_lib('centos7_cpu')
}
}
},
- 'CPU: Clang 3.9': {
+ 'GPU: CentOS 7': {
node('mxnetlinux-cpu') {
- ws('workspace/build-cpu-clang') {
+ ws('workspace/build-centos7-gpu') {
init_git()
- def flag = """ \
- USE_PROFILER=1 \
- USE_CPP_PACKAGE=1 \
- USE_BLAS=openblas \
- USE_OPENMP=0 \
- CXX=clang++-3.9 \
- CC=clang-3.9 \
- -j\$(nproc)
- """
- make("cpu_clang", flag)
- pack_lib('cpu_clang')
+ sh "ci/build.py --build --platform centos7_gpu /work/runtime_functions.sh build_centos7_gpu"
+ pack_lib('centos7_gpu')
}
}
},
- 'CPU: Clang 5': {
+ 'CPU: Openblas': {
node('mxnetlinux-cpu') {
- ws('workspace/build-cpu-clang') {
+ ws('workspace/build-cpu-openblas') {
init_git()
- def flag = """ \
- USE_PROFILER=1 \
- USE_CPP_PACKAGE=1 \
- USE_BLAS=openblas \
- USE_OPENMP=1 \
- CXX=clang++-5.0 \
- CC=clang-5.0 \
- -j\$(nproc)
- """
- make("cpu_clang", flag)
- pack_lib('cpu_clang')
+ sh "ci/build.py --build --platform ubuntu_cpu /work/runtime_functions.sh build_ubuntu_cpu_openblas"
+ pack_lib('cpu')
}
}
},
- 'CPU: MKLDNN': {
+ 'CPU: Clang 3.9': {
node('mxnetlinux-cpu') {
- ws('workspace/build-mkldnn-cpu') {
+ ws('workspace/build-cpu-clang39') {
init_git()
- def flag = """ \
- DEV=1 \
- USE_PROFILER=1 \
- USE_CPP_PACKAGE=1 \
- USE_BLAS=openblas \
- USE_MKLDNN=1 \
- -j\$(nproc)
- """
- make("cpu_mklml", flag)
- pack_lib('mkldnn_cpu', mx_mkldnn_lib)
+ sh "ci/build.py --build --platform ubuntu_cpu /work/runtime_functions.sh build_ubuntu_cpu_clang39"
}
}
},
- 'GPU: CMake MKLDNN': {
+ 'CPU: Clang 5': {
node('mxnetlinux-cpu') {
- ws('workspace/build-cmake-mkldnn-gpu') {
+ ws('workspace/build-cpu-clang50') {
init_git()
- def defines = """ \
- -DUSE_CUDA=1 \
- -DUSE_CUDNN=1 \
- -DUSE_MKLML_MKL=1 \
- -DUSE_MKLDNN=1 \
- -DCMAKE_BUILD_TYPE=Release \
- """
- def flag = "-v"
- cmake("build_cuda", defines, flag)
- pack_lib('cmake_mkldnn_gpu', mx_cmake_mkldnn_lib)
+ sh "ci/build.py --build --platform ubuntu_cpu /work/runtime_functions.sh build_ubuntu_cpu_clang50"
}
}
},
- 'GPU: CMake': {
+ 'CPU: MKLDNN': {
node('mxnetlinux-cpu') {
- ws('workspace/build-cmake-gpu') {
+ ws('workspace/build-mkldnn-cpu') {
init_git()
- def defines = """ \
- -DUSE_CUDA=1 \
- -DUSE_CUDNN=1 \
- -DUSE_MKLML_MKL=0 \
- -DUSE_MKLDNN=0 \
- -DCMAKE_BUILD_TYPE=Release \
- """
- def flag = "-v"
- cmake("build_cuda", defines, flag)
- pack_lib('cmake_gpu', mx_cmake_lib)
+ sh "ci/build.py --build --platform ubuntu_cpu /work/runtime_functions.sh build_ubuntu_cpu_mkldnn"
+ pack_lib('mkldnn_cpu', mx_mkldnn_lib)
}
}
},
@@ -299,18 +188,7 @@ try {
node('mxnetlinux-cpu') {
ws('workspace/build-mkldnn-gpu') {
init_git()
- def flag = """ \
- DEV=1 \
- USE_PROFILER=1 \
- USE_CPP_PACKAGE=1 \
- USE_BLAS=openblas \
- USE_MKLDNN=1 \
- USE_CUDA=1 \
- USE_CUDA_PATH=/usr/local/cuda \
- USE_CUDNN=1 \
- -j\$(nproc)
- """
- make("build_cuda", flag)
+ sh "ci/build.py --build --platform ubuntu_build_cuda /work/runtime_functions.sh build_ubuntu_gpu_mkldnn"
pack_lib('mkldnn_gpu', mx_mkldnn_lib)
}
}
@@ -319,17 +197,7 @@ try {
node('mxnetlinux-cpu') {
ws('workspace/build-gpu') {
init_git()
- def flag = """ \
- DEV=1 \
- USE_PROFILER=1 \
- USE_BLAS=openblas \
- USE_CUDA=1 \
- USE_CUDA_PATH=/usr/local/cuda \
- USE_CUDNN=1 \
- USE_CPP_PACKAGE=1 \
- -j\$(nproc)
- """
- make('build_cuda', flag)
+ sh "ci/build.py --build --platform ubuntu_build_cuda /work/runtime_functions.sh build_ubuntu_gpu_cuda8_cudnn5"
pack_lib('gpu')
stash includes: 'build/cpp-package/example/test_score', name: 'cpp_test_score'
stash includes: 'build/cpp-package/example/test_optimizer', name: 'cpp_test_optimizer'
@@ -340,8 +208,7 @@ try {
node('mxnetlinux-cpu') {
ws('workspace/amalgamationmin') {
init_git()
- make('cpu', '-C amalgamation/ clean')
- make('cpu', '-C amalgamation/ USE_BLAS=openblas MIN=1')
+ sh "ci/build.py --build --platform ubuntu_cpu /work/runtime_functions.sh build_ubuntu_amalgamation_min"
}
}
},
@@ -349,8 +216,26 @@ try {
node('mxnetlinux-cpu') {
ws('workspace/amalgamation') {
init_git()
- make('cpu', '-C amalgamation/ clean')
- make('cpu', '-C amalgamation/ USE_BLAS=openblas')
+ sh "ci/build.py --build --platform ubuntu_cpu /work/runtime_functions.sh build_ubuntu_amalgamation"
+ }
+ }
+ },
+
+ 'GPU: CMake MKLDNN': {
+ node('mxnetlinux-cpu') {
+ ws('workspace/build-cmake-mkldnn-gpu') {
+ init_git()
+ sh "ci/build.py --build --platform ubuntu_gpu /work/runtime_functions.sh build_ubuntu_gpu_cmake_mkldnn" //build_cuda
+ pack_lib('cmake_mkldnn_gpu', mx_cmake_mkldnn_lib)
+ }
+ }
+ },
+ 'GPU: CMake': {
+ node('mxnetlinux-cpu') {
+ ws('workspace/build-cmake-gpu') {
+ init_git()
+ sh "ci/build.py --build --platform ubuntu_gpu /work/runtime_functions.sh build_ubuntu_gpu_cmake" //build_cuda
+ pack_lib('cmake_gpu', mx_cmake_lib)
}
}
},
@@ -419,6 +304,22 @@ try {
}
}
}
+ },
+ 'NVidia Jetson / ARMv8':{
+ node('mxnetlinux-cpu') {
+ ws('workspace/build-jetson-armv8') {
+ init_git()
+ sh "ci/build.py --build --platform jetson /work/runtime_functions.sh build_jetson"
+ }
+ }
+ },
+ 'Raspberry / ARMv7':{
+ node('mxnetlinux-cpu') {
+ ws('workspace/build-raspberry-armv7') {
+ init_git()
+ sh "ci/build.py --build --platform armv7 /work/runtime_functions.sh build_armv7"
+ }
+ }
}
} // End of stage('Build')
@@ -428,7 +329,7 @@ try {
ws('workspace/ut-python2-cpu') {
init_git()
unpack_lib('cpu')
- python2_ut('cpu')
+ python2_ut('ubuntu_cpu')
}
}
},
@@ -437,7 +338,7 @@ try {
ws('workspace/ut-python3-cpu') {
init_git()
unpack_lib('cpu')
- python3_ut('cpu')
+ python3_ut('ubuntu_cpu')
}
}
},
@@ -446,7 +347,7 @@ try {
ws('workspace/ut-python2-gpu') {
init_git()
unpack_lib('gpu', mx_lib)
- python2_gpu_ut('gpu')
+ python2_gpu_ut('ubuntu_gpu')
}
}
},
@@ -455,7 +356,7 @@ try {
ws('workspace/ut-python3-gpu') {
init_git()
unpack_lib('gpu', mx_lib)
- python3_gpu_ut('gpu')
+ python3_gpu_ut('ubuntu_gpu')
}
}
},
@@ -464,8 +365,7 @@ try {
ws('workspace/ut-python2-mkldnn-cpu') {
init_git()
unpack_lib('mkldnn_cpu', mx_mkldnn_lib)
- python2_ut('cpu_mklml')
- python2_mkldnn_ut('cpu_mklml')
+ python2_ut('ubuntu_cpu')
}
}
},
@@ -474,8 +374,7 @@ try {
ws('workspace/ut-python2-mkldnn-gpu') {
init_git()
unpack_lib('mkldnn_gpu', mx_mkldnn_lib)
- python2_gpu_ut('gpu_mklml')
- python2_mkldnn_ut('gpu_mklml')
+ python2_gpu_ut('ubuntu_gpu')
}
}
},
@@ -484,8 +383,7 @@ try {
ws('workspace/ut-python3-mkldnn-cpu') {
init_git()
unpack_lib('mkldnn_cpu', mx_mkldnn_lib)
- python3_ut('cpu_mklml')
- python3_mkldnn_ut('cpu_mklml')
+ python3_ut('ubuntu_cpu')
}
}
},
@@ -494,8 +392,29 @@ try {
ws('workspace/ut-python3-mkldnn-gpu') {
init_git()
unpack_lib('mkldnn_gpu', mx_mkldnn_lib)
- python3_gpu_ut('gpu_mklml')
- python3_mkldnn_ut('gpu_mklml')
+ python3_gpu_ut('ubuntu_gpu')
+ }
+ }
+ },
+ 'Python3: CentOS 7 CPU': {
+ node('mxnetlinux-cpu') {
+ ws('workspace/build-centos7-cpu') {
+ init_git()
+ unpack_lib('centos7_cpu')
+ timeout(time: max_time, unit: 'MINUTES') {
+ sh "ci/build.py --build --platform centos7_cpu /work/runtime_functions.sh unittest_centos7_cpu"
+ }
+ }
+ }
+ },
+ 'Python3: CentOS 7 GPU': {
+ node('mxnetlinux-gpu') {
+ ws('workspace/build-centos7-gpu') {
+ init_git()
+ unpack_lib('centos7_gpu')
+ timeout(time: max_time, unit: 'MINUTES') {
+ sh "ci/build.py --nvidiadocker --build --platform centos7_gpu /work/runtime_functions.sh unittest_centos7_gpu"
+ }
}
}
},
@@ -505,8 +424,7 @@ try {
init_git()
unpack_lib('cpu')
timeout(time: max_time, unit: 'MINUTES') {
- sh "${docker_run} cpu make scalapkg USE_BLAS=openblas"
- sh "${docker_run} cpu make scalatest USE_BLAS=openblas"
+ sh "ci/build.py --build --platform ubuntu_cpu /work/runtime_functions.sh unittest_ubuntu_cpu_scala"
}
}
}
@@ -517,7 +435,7 @@ try {
init_git()
unpack_lib('cpu')
timeout(time: max_time, unit: 'MINUTES') {
- sh "${docker_run} cpu ./perl-package/test.sh"
+ sh "ci/build.py --build --platform ubuntu_cpu /work/runtime_functions.sh unittest_ubuntu_cpugpu_perl"
}
}
}
@@ -528,7 +446,7 @@ try {
init_git()
unpack_lib('gpu')
timeout(time: max_time, unit: 'MINUTES') {
- sh "${docker_run} gpu ./perl-package/test.sh"
+ sh "ci/build.py --nvidiadocker --build --platform ubuntu_gpu /work/runtime_functions.sh unittest_ubuntu_cpugpu_perl"
}
}
}
@@ -539,7 +457,7 @@ try {
init_git()
unpack_lib('cmake_gpu', mx_cmake_lib)
timeout(time: max_time, unit: 'MINUTES') {
- sh "${docker_run} gpu_mklml build/tests/mxnet_unit_tests"
+ sh "ci/build.py --nvidiadocker --build --platform ubuntu_gpu /work/runtime_functions.sh unittest_ubuntu_gpu_cpp"
}
}
}
@@ -550,11 +468,7 @@ try {
init_git()
unpack_lib('cpu')
timeout(time: max_time, unit: 'MINUTES') {
- sh "${docker_run} cpu rm -rf .Renviron"
- sh "${docker_run} cpu mkdir -p /workspace/ut-r-cpu/site-library"
- sh "${docker_run} cpu make rpkg USE_BLAS=openblas R_LIBS=/workspace/ut-r-cpu/site-library"
- sh "${docker_run} cpu R CMD INSTALL --library=/workspace/ut-r-cpu/site-library R-package"
- sh "${docker_run} cpu make rpkgtest R_LIBS=/workspace/ut-r-cpu/site-library"
+ sh "ci/build.py --build --platform ubuntu_cpu /work/runtime_functions.sh unittest_ubuntu_cpu_R"
}
}
}
@@ -565,15 +479,12 @@ try {
init_git()
unpack_lib('gpu')
timeout(time: max_time, unit: 'MINUTES') {
- sh "${docker_run} gpu rm -rf .Renviron"
- sh "${docker_run} gpu mkdir -p /workspace/ut-r-gpu/site-library"
- sh "${docker_run} gpu make rpkg USE_BLAS=openblas R_LIBS=/workspace/ut-r-gpu/site-library"
- sh "${docker_run} gpu R CMD INSTALL --library=/workspace/ut-r-gpu/site-library R-package"
- sh "${docker_run} gpu make rpkgtest R_LIBS=/workspace/ut-r-gpu/site-library R_GPU_ENABLE=1"
+ sh "ci/build.py --nvidiadocker --build --platform ubuntu_gpu /work/runtime_functions.sh unittest_ubuntu_gpu_R"
}
}
}
},
+
'Python 2: CPU Win':{
node('mxnetwindows-cpu') {
timeout(time: max_time, unit: 'MINUTES') {
@@ -590,7 +501,7 @@ try {
C:\\mxnet\\test_cpu.bat"""
}
}
- }
+ }
},
'Python 3: CPU Win': {
node('mxnetwindows-cpu') {
@@ -655,7 +566,7 @@ try {
init_git()
unpack_lib('gpu')
timeout(time: max_time, unit: 'MINUTES') {
- sh "${docker_run} gpu --dockerbinary nvidia-docker PYTHONPATH=./python/ python example/image-classification/test_score.py"
+ sh "ci/build.py --nvidiadocker --build --platform ubuntu_gpu /work/runtime_functions.sh integrationtest_ubuntu_gpu_python"
}
}
}
@@ -666,7 +577,7 @@ try {
init_git()
unpack_lib('gpu')
timeout(time: max_time, unit: 'MINUTES') {
- sh "${docker_run} caffe_gpu --dockerbinary nvidia-docker PYTHONPATH=/caffe/python:./python python tools/caffe_converter/test_converter.py"
+ sh "ci/build.py --nvidiadocker --build --platform ubuntu_gpu /work/runtime_functions.sh integrationtest_ubuntu_gpu_caffe"
}
}
}
@@ -679,7 +590,7 @@ try {
unstash 'cpp_test_score'
unstash 'cpp_test_optimizer'
timeout(time: max_time, unit: 'MINUTES') {
- sh "${docker_run} gpu --dockerbinary nvidia-docker cpp-package/tests/ci_test.sh"
+ sh "ci/build.py --nvidiadocker --build --platform ubuntu_gpu /work/runtime_functions.sh integrationtest_ubuntu_gpu_cpp_package"
}
}
}
@@ -690,9 +601,10 @@ try {
node('mxnetlinux-cpu') {
ws('workspace/docs') {
init_git()
- sh "make clean"
- sh "make docs"
- sh "tests/ci_build/deploy/ci_deploy_doc.sh ${env.BRANCH_NAME} ${env.BUILD_NUMBER}"
+ timeout(time: max_time, unit: 'MINUTES') {
+ sh "ci/build.py --build --platform ubuntu_cpu /work/runtime_functions.sh deploy_docs"
+ sh "tests/ci_build/deploy/ci_deploy_doc.sh ${env.BRANCH_NAME} ${env.BUILD_NUMBER}"
+ }
}
}
}
diff --git a/Makefile b/Makefile
index 5d81c7fbb16..50b125baaef 100644
--- a/Makefile
+++ b/Makefile
@@ -167,6 +167,7 @@ ifeq (,$(wildcard /usr/lib/liblapack.a))
ifeq (,$(wildcard /usr/lib64/liblapack.a))
ifeq (,$(wildcard $(USE_LAPACK_PATH)/liblapack.a))
USE_LAPACK = 0
+ $(warning "USE_LAPACK disabled because libraries were not found")
endif
endif
endif
@@ -473,7 +474,7 @@ pylint:
doc: docs
docs:
- tests/ci_build/ci_build.sh doc make -C docs html
+ make -C docs html
clean_docs:
make -C docs clean
diff --git a/ci/README.md b/ci/README.md
new file mode 100644
index 00000000000..98c74e31d87
--- /dev/null
+++ b/ci/README.md
@@ -0,0 +1,50 @@
+# Containerized build & test utilities
+
+This folder contains scripts and dockerfiles used to build and test MXNet using Docker containers
+
+You need docker and nvidia docker if you have a GPU.
+
+If you are in ubuntu an easy way to install Docker CE is executing the following script:
+
+
+```
+#!/bin/bash
+set -e
+set -x
+export DEBIAN_FRONTEND=noninteractive
+apt-get -y install curl
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
+add-apt-repository \
+ "deb [arch=amd64] https://download.docker.com/linux/ubuntu \
+ $(lsb_release -cs) \
+ stable"
+apt-get update
+apt-get -y install docker-ce
+service docker restart
+usermod -a -G docker $SUDO_USER
+```
+
+For detailed instructions go to the docker documentation.
+
+
+## build.py
+
+The main utility to build is build.py which will run docker and mount the mxnet folder as a volume
+to do in-place builds.
+
+The build.py script does two functions, build the docker image, and it can be also used to run
+commands inside this image with the propper mounts and paraphernalia required to build mxnet inside
+docker from the sources on the parent folder.
+
+A set of helper shell functions are in `functions.sh`. `build.py --help` will display usage
+information about the tool.
+
+To build for armv7 for example:
+
+```
+./build.py -p armv7 /work/functions.sh build_armv7
+```
+
+## Warning
+Due to current limitations of the CMake build system creating artifacts in the source 3rdparty
+folder of the parent mxnet sources concurrent builds of different platforms is NOT SUPPORTED.
diff --git a/ci/build.py b/ci/build.py
new file mode 100755
index 00000000000..8caf52b90bd
--- /dev/null
+++ b/ci/build.py
@@ -0,0 +1,179 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Multi arch dockerized build tool.
+
+"""
+
+__author__ = 'Marco de Abreu, Kellen Sunderland, Anton Chernov, Pedro Larroy'
+__version__ = '0.1'
+
+import os
+import sys
+import subprocess
+import logging
+import argparse
+from subprocess import check_call, call
+import glob
+import re
+from typing import *
+from itertools import chain
+from copy import deepcopy
+
+
+def get_platforms(path: Optional[str]="docker"):
+ """Get a list of architectures given our dockerfiles"""
+ dockerfiles = glob.glob(os.path.join(path, "Dockerfile.build.*"))
+ dockerfiles = list(filter(lambda x: x[-1] != '~', dockerfiles))
+ files = list(map(lambda x: re.sub(r"Dockerfile.build.(.*)", r"\1", x), dockerfiles))
+ files.sort()
+ platforms = list(map(lambda x: os.path.split(x)[1], files))
+ return platforms
+
+
+def get_docker_tag(platform: str) -> None:
+ return "mxnet/build.{0}".format(platform)
+
+
+def get_dockerfile(platform: str, path="docker"):
+ return os.path.join(path, "Dockerfile.build.{0}".format(platform))
+
+def get_docker_binary(use_nvidia_docker: bool):
+ if use_nvidia_docker:
+ return "nvidia-docker"
+ else:
+ return "docker"
+
+def build_docker(platform: str, docker_binary: str) -> None:
+ """Build a container for the given platform"""
+ tag = get_docker_tag(platform)
+ logging.info("Building container tagged '%s' with %s", tag, docker_binary)
+ cmd = [docker_binary, "build",
+ "-f", get_dockerfile(platform),
+ "--build-arg", "USER_ID={}".format(os.getuid()),
+ "-t", tag,
+ "docker"]
+ logging.info("Running command: '%s'", ' '.join(cmd))
+ check_call(cmd)
+
+def get_mxnet_root() -> str:
+ curpath = os.path.abspath(os.path.dirname(__file__))
+ def is_mxnet_root(path: str) -> bool:
+ return os.path.exists(os.path.join(path, ".mxnet_root"))
+ while not is_mxnet_root(curpath):
+ parent = os.path.abspath(os.path.join(curpath, os.pardir))
+ if parent == curpath:
+ raise RuntimeError("Got to the root and couldn't find a parent folder with .mxnet_root")
+ curpath = parent
+ return curpath
+
+
+def container_run(platform: str, docker_binary: str, command: List[str]) -> None:
+ tag = get_docker_tag(platform)
+ mx_root = get_mxnet_root()
+ local_build_folder = '{}/build'.format(mx_root)
+ # We need to create it first, otherwise it will be created by the docker daemon with root only permissions
+ os.makedirs(local_build_folder, exist_ok=True)
+ logging.info("Running %s in container %s", command, tag)
+ runlist = [docker_binary, 'run', '--rm',
+ '-v', "{}:/work/mxnet".format(mx_root), # mount mxnet root
+ '-v', "{}:/work/build".format(local_build_folder), # mount mxnet/build for storing build artifacts
+ '-u', '{}:{}'.format(os.getuid(), os.getgid()),
+ tag]
+ runlist.extend(command)
+ cmd = ' '.join(runlist)
+ logging.info("Executing: %s", cmd)
+ ret = call(runlist)
+ if ret != 0:
+ logging.error("Running of command in container failed: %s", cmd)
+ into_cmd = deepcopy(runlist)
+ idx = into_cmd.index('-u') + 2
+ into_cmd[idx:idx] = ['-ti', '--entrypoint', 'bash']
+ logging.error("You can try to get into the container by using the following command: %s", ' '.join(into_cmd))
+ raise subprocess.CalledProcessError(ret, cmd)
+
+def main() -> int:
+ # We need to be in the same directory than the script so the commands in the dockerfiles work as
+ # expected. But the script can be invoked from a different path
+ base = os.path.split(os.path.realpath(__file__))[0]
+ os.chdir(base)
+
+ logging.getLogger().setLevel(logging.INFO)
+ def script_name() -> str:
+ return os.path.split(sys.argv[0])[1]
+
+ logging.basicConfig(format='{}: %(asctime)-15s %(message)s'.format(script_name()))
+
+ parser = argparse.ArgumentParser()
+ parser.add_argument("-p", "--platform",
+ help="platform",
+ type=str)
+
+ parser.add_argument("-b", "--build",
+ help="Build the container",
+ action='store_true')
+
+ parser.add_argument("-n", "--nvidiadocker",
+ help="Use nvidia docker",
+ action='store_true')
+
+ parser.add_argument("-l", "--list",
+ help="List platforms",
+ action='store_true')
+
+ parser.add_argument("command",
+ help="command to run in the container",
+ nargs='*', action='append', type=str)
+
+ args = parser.parse_args()
+ command = list(chain(*args.command))
+ docker_binary = get_docker_binary(args.nvidiadocker)
+
+ if args.list:
+ platforms = get_platforms()
+ print(platforms)
+
+ elif args.platform:
+ platform = args.platform
+ if args.build:
+ build_docker(platform, docker_binary)
+ tag = get_docker_tag(platform)
+ if command:
+ container_run(platform, docker_binary, command)
+ else:
+ cmd = ["/work/mxnet/ci/docker/runtime_functions.sh", "build_{}".format(platform)]
+ logging.info("No command specified, trying default build: %s", ' '.join(cmd))
+ container_run(platform, docker_binary, cmd)
+
+ else:
+ platforms = get_platforms()
+ logging.info("Building for all architectures: {}".format(platforms))
+ logging.info("Artifacts will be produced in the build/ directory.")
+ for platform in platforms:
+ build_docker(platform, docker_binary)
+ cmd = ["/work/mxnet/ci/docker/runtime_functions.sh", "build_{}".format(platform)]
+ logging.info("No command specified, trying default build: %s", ' '.join(cmd))
+ container_run(platform, docker_binary, cmd)
+
+ return 0
+
+
+if __name__ == '__main__':
+ sys.exit(main())
diff --git a/ci/docker/Dockerfile.build.amzn_linux_cpu b/ci/docker/Dockerfile.build.amzn_linux_cpu
new file mode 100755
index 00000000000..7d6f2236af3
--- /dev/null
+++ b/ci/docker/Dockerfile.build.amzn_linux_cpu
@@ -0,0 +1,44 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to build and run MXNet for Amazon Linux on CPU
+
+FROM amazonlinux
+
+WORKDIR /work/deps
+COPY install/amzn_linux_core.sh /work/
+RUN /work/amzn_linux_core.sh
+COPY install/amzn_linux_opencv.sh /work/
+RUN /work/amzn_linux_opencv.sh
+COPY install/amzn_linux_openblas.sh /work/
+RUN /work/amzn_linux_openblas.sh
+COPY install/amzn_linux_python2.sh /work/
+RUN /work/amzn_linux_python2.sh
+COPY install/amzn_linux_python3.sh /work/
+RUN /work/amzn_linux_python3.sh
+COPY install/amzn_linux_testdeps.sh /work/
+RUN /work/amzn_linux_testdeps.sh
+COPY install/amzn_linux_julia.sh /work/
+RUN /work/amzn_linux_julia.sh
+COPY install/amzn_linux_maven.sh /work/
+RUN /work/amzn_linux_maven.sh
+COPY install/amzn_linux_library.sh /work/
+RUN /work/amzn_linux_library.sh
+WORKDIR /work/mxnet
+
+COPY runtime_functions.sh /work/
\ No newline at end of file
diff --git a/ci/docker/Dockerfile.build.android_arm64 b/ci/docker/Dockerfile.build.android_arm64
new file mode 100755
index 00000000000..d7687514d7a
--- /dev/null
+++ b/ci/docker/Dockerfile.build.android_arm64
@@ -0,0 +1,66 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to build MXNet for Android ARM64/ARMv8
+
+FROM dockcross/base:latest
+MAINTAINER Pedro Larroy "pllarroy@amazon.com"
+
+# The cross-compiling emulator
+RUN apt-get update && apt-get install -y \
+ qemu-user \
+ qemu-user-static \
+ unzip
+
+ENV CROSS_TRIPLE=aarch64-linux-android
+ENV CROSS_ROOT=/usr/${CROSS_TRIPLE}
+ENV AS=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-as \
+ AR=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-ar \
+ CC=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-gcc \
+ CPP=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-cpp \
+ CXX=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-g++ \
+ LD=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-ld
+
+
+ENV DEFAULT_DOCKCROSS_IMAGE dockcross/android-arm
+
+# Build-time metadata as defined at http://label-schema.org
+ARG BUILD_DATE
+ARG IMAGE
+ARG VCS_REF
+ARG VCS_URL
+LABEL org.label-schema.build-date=$BUILD_DATE \
+ org.label-schema.name=$IMAGE \
+ org.label-schema.vcs-ref=$VCS_REF \
+ org.label-schema.vcs-url=$VCS_URL \
+ org.label-schema.schema-version="1.0"
+
+ENV ARCH aarch64
+
+ENV CC=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-clang
+ENV CXX=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-clang++
+
+WORKDIR /work/deps
+COPY install/android_arm64_ndk.sh /work/
+RUN /work/android_arm64_ndk.sh
+COPY install/android_arm64_openblas.sh /work/
+RUN /work/android_arm64_openblas.sh
+ENV CPLUS_INCLUDE_PATH /work/deps/OpenBLAS
+WORKDIR /work/build
+
+COPY runtime_functions.sh /work/
diff --git a/docker_multiarch/Dockerfile.build.android.armv7 b/ci/docker/Dockerfile.build.android_armv7
old mode 100644
new mode 100755
similarity index 74%
rename from docker_multiarch/Dockerfile.build.android.armv7
rename to ci/docker/Dockerfile.build.android_armv7
index 12d53a41222..0074c1f9330
--- a/docker_multiarch/Dockerfile.build.android.armv7
+++ b/ci/docker/Dockerfile.build.android_armv7
@@ -1,4 +1,23 @@
# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to build MXNet for Android ARMv7
+
FROM dockcross/base:latest
MAINTAINER Pedro Larroy "pllarroy@amazon.com"
@@ -36,9 +55,6 @@ RUN mkdir -p /build && \
ENV DEFAULT_DOCKCROSS_IMAGE dockcross/android-arm
-# COPY Toolchain.cmake ${CROSS_ROOT}/
-# ENV CMAKE_TOOLCHAIN_FILE ${CROSS_ROOT}/Toolchain.cmake
-
# Build-time metadata as defined at http://label-schema.org
ARG BUILD_DATE
ARG IMAGE
@@ -76,3 +92,5 @@ RUN cd mxnet && \
WORKDIR /work/build/
RUN cp /work/mxnet/lib/* .
+
+# TODO: Bring this into the new format
\ No newline at end of file
diff --git a/ci/docker/Dockerfile.build.arm64 b/ci/docker/Dockerfile.build.arm64
new file mode 100755
index 00000000000..eb68a818ba6
--- /dev/null
+++ b/ci/docker/Dockerfile.build.arm64
@@ -0,0 +1,38 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to build MXNet for ARM64/ARMv8
+
+FROM dockcross/linux-arm64
+
+ENV ARCH aarch64
+ENV CC /usr/bin/aarch64-linux-gnu-gcc
+ENV CXX /usr/bin/aarch64-linux-gnu-g++
+ENV FC /usr/bin/aarch64-linux-gnu-gfortran-4.9
+ENV HOSTCC gcc
+
+WORKDIR /work
+
+COPY install/arm64_openblas.sh /work/
+RUN /work/arm64_openblas.sh
+
+ENV LD_LIBRARY_PATH /opt/OpenBLAS/lib
+ENV CPLUS_INCLUDE_PATH /opt/OpenBLAS/include
+WORKDIR /work/mxnet
+
+COPY runtime_functions.sh /work/
\ No newline at end of file
diff --git a/ci/docker/Dockerfile.build.armv7 b/ci/docker/Dockerfile.build.armv7
new file mode 100755
index 00000000000..c2493063518
--- /dev/null
+++ b/ci/docker/Dockerfile.build.armv7
@@ -0,0 +1,32 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to build MXNet for Android ARMv7
+
+FROM dockcross/linux-armv7
+
+ENV ARCH armv71
+ENV CC /usr/bin/arm-linux-gnueabihf-gcc
+ENV CXX /usr/bin/arm-linux-gnueabihf-g++
+
+RUN apt-get update && \
+ apt-get install -y libopenblas-dev:armhf && \
+ rm -rf /var/lib/apt/lists/*
+
+COPY runtime_functions.sh /work/
+WORKDIR /work/build
diff --git a/ci/docker/Dockerfile.build.centos7_cpu b/ci/docker/Dockerfile.build.centos7_cpu
new file mode 100755
index 00000000000..665f7ddd99a
--- /dev/null
+++ b/ci/docker/Dockerfile.build.centos7_cpu
@@ -0,0 +1,37 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to build and run MXNet on CentOS 7 for CPU
+
+FROM centos:7
+
+ARG USER_ID=0
+
+WORKDIR /work/deps
+
+COPY install/centos7_core.sh /work/
+RUN /work/centos7_core.sh
+COPY install/centos7_python.sh /work/
+RUN /work/centos7_python.sh
+COPY install/centos7_adduser.sh /work/
+RUN /work/centos7_adduser.sh
+
+ENV PYTHONPATH=./python/
+WORKDIR /work/mxnet
+
+COPY runtime_functions.sh /work/
diff --git a/ci/docker/Dockerfile.build.centos7_gpu b/ci/docker/Dockerfile.build.centos7_gpu
new file mode 100755
index 00000000000..3d7482161c4
--- /dev/null
+++ b/ci/docker/Dockerfile.build.centos7_gpu
@@ -0,0 +1,37 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to build and run MXNet on CentOS 7 for GPU
+
+FROM nvidia/cuda:8.0-cudnn5-devel-centos7
+
+ARG USER_ID=0
+
+WORKDIR /work/deps
+
+COPY install/centos7_core.sh /work/
+RUN /work/centos7_core.sh
+COPY install/centos7_python.sh /work/
+RUN /work/centos7_python.sh
+COPY install/centos7_adduser.sh /work/
+RUN /work/centos7_adduser.sh
+
+ENV PYTHONPATH=./python/
+WORKDIR /work/mxnet
+
+COPY runtime_functions.sh /work/
diff --git a/docker_multiarch/Dockerfile.build.jetson b/ci/docker/Dockerfile.build.jetson
old mode 100644
new mode 100755
similarity index 52%
rename from docker_multiarch/Dockerfile.build.jetson
rename to ci/docker/Dockerfile.build.jetson
index 8a60faef902..e49b48e7e1c
--- a/docker_multiarch/Dockerfile.build.jetson
+++ b/ci/docker/Dockerfile.build.jetson
@@ -1,5 +1,24 @@
# -*- mode: dockerfile -*-
-# dockerfile to build libmxnet.so, and a python wheel for the Jetson TX1/TX2
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to build libmxnet.so, and a python wheel for the Jetson TX1/TX2
+# This script assumes /work/mxnet exists and contains the mxnet code you wish to compile and
+# that /work/build exists and is the target for your output.
FROM nvidia/cuda:8.0-cudnn6-devel as cudabuilder
@@ -7,8 +26,6 @@ FROM dockcross/linux-arm64
ENV ARCH aarch64
ENV NVCCFLAGS "-m64"
-ENV CUDA_ARCH "-gencode arch=compute_53,code=sm_53 -gencode arch=compute_62,code=sm_62"
-ENV BUILD_OPTS "USE_OPENCV=0 USE_BLAS=openblas USE_SSE=0 USE_CUDA=1 USE_CUDNN=1 ENABLE_CUDA_RTC=0 USE_NCCL=0 USE_CUDA_PATH=/usr/local/cuda/"
ENV CC /usr/bin/aarch64-linux-gnu-gcc
ENV CXX /usr/bin/aarch64-linux-gnu-g++
ENV FC /usr/bin/aarch64-linux-gnu-gfortran-4.9
@@ -34,59 +51,22 @@ RUN JETPACK_DOWNLOAD_PREFIX=http://developer.download.nvidia.com/devzone/devcent
ARM_CUDA_INSTALLER_PACKAGE=cuda-repo-l4t-8-0-local_8.0.84-1_arm64.deb && \
ARM_CUDNN_INSTALLER_PACKAGE=libcudnn6_6.0.21-1+cuda8.0_arm64.deb && \
ARM_CUDNN_DEV_INSTALLER_PACKAGE=libcudnn6-dev_6.0.21-1+cuda8.0_arm64.deb && \
- wget $JETPACK_DOWNLOAD_PREFIX/$ARM_CUDA_INSTALLER_PACKAGE && \
- wget $JETPACK_DOWNLOAD_PREFIX/$ARM_CUDNN_INSTALLER_PACKAGE && \
- wget $JETPACK_DOWNLOAD_PREFIX/$ARM_CUDNN_DEV_INSTALLER_PACKAGE && \
+ wget -nv $JETPACK_DOWNLOAD_PREFIX/$ARM_CUDA_INSTALLER_PACKAGE && \
+ wget -nv $JETPACK_DOWNLOAD_PREFIX/$ARM_CUDNN_INSTALLER_PACKAGE && \
+ wget -nv $JETPACK_DOWNLOAD_PREFIX/$ARM_CUDNN_DEV_INSTALLER_PACKAGE && \
dpkg -i $ARM_CUDA_INSTALLER_PACKAGE && \
dpkg -i $ARM_CUDNN_INSTALLER_PACKAGE && \
dpkg -i $ARM_CUDNN_DEV_INSTALLER_PACKAGE && \
- apt update -y && \
- apt install cuda-cudart-cross-aarch64-8-0 cuda-cublas-cross-aarch64-8-0 \
+ apt update -y && \
+ apt install -y unzip cuda-cudart-cross-aarch64-8-0 cuda-cublas-cross-aarch64-8-0 \
cuda-nvml-cross-aarch64-8-0 cuda-nvrtc-cross-aarch64-8-0 cuda-cufft-cross-aarch64-8-0 \
cuda-curand-cross-aarch64-8-0 cuda-cusolver-cross-aarch64-8-0 cuda-cusparse-cross-aarch64-8-0 \
- cuda-misc-headers-cross-aarch64-8-0 cuda-npp-cross-aarch64-8-0 libcudnn6 -y && \
+ cuda-misc-headers-cross-aarch64-8-0 cuda-npp-cross-aarch64-8-0 libcudnn6 && \
cp /usr/local/cuda-8.0/targets/aarch64-linux/lib/*.so /usr/local/cuda/lib64/ && \
cp /usr/local/cuda-8.0/targets/aarch64-linux/lib/stubs/*.so /usr/local/cuda/lib64/stubs/ && \
cp -r /usr/local/cuda-8.0/targets/aarch64-linux/include/ /usr/local/cuda/include/ && \
rm $ARM_CUDA_INSTALLER_PACKAGE $ARM_CUDNN_INSTALLER_PACKAGE $ARM_CUDNN_DEV_INSTALLER_PACKAGE
-# Build MXNet
-ADD mxnet mxnet
-
WORKDIR /work/mxnet
-# Add ARM specific settings
-ADD arm.crosscompile.mk make/config.mk
-
-# Build and link
-RUN make -j$(nproc) $BUILD_OPTS
-
-# Create a binary wheel for easy installation.
-# When using tool.py output will be in the jetson folder.
-# Scp the .whl file to your target device, and install via
-# pip install
-WORKDIR /work/mxnet/python
-RUN python setup.py bdist_wheel --universal
-
-# Copy build artifacts to output folder for tool.py script
-RUN mkdir -p /work/build & cp dist/*.whl /work/build && cp ../lib/* /work/build
-
-# Fix pathing issues in the wheel. We need to move libmxnet.so from the data folder to the root
-# of the wheel, then repackage the wheel.
-# Create a temp dir to do the work.
-WORKDIR /work/build
-RUN apt-get install -y unzip && \
- mkdir temp && \
- cp *.whl temp
-
-# Extract the wheel, move the libmxnet.so file, repackage the wheel.
-WORKDIR /work/build/temp
-RUN unzip *.whl && \
- rm *.whl && \
- mv *.data/data/mxnet/libmxnet.so mxnet && \
- zip -r ../temp.zip *
-
-# Replace the existing wheel with our fixed version.
-WORKDIR /work/build
-RUN rm -rf temp && \
- for f in *.whl; do rm "$f" && mv temp.zip "$f"; done
+COPY runtime_functions.sh /work/
diff --git a/ci/docker/Dockerfile.build.ubuntu_build_cuda b/ci/docker/Dockerfile.build.ubuntu_build_cuda
new file mode 100755
index 00000000000..18c8af7deb2
--- /dev/null
+++ b/ci/docker/Dockerfile.build.ubuntu_build_cuda
@@ -0,0 +1,57 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to build MXNet on Ubuntu 16.04 for GPU but on
+# a CPU-only instance. This restriction is caused by the CPP-
+# package generation, requiring the actual CUDA library to be
+# present
+
+FROM nvidia/cuda:8.0-cudnn5-devel
+
+ARG USER_ID=0
+
+WORKDIR /work/deps
+
+COPY install/ubuntu_core.sh /work/
+RUN /work/ubuntu_core.sh
+COPY install/ubuntu_python.sh /work/
+RUN /work/ubuntu_python.sh
+COPY install/ubuntu_scala.sh /work/
+RUN /work/ubuntu_scala.sh
+COPY install/ubuntu_r.sh /work/
+RUN /work/ubuntu_r.sh
+COPY install/ubuntu_perl.sh /work/
+RUN /work/ubuntu_perl.sh
+COPY install/ubuntu_lint.sh /work/
+RUN /work/ubuntu_lint.sh
+COPY install/ubuntu_clang.sh /work/
+RUN /work/ubuntu_clang.sh
+COPY install/ubuntu_mklml.sh /work/
+RUN /work/ubuntu_mklml.sh
+COPY install/ubuntu_adduser.sh /work/
+RUN /work/ubuntu_adduser.sh
+
+# Special case because the CPP-Package requires the CUDA runtime libs
+# and not only stubs..
+COPY install/ubuntu_nvidia.sh /work/
+RUN /work/ubuntu_nvidia.sh
+
+COPY runtime_functions.sh /work/
+
+WORKDIR /work/mxnet
+ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib
\ No newline at end of file
diff --git a/ci/docker/Dockerfile.build.ubuntu_cpu b/ci/docker/Dockerfile.build.ubuntu_cpu
new file mode 100755
index 00000000000..f86c2f2e724
--- /dev/null
+++ b/ci/docker/Dockerfile.build.ubuntu_cpu
@@ -0,0 +1,53 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to build and run MXNet on Ubuntu 16.04 for CPU
+
+FROM ubuntu:16.04
+
+ARG USER_ID=0
+
+WORKDIR /work/deps
+
+COPY install/ubuntu_core.sh /work/
+RUN /work/ubuntu_core.sh
+COPY install/ubuntu_python.sh /work/
+RUN /work/ubuntu_python.sh
+COPY install/ubuntu_scala.sh /work/
+RUN /work/ubuntu_scala.sh
+COPY install/ubuntu_r.sh /work/
+RUN /work/ubuntu_r.sh
+COPY install/ubuntu_perl.sh /work/
+RUN /work/ubuntu_perl.sh
+COPY install/ubuntu_lint.sh /work/
+RUN /work/ubuntu_lint.sh
+COPY install/ubuntu_clang.sh /work/
+RUN /work/ubuntu_clang.sh
+COPY install/ubuntu_mklml.sh /work/
+RUN /work/ubuntu_mklml.sh
+COPY install/ubuntu_caffe.sh /work/
+RUN /work/ubuntu_caffe.sh
+COPY install/ubuntu_docs.sh /work/
+RUN /work/ubuntu_docs.sh
+COPY install/ubuntu_adduser.sh /work/
+RUN /work/ubuntu_adduser.sh
+
+COPY runtime_functions.sh /work/
+
+WORKDIR /work/mxnet
+ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib
diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu b/ci/docker/Dockerfile.build.ubuntu_gpu
new file mode 100755
index 00000000000..826836c78cf
--- /dev/null
+++ b/ci/docker/Dockerfile.build.ubuntu_gpu
@@ -0,0 +1,55 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to run MXNet on Ubuntu 16.04 for CPU
+
+FROM nvidia/cuda:8.0-cudnn5-devel
+
+ARG USER_ID=0
+
+WORKDIR /work/deps
+
+COPY install/ubuntu_core.sh /work/
+RUN /work/ubuntu_core.sh
+COPY install/ubuntu_python.sh /work/
+RUN /work/ubuntu_python.sh
+COPY install/ubuntu_scala.sh /work/
+RUN /work/ubuntu_scala.sh
+COPY install/ubuntu_r.sh /work/
+RUN /work/ubuntu_r.sh
+COPY install/ubuntu_perl.sh /work/
+RUN /work/ubuntu_perl.sh
+COPY install/ubuntu_lint.sh /work/
+RUN /work/ubuntu_lint.sh
+COPY install/ubuntu_clang.sh /work/
+RUN /work/ubuntu_clang.sh
+COPY install/ubuntu_mklml.sh /work/
+RUN /work/ubuntu_mklml.sh
+COPY install/ubuntu_tvm.sh /work/
+RUN /work/ubuntu_tvm.sh
+COPY install/ubuntu_llvm.sh /work/
+RUN /work/ubuntu_llvm.sh
+COPY install/ubuntu_caffe.sh /work/
+RUN /work/ubuntu_caffe.sh
+COPY install/ubuntu_adduser.sh /work/
+RUN /work/ubuntu_adduser.sh
+
+COPY runtime_functions.sh /work/
+
+WORKDIR /work/mxnet
+ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib
diff --git a/ci/docker/install/amzn_linux_core.sh b/ci/docker/install/amzn_linux_core.sh
new file mode 100755
index 00000000000..c13c9695197
--- /dev/null
+++ b/ci/docker/install/amzn_linux_core.sh
@@ -0,0 +1,45 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
+
+set -ex
+pushd .
+yum install -y git
+yum install -y wget
+yum install -y sudo
+yum install -y re2c
+yum groupinstall -y 'Development Tools'
+
+# Ninja
+git clone --recursive https://github.com/ninja-build/ninja.git
+cd ninja
+./configure.py --bootstrap
+cp ninja /usr/local/bin
+popd
+
+# CMake
+pushd .
+git clone --recursive https://github.com/Kitware/CMake.git --branch v3.10.2
+cd CMake
+./bootstrap
+make -j$(nproc)
+make install
+popd
\ No newline at end of file
diff --git a/tests/ci_build/install/install_julia.sh b/ci/docker/install/amzn_linux_julia.sh
old mode 100644
new mode 100755
similarity index 77%
rename from tests/ci_build/install/install_julia.sh
rename to ci/docker/install/amzn_linux_julia.sh
index e36dfb533b2..bfaf3c4924b
--- a/tests/ci_build/install/install_julia.sh
+++ b/ci/docker/install/amzn_linux_julia.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -16,13 +16,14 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-set -e
-set -x
-wget https://julialang.s3.amazonaws.com/bin/linux/x64/0.5/julia-0.5.0-linux-x86_64.tar.gz
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
+
+set -ex
+wget -nv https://julialang.s3.amazonaws.com/bin/linux/x64/0.5/julia-0.5.0-linux-x86_64.tar.gz
mv julia-0.5.0-linux-x86_64.tar.gz /tmp/
tar xfvz /tmp/julia-0.5.0-linux-x86_64.tar.gz
rm -f /tmp/julia-0.5.0-linux-x86_64.tar.gz
-
# tar extracted in current directory
-ln -s -f ${PWD}/julia-3c9d75391c/bin/julia /usr/bin/julia
+ln -s -f ${PWD}/julia-3c9d75391c/bin/julia /usr/bin/julia
\ No newline at end of file
diff --git a/tests/ci_build/install/install_library.sh b/ci/docker/install/amzn_linux_library.sh
old mode 100644
new mode 100755
similarity index 81%
rename from tests/ci_build/install/install_library.sh
rename to ci/docker/install/amzn_linux_library.sh
index f3d78682d4b..04708957033
--- a/tests/ci_build/install/install_library.sh
+++ b/ci/docker/install/amzn_linux_library.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -16,10 +16,11 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-set -e
-set -x
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
-yum install graphviz
+set -ex
+yum -y install graphviz
pip install graphviz
-pip install opencv-python
+pip install opencv-python
\ No newline at end of file
diff --git a/tests/ci_build/install/install_maven.sh b/ci/docker/install/amzn_linux_maven.sh
old mode 100644
new mode 100755
similarity index 75%
rename from tests/ci_build/install/install_maven.sh
rename to ci/docker/install/amzn_linux_maven.sh
index f1241864c7e..22875d0ec86
--- a/tests/ci_build/install/install_maven.sh
+++ b/ci/docker/install/amzn_linux_maven.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -16,11 +16,12 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-set -e
-set -x
-wget http://mirrors.ocf.berkeley.edu/apache/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
+
+set -ex
+wget -nv http://mirrors.ocf.berkeley.edu/apache/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz
mv apache-maven-3.3.9-bin.tar.gz /tmp/
tar xfvz /tmp/apache-maven-3.3.9-bin.tar.gz
-
-yum install -y java-1.8.0-openjdk-devel
+yum install -y java-1.8.0-openjdk-devel
\ No newline at end of file
diff --git a/tests/ci_build/install/install_openblas.sh b/ci/docker/install/amzn_linux_openblas.sh
old mode 100644
new mode 100755
similarity index 86%
rename from tests/ci_build/install/install_openblas.sh
rename to ci/docker/install/amzn_linux_openblas.sh
index 5c9282359ec..94088d6ccf1
--- a/tests/ci_build/install/install_openblas.sh
+++ b/ci/docker/install/amzn_linux_openblas.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -16,10 +16,14 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-set -e
-set -x
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
+
+set -ex
+pushd .
git clone https://github.com/xianyi/OpenBLAS
cd OpenBLAS
make FC=gfortran -j $(($(nproc) + 1))
make PREFIX=/usr/local install
+popd
\ No newline at end of file
diff --git a/tests/ci_build/install/install_opencv.sh b/ci/docker/install/amzn_linux_opencv.sh
old mode 100644
new mode 100755
similarity index 70%
rename from tests/ci_build/install/install_opencv.sh
rename to ci/docker/install/amzn_linux_opencv.sh
index 9364a5613c2..956407e8362
--- a/tests/ci_build/install/install_opencv.sh
+++ b/ci/docker/install/amzn_linux_opencv.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -16,15 +16,18 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-set -e
-set -x
-yum groupinstall -y "Development Tools"
-yum install -y cmake python27 python27-setuptools
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
+
+set -ex
+pushd .
+yum install -y python27 python27-setuptools
git clone https://github.com/opencv/opencv
cd opencv
mkdir -p build
cd build
-cmake -D BUILD_opencv_gpu=OFF -D WITH_EIGEN=ON -D WITH_TBB=ON -D WITH_CUDA=OFF -D WITH_1394=OFF -D CMAKE_BUILD_TYPE=RELEASE -D CMAKE_INSTALL_PREFIX=/usr/local ..
-make PREFIX=/usr/local install
-
+cmake -DBUILD_opencv_gpu=OFF -DWITH_EIGEN=ON -DWITH_TBB=ON -DWITH_CUDA=OFF -DWITH_1394=OFF \
+-DCMAKE_BUILD_TYPE=RELEASE -DCMAKE_INSTALL_PREFIX=/usr/local -GNinja ..
+ninja install
+popd
\ No newline at end of file
diff --git a/tests/ci_build/install/install_python2.sh b/ci/docker/install/amzn_linux_python2.sh
old mode 100644
new mode 100755
similarity index 83%
rename from tests/ci_build/install/install_python2.sh
rename to ci/docker/install/amzn_linux_python2.sh
index 5c7c4538098..e099ad6d6c4
--- a/tests/ci_build/install/install_python2.sh
+++ b/ci/docker/install/amzn_linux_python2.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -16,20 +16,21 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-set -e
-set -x
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
+
+set -ex
yum groupinstall -y "Development Tools"
yum install -y mlocate python27 python27-setuptools python27-tools python27-numpy python27-scipy python27-nose python27-matplotlib unzip
ln -s -f /usr/bin/python2.7 /usr/bin/python2
-wget https://bootstrap.pypa.io/get-pip.py
+wget -nv https://bootstrap.pypa.io/get-pip.py
python2 get-pip.py
$(which easy_install-2.7) --upgrade pip
if [ -f /usr/local/bin/pip ] && [ -f /usr/bin/pip ]; then
- mv /usr/bin/pip /usr/bin/pip.bak
- ln /usr/local/bin/pip /usr/bin/pip
+ mv /usr/bin/pip /usr/bin/pip.bak
+ ln /usr/local/bin/pip /usr/bin/pip
fi
ln -s -f /usr/local/bin/pip /usr/bin/pip
for i in ipython[all] jupyter pandas scikit-image h5py pandas sklearn sympy; do echo "${i}..."; pip install -U $i >/dev/null; done
-
diff --git a/tests/ci_build/install/install_python3.sh b/ci/docker/install/amzn_linux_python3.sh
old mode 100644
new mode 100755
similarity index 78%
rename from tests/ci_build/install/install_python3.sh
rename to ci/docker/install/amzn_linux_python3.sh
index 0df12eb0850..3f80d7d98d8
--- a/tests/ci_build/install/install_python3.sh
+++ b/ci/docker/install/amzn_linux_python3.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -16,18 +16,21 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-set -e
-set -x
-wget https://bootstrap.pypa.io/get-pip.py || exit 1
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
+
+set -ex
+pushd .
+wget -nv https://bootstrap.pypa.io/get-pip.py
mkdir py3
cd py3
-wget https://www.python.org/ftp/python/3.5.2/Python-3.5.2.tgz || exit 1
+wget -nv https://www.python.org/ftp/python/3.5.2/Python-3.5.2.tgz
tar -xvzf Python-3.5.2.tgz
cd Python-3.5.2
yum install -y zlib-devel openssl-devel sqlite-devel bzip2-devel gdbm-devel ncurses-devel xz-devel readline-devel
./configure --prefix=/opt/ --with-zlib-dir=/usr/lib64
-make || exit 1
+make -j$(nproc)
mkdir /opt/bin
mkdir /opt/lib
make install
@@ -36,5 +39,6 @@ cd ../..
python3 get-pip.py
ln -s -f /opt/bin/pip /usr/bin/pip3
-mkdir -p /home/jenkins/.local/lib/python3.5/site-packages/
+mkdir -p ~/.local/lib/python3.5/site-packages/
pip3 install numpy
+popd
\ No newline at end of file
diff --git a/tests/ci_build/install/install_testdeps.sh b/ci/docker/install/amzn_linux_testdeps.sh
old mode 100644
new mode 100755
similarity index 82%
rename from tests/ci_build/install/install_testdeps.sh
rename to ci/docker/install/amzn_linux_testdeps.sh
index 26da186dd27..f5c49d9e37b
--- a/tests/ci_build/install/install_testdeps.sh
+++ b/ci/docker/install/amzn_linux_testdeps.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -16,11 +16,12 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-set -e
-set -x
-pip install cpplint 'pylint==1.4.4' 'astroid==1.3.6'
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
+set -ex
+pip install cpplint 'pylint==1.4.4' 'astroid==1.3.6'
pip3 install nose
ln -s -f /opt/bin/nosetests /usr/local/bin/nosetests3
-ln -s -f /opt/bin/nosetests-3.4 /usr/local/bin/nosetests-3.4
+ln -s -f /opt/bin/nosetests-3.4 /usr/local/bin/nosetests-3.4
\ No newline at end of file
diff --git a/ci/docker/install/android_arm64_ndk.sh b/ci/docker/install/android_arm64_ndk.sh
new file mode 100755
index 00000000000..23d9ea2c0e9
--- /dev/null
+++ b/ci/docker/install/android_arm64_ndk.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
+
+set -ex
+pushd .
+export ANDROID_NDK_REVISION=15c
+curl -O https://dl.google.com/android/repository/android-ndk-r${ANDROID_NDK_REVISION}-linux-x86_64.zip && \
+unzip ./android-ndk-r${ANDROID_NDK_REVISION}-linux-x86_64.zip && \
+cd android-ndk-r${ANDROID_NDK_REVISION} && \
+./build/tools/make_standalone_toolchain.py \
+ --stl=libc++ \
+ --arch arm64 \
+ --api 21 \
+ --install-dir=${CROSS_ROOT} && \
+
+popd
\ No newline at end of file
diff --git a/ci/docker/install/android_arm64_openblas.sh b/ci/docker/install/android_arm64_openblas.sh
new file mode 100755
index 00000000000..87e0bf9e8ed
--- /dev/null
+++ b/ci/docker/install/android_arm64_openblas.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
+
+set -ex
+pushd .
+git clone https://github.com/xianyi/OpenBLAS.git
+cd OpenBLAS
+make -j$(nproc) TARGET=ARMV8 ARM_SOFTFP_ABI=1 HOSTCC=gcc NOFORTRAN=1 libs
+# Can't be run (utility not compiled for the target platform)
+#make install
+cp *.h /usr/include
+cp libopenblas.a /usr/local/lib
+popd
diff --git a/ci/docker/install/arm64_openblas.sh b/ci/docker/install/arm64_openblas.sh
new file mode 100755
index 00000000000..3151a4b5c2a
--- /dev/null
+++ b/ci/docker/install/arm64_openblas.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
+
+set -ex
+pushd .
+wget -nv https://api.github.com/repos/xianyi/OpenBLAS/git/refs/heads/master -O openblas_version.json
+echo "Using openblas:"
+cat openblas_version.json
+git clone https://github.com/xianyi/OpenBLAS.git
+cd OpenBLAS
+make -j$(nproc) TARGET=ARMV8
+make install
+ln -s /opt/OpenBLAS/lib/libopenblas.so /usr/lib/libopenblas.so
+ln -s /opt/OpenBLAS/lib/libopenblas.a /usr/lib/libopenblas.a
+ln -s /opt/OpenBLAS/lib/libopenblas.a /usr/lib/liblapack.a
+popd
\ No newline at end of file
diff --git a/ci/docker/install/centos7_adduser.sh b/ci/docker/install/centos7_adduser.sh
new file mode 100755
index 00000000000..7ed64c2dbd3
--- /dev/null
+++ b/ci/docker/install/centos7_adduser.sh
@@ -0,0 +1,37 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Add user in order to make sure the assumed user the container is running under
+# actually exists inside the container to avoid problems like missing home dir
+
+
+set -ex
+
+# $USER_ID is coming from build.py:build_docker passed as --build-arg
+if [[ "$USER_ID" -gt 0 ]]
+then
+ # -no-log-init required due to https://github.com/moby/moby/issues/5419
+ useradd -m --no-log-init --uid $USER_ID --system jenkins_slave
+ usermod -aG wheel jenkins_slave
+
+ # By default, docker creates all WORK_DIRs with root owner
+ mkdir /work/mxnet
+ mkdir /work/build
+ chown -R jenkins_slave /work/
+fi
diff --git a/ci/docker/install/centos7_core.sh b/ci/docker/install/centos7_core.sh
new file mode 100755
index 00000000000..1688b81ba89
--- /dev/null
+++ b/ci/docker/install/centos7_core.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
+
+set -ex
+
+# Multipackage installation does not fail in yum
+yum -y install epel-release
+yum -y install git
+yum -y install wget
+yum -y install atlas-devel # Provide clbas headerfiles
+yum -y install openblas-devel
+yum -y install lapack-devel
+yum -y install opencv-devel
+yum -y install openssl-devel
+yum -y install gcc-c++
+yum -y install make
+yum -y install cmake
+yum -y install wget
+yum -y install unzip
+yum -y install ninja-build
\ No newline at end of file
diff --git a/ci/docker/install/centos7_python.sh b/ci/docker/install/centos7_python.sh
new file mode 100755
index 00000000000..9e076b61e87
--- /dev/null
+++ b/ci/docker/install/centos7_python.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
+
+set -ex
+
+ # Python 2.7 is installed by default, install 3.6 on top
+yum -y install https://centos7.iuscommunity.org/ius-release.rpm
+yum -y install python36u
+
+# Install PIP
+curl "https://bootstrap.pypa.io/get-pip.py" -o "get-pip.py"
+python2.7 get-pip.py
+python3.6 get-pip.py
+
+pip2 install nose pylint numpy nose-timer requests h5py scipy
+pip3 install nose pylint numpy nose-timer requests h5py scipy
\ No newline at end of file
diff --git a/ci/docker/install/ubuntu_adduser.sh b/ci/docker/install/ubuntu_adduser.sh
new file mode 100755
index 00000000000..fd9a3f8d450
--- /dev/null
+++ b/ci/docker/install/ubuntu_adduser.sh
@@ -0,0 +1,37 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Add user in order to make sure the assumed user the container is running under
+# actually exists inside the container to avoid problems like missing home dir
+
+
+set -ex
+
+# $USER_ID is coming from build.py:build_docker passed as --build-arg
+if [[ "$USER_ID" -gt 0 ]]
+then
+ # -no-log-init required due to https://github.com/moby/moby/issues/5419
+ useradd -m --no-log-init --uid $USER_ID --system jenkins_slave
+ usermod -aG sudo jenkins_slave
+
+ # By default, docker creates all WORK_DIRs with root owner
+ mkdir /work/mxnet
+ mkdir /work/build
+ chown -R jenkins_slave /work/
+fi
diff --git a/ci/docker/install/ubuntu_caffe.sh b/ci/docker/install/ubuntu_caffe.sh
new file mode 100755
index 00000000000..50e257b2ef3
--- /dev/null
+++ b/ci/docker/install/ubuntu_caffe.sh
@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+set -ex
+
+apt-get install -y libprotobuf-dev libleveldb-dev \
+ libsnappy-dev libopencv-dev libhdf5-serial-dev protobuf-compiler \
+ libatlas-base-dev python-dev libgflags-dev libgoogle-glog-dev liblmdb-dev \
+ python-numpy python-opencv
+
+apt-get install -y --no-install-recommends libboost-all-dev
+
+cd /work/deps
+git clone http://github.com/BVLC/caffe.git
+
+cd caffe
+cp Makefile.config.example Makefile.config
+
+echo "CPU_ONLY := 1" >> Makefile.config
+
+# Fixes https://github.com/BVLC/caffe/issues/5658 See https://github.com/intel/caffe/wiki/Ubuntu-16.04-or-15.10-Installation-Guide
+echo "INCLUDE_DIRS += /usr/lib /usr/lib/x86_64-linux-gnu /usr/include/hdf5/serial/ " >> Makefile.config
+echo "LIBRARY_DIRS += /usr/lib /usr/lib/x86_64-linux-gnu /usr/lib/x86_64-linux-gnu/hdf5/serial " >> Makefile.config
+
+# Fixes https://github.com/BVLC/caffe/issues/4333 See https://github.com/intel/caffe/wiki/Ubuntu-16.04-or-15.10-Installation-Guide
+# Note: This is only valid on Ubuntu16.04 - the version numbers are bound to the distribution
+ln -s /usr/lib/x86_64-linux-gnu/libhdf5_serial.so.10.0.2 /usr/lib/x86_64-linux-gnu/libhdf5.so
+ln -s /usr/lib/x86_64-linux-gnu/libhdf5_serial_hl.so.10.0.2 /usr/lib/x86_64-linux-gnu/libhdf5_hl.so
+
+make all pycaffe -j$(nproc)
+
+cd python
+for req in $(cat requirements.txt); do pip2 install $req; done
+
diff --git a/ci/docker/install/ubuntu_clang.sh b/ci/docker/install/ubuntu_clang.sh
new file mode 100755
index 00000000000..08b95bceddc
--- /dev/null
+++ b/ci/docker/install/ubuntu_clang.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
+
+set -ex
+# Install clang 3.9 (the same version as in XCode 8.*) and 5.0 (latest major release)
+wget -O - http://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
+ apt-add-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-3.9 main" && \
+ apt-add-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-5.0 main" && \
+ apt-get update && \
+ apt-get install -y clang-3.9 clang-5.0 && \
+ clang-3.9 --version && \
+ clang-5.0 --version
\ No newline at end of file
diff --git a/tests/ci_build/install/ubuntu_install_core.sh b/ci/docker/install/ubuntu_core.sh
similarity index 67%
rename from tests/ci_build/install/ubuntu_install_core.sh
rename to ci/docker/install/ubuntu_core.sh
index 74fb05a1590..e78f29ae0a9 100755
--- a/tests/ci_build/install/ubuntu_install_core.sh
+++ b/ci/docker/install/ubuntu_core.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -16,14 +16,27 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-set -e
-set -x
-# install libraries for building mxnet c++ core on ubuntu
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
-apt-get update && apt-get install -y \
- build-essential git libopenblas-dev liblapack-dev libopencv-dev \
- libcurl4-openssl-dev cmake wget unzip sudo ninja-build
+set -ex
+apt-get update
+apt-get install -y \
+ build-essential \
+ git \
+ libopenblas-dev \
+ liblapack-dev \
+ libopencv-dev \
+ libcurl4-openssl-dev \
+ cmake \
+ wget \
+ unzip \
+ sudo \
+ software-properties-common \
+ ninja-build \
+ python-pip
# Link Openblas to Cblas as this link does not exist on ubuntu16.04
ln -s /usr/lib/libopenblas.so /usr/lib/libcblas.so
+pip install cpplint==1.3.0 pylint==1.8.2
\ No newline at end of file
diff --git a/ci/docker/install/ubuntu_docs.sh b/ci/docker/install/ubuntu_docs.sh
new file mode 100755
index 00000000000..bb8026f0daa
--- /dev/null
+++ b/ci/docker/install/ubuntu_docs.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
+
+set -ex
+wget http://downloads.lightbend.com/scala/2.11.8/scala-2.11.8.deb && \
+ dpkg -i scala-2.11.8.deb && rm scala-2.11.8.deb
+
+apt-get install -y doxygen libatlas-base-dev graphviz pandoc
+pip install sphinx==1.3.5 CommonMark==0.5.4 breathe mock recommonmark pypandoc beautifulsoup4
diff --git a/ci/docker/install/ubuntu_lint.sh b/ci/docker/install/ubuntu_lint.sh
new file mode 100755
index 00000000000..f3a1d6ba244
--- /dev/null
+++ b/ci/docker/install/ubuntu_lint.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
+
+set -ex
+apt-get update
+apt-get install -y python-pip sudo
\ No newline at end of file
diff --git a/tests/ci_build/install/ubuntu_install_llvm.sh b/ci/docker/install/ubuntu_llvm.sh
similarity index 94%
rename from tests/ci_build/install/ubuntu_install_llvm.sh
rename to ci/docker/install/ubuntu_llvm.sh
index d3282e7a5fc..09e13d3d1ed 100755
--- a/tests/ci_build/install/ubuntu_install_llvm.sh
+++ b/ci/docker/install/ubuntu_llvm.sh
@@ -17,12 +17,10 @@
# specific language governing permissions and limitations
# under the License.
-
-
echo deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-5.0 main\
>> /etc/apt/sources.list.d/llvm.list
echo deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial-5.0 main\
>> /etc/apt/sources.list.d/llvm.list
wget -O - http://apt.llvm.org/llvm-snapshot.gpg.key|sudo apt-key add -
-apt-get update && apt-get install -y --force-yes llvm-5.0
+apt-get update && apt-get install -y --force-yes llvm-5.0
\ No newline at end of file
diff --git a/ci/docker/install/ubuntu_mklml.sh b/ci/docker/install/ubuntu_mklml.sh
new file mode 100755
index 00000000000..4f68fa79517
--- /dev/null
+++ b/ci/docker/install/ubuntu_mklml.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
+
+set -ex
+wget --no-check-certificate -O /tmp/mklml.tgz https://github.com/01org/mkl-dnn/releases/download/v0.12/mklml_lnx_2018.0.1.20171227.tgz
+tar -zxvf /tmp/mklml.tgz && cp -rf mklml_*/* /usr/local/ && rm -rf mklml_*
\ No newline at end of file
diff --git a/tests/ci_build/install/ubuntu_install_nvidia.sh b/ci/docker/install/ubuntu_nvidia.sh
similarity index 74%
rename from tests/ci_build/install/ubuntu_install_nvidia.sh
rename to ci/docker/install/ubuntu_nvidia.sh
index d34980946ea..bb1c73eec76 100755
--- a/tests/ci_build/install/ubuntu_install_nvidia.sh
+++ b/ci/docker/install/ubuntu_nvidia.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -17,17 +17,13 @@
# specific language governing permissions and limitations
# under the License.
-# install nvidia libraries to compile and run CUDA without
-# the necessity of nvidia-docker and a GPU
-set -e
-set -x
-
-# Needed to run add-apt-repository
-apt update && apt install -y software-properties-common
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
+set -ex
+apt install -y software-properties-common
add-apt-repository -y ppa:graphics-drivers
-
# Retrieve ppa:graphics-drivers and install nvidia-drivers.
# Note: DEBIAN_FRONTEND required to skip the interactive setup steps
-apt update && \
- DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends cuda-8-0
+apt update
+DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends cuda-8-0
\ No newline at end of file
diff --git a/tests/ci_build/install/ubuntu_install_perl.sh b/ci/docker/install/ubuntu_perl.sh
similarity index 81%
rename from tests/ci_build/install/ubuntu_install_perl.sh
rename to ci/docker/install/ubuntu_perl.sh
index bb0fa9d68e6..52bd010ee0b 100755
--- a/tests/ci_build/install/ubuntu_install_perl.sh
+++ b/ci/docker/install/ubuntu_perl.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -16,9 +16,11 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-set -e
-set -x
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
+
+set -ex
# install libraries for mxnet's perl package on ubuntu
-apt-get update && apt-get install -y libmouse-perl pdl cpanminus swig libgraphviz-perl
+apt-get install -y libmouse-perl pdl cpanminus swig libgraphviz-perl
cpanm -q Function::Parameters Hash::Ordered PDL::CCS
diff --git a/tests/ci_build/install/ubuntu_install_python.sh b/ci/docker/install/ubuntu_python.sh
similarity index 75%
rename from tests/ci_build/install/ubuntu_install_python.sh
rename to ci/docker/install/ubuntu_python.sh
index 0ba98004a3e..b19448eecab 100755
--- a/tests/ci_build/install/ubuntu_install_python.sh
+++ b/ci/docker/install/ubuntu_python.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -16,14 +16,18 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-set -e
-set -x
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
+
+set -ex
# install libraries for mxnet's python package on ubuntu
-apt-get update && apt-get install -y python-dev python3-dev
+apt-get install -y python-dev python3-dev virtualenv
# the version of the pip shipped with ubuntu may be too lower, install a recent version here
-cd /tmp && wget https://bootstrap.pypa.io/get-pip.py && python3 get-pip.py && python2 get-pip.py
+wget -nv https://bootstrap.pypa.io/get-pip.py
+python3 get-pip.py
+python2 get-pip.py
pip2 install nose pylint numpy nose-timer requests h5py scipy
-pip3 install nose pylint numpy nose-timer requests h5py scipy
+pip3 install nose pylint numpy nose-timer requests h5py scipy
\ No newline at end of file
diff --git a/tests/ci_build/install/ubuntu_install_r.sh b/ci/docker/install/ubuntu_r.sh
similarity index 89%
rename from tests/ci_build/install/ubuntu_install_r.sh
rename to ci/docker/install/ubuntu_r.sh
index 6855b33a45c..e04e94d6486 100755
--- a/tests/ci_build/install/ubuntu_install_r.sh
+++ b/ci/docker/install/ubuntu_r.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -16,15 +16,15 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-set -e
-set -x
-# install libraries for mxnet's r package on ubuntu
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
+set -ex
+# install libraries for mxnet's r package on ubuntu
echo "deb http://cran.rstudio.com/bin/linux/ubuntu trusty/" >> /etc/apt/sources.list
gpg --keyserver keyserver.ubuntu.com --recv-key E084DAB9
gpg -a --export E084DAB9 | apt-key add -
apt-get update
-apt-get install -y r-base r-base-dev libxml2-dev libssl-dev libxt-dev
-
+apt-get install -y r-base r-base-dev libxml2-dev libssl-dev libxt-dev
\ No newline at end of file
diff --git a/tests/ci_build/install/ubuntu_install_scala.sh b/ci/docker/install/ubuntu_scala.sh
similarity index 86%
rename from tests/ci_build/install/ubuntu_install_scala.sh
rename to ci/docker/install/ubuntu_scala.sh
index 7dd343acde3..ee5554d218a 100755
--- a/tests/ci_build/install/ubuntu_install_scala.sh
+++ b/ci/docker/install/ubuntu_scala.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -16,15 +16,16 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-set -e
-set -x
-# install libraries for mxnet's scala package on ubuntu
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
+set -ex
+# install libraries for mxnet's scala package on ubuntu
apt-get install -y software-properties-common
add-apt-repository -y ppa:webupd8team/java
apt-get update
echo "oracle-java8-installer shared/accepted-oracle-license-v1-1 select true" | debconf-set-selections
apt-get install -y oracle-java8-installer
apt-get install -y oracle-java8-set-default
-apt-get update && apt-get install -y maven
+apt-get update && apt-get install -y maven
\ No newline at end of file
diff --git a/tests/ci_build/install/ubuntu_install_tvm.sh b/ci/docker/install/ubuntu_tvm.sh
similarity index 98%
rename from tests/ci_build/install/ubuntu_install_tvm.sh
rename to ci/docker/install/ubuntu_tvm.sh
index 2729c7fe3be..9ab359b7207 100755
--- a/tests/ci_build/install/ubuntu_install_tvm.sh
+++ b/ci/docker/install/ubuntu_tvm.sh
@@ -33,7 +33,7 @@ echo LLVM_CONFIG=llvm-config-5.0 >> config.mk
echo USE_RPC=1 >> config.mk
echo USE_GRAPH_RUNTIME=1 >> config.mk
echo CUDA_PATH=/usr/local/cuda >> config.mk
-make -j`nproc`
+make -j$(nproc)
cd python
python setup.py install
@@ -41,4 +41,4 @@ cd -
cd topi/python
python setup.py install
-cd -
+cd -
\ No newline at end of file
diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
new file mode 100755
index 00000000000..89ea44fa1ef
--- /dev/null
+++ b/ci/docker/runtime_functions.sh
@@ -0,0 +1,471 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# build and install are separated so changes to build don't invalidate
+# the whole docker cache for the image
+
+set -ex
+
+clean_repo() {
+ set -ex
+ git clean -xfd
+ git submodule foreach --recursive git clean -xfd
+ git reset --hard
+ git submodule foreach --recursive git reset --hard
+ git submodule update --init --recursive
+}
+
+
+# Build commands: Every platform in docker/Dockerfile.build.<platform> should have a corresponding
+# function here with the same suffix:
+
+build_jetson() {
+ set -ex
+ pushd .
+
+ #cd /work/mxnet
+ #make -j$(nproc) USE_OPENCV=0 USE_BLAS=openblas USE_SSE=0 USE_CUDA=1 USE_CUDNN=1 ENABLE_CUDA_RTC=0 USE_NCCL=0 USE_CUDA_PATH=/usr/local/cuda/
+ cd /work/build
+ cmake\
+ -DUSE_CUDA=OFF\
+ -DUSE_OPENCV=OFF\
+ -DUSE_OPENMP=ON\
+ -DUSE_SIGNAL_HANDLER=ON\
+ -DUSE_MKL_IF_AVAILABLE=OFF\
+ -DUSE_LAPACK=OFF\
+ -DCMAKE_BUILD_TYPE=RelWithDebInfo\
+ -G Ninja /work/mxnet
+ ninja
+ export MXNET_LIBRARY_PATH=`pwd`/libmxnet.so
+ cd /work/mxnet/python
+ python setup.py bdist_wheel --universal
+
+
+ # Fix pathing issues in the wheel. We need to move libmxnet.so from the data folder to the root
+ # of the wheel, then repackage the wheel.
+ # Create a temp dir to do the work.
+ # TODO: move apt call to install
+ WHEEL=`readlink -f dist/*.whl`
+ TMPDIR=`mktemp -d`
+ unzip -d $TMPDIR $WHEEL
+ rm $WHEEL
+ cd $TMPDIR
+ mv *.data/data/mxnet/libmxnet.so mxnet
+ zip -r $WHEEL $TMPDIR
+ cp $WHEEL /work/build
+ rm -rf $TMPDIR
+ popd
+}
+
+build_armv7() {
+ set -ex
+ pushd .
+ cd /work/build
+ cmake\
+ -DUSE_CUDA=OFF\
+ -DUSE_OPENCV=OFF\
+ -DUSE_OPENMP=OFF\
+ -DUSE_SIGNAL_HANDLER=ON\
+ -DCMAKE_BUILD_TYPE=RelWithDebInfo\
+ -DUSE_MKL_IF_AVAILABLE=OFF\
+ -G Ninja /work/mxnet
+ ninja
+ export MXNET_LIBRARY_PATH=`pwd`/libmxnet.so
+ cd /work/mxnet/python
+ python setup.py bdist_wheel --universal
+ cp dist/*.whl /work/build
+ popd
+}
+
+build_amzn_linux_cpu() {
+ cd /work/build
+ cmake\
+ -DUSE_CUDA=OFF\
+ -DUSE_OPENCV=ON\
+ -DUSE_OPENMP=ON\
+ -DUSE_SIGNAL_HANDLER=ON\
+ -DCMAKE_BUILD_TYPE=RelWithDebInfo\
+ -DUSE_MKL_IF_AVAILABLE=OFF\
+ -DUSE_LAPACK=OFF\
+ -G Ninja /work/mxnet
+ ninja
+ export MXNET_LIBRARY_PATH=`pwd`/libmxnet.so
+}
+
+build_arm64() {
+ cmake\
+ -DUSE_CUDA=OFF\
+ -DUSE_OPENCV=OFF\
+ -DUSE_OPENMP=OFF\
+ -DUSE_SIGNAL_HANDLER=ON\
+ -DCMAKE_BUILD_TYPE=RelWithDebInfo\
+ -DUSE_MKL_IF_AVAILABLE=OFF\
+ -G Ninja /work/mxnet
+ ninja
+ export MXNET_LIBRARY_PATH=`pwd`/libmxnet.so
+ cd /work/mxnet/python
+ python setup.py bdist_wheel --universal
+ cp dist/*.whl /work/build
+}
+
+build_android_arm64() {
+ set -ex
+ cd /work/build
+ cmake\
+ -DUSE_CUDA=OFF\
+ -DUSE_SSE=OFF\
+ -DUSE_LAPACK=OFF\
+ -DUSE_OPENCV=OFF\
+ -DUSE_OPENMP=OFF\
+ -DUSE_SIGNAL_HANDLER=ON\
+ -DCMAKE_BUILD_TYPE=RelWithDebInfo\
+ -DUSE_MKL_IF_AVAILABLE=OFF\
+ -G Ninja /work/mxnet
+ ninja
+ export MXNET_LIBRARY_PATH=`pwd`/libmxnet.so
+ cd /work/mxnet/python
+ python setup.py bdist_wheel --universal
+ cp dist/*.whl /work/build
+}
+
+build_centos7_cpu() {
+ set -ex
+ cd /work/mxnet
+ make \
+ DEV=1 \
+ USE_LAPACK=1 \
+ USE_LAPACK_PATH=/usr/lib64/liblapack.so \
+ USE_PROFILER=1 \
+ USE_BLAS=openblas \
+ -j$(nproc)
+}
+
+build_centos7_gpu() {
+ set -ex
+ cd /work/mxnet
+ make \
+ DEV=1 \
+ USE_LAPACK=1 \
+ USE_LAPACK_PATH=/usr/lib64/liblapack.so \
+ USE_PROFILER=1 \
+ USE_BLAS=openblas \
+ USE_CUDA=1 \
+ USE_CUDA_PATH=/usr/local/cuda \
+ USE_CUDNN=1 \
+ -j$(nproc)
+}
+
+build_ubuntu_cpu_openblas() {
+ set -ex
+ make \
+ DEV=1 \
+ USE_PROFILER=1 \
+ USE_CPP_PACKAGE=1 \
+ USE_BLAS=openblas \
+ -j$(nproc)
+}
+
+build_ubuntu_cpu_clang39() {
+ set -ex
+ make \
+ USE_PROFILER=1 \
+ USE_CPP_PACKAGE=1 \
+ USE_BLAS=openblas \
+ USE_OPENMP=0 \
+ CXX=clang++-3.9 \
+ CC=clang-3.9 \
+ -j$(nproc)
+}
+
+build_ubuntu_cpu_clang50() {
+ set -ex
+ make \
+ USE_PROFILER=1 \
+ USE_CPP_PACKAGE=1 \
+ USE_BLAS=openblas \
+ USE_OPENMP=1 \
+ CXX=clang++-5.0 \
+ CC=clang-5.0 \
+ -j$(nproc)
+}
+
+build_ubuntu_cpu_mkldnn() {
+ set -ex
+ make \
+ DEV=1 \
+ USE_PROFILER=1 \
+ USE_CPP_PACKAGE=1 \
+ USE_BLAS=openblas \
+ USE_MKLDNN=1 \
+ -j$(nproc)
+}
+
+build_ubuntu_gpu_mkldnn() {
+ set -ex
+ make \
+ DEV=1 \
+ USE_PROFILER=1 \
+ USE_CPP_PACKAGE=1 \
+ USE_BLAS=openblas \
+ USE_MKLDNN=1 \
+ USE_CUDA=1 \
+ USE_CUDA_PATH=/usr/local/cuda \
+ USE_CUDNN=1 \
+ -j$(nproc)
+}
+
+build_ubuntu_gpu_cuda8_cudnn5() {
+ set -ex
+ make \
+ DEV=1 \
+ USE_PROFILER=1 \
+ USE_BLAS=openblas \
+ USE_CUDA=1 \
+ USE_CUDA_PATH=/usr/local/cuda \
+ USE_CUDNN=1 \
+ USE_CPP_PACKAGE=1 \
+ -j$(nproc)
+}
+
+build_ubuntu_amalgamation() {
+ set -ex
+ # Amalgamation can not be run with -j nproc
+ make -C amalgamation/ clean
+ make -C amalgamation/ USE_BLAS=openblas
+}
+
+build_ubuntu_amalgamation_min() {
+ set -ex
+ # Amalgamation can not be run with -j nproc
+ make -C amalgamation/ clean
+ make -C amalgamation/ USE_BLAS=openblas MIN=1
+}
+
+build_ubuntu_gpu_cmake_mkldnn() {
+ set -ex
+ cd /work/build
+ cmake \
+ -DUSE_CUDA=1 \
+ -DUSE_CUDNN=1 \
+ -DUSE_MKLML_MKL=1 \
+ -DUSE_MKLDNN=1 \
+ -DCMAKE_BUILD_TYPE=Release \
+ -G Ninja \
+ /work/mxnet
+
+ ninja -v
+}
+
+build_ubuntu_gpu_cmake() {
+ set -ex
+ cd /work/build
+ cmake \
+ -DUSE_CUDA=1 \
+ -DUSE_CUDNN=1 \
+ -DUSE_MKLML_MKL=0 \
+ -DUSE_MKLDNN=0 \
+ -DCMAKE_BUILD_TYPE=Release \
+ -G Ninja \
+ /work/mxnet
+
+ ninja -v
+}
+
+
+# Testing
+
+sanity_check() {
+ set -ex
+ tools/license_header.py check
+ make cpplint rcpplint jnilint
+ make pylint
+}
+
+
+unittest_ubuntu_python2_cpu() {
+ set -ex
+ export PYTHONPATH=./python/
+ # MXNET_MKLDNN_DEBUG is buggy and produces false positives
+ # https://github.com/apache/incubator-mxnet/issues/10026
+ #export MXNET_MKLDNN_DEBUG=1 # Ignored if not present
+ export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
+ nosetests-2.7 --verbose tests/python/unittest
+ nosetests-2.7 --verbose tests/python/train
+}
+
+unittest_ubuntu_python3_cpu() {
+ set -ex
+ export PYTHONPATH=./python/
+ # MXNET_MKLDNN_DEBUG is buggy and produces false positives
+ # https://github.com/apache/incubator-mxnet/issues/10026
+ #export MXNET_MKLDNN_DEBUG=1 # Ignored if not present
+ export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
+ nosetests-3.4 --verbose tests/python/unittest
+}
+
+unittest_ubuntu_python2_gpu() {
+ set -ex
+ export PYTHONPATH=./python/
+ # MXNET_MKLDNN_DEBUG is buggy and produces false positives
+ # https://github.com/apache/incubator-mxnet/issues/10026
+ #export MXNET_MKLDNN_DEBUG=1 # Ignored if not present
+ export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
+ nosetests-2.7 --verbose tests/python/gpu
+}
+
+unittest_ubuntu_python3_gpu() {
+ set -ex
+ export PYTHONPATH=./python/
+ # MXNET_MKLDNN_DEBUG is buggy and produces false positives
+ # https://github.com/apache/incubator-mxnet/issues/10026
+ #export MXNET_MKLDNN_DEBUG=1 # Ignored if not present
+ export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
+ nosetests-3.4 --verbose tests/python/gpu
+}
+
+unittest_ubuntu_cpu_scala() {
+ set -ex
+ make scalapkg USE_BLAS=openblas
+ make scalatest USE_BLAS=openblas
+}
+
+unittest_ubuntu_cpugpu_perl() {
+ set -ex
+ ./perl-package/test.sh
+}
+
+unittest_ubuntu_gpu_cpp() {
+ set -ex
+ build/tests/mxnet_unit_tests
+}
+
+unittest_ubuntu_cpu_R() {
+ set -ex
+ mkdir -p /tmp/r-site-library
+ # make -j not supported
+ make rpkg USE_BLAS=openblas R_LIBS=/tmp/r-site-library
+ R CMD INSTALL --library=/tmp/r-site-library R-package
+ make rpkgtest R_LIBS=/tmp/r-site-library
+}
+
+unittest_ubuntu_gpu_R() {
+ set -ex
+ mkdir -p /tmp/r-site-library
+ # make -j not supported
+ make rpkg USE_BLAS=openblas R_LIBS=/tmp/r-site-library
+ R CMD INSTALL --library=/tmp/r-site-library R-package
+ make rpkgtest R_LIBS=/tmp/r-site-library R_GPU_ENABLE=1
+}
+
+unittest_centos7_cpu() {
+ set -ex
+ cd /work/mxnet
+ python3.6 -m "nose" --with-timer --verbose tests/python/unittest
+ python3.6 -m "nose" --with-timer --verbose tests/python/train
+}
+
+unittest_centos7_gpu() {
+ set -ex
+ cd /work/mxnet
+ python3.6 -m "nose" --with-timer --verbose tests/python/gpu
+}
+
+integrationtest_ubuntu_gpu_python() {
+ set -ex
+ export PYTHONPATH=./python/
+ export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
+ python example/image-classification/test_score.py
+}
+
+integrationtest_ubuntu_gpu_caffe() {
+ set -ex
+ export PYTHONPATH=/work/deps/caffe/python:./python
+ python tools/caffe_converter/test_converter.py
+}
+
+integrationtest_ubuntu_gpu_cpp_package() {
+ set -ex
+ cpp-package/tests/ci_test.sh
+}
+
+
+test_ubuntu_cpu_python2() {
+ set -ex
+ pushd .
+ export MXNET_LIBRARY_PATH=/work/build/libmxnet.so
+
+ VENV=mxnet_py2_venv
+ virtualenv -p `which python2` $VENV
+ source $VENV/bin/activate
+ pip install nose nose-timer
+
+ cd /work/mxnet/python
+ pip install -e .
+ cd /work/mxnet
+ python -m "nose" --with-timer --verbose tests/python/unittest
+ popd
+}
+
+test_ubuntu_cpu_python3() {
+ set -ex
+ pushd .
+ export MXNET_LIBRARY_PATH=/work/build/libmxnet.so
+ VENV=mxnet_py3_venv
+ virtualenv -p `which python3` $VENV
+ source $VENV/bin/activate
+
+ cd /work/mxnet/python
+ pip3 install nose nose-timer
+ pip3 install -e .
+ cd /work/mxnet
+ python3 -m "nose" --with-timer --verbose tests/python/unittest
+
+ popd
+}
+
+# Deploy
+
+deploy_docs() {
+ set -ex
+ pushd .
+
+ make docs
+
+ popd
+}
+
+##############################################################
+# MAIN
+#
+# Run function passed as argument
+set +x
+if [ $# -gt 0 ]
+then
+ $@
+else
+ cat<<EOF
+
+$0: Execute a function by passing it as an argument to the script:
+
+Possible commands:
+
+EOF
+ declare -F | cut -d' ' -f3
+ echo
+fi
diff --git a/docker_multiarch/.gitignore b/docker_multiarch/.gitignore
deleted file mode 100644
index 2a07fbf7c51..00000000000
--- a/docker_multiarch/.gitignore
+++ /dev/null
@@ -1,2 +0,0 @@
-mxnet/
-build/
diff --git a/docker_multiarch/Dockerfile.build.android.arm64 b/docker_multiarch/Dockerfile.build.android.arm64
deleted file mode 100644
index 995e718bfb5..00000000000
--- a/docker_multiarch/Dockerfile.build.android.arm64
+++ /dev/null
@@ -1,77 +0,0 @@
-# -*- mode: dockerfile -*-
-FROM dockcross/base:latest
-MAINTAINER Pedro Larroy "pllarroy@amazon.com"
-
-# The cross-compiling emulator
-RUN apt-get update && apt-get install -y \
- qemu-user \
- qemu-user-static \
- unzip
-
-ENV CROSS_TRIPLE=aarch64-linux-android
-ENV CROSS_ROOT=/usr/${CROSS_TRIPLE}
-ENV AS=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-as \
- AR=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-ar \
- CC=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-gcc \
- CPP=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-cpp \
- CXX=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-g++ \
- LD=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-ld
-
-ENV ANDROID_NDK_REVISION 15c
-RUN mkdir -p /build && \
- cd /build && \
- curl -O https://dl.google.com/android/repository/android-ndk-r${ANDROID_NDK_REVISION}-linux-x86_64.zip && \
- unzip ./android-ndk-r${ANDROID_NDK_REVISION}-linux-x86_64.zip && \
- cd android-ndk-r${ANDROID_NDK_REVISION} && \
- ./build/tools/make_standalone_toolchain.py \
- --stl=libc++ \
- --arch arm64 \
- --api 21 \
- --install-dir=${CROSS_ROOT} && \
- cd / && \
- rm -rf /build && \
- find ${CROSS_ROOT} -exec chmod a+r '{}' \; && \
- find ${CROSS_ROOT} -executable -exec chmod a+x '{}' \;
-
-
-ENV DEFAULT_DOCKCROSS_IMAGE dockcross/android-arm
-
-# COPY Toolchain.cmake ${CROSS_ROOT}/
-# ENV CMAKE_TOOLCHAIN_FILE ${CROSS_ROOT}/Toolchain.cmake
-
-# Build-time metadata as defined at http://label-schema.org
-ARG BUILD_DATE
-ARG IMAGE
-ARG VCS_REF
-ARG VCS_URL
-LABEL org.label-schema.build-date=$BUILD_DATE \
- org.label-schema.name=$IMAGE \
- org.label-schema.vcs-ref=$VCS_REF \
- org.label-schema.vcs-url=$VCS_URL \
- org.label-schema.schema-version="1.0"
-
-ENV ARCH aarch64
-
-# Build OpenBLAS
-# https://github.com/xianyi/OpenBLAS/wiki/How-to-build-OpenBLAS-for-Android
-RUN git clone https://github.com/xianyi/OpenBLAS.git && \
- cd OpenBLAS && \
- make -j$(nproc) TARGET=ARMV8 ARM_SOFTFP_ABI=1 HOSTCC=gcc NOFORTRAN=1 libs
-
-ENV OPENBLAS_ROOT /work/OpenBLAS
-ENV LIBRARY_PATH /work/OpenBLAS/lib/:/work/OpenBLAS/:$LIBRARY_PATH
-ENV CPLUS_INCLUDE_PATH /work/OpenBLAS/include/:/work/OpenBLAS/:$CPLUS_INCLUDE_PATH
-WORKDIR /work
-
-ENV CC=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-clang
-ENV CXX=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-clang++
-ENV BUILD_OPTS "USE_BLAS=openblas USE_SSE=0 DMLC_LOG_STACK_TRACE=0 USE_OPENCV=0 USE_LAPACK=0"
-
-# Build MXNet
-ADD mxnet mxnet
-ADD arm.crosscompile.android.mk /work/mxnet/make/config.mk
-RUN cd mxnet && \
- make -j$(nproc) $BUILD_OPTS
-
-WORKDIR /work/build/
-RUN cp /work/mxnet/lib/* .
diff --git a/docker_multiarch/Dockerfile.build.arm64 b/docker_multiarch/Dockerfile.build.arm64
deleted file mode 100644
index 33f7b3f45e9..00000000000
--- a/docker_multiarch/Dockerfile.build.arm64
+++ /dev/null
@@ -1,37 +0,0 @@
-# -*- mode: dockerfile -*-
-# dockerfile to build libmxnet.so for armv7
-FROM dockcross/linux-arm64
-
-ENV ARCH aarch64
-ENV BUILD_OPTS "USE_BLAS=openblas USE_SSE=0 USE_OPENCV=0"
-ENV CC /usr/bin/aarch64-linux-gnu-gcc
-ENV CXX /usr/bin/aarch64-linux-gnu-g++
-ENV FC /usr/bin/aarch64-linux-gnu-gfortran-4.9
-ENV HOSTCC gcc
-
-WORKDIR /work
-
-# Build OpenBLAS
-ADD https://api.github.com/repos/xianyi/OpenBLAS/git/refs/heads/master /tmp/openblas_version.json
-RUN git clone https://github.com/xianyi/OpenBLAS.git && \
- cd OpenBLAS && \
- make -j$(nproc) TARGET=ARMV8 && \
- make install && \
- ln -s /opt/OpenBLAS/lib/libopenblas.so /usr/lib/libopenblas.so && \
- ln -s /opt/OpenBLAS/lib/libopenblas.a /usr/lib/libopenblas.a && \
- ln -s /opt/OpenBLAS/lib/libopenblas.a /usr/lib/liblapack.a
-
-ENV LD_LIBRARY_PATH /opt/OpenBLAS/lib
-ENV CPLUS_INCLUDE_PATH /opt/OpenBLAS/include
-
-# Build MXNet
-#ADD https://api.github.com/repos/apache/incubator-mxnet/git/refs/heads/master mxnet_version.json
-#RUN git clone --recursive https://github.com/apache/incubator-mxnet.git mxnet
-ADD mxnet mxnet
-
-WORKDIR /work/mxnet
-ADD arm.crosscompile.mk make/config.mk
-RUN make -j$(nproc) $BUILD_OPTS
-
-WORKDIR /work/build/
-RUN cp /work/mxnet/lib/* .
diff --git a/docker_multiarch/Dockerfile.build.armv6 b/docker_multiarch/Dockerfile.build.armv6
deleted file mode 100644
index 9adfc5b49d4..00000000000
--- a/docker_multiarch/Dockerfile.build.armv6
+++ /dev/null
@@ -1,38 +0,0 @@
-# -*- mode: dockerfile -*-
-# dockerfile to build libmxnet.so for armv7
-FROM dockcross/linux-armv6
-
-ENV ARCH armv6l
-ENV BUILD_OPTS "USE_BLAS=openblas USE_SSE=0 USE_OPENCV=0"
-ENV CC /usr/bin/arm-linux-gnueabihf-gcc
-ENV CXX /usr/bin/arm-linux-gnueabihf-g++
-ENV FC /usr/bin/arm-linux-gnueabihf-gfortran
-ENV HOSTCC gcc
-
-WORKDIR /work
-
-# Build OpenBLAS
-ADD https://api.github.com/repos/xianyi/OpenBLAS/git/refs/heads/master openblas_version.json
-RUN git clone https://github.com/xianyi/OpenBLAS.git && \
- cd OpenBLAS && \
- make -j$(nproc) TARGET=ARMV6 && \
- make install && \
- ln -s /opt/OpenBLAS/lib/libopenblas.so /usr/lib/gcc/arm-linux-gnueabihf/libopenblas.so && \
- ln -s /opt/OpenBLAS/lib/libopenblas.a /usr/lib/gcc/arm-linux-gnueabihf/libopenblas.a && \
- ln -s /opt/OpenBLAS/lib/libopenblas.a /usr/lib/gcc/arm-linux-gnueabihf/liblapack.a && \
- ln -s /opt/OpenBLAS/lib/libopenblas.a /usr/lib/liblapack.a
-
-ENV LD_LIBRARY_PATH /opt/OpenBLAS/lib
-ENV CPLUS_INCLUDE_PATH /opt/OpenBLAS/include
-
-# Build MXNet
-#ADD https://api.github.com/repos/apache/incubator-mxnet/git/refs/heads/master mxnet_version.json
-#RUN git clone --recursive https://github.com/apache/incubator-mxnet.git mxnet
-ADD mxnet mxnet
-
-WORKDIR /work/mxnet
-ADD arm.crosscompile.mk make/config.mk
-RUN make -j$(nproc) $BUILD_OPTS
-
-WORKDIR /work/build/
-RUN cp /work/mxnet/lib/* .
diff --git a/docker_multiarch/Dockerfile.build.armv7 b/docker_multiarch/Dockerfile.build.armv7
deleted file mode 100644
index 740f2b22da6..00000000000
--- a/docker_multiarch/Dockerfile.build.armv7
+++ /dev/null
@@ -1,26 +0,0 @@
-# -*- mode: dockerfile -*-
-# dockerfile to build libmxnet.so for armv7
-FROM dockcross/linux-armv7
-
-RUN apt-get update && \
- apt-get install -y libopenblas-dev:armhf && \
- rm -rf /var/lib/apt/lists/*
-
-ENV ARCH armv71
-ENV CC /usr/bin/arm-linux-gnueabihf-gcc
-ENV CXX /usr/bin/arm-linux-gnueabihf-g++
-ENV BUILD_OPTS "USE_OPENCV=0 USE_BLAS=openblas USE_SSE=0"
-
-# Build MXNet
-
-WORKDIR /work
-#ADD https://api.github.com/repos/apache/incubator-mxnet/git/refs/heads/master mxnet_version.json
-#RUN git clone --recursive https://github.com/apache/incubator-mxnet.git mxnet
-ADD mxnet mxnet
-
-WORKDIR /work/mxnet
-ADD arm.crosscompile.mk make/config.mk
-RUN make -j$(nproc) $BUILD_OPTS
-
-WORKDIR /work/build/
-RUN cp /work/mxnet/lib/* .
diff --git a/docker_multiarch/Dockerfile.build.cmake.ubuntu-17.04 b/docker_multiarch/Dockerfile.build.cmake.ubuntu-17.04
deleted file mode 100644
index cf0a981e04f..00000000000
--- a/docker_multiarch/Dockerfile.build.cmake.ubuntu-17.04
+++ /dev/null
@@ -1,37 +0,0 @@
-FROM ubuntu:17.04
-
-
-RUN apt-get update &&\
- apt-get install -y wget python3.5 gcc-4.9 gcc-5 g++-4.9 g++-5 cmake less python3-pip python3-dev\
- build-essential git pkgconf\
- libopenblas-dev liblapack-dev\
- maven default-jdk\
- ninja-build\
- libgtest-dev\
- &&\
- rm -rf /var/lib/apt/lists/*
-
-
-
-###########################
-# Build gtest
-WORKDIR /work/googletest
-RUN cmake /usr/src/googletest/googletest/ -GNinja
-RUN ninja
-RUN cp libgtest.a /usr/lib
-###########################
-
-
-
-WORKDIR /work
-#RUN git clone --recursive https://github.com/apache/incubator-mxnet.git mxnet
-ADD mxnet mxnet
-
-WORKDIR mxnet/build
-RUN cmake -DUSE_CUDA=OFF -DUSE_OPENCV=OFF -GNinja ..
-RUN ninja
-
-
-# Copy artifacts
-RUN mkdir -p /work/build
-RUN cp *.a *.so /work/build
diff --git a/docker_multiarch/Dockerfile.build.ubuntu-16.04-cuda_8.0_cudnn5 b/docker_multiarch/Dockerfile.build.ubuntu-16.04-cuda_8.0_cudnn5
deleted file mode 100644
index 071a351b312..00000000000
--- a/docker_multiarch/Dockerfile.build.ubuntu-16.04-cuda_8.0_cudnn5
+++ /dev/null
@@ -1,32 +0,0 @@
-FROM nvidia/cuda:8.0-cudnn5-devel
-
-RUN apt-get update &&\
- apt-get install -y wget python3.5 gcc-4.9 gcc-5 g++-4.9 g++-5 cmake less python3-pip python3-dev\
- build-essential git pkgconf\
- libopenblas-dev liblapack-dev\
- maven default-jdk\
- &&\
- rm -rf /var/lib/apt/lists/*
-
-WORKDIR /work
-#RUN git clone --recursive https://github.com/apache/incubator-mxnet.git mxnet
-ADD mxnet mxnet
-
-# Compile MxNet
-ENV BUILD_OPTS "USE_OPENCV=0 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1"
-WORKDIR /work/mxnet
-RUN make -j$(nproc) $BUILD_OPTS
-
-WORKDIR /work/build/
-RUN cp /work/mxnet/lib/* .
-
-# Scala packag
-#WORKDIR /work
-#RUN wget --quiet http://downloads.lightbend.com/scala/2.11.8/scala-2.11.8.deb
-#RUN dpkg -i scala-2.11.8.deb && rm scala-2.11.8.deb
-
-#WORKDIR /work/mxnet
-#RUN make scalapkg $BUILD_OPTS
-
-#WORKDIR /work/build/scala_gpu
-#RUN cp /work/mxnet/scala-package/assembly/linux-x86_64-gpu/target/*.jar .
diff --git a/docker_multiarch/Dockerfile.build.ubuntu-17.04 b/docker_multiarch/Dockerfile.build.ubuntu-17.04
deleted file mode 100644
index 63b3c0716d9..00000000000
--- a/docker_multiarch/Dockerfile.build.ubuntu-17.04
+++ /dev/null
@@ -1,30 +0,0 @@
-#
-# Base image to build MXNet from source in ubuntu
-#
-# Other images depend on it, so build it like:
-#
-# docker build -f Dockerfile.build.ubuntu-17.04 -t mxnet.build.ubuntu-17.04 .
-#
-FROM ubuntu:17.04
-
-
-RUN apt-get update &&\
- apt-get install -y wget python3.5 gcc-4.9 gcc-5 g++-4.9 g++-5 cmake less python3-pip python3-dev\
- build-essential git pkgconf\
- libopenblas-dev liblapack-dev\
- maven default-jdk
-
-RUN rm -rf /var/lib/apt/lists/*
-
-WORKDIR /work
-#RUN git clone --recursive https://github.com/apache/incubator-mxnet.git mxnet
-ADD mxnet mxnet
-
-# Compile MxNet
-ENV BUILD_OPTS "USE_OPENCV=0 USE_BLAS=openblas"
-WORKDIR /work/mxnet
-RUN make -j$(nproc) $BUILD_OPTS
-
-# Copy artifacts
-WORKDIR /work/build/
-RUN cp /work/mxnet/lib/* .
diff --git a/docker_multiarch/Dockerfile.build.ubuntu-17.04.scala.docker b/docker_multiarch/Dockerfile.build.ubuntu-17.04.scala.docker
deleted file mode 100644
index a31ce893783..00000000000
--- a/docker_multiarch/Dockerfile.build.ubuntu-17.04.scala.docker
+++ /dev/null
@@ -1,17 +0,0 @@
-# Before building this image you would need to build MXNet by executing:
-# docker build -f Dockerfile.build.ubuntu-17.04 -t mxnet.build.ubuntu-17.04 .
-# if you haven't done it before.
-
-FROM mxnet.build.ubuntu-17.04
-
-# Scala package
-WORKDIR /work
-RUN wget --quiet http://downloads.lightbend.com/scala/2.11.8/scala-2.11.8.deb
-RUN dpkg -i scala-2.11.8.deb && rm scala-2.11.8.deb
-
-WORKDIR /work/mxnet
-RUN make scalapkg $BUILD_OPTS
-
-WORKDIR /work/build/scala
-RUN cp /work/mxnet/scala-package/core/target/*.jar .
-RUN cp /work/mxnet/scala-package/assembly/linux-x86_64-cpu/target/*.jar .
diff --git a/docker_multiarch/Dockerfile.run.ubuntu-17.04.julia b/docker_multiarch/Dockerfile.run.ubuntu-17.04.julia
deleted file mode 100644
index df3a036d83f..00000000000
--- a/docker_multiarch/Dockerfile.run.ubuntu-17.04.julia
+++ /dev/null
@@ -1,24 +0,0 @@
-# Before building this image you would need to build MXNet by executing:
-# docker build -f Dockerfile.build.ubuntu-17.04 -t mxnet.build.ubuntu-17.04 .
-# if you haven't done it before.
-
-FROM mxnet.build.ubuntu-17.04
-
-ENV DEBIAN_FRONTEND=noninteractive
-
-##################
-# Julia installation
-RUN wget -q https://julialang.s3.amazonaws.com/bin/linux/x64/0.5/julia-0.5.1-linux-x86_64.tar.gz\
- && tar -zxf julia-0.5.1-linux-x86_64.tar.gz\
- && rm julia-0.5.1-linux-x86_64.tar.gz\
- && ln -s $(pwd)/julia-6445c82d00/bin/julia /usr/bin/julia
-##################
-
-
-ENV MXNET_HOME /work/mxnet
-WORKDIR /work/mxnet
-RUN julia -e 'Pkg.add("MXNet")'
-
-
-
-
diff --git a/docker_multiarch/Dockerfile.run.ubuntu-17.04.perl b/docker_multiarch/Dockerfile.run.ubuntu-17.04.perl
deleted file mode 100644
index 17c5d8d8c71..00000000000
--- a/docker_multiarch/Dockerfile.run.ubuntu-17.04.perl
+++ /dev/null
@@ -1,23 +0,0 @@
-# Before building this image you would need to build MXNet by executing:
-# docker build -f Dockerfile.build.ubuntu-17.04 -t mxnet.build.ubuntu-17.04 .
-# if you haven't done it before.
-
-FROM mxnet.build.ubuntu-17.04
-
-ENV DEBIAN_FRONTEND=noninteractive
-
-RUN apt-get update
-RUN apt-get install -y\
- libmouse-perl pdl cpanminus swig libgraphviz-perl
-RUN rm -rf /var/lib/apt/lists/*
-
-RUN cpanm -q Function::Parameters Hash::Ordered PDL::CCS
-
-WORKDIR /work/mxnet/perl-package/AI-MXNetCAPI
-RUN perl Makefile.PL && make install
-
-WORKDIR /work/mxnet/perl-package/AI-NNVMCAPI/
-RUN perl Makefile.PL && make install
-
-WORKDIR /work/mxnet/perl-package/AI-MXNet/
-RUN perl Makefile.PL && make install
diff --git a/docker_multiarch/Dockerfile.run.ubuntu-17.04.python b/docker_multiarch/Dockerfile.run.ubuntu-17.04.python
deleted file mode 100644
index 8bd262b891c..00000000000
--- a/docker_multiarch/Dockerfile.run.ubuntu-17.04.python
+++ /dev/null
@@ -1,17 +0,0 @@
-# Before building this image you would need to build MXNet by executing:
-# docker build -f Dockerfile.build.ubuntu-17.04 -t mxnet.build.ubuntu-17.04 .
-# if you haven't done it before.
-
-FROM mxnet.build.ubuntu-17.04
-
-ENV DEBIAN_FRONTEND=noninteractive
-
-
-RUN apt-get update
-RUN apt-get install -y python-pip
-RUN rm -rf /var/lib/apt/lists/*
-
-WORKDIR /work/mxnet/python
-RUN pip3 install -e .
-RUN pip install -e .
-
diff --git a/docker_multiarch/Dockerfile.run.ubuntu-17.04.r b/docker_multiarch/Dockerfile.run.ubuntu-17.04.r
deleted file mode 100644
index 493c7f0ef17..00000000000
--- a/docker_multiarch/Dockerfile.run.ubuntu-17.04.r
+++ /dev/null
@@ -1,36 +0,0 @@
-# Before building this image you would need to build MXNet by executing:
-# docker build -f Dockerfile.build.ubuntu-17.04 -t mxnet.build.ubuntu-17.04 .
-# if you haven't done it before.
-
-FROM mxnet.build.ubuntu-17.04
-
-ENV DEBIAN_FRONTEND=noninteractive
-#ENV BUILD_OPTS "USE_OPENCV=0 USE_BLAS=openblas GTEST_PATH=/usr/src/googletest/googletest"
-
-##################
-# R installation
-RUN apt-get update
-#RUN apt-get remove -y gnupg
-#RUN apt-get install -y --reinstall\
-# gnupg2 dirmngr
-
-RUN apt-get install -y dirmngr libopencv-dev
-RUN echo "deb http://cran.rstudio.com/bin/linux/ubuntu zesty/" >> /etc/apt/sources.list
-RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E084DAB9
-
-RUN apt-get install -y\
- r-base r-base-core r-recommended r-base-dev libxml2-dev libxt-dev libssl-dev libcurl4-openssl-dev
-
-
-WORKDIR /work/mxnet
-RUN cp R-package/DESCRIPTION .
-RUN Rscript -e "install.packages('devtools', repo = 'https://cran.rstudio.com')"
-RUN Rscript -e "library(devtools); library(methods); options(repos=c(CRAN='https://cran.rstudio.com')); install_deps(dependencies = TRUE)"
-
-
-##################
-# MXNet R package
-RUN make rpkg
-RUN R CMD INSTALL mxnet_current_r.tar.gz
-##################
-
diff --git a/docker_multiarch/Dockerfile.test.ubuntu-17.04 b/docker_multiarch/Dockerfile.test.ubuntu-17.04
deleted file mode 100644
index 1b0c145f488..00000000000
--- a/docker_multiarch/Dockerfile.test.ubuntu-17.04
+++ /dev/null
@@ -1,41 +0,0 @@
-FROM ubuntu-17.04
-RUN apt-get update &&\
- apt-get install -y python3-nose python-nose python-pip libgtest-dev valgrind ninja-build\
- &&\
- rm -rf /var/lib/apt/lists/*
-
-###########################
-# Unit tests
-# Build google test
-WORKDIR /work/googletest
-RUN cmake /usr/src/googletest/googletest/ -GNinja
-RUN ninja
-# FIXME
-RUN mkdir -p /usr/src/googletest/googletest/lib/
-RUN cp libgtest.a /usr/src/googletest/googletest/lib/
-
-ENV BUILD_OPTS "USE_OPENCV=0 USE_BLAS=openblas GTEST_PATH=/usr/src/googletest/googletest"
-
-WORKDIR /work/mxnet
-RUN make -j$(nproc) test $BUILD_OPTS
-ENV MXNET_ENGINE_INFO=true
-RUN build/tests/cpp/mxnet_test
-RUN valgrind build/tests/cpp/mxnet_test
-############################
-
-############################
-# Python tests
-WORKDIR /work/mxnet/python
-RUN pip3 install -e .
-RUN pip install -e .
-
-WORKDIR /work/mxnet
-RUN nosetests3 --verbose tests/python/unittest
-RUN nosetests --verbose tests/python/unittest
-############################
-
-
-############################
-# Scala tests
-RUN make scalatest $BUILD_OPTS
-############################
diff --git a/docker_multiarch/License.md b/docker_multiarch/License.md
deleted file mode 100644
index 614c582aefa..00000000000
--- a/docker_multiarch/License.md
+++ /dev/null
@@ -1,20 +0,0 @@
-## Dockerfiles License
-
-### All Dockerfiles in this directory are provided under an Apache 2.0 license.
-
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
\ No newline at end of file
diff --git a/docker_multiarch/README.md b/docker_multiarch/README.md
deleted file mode 100644
index a463d15bc82..00000000000
--- a/docker_multiarch/README.md
+++ /dev/null
@@ -1,42 +0,0 @@
-# Dockerized multi-architecture build
-
-These docker files and utilities will build mxnet and run tests for different architectures using cross compilation and produce
-runtime binary artifacts.
-
-This utilities require that you have docker installed. [Docker CE](https://docs.docker.com/engine/installation/linux/docker-ce/ubuntu/#install-docker) is recommended.
-
-
-To compile for all the supported architectures you can run the script
-```
-$ ./tool.py
-```
-
-To build a single arch, you can invoke docker directly:
-
-```
-$ docker build -f Dockerfile.build.<arch> -t <tag> .
-```
-
-Or call the dockerfile directly:
-
-```
-docker build -f <dockerfile> -t <tag> .
-```
-
-Or pass the architecture id to the tool:
-```
-$ ./tool.py -a ubuntu-17.04
-```
-
-By convention all the Dockerfiles produce the build artifacts on /work/build so they can be copied
-after.
-
-
-The tool will leave the resulting artifacts on the build/ directory
-
-# TODO
-
-- Handle dependencies between docker files, for example having a yaml file with the dependency graph
- so they can be built in the right order. Right now the dependency is very simple so simple
- alphabetical sorting of the images does the trick.
-
diff --git a/docker_multiarch/arm.crosscompile.android.mk b/docker_multiarch/arm.crosscompile.android.mk
deleted file mode 100644
index 0302c5cf25a..00000000000
--- a/docker_multiarch/arm.crosscompile.android.mk
+++ /dev/null
@@ -1,161 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-#-------------------------------------------------------------------------------
-# Template configuration for compiling mxnet
-#
-# If you want to change the configuration, please use the following
-# steps. Assume you are on the root directory of mxnet. First copy the this
-# file so that any local changes will be ignored by git
-#
-# $ cp make/config.mk .
-#
-# Next modify the according entries, and then compile by
-#
-# $ make
-#
-# or build in parallel with 8 threads
-#
-# $ make -j8
-#-------------------------------------------------------------------------------
-
-#---------------------
-# We do not assign compilers here. Often when cross-compiling these will already
-# be set correctly.
-#--------------------
-
-export NVCC = nvcc
-
-# whether compile with options for MXNet developer
-DEV = 0
-
-# whether compile with debug
-DEBUG = 0
-
-# whether compiler with profiler
-USE_PROFILER =
-
-# the additional link flags you want to add
-# TODO: Move flags here
-ADD_LDFLAGS=-static-libstdc++ -L/work/OpenBLAS/
-
-# the additional compile flags you want to add
-ADD_CFLAGS =
-
-#---------------------------------------------
-# matrix computation libraries for CPU/GPU
-#---------------------------------------------
-
-# whether use CUDA during compile
-USE_CUDA = 0
-
-# add the path to CUDA library to link and compile flag
-# if you have already add them to environment variable, leave it as NONE
-# USE_CUDA_PATH = /usr/local/cuda
-USE_CUDA_PATH = NONE
-
-# whether use CuDNN R3 library
-USE_CUDNN = 0
-
-# whether use cuda runtime compiling for writing kernels in native language (i.e. Python)
-USE_NVRTC = 0
-
-# whether use opencv during compilation
-# you can disable it, however, you will not able to use
-# imbin iterator
-USE_OPENCV = 0
-
-# use openmp for parallelization
-USE_OPENMP = 1
-
-# whether use NNPACK library
-USE_NNPACK = 0
-
-# For arm builds we're using openblas
-USE_BLAS = openblas
-
-# whether use lapack during compilation
-# only effective when compiled with blas versions openblas/apple/atlas/mkl
-USE_LAPACK = 0
-
-# path to lapack library in case of a non-standard installation
-USE_LAPACK_PATH =
-
-# add path to intel library, you may need it for MKL, if you did not add the path
-# to environment variable
-USE_INTEL_PATH = NONE
-
-# If use MKL only for BLAS, choose static link automatically to allow python wrapper
-USE_STATIC_MKL = NONE
-ifeq ($(USE_BLAS), mkl)
-USE_STATIC_MKL = 1
-endif
-
-#----------------------------
-# distributed computing
-#----------------------------
-
-# whether or not to enable multi-machine supporting
-USE_DIST_KVSTORE = 0
-
-# whether or not allow to read and write HDFS directly. If yes, then hadoop is
-# required
-USE_HDFS = 0
-
-# path to libjvm.so. required if USE_HDFS=1
-LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server
-
-# whether or not allow to read and write AWS S3 directly. If yes, then
-# libcurl4-openssl-dev is required, it can be installed on Ubuntu by
-# sudo apt-get install -y libcurl4-openssl-dev
-USE_S3 = 0
-
-#----------------------------
-# additional operators
-#----------------------------
-
-# path to folders containing projects specific operators that you don't want to put in src/operators
-EXTRA_OPERATORS =
-
-#----------------------------
-# other features
-#----------------------------
-
-# Create C++ interface package
-USE_CPP_PACKAGE = 0
-
-#----------------------------
-# plugins
-#----------------------------
-
-# whether to use caffe integration. This requires installing caffe.
-# You also need to add CAFFE_PATH/build/lib to your LD_LIBRARY_PATH
-# CAFFE_PATH = $(HOME)/caffe
-# MXNET_PLUGINS += plugin/caffe/caffe.mk
-
-# whether to use torch integration. This requires installing torch.
-# You also need to add TORCH_PATH/install/lib to your LD_LIBRARY_PATH
-# TORCH_PATH = $(HOME)/torch
-# MXNET_PLUGINS += plugin/torch/torch.mk
-
-# WARPCTC_PATH = $(HOME)/warp-ctc
-# MXNET_PLUGINS += plugin/warpctc/warpctc.mk
-
-# whether to use sframe integration. This requires build sframe
-# git@github.com:dato-code/SFrame.git
-# SFRAME_PATH = $(HOME)/SFrame
-# MXNET_PLUGINS += plugin/sframe/plugin.mk
diff --git a/docker_multiarch/arm.crosscompile.mk b/docker_multiarch/arm.crosscompile.mk
deleted file mode 100644
index 2bca9e396d7..00000000000
--- a/docker_multiarch/arm.crosscompile.mk
+++ /dev/null
@@ -1,161 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-#-------------------------------------------------------------------------------
-# Template configuration for compiling mxnet
-#
-# If you want to change the configuration, please use the following
-# steps. Assume you are on the root directory of mxnet. First copy the this
-# file so that any local changes will be ignored by git
-#
-# $ cp make/config.mk .
-#
-# Next modify the according entries, and then compile by
-#
-# $ make
-#
-# or build in parallel with 8 threads
-#
-# $ make -j8
-#-------------------------------------------------------------------------------
-
-#---------------------
-# We do not assign compilers here. Often when cross-compiling these will already
-# be set correctly.
-#--------------------
-
-export NVCC = nvcc
-
-# whether compile with options for MXNet developer
-DEV = 0
-
-# whether compile with debug
-DEBUG = 0
-
-# whether compiler with profiler
-USE_PROFILER =
-
-# the additional link flags you want to add
-# TODO: Move flags here
-ADD_LDFLAGS=-static-libstdc++
-
-# the additional compile flags you want to add
-ADD_CFLAGS =
-
-#---------------------------------------------
-# matrix computation libraries for CPU/GPU
-#---------------------------------------------
-
-# whether use CUDA during compile
-USE_CUDA = 0
-
-# add the path to CUDA library to link and compile flag
-# if you have already add them to environment variable, leave it as NONE
-# USE_CUDA_PATH = /usr/local/cuda
-USE_CUDA_PATH = NONE
-
-# whether use CuDNN R3 library
-USE_CUDNN = 0
-
-# whether use cuda runtime compiling for writing kernels in native language (i.e. Python)
-USE_NVRTC = 0
-
-# whether use opencv during compilation
-# you can disable it, however, you will not able to use
-# imbin iterator
-USE_OPENCV = 0
-
-# use openmp for parallelization
-USE_OPENMP = 1
-
-# whether use NNPACK library
-USE_NNPACK = 0
-
-# For arm builds we're using openblas
-USE_BLAS = openblas
-
-# whether use lapack during compilation
-# only effective when compiled with blas versions openblas/apple/atlas/mkl
-USE_LAPACK = 1
-
-# path to lapack library in case of a non-standard installation
-USE_LAPACK_PATH =
-
-# add path to intel library, you may need it for MKL, if you did not add the path
-# to environment variable
-USE_INTEL_PATH = NONE
-
-# If use MKL only for BLAS, choose static link automatically to allow python wrapper
-USE_STATIC_MKL = NONE
-ifeq ($(USE_BLAS), mkl)
-USE_STATIC_MKL = 1
-endif
-
-#----------------------------
-# distributed computing
-#----------------------------
-
-# whether or not to enable multi-machine supporting
-USE_DIST_KVSTORE = 0
-
-# whether or not allow to read and write HDFS directly. If yes, then hadoop is
-# required
-USE_HDFS = 0
-
-# path to libjvm.so. required if USE_HDFS=1
-LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server
-
-# whether or not allow to read and write AWS S3 directly. If yes, then
-# libcurl4-openssl-dev is required, it can be installed on Ubuntu by
-# sudo apt-get install -y libcurl4-openssl-dev
-USE_S3 = 0
-
-#----------------------------
-# additional operators
-#----------------------------
-
-# path to folders containing projects specific operators that you don't want to put in src/operators
-EXTRA_OPERATORS =
-
-#----------------------------
-# other features
-#----------------------------
-
-# Create C++ interface package
-USE_CPP_PACKAGE = 0
-
-#----------------------------
-# plugins
-#----------------------------
-
-# whether to use caffe integration. This requires installing caffe.
-# You also need to add CAFFE_PATH/build/lib to your LD_LIBRARY_PATH
-# CAFFE_PATH = $(HOME)/caffe
-# MXNET_PLUGINS += plugin/caffe/caffe.mk
-
-# whether to use torch integration. This requires installing torch.
-# You also need to add TORCH_PATH/install/lib to your LD_LIBRARY_PATH
-# TORCH_PATH = $(HOME)/torch
-# MXNET_PLUGINS += plugin/torch/torch.mk
-
-# WARPCTC_PATH = $(HOME)/warp-ctc
-# MXNET_PLUGINS += plugin/warpctc/warpctc.mk
-
-# whether to use sframe integration. This requires build sframe
-# git@github.com:dato-code/SFrame.git
-# SFRAME_PATH = $(HOME)/SFrame
-# MXNET_PLUGINS += plugin/sframe/plugin.mk
diff --git a/docker_multiarch/tool.py b/docker_multiarch/tool.py
deleted file mode 100755
index d0003ec0582..00000000000
--- a/docker_multiarch/tool.py
+++ /dev/null
@@ -1,155 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-"""Multi arch dockerized build tool.
-
-"""
-
-__author__ = 'Pedro Larroy'
-__version__ = '0.1'
-
-import os
-import sys
-import subprocess
-import logging
-import argparse
-from subprocess import check_call
-import glob
-import re
-
-class CmdResult(object):
- def __init__(self, std_out, std_err, status_code):
- self.std_out = std_out
- self.std_err = std_err
- self.status_code = status_code if status_code is not None else 0
-
- def __str__(self):
- return "%s, %s, %s" % (self.std_out, self.std_err, self.status_code)
-
-def run(cmd, fail_on_error=True):
- logging.debug("executing shell command:\n" + cmd)
- proc = subprocess.Popen(
- cmd,
- shell=True,
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE,
- )
- std_out, std_err = proc.communicate()
- if fail_on_error:
- if proc.returncode != 0:
- logging.warn('Error running command: {}'.format(cmd))
- assert proc.returncode == 0, std_err
- res = CmdResult(std_out.decode('utf-8'), std_err.decode('utf-8'), proc.returncode)
- return res
-
-
-def mkdir_p(d):
- rev_path_list = list()
- head = d
- while len(head) and head != os.sep:
- rev_path_list.append(head)
- (head, tail) = os.path.split(head)
-
- rev_path_list.reverse()
- for p in rev_path_list:
- try:
- os.mkdir(p)
- except OSError as e:
- if e.errno != 17:
- raise
-
-def get_arches():
- """Get a list of architectures given our dockerfiles"""
- dockerfiles = glob.glob("Dockerfile.build.*")
- dockerfiles = list(filter(lambda x: x[-1] != '~', dockerfiles))
- arches = list(map(lambda x: re.sub(r"Dockerfile.build.(.*)", r"\1", x), dockerfiles))
- arches.sort()
- return arches
-
-def sync_source():
- logging.info("Copying sources")
- check_call(["rsync","-a","--delete","--exclude=\".git/\"",'--exclude=/docker_multiarch/',"../","mxnet"])
-
-def get_docker_tag(arch):
- return "mxnet.build.{0}".format(arch)
-
-def get_dockerfile(arch):
- return "Dockerfile.build.{0}".format(arch)
-
-def build(arch):
- """Build the given architecture in the container"""
- assert arch in get_arches(), "No such architecture {0}, Dockerfile.build.{0} not found".format(arch)
- logging.info("Building for target platform {0}".format(arch))
- check_call(["docker", "build",
- "-f", get_dockerfile(arch),
- "-t", get_docker_tag(arch),
- "."])
-
-def collect_artifacts(arch):
- """Collects the artifacts built inside the docker container to the local fs"""
- def artifact_path(arch):
- return "{}/build/{}".format(os.getcwd(), arch)
- logging.info("Collect artifacts from build in {0}".format(artifact_path(arch)))
- mkdir_p("build/{}".format(arch))
-
- # Mount artifact_path on /$arch inside the container and copy the build output so we can access
- # locally from the host fs
- check_call(["docker","run",
- "-v", "{}:/{}".format(artifact_path(arch), arch),
- get_docker_tag(arch),
- "bash", "-c", "cp -r /work/build/* /{}".format(arch)])
-
-def main():
- logging.getLogger().setLevel(logging.INFO)
- logging.basicConfig(format='%(asctime)-15s %(message)s')
-
- parser = argparse.ArgumentParser()
- parser.add_argument("-a", "--arch",
- help="Architecture",
- type=str)
-
- parser.add_argument("-l", "--list_arch",
- help="List architectures",
- action='store_true')
- args = parser.parse_args()
-
- if args.list_arch:
- arches = get_arches()
- print(arches)
-
- elif args.arch:
- sync_source()
- build(args.arch)
- collect_artifacts(args.arch)
-
- else:
- arches = get_arches()
- logging.info("Building for all architectures: {}".format(arches))
- logging.info("Artifacts will be produced in the build/ directory.")
- sync_source()
- for arch in arches:
- build(arch)
- collect_artifacts(arch)
-
- return 0
-
-if __name__ == '__main__':
- sys.exit(main())
-
diff --git a/python/mxnet/gluon/data/dataloader.py b/python/mxnet/gluon/data/dataloader.py
index 02aa5c04143..6efaf35e881 100644
--- a/python/mxnet/gluon/data/dataloader.py
+++ b/python/mxnet/gluon/data/dataloader.py
@@ -214,6 +214,7 @@ def __iter__(self):
worker.start()
workers.append(worker)
+ idx = 0
for idx, batch in enumerate(self._batch_sampler):
key_queue.put((idx, batch))
num_batches = idx + 1
diff --git a/python/mxnet/image/detection.py b/python/mxnet/image/detection.py
index 191a2bec31e..63a44ab4643 100644
--- a/python/mxnet/image/detection.py
+++ b/python/mxnet/image/detection.py
@@ -842,7 +842,7 @@ def draw_next(self, color=None, thickness=2, mean=None, std=None, clip=True,
import cv2
except ImportError as e:
warnings.warn('Unable to import cv2, skip drawing: %s', str(e))
- raise StopIteration
+ return
count = 0
try:
while True:
@@ -896,7 +896,7 @@ def draw_next(self, color=None, thickness=2, mean=None, std=None, clip=True,
yield image
except StopIteration:
if not count:
- raise StopIteration
+ return
def sync_label_shape(self, it, verbose=False):
"""Synchronize label shape with the input iterator. This is useful when
diff --git a/python/mxnet/libinfo.py b/python/mxnet/libinfo.py
index 9a82b797a90..3220b5a3352 100644
--- a/python/mxnet/libinfo.py
+++ b/python/mxnet/libinfo.py
@@ -20,6 +20,7 @@
from __future__ import absolute_import
import os
import platform
+import logging
def find_lib_path():
"""Find MXNet dynamic library files.
@@ -29,6 +30,17 @@ def find_lib_path():
lib_path : list(string)
List of all found path to the libraries.
"""
+ lib_from_env = os.environ.get('MXNET_LIBRARY_PATH')
+ if lib_from_env:
+ if os.path.isfile(lib_from_env):
+ if not os.path.isabs(lib_from_env):
+ logging.warning("MXNET_LIBRARY_PATH should be an absolute path, instead of: %s",
+ lib_from_env)
+ else:
+ return [lib_from_env]
+ else:
+ logging.warning("MXNET_LIBRARY_PATH '%s' doesn't exist", lib_from_env)
+
curr_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__)))
api_path = os.path.join(curr_path, '../../lib/')
cmake_build_path = os.path.join(curr_path, '../../build/')
diff --git a/tests/ci_build/Dockerfile.amzn_linux_cpu b/tests/ci_build/Dockerfile.amzn_linux_cpu
deleted file mode 100644
index d1fb4fc8ab0..00000000000
--- a/tests/ci_build/Dockerfile.amzn_linux_cpu
+++ /dev/null
@@ -1,16 +0,0 @@
-FROM amazonlinux
-MAINTAINER Ly Nguyen <ly...@amazon.com>
-
-COPY install/* /install/
-
-RUN yum install -y git wget sudo
-
-RUN chmod -R 755 install
-RUN /install/install_opencv.sh
-RUN /install/install_openblas.sh
-RUN /install/install_python2.sh
-RUN /install/install_python3.sh
-RUN /install/install_testdeps.sh
-RUN /install/install_julia.sh
-RUN /install/install_maven.sh
-RUN /install/install_library.sh
diff --git a/tests/ci_build/Dockerfile.build_cuda b/tests/ci_build/Dockerfile.build_cuda
deleted file mode 100644
index d659a1a8ade..00000000000
--- a/tests/ci_build/Dockerfile.build_cuda
+++ /dev/null
@@ -1,26 +0,0 @@
-FROM nvidia/cuda:8.0-cudnn5-devel
-# cuda8.0 has to be used because this is the first ubuntu16.04 container
-# which is required due to OpenBLAS being incompatible with ubuntu14.04
-# the reason we used a gpu base container because we are going to test MKLDNN
-# operator implementation against GPU implementation
-
-COPY install/ubuntu_install_core.sh /install/
-RUN /install/ubuntu_install_core.sh
-COPY install/ubuntu_install_python.sh /install/
-RUN /install/ubuntu_install_python.sh
-COPY install/ubuntu_install_scala.sh /install/
-RUN /install/ubuntu_install_scala.sh
-COPY install/ubuntu_install_r.sh /install/
-RUN /install/ubuntu_install_r.sh
-COPY install/ubuntu_install_perl.sh /install/
-RUN /install/ubuntu_install_perl.sh
-
-# Allows to run tasks on a CPU without nvidia-docker and GPU
-COPY install/ubuntu_install_nvidia.sh /install/
-RUN /install/ubuntu_install_nvidia.sh
-
-# Add MKLML libraries
-RUN wget --no-check-certificate -O /tmp/mklml.tgz https://github.com/01org/mkl-dnn/releases/download/v0.12/mklml_lnx_2018.0.1.20171227.tgz
-RUN tar -zxvf /tmp/mklml.tgz && cp -rf mklml_*/* /usr/local/ && rm -rf mklml_*
-
-ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib
diff --git a/tests/ci_build/Dockerfile.caffe_gpu b/tests/ci_build/Dockerfile.caffe_gpu
deleted file mode 100644
index 34c4625bdcd..00000000000
--- a/tests/ci_build/Dockerfile.caffe_gpu
+++ /dev/null
@@ -1,36 +0,0 @@
-FROM nvidia/cuda:8.0-cudnn5-devel
-# cuda8.0 has to be used because this is the first ubuntu16.04 container
-# which is required due to OpenBLAS being incompatible with ubuntu14.04
-
-COPY install/ubuntu_install_core.sh /install/
-RUN /install/ubuntu_install_core.sh
-
-COPY install/ubuntu_install_python.sh /install/
-RUN /install/ubuntu_install_python.sh
-
-RUN apt-get install -y libprotobuf-dev libleveldb-dev \
- libsnappy-dev libopencv-dev libhdf5-serial-dev protobuf-compiler \
- libatlas-base-dev python-dev libgflags-dev libgoogle-glog-dev liblmdb-dev \
- python-numpy python-opencv
-
-RUN apt-get install -y --no-install-recommends libboost-all-dev
-
-RUN cd /; git clone http://github.com/BVLC/caffe.git; cd caffe; \
- cp Makefile.config.example Makefile.config
-
-RUN echo "CPU_ONLY := 1" >> /caffe/Makefile.config
-
-# Fixes https://github.com/BVLC/caffe/issues/5658 See https://github.com/intel/caffe/wiki/Ubuntu-16.04-or-15.10-Installation-Guide
-RUN echo "INCLUDE_DIRS += /usr/lib /usr/lib/x86_64-linux-gnu /usr/include/hdf5/serial/ " >> /caffe/Makefile.config
-RUN echo "LIBRARY_DIRS += /usr/lib /usr/lib/x86_64-linux-gnu /usr/lib/x86_64-linux-gnu/hdf5/serial " >> /caffe/Makefile.config
-
-# Fixes https://github.com/BVLC/caffe/issues/4333 See https://github.com/intel/caffe/wiki/Ubuntu-16.04-or-15.10-Installation-Guide
-# Note: This is only valid on Ubuntu16.04 - the version numbers are bound to the distribution
-RUN ln -s /usr/lib/x86_64-linux-gnu/libhdf5_serial.so.10.0.2 /usr/lib/x86_64-linux-gnu/libhdf5.so
-RUN ln -s /usr/lib/x86_64-linux-gnu/libhdf5_serial_hl.so.10.0.2 /usr/lib/x86_64-linux-gnu/libhdf5_hl.so
-
-RUN cd caffe; make all pycaffe -j$(nproc)
-
-RUN cd caffe/python; for req in $(cat requirements.txt); do pip2 install $req; done
-
-ENV PYTHONPATH=${PYTHONPATH}:/caffe/python
diff --git a/tests/ci_build/Dockerfile.cpu b/tests/ci_build/Dockerfile.cpu
deleted file mode 100644
index 226054a5974..00000000000
--- a/tests/ci_build/Dockerfile.cpu
+++ /dev/null
@@ -1,12 +0,0 @@
-FROM ubuntu:16.04
-
-COPY install/ubuntu_install_core.sh /install/
-RUN /install/ubuntu_install_core.sh
-COPY install/ubuntu_install_python.sh /install/
-RUN /install/ubuntu_install_python.sh
-COPY install/ubuntu_install_scala.sh /install/
-RUN /install/ubuntu_install_scala.sh
-COPY install/ubuntu_install_r.sh /install/
-RUN /install/ubuntu_install_r.sh
-COPY install/ubuntu_install_perl.sh /install/
-RUN /install/ubuntu_install_perl.sh
diff --git a/tests/ci_build/Dockerfile.cpu_clang b/tests/ci_build/Dockerfile.cpu_clang
deleted file mode 100644
index 5bcdee563eb..00000000000
--- a/tests/ci_build/Dockerfile.cpu_clang
+++ /dev/null
@@ -1,21 +0,0 @@
-FROM ubuntu:16.04
-
-COPY install/ubuntu_install_core.sh /install/
-RUN /install/ubuntu_install_core.sh
-COPY install/ubuntu_install_python.sh /install/
-RUN /install/ubuntu_install_python.sh
-COPY install/ubuntu_install_scala.sh /install/
-RUN /install/ubuntu_install_scala.sh
-COPY install/ubuntu_install_r.sh /install/
-RUN /install/ubuntu_install_r.sh
-COPY install/ubuntu_install_perl.sh /install/
-RUN /install/ubuntu_install_perl.sh
-
-# Install clang 3.9 (the same version as in XCode 8.*) and 5.0 (latest major release)
-RUN wget -O - http://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
- apt-add-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-3.9 main" && \
- apt-add-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-5.0 main" && \
- apt-get update && \
- apt-get install -y clang-3.9 clang-5.0 && \
- clang-3.9 --version && \
- clang-5.0 --version
diff --git a/tests/ci_build/Dockerfile.cpu_mklml b/tests/ci_build/Dockerfile.cpu_mklml
deleted file mode 100644
index 9f235b46119..00000000000
--- a/tests/ci_build/Dockerfile.cpu_mklml
+++ /dev/null
@@ -1,18 +0,0 @@
-FROM ubuntu:16.04
-
-COPY install/ubuntu_install_core.sh /install/
-RUN /install/ubuntu_install_core.sh
-COPY install/ubuntu_install_python.sh /install/
-RUN /install/ubuntu_install_python.sh
-COPY install/ubuntu_install_scala.sh /install/
-RUN /install/ubuntu_install_scala.sh
-COPY install/ubuntu_install_r.sh /install/
-RUN /install/ubuntu_install_r.sh
-COPY install/ubuntu_install_perl.sh /install/
-RUN /install/ubuntu_install_perl.sh
-
-# Add MKLML library, compatiable with Ubuntu16.04
-RUN wget --no-check-certificate -O /tmp/mklml.tgz https://github.com/01org/mkl-dnn/releases/download/v0.12/mklml_lnx_2018.0.1.20171227.tgz
-RUN tar -zxvf /tmp/mklml.tgz && cp -rf mklml_*/* /usr/local/ && rm -rf mklml_*
-
-ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib
diff --git a/tests/ci_build/Dockerfile.crosstool b/tests/ci_build/Dockerfile.crosstool
deleted file mode 100644
index 371cdc730bb..00000000000
--- a/tests/ci_build/Dockerfile.crosstool
+++ /dev/null
@@ -1,23 +0,0 @@
-FROM ubuntu
-MAINTAINER Aran Khanna <ar...@amazon.com>
-
-# UPDATE BOX
-RUN apt-get update && apt-get -y upgrade
-
-# TOOLCHAIN DEPS
-RUN apt-get install -y python python-setuptools python-pip python-dev unzip gfortran
-RUN apt-get install -y git bison cvs flex gperf texinfo automake libtool help2man make libtool-bin libncurses5-dev g++ cmake wget gawk
-RUN pip install numpy nose
-
-# BUILD TOOLCHAIN
-RUN git clone https://github.com/arank/crosstool-NG
-RUN cd crosstool-NG && ./bootstrap && ./configure && make && make install
-
-RUN useradd -ms /bin/bash aran
-RUN cd && cp -R .profile .bashrc /home/aran
-ADD . /home/aran/build
-
-RUN chown -R aran:aran /home/aran
-
-RUN cd /home/aran/build && su -m aran -c "export HOME=/home/aran;ct-ng arm-unknown-linux-gnueabi;ct-ng build"
-
diff --git a/tests/ci_build/Dockerfile.doc b/tests/ci_build/Dockerfile.doc
deleted file mode 100644
index 43d1fa97ac3..00000000000
--- a/tests/ci_build/Dockerfile.doc
+++ /dev/null
@@ -1,15 +0,0 @@
-FROM ubuntu:14.04
-
-
-COPY install/ubuntu_install_core.sh /install/
-RUN /install/ubuntu_install_core.sh
-COPY install/ubuntu_install_python.sh /install/
-RUN /install/ubuntu_install_python.sh
-COPY install/ubuntu_install_scala.sh /install/
-RUN /install/ubuntu_install_scala.sh
-
-RUN wget http://downloads.lightbend.com/scala/2.11.8/scala-2.11.8.deb && \
- dpkg -i scala-2.11.8.deb && rm scala-2.11.8.deb
-
-RUN apt-get install -y doxygen libatlas-base-dev graphviz pandoc
-RUN pip install sphinx==1.3.5 CommonMark==0.5.4 breathe mock recommonmark pypandoc beautifulsoup4
diff --git a/tests/ci_build/Dockerfile.emscripten b/tests/ci_build/Dockerfile.emscripten
deleted file mode 100644
index e359e6cf5a6..00000000000
--- a/tests/ci_build/Dockerfile.emscripten
+++ /dev/null
@@ -1,19 +0,0 @@
-FROM ubuntu
-LABEL maintainer "arankhan@amazon.com"
-
-# UPDATE BOX
-RUN apt-get update && apt-get -y upgrade
-
-# TOOLCHAIN DEPS
-RUN apt-get install -y python python-setuptools python-pip python-dev unzip gfortran
-RUN apt-get install -y git nodejs build-essential cmake
-
-# BUILD EMSCRIPTEN
-RUN git clone https://github.com/kripken/emscripten.git
-RUN git clone https://github.com/kripken/emscripten-fastcomp
-RUN cd emscripten-fastcomp && \
-git clone https://github.com/kripken/emscripten-fastcomp-clang tools/clang && \
-mkdir build && cd build && \
-cmake .. -DCMAKE_BUILD_TYPE=Release -DLLVM_TARGETS_TO_BUILD="X86;JSBackend" \
--DLLVM_INCLUDE_EXAMPLES=OFF -DLLVM_INCLUDE_TESTS=OFF -DCLANG_INCLUDE_EXAMPLES=OFF \
--DCLANG_INCLUDE_TESTS=OFF && make
diff --git a/tests/ci_build/Dockerfile.gpu b/tests/ci_build/Dockerfile.gpu
deleted file mode 100644
index bd1a0083916..00000000000
--- a/tests/ci_build/Dockerfile.gpu
+++ /dev/null
@@ -1,20 +0,0 @@
-FROM nvidia/cuda:8.0-cudnn5-devel
-# cuda8.0 has to be used because this is the first ubuntu16.04 container
-# which is required due to OpenBLAS being incompatible with ubuntu14.04
-
-COPY install/ubuntu_install_core.sh /install/
-RUN /install/ubuntu_install_core.sh
-COPY install/ubuntu_install_python.sh /install/
-RUN /install/ubuntu_install_python.sh
-COPY install/ubuntu_install_scala.sh /install/
-RUN /install/ubuntu_install_scala.sh
-COPY install/ubuntu_install_r.sh /install/
-RUN /install/ubuntu_install_r.sh
-COPY install/ubuntu_install_perl.sh /install/
-RUN /install/ubuntu_install_perl.sh
-
-COPY install/ubuntu_install_llvm.sh /install/
-RUN /install/ubuntu_install_llvm.sh
-
-COPY install/ubuntu_install_tvm.sh /install/
-RUN /install/ubuntu_install_tvm.sh
diff --git a/tests/ci_build/Dockerfile.gpu_mklml b/tests/ci_build/Dockerfile.gpu_mklml
deleted file mode 100644
index f594b1c9e48..00000000000
--- a/tests/ci_build/Dockerfile.gpu_mklml
+++ /dev/null
@@ -1,18 +0,0 @@
-FROM nvidia/cuda:8.0-cudnn5-devel
-# cuda8.0 has to be used because this is the first ubuntu16.04 container
-# # which is required due to OpenBLAS being incompatible with ubuntu14.04
-# the reason we used a gpu base container because we are going to test MKLDNN
-# operator implementation against GPU implementation
-
-COPY install/ubuntu_install_core.sh /install/
-RUN /install/ubuntu_install_core.sh
-COPY install/ubuntu_install_python.sh /install/
-RUN /install/ubuntu_install_python.sh
-COPY install/ubuntu_install_scala.sh /install/
-RUN /install/ubuntu_install_scala.sh
-
-# Add MKLML library, compatible with Ubuntu16.04
-RUN wget --no-check-certificate -O /tmp/mklml.tgz https://github.com/01org/mkl-dnn/releases/download/v0.12/mklml_lnx_2018.0.1.20171227.tgz
-RUN tar -zxvf /tmp/mklml.tgz && cp -rf mklml_*/* /usr/local/ && rm -rf mklml_*
-
-ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib/:/usr/lib/gcc/x86_64-linux-gnu/5/
diff --git a/tests/ci_build/Dockerfile.lint b/tests/ci_build/Dockerfile.lint
deleted file mode 100644
index 7fde1f5daa5..00000000000
--- a/tests/ci_build/Dockerfile.lint
+++ /dev/null
@@ -1,5 +0,0 @@
-# For lint test
-FROM ubuntu:16.04
-
-RUN apt-get update && apt-get install -y python-pip sudo
-RUN pip install cpplint==1.3.0 pylint==1.8.2
diff --git a/tests/ci_build/Dockerfile.spell_checker b/tests/ci_build/Dockerfile.spell_checker
deleted file mode 100644
index c487959c7dc..00000000000
--- a/tests/ci_build/Dockerfile.spell_checker
+++ /dev/null
@@ -1,6 +0,0 @@
-# For spell checker
-FROM mxnet/aml:latest
-
-RUN yum install -y enchant
-RUN pip install pyenchant grammar-check html2text
-RUN pip install sphinx==1.5.1 CommonMark==0.5.4 breathe mock==1.0.1 recommonmark
diff --git a/tests/ci_build/Dockerfile.ubuntu1404_cuda75_cudnn5 b/tests/ci_build/Dockerfile.ubuntu1404_cuda75_cudnn5
deleted file mode 100644
index 88fd7cea6fc..00000000000
--- a/tests/ci_build/Dockerfile.ubuntu1404_cuda75_cudnn5
+++ /dev/null
@@ -1,40 +0,0 @@
-FROM nvidia/cuda:7.5-cudnn5-devel
-MAINTAINER Ly Nguyen <ly...@amazon.com>
-
-# OPENCV
-RUN apt-mark hold libcudnn5 libcudnn5-dev
-RUN apt-get update && apt-get -y upgrade
-RUN apt-get install -y libopencv-dev
-
-# BLAS
-RUN apt-get install -y libatlas-base-dev
-
-# PYTHON2
-RUN apt-get install -y python-setuptools python-pip python-dev unzip gfortran
-RUN pip install numpy nose scipy
-
-# PYTHON3
-RUN apt-get install -y python3-setuptools python3-pip
-RUN pip3 install numpy nose scipy && ln -s -f /usr/local/bin/nosetests-3.4 /usr/local/bin/nosetests3
-
-# TESTDEPS
-RUN apt-get install -y libgtest-dev cmake wget
-RUN cd /usr/src/gtest && cmake CMakeLists.txt && make && cp *.a /usr/lib
-RUN pip install nose cpplint 'pylint==1.4.4' 'astroid==1.3.6'
-
-# MAVEN
-RUN apt-get install -y software-properties-common
-RUN add-apt-repository ppa:webupd8team/java -y
-RUN apt-get update
-RUN echo "oracle-java8-installer shared/accepted-oracle-license-v1-1 select true" | debconf-set-selections
-RUN apt-get install -y oracle-java8-installer
-RUN apt-get install -y oracle-java8-set-default
-RUN apt-get install -y maven
-
-# R
-RUN apt-get install -y software-properties-common r-base-core libcurl4-openssl-dev libssl-dev libxml2-dev
-RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E084DAB9
-RUN add-apt-repository -y ppa:marutter/rdev
-RUN apt-get update && apt-get -y upgrade
-RUN DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confnew" install r-base r-base-dev
-RUN Rscript -e "install.packages('devtools', repo = 'https://cran.rstudio.com')"
diff --git a/tests/ci_build/README.md b/tests/ci_build/README.md
deleted file mode 100644
index 4d647f2631c..00000000000
--- a/tests/ci_build/README.md
+++ /dev/null
@@ -1,41 +0,0 @@
-# MXNET Builds
-
-This directory contains the files and setup instructions to run all tests. They
-are running on [ci.mxnet.io](http://ci.mxnet.io/blue/pipelines). But you can also
-run them locally easily.
-
-## Run locally
-
-To run locally, we need to first install
-[docker](https://docs.docker.com/engine/installation/) and
-[nvidia-docker](https://github.com/NVIDIA/nvidia-docker/wiki).
-
-We may use the AWS EC2 AMI `ami-d73bb4b7` available at US West (Oregon) which
-has both pre-installed.
-
-Then we can run the tasks defined in the [Jenkinsfile](../../Jenkinsfile) by
-using (`ci_build.sh`)[./ci_build.sh]. For example
-
-- lint the python codes
-
- ```bash
- ./ci_build.sh lint make pylint
- ```
-
-- build codes with CUDA supports
-
- ```bash
- ./ci_build.sh gpu make -j$(nproc) USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1
- ```
-
-- do the python unittest
-
- ```bash
- ./ci_build.sh gpu PYTHONPATH=./python/ nosetests --with-timer --verbose tests/python/unittest'
- ```
-
-- build the documents. The results will be available at `docs/_build/html`
-
- ```bash
- tests/ci_build/ci_build.sh doc make -C docs html
- ```
diff --git a/tests/ci_build/ci_build.sh b/tests/ci_build/ci_build.sh
deleted file mode 100755
index 768a2ed3028..00000000000
--- a/tests/ci_build/ci_build.sh
+++ /dev/null
@@ -1,188 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-#
-# Execute command within a docker container
-#
-# Usage: ci_build.sh <CONTAINER_TYPE> [--dockerfile <DOCKERFILE_PATH>] [-it]
-# <COMMAND>
-#
-# CONTAINER_TYPE: Type of the docker container used the run the build: e.g.,
-# (cpu | gpu)
-#
-# DOCKERFILE_PATH: (Optional) Path to the Dockerfile used for docker build. If
-# this optional value is not supplied (via the --dockerfile
-# flag), will use Dockerfile.CONTAINER_TYPE in default
-#
-# COMMAND: Command to be executed in the docker container
-#
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-
-# Get the command line arguments.
-CONTAINER_TYPE=$( echo "$1" | tr '[:upper:]' '[:lower:]' )
-shift 1
-
-# Dockerfile to be used in docker build
-DOCKERFILE_PATH="${SCRIPT_DIR}/Dockerfile.${CONTAINER_TYPE}"
-DOCKER_CONTEXT_PATH="${SCRIPT_DIR}"
-
-if [[ "$1" == "--dockerfile" ]]; then
- DOCKERFILE_PATH="$2"
- DOCKER_CONTEXT_PATH=$(dirname "${DOCKERFILE_PATH}")
- echo "Using custom Dockerfile path: ${DOCKERFILE_PATH}"
- echo "Using custom docker build context path: ${DOCKER_CONTEXT_PATH}"
- shift 2
-fi
-
-if [[ "$1" == "-it" ]]; then
- CI_DOCKER_EXTRA_PARAMS+=('-it')
- shift 1
-fi
-
-if [[ "$1" == "--dockerbinary" ]]; then
- DOCKER_BINARY="$2"
- echo "Using custom Docker Engine: ${DOCKER_BINARY}"
- shift 2
-fi
-
-if [[ ! -f "${DOCKERFILE_PATH}" ]]; then
- echo "Invalid Dockerfile path: \"${DOCKERFILE_PATH}\""
- exit 1
-fi
-
-COMMAND=("$@")
-
-# Validate command line arguments.
-if [ "$#" -lt 1 ] || [ ! -e "${SCRIPT_DIR}/Dockerfile.${CONTAINER_TYPE}" ]; then
- supported_container_types=$( ls -1 ${SCRIPT_DIR}/Dockerfile.* | \
- sed -n 's/.*Dockerfile\.\([^\/]*\)/\1/p' | tr '\n' ' ' )
- echo "Usage: $(basename $0) CONTAINER_TYPE COMMAND"
- echo " CONTAINER_TYPE can be one of [${supported_container_types}]"
- echo " COMMAND is a command (with arguments) to run inside"
- echo " the container."
- exit 1
-fi
-
-# Only set docker binary automatically if it has not been specified
-if [[ -z "${DOCKER_BINARY}" ]]; then
- # Use nvidia-docker if the container is GPU.
- if [[ "${CONTAINER_TYPE}" == *"gpu"* ]]; then
- DOCKER_BINARY="nvidia-docker"
- else
- DOCKER_BINARY="docker"
- fi
- echo "Automatically assuming ${DOCKER_BINARY} as docker binary"
-fi
-
-# Helper function to traverse directories up until given file is found.
-function upsearch () {
- test / == "$PWD" && return || \
- test -e "$1" && echo "$PWD" && return || \
- cd .. && upsearch "$1"
-}
-
-# Set up WORKSPACE. Jenkins will set them for you or we pick
-# reasonable defaults if you run it outside of Jenkins.
-WORKSPACE="${WORKSPACE:-${SCRIPT_DIR}/../../}"
-
-# Set up a default WORKDIR, unless otherwise specified.
-if [[ -z "${WORKDIR}" ]]; then
- WORKDIR="/workspace"
-fi
-
-# Determine the docker image name
-DOCKER_IMG_NAME="mx-ci.${CONTAINER_TYPE}"
-
-# Under Jenkins matrix build, the build tag may contain characters such as
-# commas (,) and equal signs (=), which are not valid inside docker image names.
-DOCKER_IMG_NAME=$(echo "${DOCKER_IMG_NAME}" | sed -e 's/=/_/g' -e 's/,/-/g')
-
-# Convert to all lower-case, as per requirement of Docker image names
-DOCKER_IMG_NAME=$(echo "${DOCKER_IMG_NAME}" | tr '[:upper:]' '[:lower:]')
-
-# Skip with_the_same_user for non-linux
-uname=`uname`
-if [[ "$uname" == "Linux" ]]; then
- # By convention the root of our source dir is always mapped to the /workspace folder
- # inside the docker container. Our working directory when we start the container
- # is variable, so we should ensure we call commands such as with_the_same_user
- # with their absolute paths.
- PRE_COMMAND="/workspace/tests/ci_build/with_the_same_user"
-else
- PRE_COMMAND=""
-fi
-
-# Print arguments.
-echo "WORKSPACE: ${WORKSPACE}"
-echo "WORKDIR: ${WORKDIR}"
-echo "CI_DOCKER_EXTRA_PARAMS: ${CI_DOCKER_EXTRA_PARAMS[@]}"
-echo "COMMAND: ${COMMAND[@]}"
-echo "CONTAINER_TYPE: ${CONTAINER_TYPE}"
-echo "BUILD_TAG: ${BUILD_TAG}"
-echo "NODE_NAME: ${NODE_NAME}"
-echo "DOCKER CONTAINER NAME: ${DOCKER_IMG_NAME}"
-echo "PRE_COMMAND: ${PRE_COMMAND}"
-echo ""
-
-
-# Build the docker container.
-echo "Building container (${DOCKER_IMG_NAME})..."
-docker build -t ${DOCKER_IMG_NAME} \
- -f "${DOCKERFILE_PATH}" "${DOCKER_CONTEXT_PATH}"
-
-# Check docker build status
-if [[ $? != "0" ]]; then
- echo "ERROR: docker build failed."
- exit 1
-fi
-
-
-# Run the command inside the container.
-echo "Running '${COMMAND[@]}' inside ${DOCKER_IMG_NAME}..."
-
-# By default we cleanup - remove the container once it finish running (--rm)
-# and share the PID namespace (--pid=host) so the process inside does not have
-# pid 1 and SIGKILL is propagated to the process inside (jenkins can kill it).
-
-# Turning off MXNET_STORAGE_FALLBACK_LOG_WARNING temporarily per this issue:
-# https://github.com/apache/incubator-mxnet/issues/8980
-
-# By convention always map the root of the MXNet source directory into /workspace.
-# ${WORKSPACE} represents the path to the source folder on the host system.
-# ${WORKDIR} is the working directory we start the container in. By default this
-# is /workspace, but for example when running cmake it is sometimes /workspace/build.
-
-# Currently compiling for sm52 (g3s) and sm61(p3s). sm61 is required to test DP4A support on p3
-# instances. In the long term we should enable sm70 for p3 instances, but this will require
-# CUDA 9(.1) support with our base AMI drivers.
-
-${DOCKER_BINARY} run --rm --pid=host \
- -v ${WORKSPACE}:/workspace \
- -w ${WORKDIR} \
- -e "CI_BUILD_HOME=${WORKSPACE}" \
- -e "CI_BUILD_USER=$(id -u -n)" \
- -e "CI_BUILD_UID=$(id -u)" \
- -e "CI_BUILD_GROUP=$(id -g -n)" \
- -e "CI_BUILD_GID=$(id -g)" \
- -e "CUDA_ARCH=-gencode arch=compute_52,code=[sm_52,compute_52] -gencode arch=compute_61,code=sm_61 --fatbin-options -compress-all" \
- -e "MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0" \
- ${CI_DOCKER_EXTRA_PARAMS[@]} \
- ${DOCKER_IMG_NAME} \
- ${PRE_COMMAND} \
- ${COMMAND[@]}
diff --git a/tests/ci_build/with_the_same_user b/tests/ci_build/with_the_same_user
deleted file mode 100755
index 19f0de1e1d6..00000000000
--- a/tests/ci_build/with_the_same_user
+++ /dev/null
@@ -1,47 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# This script is a wrapper creating the same user inside container as the one
-# running the ci_build.sh outside the container. It also set the home directory
-# for the user inside container to match the same absolute path as the workspace
-# outside of container. Do not run this manually. It does not make sense. It is
-# intended to be called by ci_build.sh only.
-
-set -e
-
-COMMAND=("$@")
-
-if ! touch /this_is_writable_file_system; then
- echo "You can't write to your filesystem!"
- echo "If you are in Docker you should check you do not have too many images" \
- "with too many files in them. Docker has some issue with it."
- exit 1
-else
- rm /this_is_writable_file_system
-fi
-
-getent group "${CI_BUILD_GID}" || addgroup --gid "${CI_BUILD_GID}" "${CI_BUILD_GROUP}"
-getent passwd "${CI_BUILD_UID}" || adduser --gid "${CI_BUILD_GID}" --uid "${CI_BUILD_UID}" \
- --gecos "${CI_BUILD_USER} (generated by with_the_same_user script)" \
- --disabled-password --home "${CI_BUILD_HOME}" --quiet "${CI_BUILD_USER}"
-usermod -a -G sudo "${CI_BUILD_USER}"
-echo "${CI_BUILD_USER} ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/90-nopasswd-sudo
-
-sudo -u "#${CI_BUILD_UID}" --preserve-env "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" \
-"HOME=${CI_BUILD_HOME}" ${COMMAND[@]}
diff --git a/tests/python/cpu/test_mkldnn.py b/tests/python/cpu/test_mkldnn.py
deleted file mode 100644
index 16b30039737..00000000000
--- a/tests/python/cpu/test_mkldnn.py
+++ /dev/null
@@ -1,58 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-"""
-MKL-DNN related test cases
-"""
-
-import logging
-import os
-from sys import platform
-
-def test_mkldnn_install():
- """
- This test will verify that MXNet is built/installed correctly when
- compiled with Intel MKL-DNN library. The method will try to import
- the mxnet module and see if the mkldnn library is mapped to this
- process's address space.
- """
- logging.basicConfig(level=logging.INFO)
-
- if not platform.startswith('linux'):
- logging.info("Bypass mkldnn install test for non-Linux OS")
- return
-
- try:
- #pylint: disable=unused-variable
- import mxnet as mx
- except (ImportError, OSError) as e:
- assert 0, "Import mxnet error: %s. Please double check your build/" \
- "install steps or environment variable settings" % str(e)
-
- pid = os.getpid()
- rc = os.system("cat /proc/" + str(pid) + \
- "/maps | grep libmkldnn > /dev/null")
-
- if rc == 0:
- logging.info("MXNet is built/installed correctly with MKL-DNN")
- else:
- assert 0, "MXNet is built/installed incorrectly with MKL-DNN, please " \
- "double check your build/install steps or environment " \
- "variable settings"
-
-if __name__ == '__main__':
- test_mkldnn_install()
diff --git a/tools/caffe_converter/compare_layers.py b/tools/caffe_converter/compare_layers.py
index 3f6883e1c3c..6cb82e8c2c8 100644
--- a/tools/caffe_converter/compare_layers.py
+++ b/tools/caffe_converter/compare_layers.py
@@ -22,9 +22,9 @@
import os
import warnings
+import numpy as np
import cv2
import mxnet as mx
-import numpy as np
logging.basicConfig(level=logging.INFO)
diff --git a/tools/caffe_converter/convert_mean.py b/tools/caffe_converter/convert_mean.py
index 3b6dc42a7af..1a3df71d3e5 100644
--- a/tools/caffe_converter/convert_mean.py
+++ b/tools/caffe_converter/convert_mean.py
@@ -18,8 +18,8 @@
"""Convert caffe mean
"""
import argparse
-import mxnet as mx
import numpy as np
+import mxnet as mx
import caffe_parser
def convert_mean(binaryproto_fname, output=None):
diff --git a/tools/caffe_converter/convert_model.py b/tools/caffe_converter/convert_model.py
index b2ea57ea33f..d5c069b5706 100644
--- a/tools/caffe_converter/convert_model.py
+++ b/tools/caffe_converter/convert_model.py
@@ -21,9 +21,9 @@
import argparse
import sys
import re
+import numpy as np
import caffe_parser
import mxnet as mx
-import numpy as np
from convert_symbol import convert_symbol
def prob_label(arg_names):
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services