You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by GitBox <gi...@apache.org> on 2018/08/08 12:04:59 UTC
[GitHub] marcoabreu closed pull request #12084: [Do not merge] Testing a
segfault in CI
marcoabreu closed pull request #12084: [Do not merge] Testing a segfault in CI
URL: https://github.com/apache/incubator-mxnet/pull/12084
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/Jenkinsfile b/Jenkinsfile
index 9d7792066e3..c158bb4c682 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -22,14 +22,8 @@
// mxnet libraries
mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a'
-// for scala build, need to pass extra libs when run with dist_kvstore
-mx_dist_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a'
// mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default.
mx_cmake_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so'
-// mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default.
-mx_cmake_lib_debug = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests'
-mx_cmake_mkldnn_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, build/3rdparty/mkldnn/src/libmkldnn.so.0'
-mx_mkldnn_lib = 'lib/libmxnet.so, lib/libmxnet.a, lib/libiomp5.so, lib/libmkldnn.so.0, lib/libmklml_intel.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a'
// timeout in minutes
max_time = 120
// assign any caught errors here
@@ -55,25 +49,6 @@ def init_git() {
}
}
-def init_git_win() {
- deleteDir()
- retry(5) {
- try {
- // Make sure wait long enough for api.github.com request quota. Important: Don't increase the amount of
- // retries as this will increase the amount of requests and worsen the throttling
- timeout(time: 15, unit: 'MINUTES') {
- checkout scm
- bat 'git submodule update --init --recursive'
- bat 'git clean -d -f'
- }
- } catch (exc) {
- deleteDir()
- error "Failed to fetch source codes with ${exc}"
- sleep 2
- }
- }
-}
-
// pack libraries for later use
def pack_lib(name, libs=mx_lib) {
sh """
@@ -115,7 +90,6 @@ def collect_test_results_windows(original_file_name, new_file_name) {
}
}
-
def docker_run(platform, function_name, use_nvidia, shared_mem = '500m') {
def command = "ci/build.py --docker-registry ${env.DOCKER_CACHE_REGISTRY} %USE_NVIDIA% --platform %PLATFORM% --docker-build-retries 3 --shm-size %SHARED_MEM% /work/runtime_functions.sh %FUNCTION_NAME%"
command = command.replaceAll('%USE_NVIDIA%', use_nvidia ? '--nvidiadocker' : '')
@@ -126,49 +100,12 @@ def docker_run(platform, function_name, use_nvidia, shared_mem = '500m') {
sh command
}
-// Python unittest for CPU
-// Python 2
-def python2_ut(docker_container_name) {
- timeout(time: max_time, unit: 'MINUTES') {
- docker_run(docker_container_name, 'unittest_ubuntu_python2_cpu', false)
- }
-}
-
-// Python 3
-def python3_ut(docker_container_name) {
- timeout(time: max_time, unit: 'MINUTES') {
- docker_run(docker_container_name, 'unittest_ubuntu_python3_cpu', false)
- }
-}
-
-def python3_ut_mkldnn(docker_container_name) {
- timeout(time: max_time, unit: 'MINUTES') {
- docker_run(docker_container_name, 'unittest_ubuntu_python3_cpu_mkldnn', false)
- }
-}
-
-// GPU test has two parts. 1) run unittest on GPU, 2) compare the results on
-// both CPU and GPU
-// Python 2
-def python2_gpu_ut(docker_container_name) {
- timeout(time: max_time, unit: 'MINUTES') {
- docker_run(docker_container_name, 'unittest_ubuntu_python2_gpu', true)
- }
-}
-
-// Python 3
def python3_gpu_ut(docker_container_name) {
timeout(time: max_time, unit: 'MINUTES') {
docker_run(docker_container_name, 'unittest_ubuntu_python3_gpu', true)
}
}
-// Python 3 NOCUDNN
-def python3_gpu_ut_nocudnn(docker_container_name) {
- timeout(time: max_time, unit: 'MINUTES') {
- docker_run(docker_container_name, 'unittest_ubuntu_python3_gpu_nocudnn', true)
- }
-}
try {
stage('Sanity Check') {
@@ -191,192 +128,20 @@ try {
}
stage('Build') {
- parallel 'CPU: CentOS 7': {
- node('mxnetlinux-cpu') {
- ws('workspace/build-centos7-cpu') {
- timeout(time: max_time, unit: 'MINUTES') {
- init_git()
- docker_run('centos7_cpu', 'build_centos7_cpu', false)
- pack_lib('centos7_cpu')
- }
- }
- }
- },
- 'CPU: CentOS 7 MKLDNN': {
- node('mxnetlinux-cpu') {
- ws('workspace/build-centos7-mkldnn') {
- timeout(time: max_time, unit: 'MINUTES') {
- init_git()
- docker_run('centos7_cpu', 'build_centos7_mkldnn', false)
- pack_lib('centos7_mkldnn')
- }
- }
- }
- },
- 'GPU: CentOS 7': {
- node('mxnetlinux-cpu') {
- ws('workspace/build-centos7-gpu') {
- timeout(time: max_time, unit: 'MINUTES') {
- init_git()
- docker_run('centos7_gpu', 'build_centos7_gpu', false)
- pack_lib('centos7_gpu')
- }
- }
- }
- },
- 'CPU: Openblas': {
- node('mxnetlinux-cpu') {
- ws('workspace/build-cpu-openblas') {
- timeout(time: max_time, unit: 'MINUTES') {
- init_git()
- docker_run('ubuntu_cpu', 'build_ubuntu_cpu_openblas', false)
- pack_lib('cpu', mx_dist_lib)
- }
- }
- }
- },
- 'CPU: Openblas, debug': {
- node('mxnetlinux-cpu') {
- ws('workspace/build-cpu-openblas') {
- timeout(time: max_time, unit: 'MINUTES') {
- init_git()
- docker_run('ubuntu_cpu', 'build_ubuntu_cpu_cmake_debug', false)
- pack_lib('cpu_debug', mx_cmake_lib_debug)
- }
- }
- }
- },
- 'CPU: Clang 3.9': {
- node('mxnetlinux-cpu') {
- ws('workspace/build-cpu-clang39') {
- timeout(time: max_time, unit: 'MINUTES') {
- init_git()
- docker_run('ubuntu_cpu', 'build_ubuntu_cpu_clang39', false)
- }
- }
- }
- },
- 'CPU: Clang 5': {
- node('mxnetlinux-cpu') {
- ws('workspace/build-cpu-clang50') {
- timeout(time: max_time, unit: 'MINUTES') {
- init_git()
- docker_run('ubuntu_cpu', 'build_ubuntu_cpu_clang50', false)
- }
- }
- }
- },
- 'CPU: Clang 3.9 MKLDNN': {
- node('mxnetlinux-cpu') {
- ws('workspace/build-cpu-mkldnn-clang39') {
- timeout(time: max_time, unit: 'MINUTES') {
- init_git()
- docker_run('ubuntu_cpu', 'build_ubuntu_cpu_clang39_mkldnn', false)
- pack_lib('mkldnn_cpu_clang3', mx_mkldnn_lib)
- }
- }
- }
- },
- 'CPU: Clang 5 MKLDNN': {
+ parallel 'GPU: Make': {
node('mxnetlinux-cpu') {
- ws('workspace/build-cpu-mkldnn-clang50') {
+ ws('workspace/build-gpu-make') {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
- docker_run('ubuntu_cpu', 'build_ubuntu_cpu_clang50_mkldnn', false)
- pack_lib('mkldnn_cpu_clang5', mx_mkldnn_lib)
- }
- }
- }
- },
- 'CPU: MKLDNN': {
- node('mxnetlinux-cpu') {
- ws('workspace/build-mkldnn-cpu') {
- timeout(time: max_time, unit: 'MINUTES') {
- init_git()
- docker_run('ubuntu_cpu', 'build_ubuntu_cpu_mkldnn', false)
- pack_lib('mkldnn_cpu', mx_mkldnn_lib)
- }
- }
- }
- },
- 'GPU: MKLDNN': {
- node('mxnetlinux-cpu') {
- ws('workspace/build-mkldnn-gpu') {
- timeout(time: max_time, unit: 'MINUTES') {
- init_git()
- docker_run('ubuntu_build_cuda', 'build_ubuntu_gpu_mkldnn', false)
- pack_lib('mkldnn_gpu', mx_mkldnn_lib)
- }
- }
- }
- },
- 'GPU: MKLDNN_CUDNNOFF': {
- node('mxnetlinux-cpu') {
- ws('workspace/build-mkldnn-gpu-nocudnn') {
- timeout(time: max_time, unit: 'MINUTES') {
- init_git()
- docker_run('ubuntu_build_cuda', 'build_ubuntu_gpu_mkldnn_nocudnn', false)
- pack_lib('mkldnn_gpu_nocudnn', mx_mkldnn_lib)
- }
- }
- }
- },
- 'GPU: CUDA9.1+cuDNN7': {
- node('mxnetlinux-cpu') {
- ws('workspace/build-gpu') {
- timeout(time: max_time, unit: 'MINUTES') {
- init_git()
- docker_run('ubuntu_build_cuda', 'build_ubuntu_gpu_cuda91_cudnn7', false)
- pack_lib('gpu', mx_dist_lib)
- stash includes: 'build/cpp-package/example/lenet', name: 'cpp_lenet'
- stash includes: 'build/cpp-package/example/alexnet', name: 'cpp_alexnet'
- stash includes: 'build/cpp-package/example/googlenet', name: 'cpp_googlenet'
- stash includes: 'build/cpp-package/example/lenet_with_mxdataiter', name: 'cpp_lenet_with_mxdataiter'
- stash includes: 'build/cpp-package/example/resnet', name: 'cpp_resnet'
- stash includes: 'build/cpp-package/example/mlp', name: 'cpp_mlp'
- stash includes: 'build/cpp-package/example/mlp_cpu', name: 'cpp_mlp_cpu'
- stash includes: 'build/cpp-package/example/mlp_gpu', name: 'cpp_mlp_gpu'
- stash includes: 'build/cpp-package/example/test_score', name: 'cpp_test_score'
- stash includes: 'build/cpp-package/example/test_optimizer', name: 'cpp_test_optimizer'
- }
- }
- }
- },
- 'Amalgamation MIN': {
- node('mxnetlinux-cpu') {
- ws('workspace/amalgamationmin') {
- timeout(time: max_time, unit: 'MINUTES') {
- init_git()
- docker_run('ubuntu_cpu', 'build_ubuntu_amalgamation_min', false)
- }
- }
- }
- },
- 'Amalgamation': {
- node('mxnetlinux-cpu') {
- ws('workspace/amalgamation') {
- timeout(time: max_time, unit: 'MINUTES') {
- init_git()
- docker_run('ubuntu_cpu', 'build_ubuntu_amalgamation', false)
- }
- }
- }
- },
-
- 'GPU: CMake MKLDNN': {
- node('mxnetlinux-cpu') {
- ws('workspace/build-cmake-mkldnn-gpu') {
- timeout(time: max_time, unit: 'MINUTES') {
- init_git()
- docker_run('ubuntu_gpu', 'build_ubuntu_gpu_cmake_mkldnn', false)
- pack_lib('cmake_mkldnn_gpu', mx_cmake_mkldnn_lib)
+ docker_run('ubuntu_gpu', 'build_ubuntu_gpu_make', false)
+ pack_lib('make_gpu', mx_lib)
}
}
}
},
'GPU: CMake': {
node('mxnetlinux-cpu') {
- ws('workspace/build-cmake-gpu') {
+ ws('workspace/build-gpu-cmake') {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
docker_run('ubuntu_gpu', 'build_ubuntu_gpu_cmake', false)
@@ -384,176 +149,16 @@ try {
}
}
}
- },
- 'Build CPU windows':{
- node('mxnetwindows-cpu') {
- timeout(time: max_time, unit: 'MINUTES') {
- ws('workspace/build-cpu') {
- withEnv(['OpenBLAS_HOME=C:\\mxnet\\openblas', 'OpenCV_DIR=C:\\mxnet\\opencv_vc14', 'CUDA_PATH=C:\\CUDA\\v8.0']) {
- init_git_win()
- powershell 'python ci/build_windows.py -f WIN_CPU'
- stash includes: 'windows_package.7z', name: 'windows_package_cpu'
- }
- }
- }
- }
- },
-
- 'Build GPU windows':{
- node('mxnetwindows-cpu') {
- timeout(time: max_time, unit: 'MINUTES') {
- ws('workspace/build-gpu') {
- withEnv(['OpenBLAS_HOME=C:\\mxnet\\openblas', 'OpenCV_DIR=C:\\mxnet\\opencv_vc14', 'CUDA_PATH=C:\\CUDA\\v8.0']) {
- init_git_win()
- powershell 'python ci/build_windows.py -f WIN_GPU'
- stash includes: 'windows_package.7z', name: 'windows_package_gpu'
- }
- }
- }
- }
- },
- 'Build GPU MKLDNN windows':{
- node('mxnetwindows-cpu') {
- timeout(time: max_time, unit: 'MINUTES') {
- ws('workspace/build-gpu') {
- withEnv(['OpenBLAS_HOME=C:\\mxnet\\openblas', 'OpenCV_DIR=C:\\mxnet\\opencv_vc14', 'CUDA_PATH=C:\\CUDA\\v8.0','BUILD_NAME=vc14_gpu_mkldnn']) {
- init_git_win()
- powershell 'python ci/build_windows.py -f WIN_GPU_MKLDNN'
- stash includes: 'windows_package.7z', name: 'windows_package_gpu_mkldnn'
- }
- }
- }
- }
- },
- 'NVidia Jetson / ARMv8':{
- node('mxnetlinux-cpu') {
- ws('workspace/build-jetson-armv8') {
- timeout(time: max_time, unit: 'MINUTES') {
- init_git()
- docker_run('jetson', 'build_jetson', false)
- }
- }
- }
- },
- 'ARMv7':{
- node('mxnetlinux-cpu') {
- ws('workspace/build-ARMv7') {
- timeout(time: max_time, unit: 'MINUTES') {
- init_git()
- docker_run('armv7', 'build_armv7', false)
- }
- }
- }
- },
- 'ARMv6':{
- node('mxnetlinux-cpu') {
- ws('workspace/build-ARMv6') {
- timeout(time: max_time, unit: 'MINUTES') {
- init_git()
- docker_run('armv6', 'build_armv6', false)
- }
- }
- }
- },
- 'ARMv8':{
- node('mxnetlinux-cpu') {
- ws('workspace/build-ARMv8') {
- timeout(time: max_time, unit: 'MINUTES') {
- init_git()
- docker_run('armv8', 'build_armv8', false)
- }
- }
- }
- },
- 'Android / ARMv8':{
- node('mxnetlinux-cpu') {
- ws('workspace/android64') {
- timeout(time: max_time, unit: 'MINUTES') {
- init_git()
- docker_run('android_armv8', 'build_android_armv8', false)
- }
- }
- }
- },
- 'Android / ARMv7':{
- node('mxnetlinux-cpu') {
- ws('workspace/androidv7') {
- timeout(time: max_time, unit: 'MINUTES') {
- init_git()
- docker_run('android_armv7', 'build_android_armv7', false)
- }
- }
- }
}
-
} // End of stage('Build')
stage('Tests') {
- parallel 'Python2: CPU': {
- node('mxnetlinux-cpu') {
- ws('workspace/ut-python2-cpu') {
- try {
- init_git()
- unpack_lib('cpu')
- python2_ut('ubuntu_cpu')
- publish_test_coverage()
- } finally {
- collect_test_results_unix('nosetests_unittest.xml', 'nosetests_python2_cpu_unittest.xml')
- collect_test_results_unix('nosetests_train.xml', 'nosetests_python2_cpu_train.xml')
- collect_test_results_unix('nosetests_quantization.xml', 'nosetests_python2_cpu_quantization.xml')
- }
- }
- }
- },
- 'Python3: CPU': {
- node('mxnetlinux-cpu') {
- ws('workspace/ut-python3-cpu') {
- try {
- init_git()
- unpack_lib('cpu')
- python3_ut('ubuntu_cpu')
- publish_test_coverage()
- } finally {
- collect_test_results_unix('nosetests_unittest.xml', 'nosetests_python3_cpu_unittest.xml')
- collect_test_results_unix('nosetests_quantization.xml', 'nosetests_python3_cpu_quantization.xml')
- }
- }
- }
- },
- 'Python3: CPU debug': {
- node('mxnetlinux-cpu') {
- ws('workspace/ut-python3-cpu-debug') {
- try {
- init_git()
- unpack_lib('cpu_debug', mx_cmake_lib_debug)
- python3_ut('ubuntu_cpu')
- } finally {
- collect_test_results_unix('nosetests_unittest.xml', 'nosetests_python3_cpu_debug_unittest.xml')
- collect_test_results_unix('nosetests_quantization.xml', 'nosetests_python3_cpu_debug_quantization.xml')
- }
- }
- }
- },
- 'Python2: GPU': {
+ parallel 'Python3: GPU, Make': {
node('mxnetlinux-gpu') {
- ws('workspace/ut-python2-gpu') {
+ ws('workspace/ut-python3-gpu-make') {
try {
init_git()
- unpack_lib('gpu', mx_lib)
- python2_gpu_ut('ubuntu_gpu')
- publish_test_coverage()
- } finally {
- collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python2_gpu.xml')
- }
- }
- }
- },
- 'Python3: GPU': {
- node('mxnetlinux-gpu') {
- ws('workspace/ut-python3-gpu') {
- try {
- init_git()
- unpack_lib('gpu', mx_lib)
+ unpack_lib('make_gpu', mx_lib)
python3_gpu_ut('ubuntu_gpu')
publish_test_coverage()
} finally {
@@ -562,414 +167,16 @@ try {
}
}
},
- 'Python2: Quantize GPU': {
- node('mxnetlinux-gpu-p3') {
- ws('workspace/ut-python2-quantize-gpu') {
- timeout(time: max_time, unit: 'MINUTES') {
- try {
- init_git()
- unpack_lib('gpu', mx_lib)
- docker_run('ubuntu_gpu', 'unittest_ubuntu_python2_quantization_gpu', true)
- publish_test_coverage()
- } finally {
- collect_test_results_unix('nosetests_quantization_gpu.xml', 'nosetests_python2_quantize_gpu.xml')
- }
- }
- }
- }
- },
- 'Python3: Quantize GPU': {
- node('mxnetlinux-gpu-p3') {
- ws('workspace/ut-python3-quantize-gpu') {
- timeout(time: max_time, unit: 'MINUTES') {
- try {
- init_git()
- unpack_lib('gpu', mx_lib)
- docker_run('ubuntu_gpu', 'unittest_ubuntu_python3_quantization_gpu', true)
- publish_test_coverage()
- } finally {
- collect_test_results_unix('nosetests_quantization_gpu.xml', 'nosetests_python3_quantize_gpu.xml')
- }
- }
- }
- }
- },
- 'Python2: MKLDNN-CPU': {
- node('mxnetlinux-cpu') {
- ws('workspace/ut-python2-mkldnn-cpu') {
- try {
- init_git()
- unpack_lib('mkldnn_cpu', mx_mkldnn_lib)
- python2_ut('ubuntu_cpu')
- publish_test_coverage()
- } finally {
- collect_test_results_unix('nosetests_unittest.xml', 'nosetests_python2_mkldnn_cpu_unittest.xml')
- collect_test_results_unix('nosetests_train.xml', 'nosetests_python2_mkldnn_cpu_train.xml')
- collect_test_results_unix('nosetests_quantization.xml', 'nosetests_python2_mkldnn_cpu_quantization.xml')
- }
- }
- }
- },
- 'Python2: MKLDNN-GPU': {
+ 'Python3: GPU, CMake': {
node('mxnetlinux-gpu') {
- ws('workspace/ut-python2-mkldnn-gpu') {
+ ws('workspace/ut-python3-gpu-cmake') {
try {
init_git()
- unpack_lib('mkldnn_gpu', mx_mkldnn_lib)
- python2_gpu_ut('ubuntu_gpu')
- publish_test_coverage()
- } finally {
- collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python2_mkldnn_gpu.xml')
- }
- }
- }
- },
- 'Python3: MKLDNN-CPU': {
- node('mxnetlinux-cpu') {
- ws('workspace/ut-python3-mkldnn-cpu') {
- try {
- init_git()
- unpack_lib('mkldnn_cpu', mx_mkldnn_lib)
- python3_ut_mkldnn('ubuntu_cpu')
- publish_test_coverage()
- } finally {
- collect_test_results_unix('nosetests_unittest.xml', 'nosetests_python3_mkldnn_cpu_unittest.xml')
- collect_test_results_unix('nosetests_mkl.xml', 'nosetests_python3_mkldnn_cpu_mkl.xml')
- }
- }
- }
- },
- 'Python3: MKLDNN-GPU': {
- node('mxnetlinux-gpu') {
- ws('workspace/ut-python3-mkldnn-gpu') {
- try {
- init_git()
- unpack_lib('mkldnn_gpu', mx_mkldnn_lib)
+ unpack_lib('cmake_gpu', mx_cmake_lib)
python3_gpu_ut('ubuntu_gpu')
publish_test_coverage()
} finally {
- collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python3_mkldnn_gpu.xml')
- }
- }
- }
- },
- 'Python3: MKLDNN-GPU-NOCUDNN': {
- node('mxnetlinux-gpu') {
- ws('workspace/ut-python3-mkldnn-gpu-nocudnn') {
- try {
- init_git()
- unpack_lib('mkldnn_gpu_nocudnn', mx_mkldnn_lib)
- python3_gpu_ut_nocudnn('ubuntu_gpu')
- publish_test_coverage()
- } finally {
- collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python3_mkldnn_gpu_nocudnn.xml')
- }
- }
- }
- },
- 'Python3: CentOS 7 CPU': {
- node('mxnetlinux-cpu') {
- ws('workspace/build-centos7-cpu') {
- timeout(time: max_time, unit: 'MINUTES') {
- try {
- init_git()
- unpack_lib('centos7_cpu')
- docker_run('centos7_cpu', 'unittest_centos7_cpu', false)
- publish_test_coverage()
- } finally {
- collect_test_results_unix('nosetests_unittest.xml', 'nosetests_python3_centos7_cpu_unittest.xml')
- collect_test_results_unix('nosetests_train.xml', 'nosetests_python3_centos7_cpu_train.xml')
- }
- }
- }
- }
- },
- 'Python3: CentOS 7 GPU': {
- node('mxnetlinux-gpu') {
- ws('workspace/build-centos7-gpu') {
- timeout(time: max_time, unit: 'MINUTES') {
- try {
- init_git()
- unpack_lib('centos7_gpu')
- docker_run('centos7_gpu', 'unittest_centos7_gpu', true)
- publish_test_coverage()
- } finally {
- collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python3_centos7_gpu.xml')
- }
- }
- }
- }
- },
- 'Scala: CPU': {
- node('mxnetlinux-cpu') {
- ws('workspace/ut-scala-cpu') {
- timeout(time: max_time, unit: 'MINUTES') {
- init_git()
- unpack_lib('cpu', mx_dist_lib)
- docker_run('ubuntu_cpu', 'unittest_ubuntu_cpu_scala', false)
- publish_test_coverage()
- }
- }
- }
- },
- 'Clojure: CPU': {
- node('mxnetlinux-cpu') {
- ws('workspace/ut-clojure-cpu') {
- timeout(time: max_time, unit: 'MINUTES') {
- init_git()
- unpack_lib('cpu', mx_dist_lib)
- docker_run('ubuntu_cpu', 'unittest_ubuntu_cpu_clojure', false)
- publish_test_coverage()
- }
- }
- }
- },
- 'Perl: CPU': {
- node('mxnetlinux-cpu') {
- ws('workspace/ut-perl-cpu') {
- timeout(time: max_time, unit: 'MINUTES') {
- init_git()
- unpack_lib('cpu')
- docker_run('ubuntu_cpu', 'unittest_ubuntu_cpugpu_perl', false)
- publish_test_coverage()
- }
- }
- }
- },
- 'Perl: GPU': {
- node('mxnetlinux-gpu') {
- ws('workspace/ut-perl-gpu') {
- timeout(time: max_time, unit: 'MINUTES') {
- init_git()
- unpack_lib('gpu')
- docker_run('ubuntu_gpu', 'unittest_ubuntu_cpugpu_perl', true)
- publish_test_coverage()
- }
- }
- }
- },
- 'Cpp: GPU': {
- node('mxnetlinux-gpu') {
- ws('workspace/ut-cpp-gpu') {
- timeout(time: max_time, unit: 'MINUTES') {
- init_git()
- unpack_lib('cmake_gpu', mx_cmake_lib)
- docker_run('ubuntu_gpu', 'unittest_ubuntu_gpu_cpp', true)
- publish_test_coverage()
- }
- }
- }
- },
- 'Cpp: MKLDNN+GPU': {
- node('mxnetlinux-gpu') {
- ws('workspace/ut-cpp-mkldnn-gpu') {
- timeout(time: max_time, unit: 'MINUTES') {
- init_git()
- unpack_lib('cmake_mkldnn_gpu', mx_cmake_mkldnn_lib)
- docker_run('ubuntu_gpu', 'unittest_ubuntu_gpu_cpp', true)
- publish_test_coverage()
- }
- }
- }
- },
- 'R: CPU': {
- node('mxnetlinux-cpu') {
- ws('workspace/ut-r-cpu') {
- timeout(time: max_time, unit: 'MINUTES') {
- init_git()
- unpack_lib('cpu')
- docker_run('ubuntu_cpu', 'unittest_ubuntu_cpu_R', false)
- publish_test_coverage()
- }
- }
- }
- },
- 'R: GPU': {
- node('mxnetlinux-gpu') {
- ws('workspace/ut-r-gpu') {
- timeout(time: max_time, unit: 'MINUTES') {
- init_git()
- unpack_lib('gpu')
- docker_run('ubuntu_gpu', 'unittest_ubuntu_gpu_R', true)
- publish_test_coverage()
- }
- }
- }
- },
-
- 'Python 2: CPU Win':{
- node('mxnetwindows-cpu') {
- timeout(time: max_time, unit: 'MINUTES') {
- ws('workspace/ut-python-cpu') {
- try {
- init_git_win()
- unstash 'windows_package_cpu'
- powershell 'ci/windows/test_py2_cpu.ps1'
- } finally {
- collect_test_results_windows('nosetests_unittest.xml', 'nosetests_unittest_windows_python2_cpu.xml')
- }
- }
- }
- }
- },
- 'Python 3: CPU Win': {
- node('mxnetwindows-cpu') {
- timeout(time: max_time, unit: 'MINUTES') {
- ws('workspace/ut-python-cpu') {
- try {
- init_git_win()
- unstash 'windows_package_cpu'
- powershell 'ci/windows/test_py3_cpu.ps1'
- } finally {
- collect_test_results_windows('nosetests_unittest.xml', 'nosetests_unittest_windows_python3_cpu.xml')
- }
- }
- }
- }
- },
- 'Python 2: GPU Win':{
- node('mxnetwindows-gpu') {
- timeout(time: max_time, unit: 'MINUTES') {
- ws('workspace/ut-python-gpu') {
- try {
- init_git_win()
- unstash 'windows_package_gpu'
- powershell 'ci/windows/test_py2_gpu.ps1'
- } finally {
- collect_test_results_windows('nosetests_forward.xml', 'nosetests_gpu_forward_windows_python2_gpu.xml')
- collect_test_results_windows('nosetests_operator.xml', 'nosetests_gpu_operator_windows_python2_gpu.xml')
- }
- }
- }
- }
- },
- 'Python 3: GPU Win':{
- node('mxnetwindows-gpu') {
- timeout(time: max_time, unit: 'MINUTES') {
- ws('workspace/ut-python-gpu') {
- try {
- init_git_win()
- unstash 'windows_package_gpu'
- powershell 'ci/windows/test_py3_gpu.ps1'
- } finally {
- collect_test_results_windows('nosetests_forward.xml', 'nosetests_gpu_forward_windows_python3_gpu.xml')
- collect_test_results_windows('nosetests_operator.xml', 'nosetests_gpu_operator_windows_python3_gpu.xml')
- }
- }
- }
- }
- },
- 'Python 3: MKLDNN-GPU Win':{
- node('mxnetwindows-gpu') {
- timeout(time: max_time, unit: 'MINUTES') {
- ws('workspace/ut-python-gpu') {
- try {
- init_git_win()
- unstash 'windows_package_gpu_mkldnn'
- powershell 'ci/windows/test_py3_gpu.ps1'
- } finally {
- collect_test_results_windows('nosetests_forward.xml', 'nosetests_gpu_forward_windows_python3_gpu_mkldnn.xml')
- collect_test_results_windows('nosetests_operator.xml', 'nosetests_gpu_operator_windows_python3_gpu_mkldnn.xml')
- }
- }
- }
- }
- },
- 'Onnx CPU': {
- node('mxnetlinux-cpu') {
- ws('workspace/it-onnx-cpu') {
- timeout(time: max_time, unit: 'MINUTES') {
- init_git()
- unpack_lib('cpu')
- docker_run('ubuntu_cpu', 'integrationtest_ubuntu_cpu_onnx', false)
- publish_test_coverage()
- }
- }
- }
- },
- 'Python GPU': {
- node('mxnetlinux-gpu') {
- ws('workspace/it-python-gpu') {
- timeout(time: max_time, unit: 'MINUTES') {
- init_git()
- unpack_lib('gpu')
- docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_python', true)
- publish_test_coverage()
- }
- }
- }
- },
- // Disabled due to: https://github.com/apache/incubator-mxnet/issues/11407
- // 'Caffe GPU': {
- // node('mxnetlinux-gpu') {
- // ws('workspace/it-caffe') {
- // timeout(time: max_time, unit: 'MINUTES') {
- // init_git()
- // unpack_lib('gpu')
- // docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_caffe', true)
- // publish_test_coverage()
- // }
- // }
- // }
- // },
- 'cpp-package GPU': {
- node('mxnetlinux-gpu') {
- ws('workspace/it-cpp-package') {
- timeout(time: max_time, unit: 'MINUTES') {
- init_git()
- unpack_lib('gpu')
- unstash 'cpp_lenet'
- unstash 'cpp_alexnet'
- unstash 'cpp_googlenet'
- unstash 'cpp_lenet_with_mxdataiter'
- unstash 'cpp_resnet'
- unstash 'cpp_mlp'
- unstash 'cpp_mlp_cpu'
- unstash 'cpp_mlp_gpu'
- unstash 'cpp_test_score'
- unstash 'cpp_test_optimizer'
- docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_cpp_package', true)
- publish_test_coverage()
- }
- }
- }
- },
- 'dist-kvstore tests GPU': {
- node('mxnetlinux-gpu') {
- ws('workspace/it-dist-kvstore') {
- timeout(time: max_time, unit: 'MINUTES') {
- init_git()
- unpack_lib('gpu')
- docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_dist_kvstore', true)
- publish_test_coverage()
- }
- }
- }
- },
- /* Disabled due to master build failure:
- * http://jenkins.mxnet-ci.amazon-ml.com/blue/organizations/jenkins/incubator-mxnet/detail/master/1221/pipeline/
- * https://github.com/apache/incubator-mxnet/issues/11801
-
- 'dist-kvstore tests CPU': {
- node('mxnetlinux-cpu') {
- ws('workspace/it-dist-kvstore') {
- timeout(time: max_time, unit: 'MINUTES') {
- init_git()
- unpack_lib('cpu')
- docker_run('ubuntu_cpu', 'integrationtest_ubuntu_cpu_dist_kvstore', false)
- publish_test_coverage()
- }
- }
- }
- }, */
- 'Scala: GPU': {
- node('mxnetlinux-gpu') {
- ws('workspace/ut-scala-gpu') {
- timeout(time: max_time, unit: 'MINUTES') {
- init_git()
- unpack_lib('gpu', mx_dist_lib)
- docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_scala', true)
- publish_test_coverage()
+ collect_test_results_unix('nosetests_gpu.xml', 'nosetests_python3_gpu.xml')
}
}
}
diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index 1c861beb916..05acc4c0ea9 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -96,410 +96,22 @@ build_wheel() {
popd
}
-# Build commands: Every platform in docker/Dockerfile.build.<platform> should have a corresponding
-# function here with the same suffix:
-
-build_jetson() {
- set -ex
- pushd .
-
- cp make/crosscompile.jetson.mk ./config.mk
- make -j$(nproc)
-
- build_wheel /work/mxnet/python /work/mxnet/lib
- popd
-}
-
-#
-# ARM builds
-#
-
-build_armv6() {
- set -ex
- pushd .
- cd /work/build
-
- # Lapack functionality will be included and statically linked to openblas.
- # But USE_LAPACK needs to be set to OFF, otherwise the main CMakeLists.txt
- # file tries to add -llapack. Lapack functionality though, requires -lgfortran
- # to be linked additionally.
-
- # We do not need OpenMP, since most armv6 systems have only 1 core
-
- cmake \
- -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} \
- -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
- -DCMAKE_C_COMPILER_LAUNCHER=ccache \
- -DUSE_CUDA=OFF \
- -DUSE_OPENCV=OFF \
- -DUSE_OPENMP=OFF \
- -DUSE_SIGNAL_HANDLER=ON \
- -DCMAKE_BUILD_TYPE=Release \
- -DUSE_MKL_IF_AVAILABLE=OFF \
- -DUSE_LAPACK=OFF \
- -DBUILD_CPP_EXAMPLES=OFF \
- -Dmxnet_LINKER_LIBS=-lgfortran \
- -G Ninja /work/mxnet
-
- ninja -v
- build_wheel
- popd
-}
-
-build_armv7() {
- set -ex
- pushd .
- cd /work/build
-
- # Lapack functionality will be included and statically linked to openblas.
- # But USE_LAPACK needs to be set to OFF, otherwise the main CMakeLists.txt
- # file tries to add -llapack. Lapack functionality though, requires -lgfortran
- # to be linked additionally.
-
- cmake \
- -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} \
- -DCMAKE_CROSSCOMPILING=ON \
- -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
- -DCMAKE_C_COMPILER_LAUNCHER=ccache \
- -DUSE_CUDA=OFF \
- -DUSE_OPENCV=OFF \
- -DUSE_OPENMP=ON \
- -DUSE_SIGNAL_HANDLER=ON \
- -DCMAKE_BUILD_TYPE=Release \
- -DUSE_MKL_IF_AVAILABLE=OFF \
- -DUSE_LAPACK=OFF \
- -DBUILD_CPP_EXAMPLES=OFF \
- -Dmxnet_LINKER_LIBS=-lgfortran \
- -G Ninja /work/mxnet
-
- ninja -v
- build_wheel
- popd
-}
-
-build_armv8() {
- cmake \
- -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
- -DCMAKE_C_COMPILER_LAUNCHER=ccache \
- -DUSE_CUDA=OFF\
- -DSUPPORT_F16C=OFF\
- -DUSE_OPENCV=OFF\
- -DUSE_OPENMP=OFF\
- -DUSE_LAPACK=OFF\
- -DUSE_SIGNAL_HANDLER=ON\
- -DCMAKE_BUILD_TYPE=Release\
- -DUSE_MKL_IF_AVAILABLE=OFF\
- -G Ninja /work/mxnet
- ninja -v
- build_wheel
-}
-
-
-#
-# ANDROID builds
-#
-
-build_android_armv7() {
- set -ex
- cd /work/build
- cmake \
- -DANDROID=ON\
- -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
- -DCMAKE_C_COMPILER_LAUNCHER=ccache \
- -DUSE_CUDA=OFF\
- -DUSE_SSE=OFF\
- -DSUPPORT_F16C=OFF\
- -DUSE_LAPACK=OFF\
- -DUSE_OPENCV=OFF\
- -DUSE_OPENMP=OFF\
- -DUSE_SIGNAL_HANDLER=ON\
- -DCMAKE_BUILD_TYPE=RelWithDebInfo\
- -DUSE_MKL_IF_AVAILABLE=OFF\
- -G Ninja /work/mxnet
- ninja -v
-}
-
-build_android_armv8() {
- set -ex
- cd /work/build
- cmake\
- -DANDROID=ON \
- -DUSE_CUDA=OFF\
- -DUSE_SSE=OFF\
- -DUSE_LAPACK=OFF\
- -DUSE_OPENCV=OFF\
- -DUSE_OPENMP=OFF\
- -DUSE_SIGNAL_HANDLER=ON\
- -DCMAKE_BUILD_TYPE=RelWithDebInfo\
- -DUSE_MKL_IF_AVAILABLE=OFF\
- -G Ninja /work/mxnet
- ninja -v
-}
-
-build_centos7_cpu() {
- set -ex
- cd /work/mxnet
- export CC="ccache gcc"
- export CXX="ccache g++"
-
- make \
- DEV=1 \
- USE_LAPACK=1 \
- USE_LAPACK_PATH=/usr/lib64/liblapack.so \
- USE_BLAS=openblas \
- USE_DIST_KVSTORE=1 \
- -j$(nproc)
-}
-
-build_amzn_linux_cpu() {
- cd /work/build
- cmake \
- -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
- -DCMAKE_C_COMPILER_LAUNCHER=ccache \
- -DUSE_CUDA=OFF\
- -DUSE_OPENCV=ON\
- -DUSE_OPENMP=ON\
- -DUSE_SIGNAL_HANDLER=ON\
- -DCMAKE_BUILD_TYPE=RelWithDebInfo\
- -DUSE_MKL_IF_AVAILABLE=OFF\
- -DUSE_LAPACK=OFF\
- -DUSE_DIST_KVSTORE=ON\
- -G Ninja /work/mxnet
- ninja -v
-}
-
-
-build_centos7_mkldnn() {
- set -ex
- cd /work/mxnet
- export CC="ccache gcc"
- export CXX="ccache g++"
-
- make \
- DEV=1 \
- USE_LAPACK=1 \
- USE_LAPACK_PATH=/usr/lib64/liblapack.so \
- USE_MKLDNN=1 \
- USE_BLAS=openblas \
- -j$(nproc)
-}
-
-build_centos7_gpu() {
- set -ex
- cd /work/mxnet
- # unfortunately this build has problems in 3rdparty dependencies with ccache and make
- # build_ccache_wrappers
- make \
- DEV=1 \
- USE_LAPACK=1 \
- USE_LAPACK_PATH=/usr/lib64/liblapack.so \
- USE_BLAS=openblas \
- USE_CUDA=1 \
- USE_CUDA_PATH=/usr/local/cuda \
- USE_CUDNN=1 \
- USE_DIST_KVSTORE=1 \
- -j$(nproc)
-}
-
-build_ubuntu_cpu() {
- build_ubuntu_cpu_openblas
-}
-
-build_ubuntu_cpu_openblas() {
- set -ex
- export CC="ccache gcc"
- export CXX="ccache g++"
- make \
- DEV=1 \
- USE_CPP_PACKAGE=1 \
- USE_BLAS=openblas \
- USE_DIST_KVSTORE=1 \
- -j$(nproc)
-}
-
-build_ubuntu_cpu_cmake_debug() {
- set -ex
- pushd .
- cd /work/build
- cmake \
- -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
- -DCMAKE_C_COMPILER_LAUNCHER=ccache \
- -DUSE_CUDA=OFF \
- -DUSE_MKL_IF_AVAILABLE=OFF \
- -DUSE_OPENMP=OFF \
- -DUSE_OPENCV=ON \
- -DCMAKE_BUILD_TYPE=Debug \
- -G Ninja \
- /work/mxnet
-
- ninja -v
- popd
-}
-
-build_ubuntu_cpu_clang39() {
- set -ex
- export CXX=clang++-3.9
- export CC=clang-3.9
- build_ccache_wrappers
- make \
- USE_CPP_PACKAGE=1 \
- USE_BLAS=openblas \
- USE_OPENMP=0 \
- USE_DIST_KVSTORE=1 \
- -j$(nproc)
-}
-
-build_ubuntu_cpu_clang50() {
- set -ex
-
- export CXX=clang++-5.0
- export CC=clang-5.0
-
- build_ccache_wrappers
-
- make \
- USE_CPP_PACKAGE=1 \
- USE_BLAS=openblas \
- USE_OPENMP=1 \
- USE_DIST_KVSTORE=1 \
- -j$(nproc)
-}
-
-build_ubuntu_cpu_clang39_mkldnn() {
+build_ubuntu_gpu_make() {
set -ex
-
- export CXX=clang++-3.9
- export CC=clang-3.9
-
- build_ccache_wrappers
-
make \
- USE_CPP_PACKAGE=1 \
- USE_BLAS=openblas \
- USE_MKLDNN=1 \
- USE_OPENMP=0 \
- -j$(nproc)
-}
-
-build_ubuntu_cpu_clang50_mkldnn() {
- set -ex
-
- export CXX=clang++-5.0
- export CC=clang-5.0
-
- build_ccache_wrappers
-
- make \
- USE_CPP_PACKAGE=1 \
- USE_BLAS=openblas \
- USE_MKLDNN=1 \
- USE_OPENMP=1 \
- -j$(nproc)
-}
-
-build_ubuntu_cpu_mkldnn() {
- set -ex
-
- build_ccache_wrappers
-
- make \
- DEV=1 \
- USE_CPP_PACKAGE=1 \
- USE_BLAS=openblas \
- USE_MKLDNN=1 \
- -j$(nproc)
-}
-
-build_ubuntu_gpu() {
- build_ubuntu_gpu_cuda91_cudnn7
-}
-
-build_ubuntu_gpu_mkldnn() {
- set -ex
-
- build_ccache_wrappers
-
- make \
- DEV=1 \
- USE_CPP_PACKAGE=1 \
- USE_BLAS=openblas \
- USE_MKLDNN=1 \
USE_CUDA=1 \
- USE_CUDA_PATH=/usr/local/cuda \
USE_CUDNN=1 \
- -j$(nproc)
-}
-
-build_ubuntu_gpu_mkldnn_nocudnn() {
- set -ex
-
- build_ccache_wrappers
-
- make \
- DEV=1 \
- USE_BLAS=openblas \
- USE_MKLDNN=1 \
- USE_CUDA=1 \
USE_CUDA_PATH=/usr/local/cuda \
- USE_CUDNN=0 \
- -j$(nproc)
-}
-
-build_ubuntu_gpu_cuda91_cudnn7() {
- set -ex
- # unfortunately this build has problems in 3rdparty dependencies with ccache and make
- # build_ccache_wrappers
- make \
- DEV=1 \
- USE_BLAS=openblas \
- USE_CUDA=1 \
- USE_CUDA_PATH=/usr/local/cuda \
- USE_CUDNN=1 \
- USE_CPP_PACKAGE=1 \
+ USE_MKLML_MKL=0 \
+ USE_MKLDNN=0 \
USE_DIST_KVSTORE=1 \
-j$(nproc)
}
-build_ubuntu_amalgamation() {
- set -ex
- # Amalgamation can not be run with -j nproc
- make -C amalgamation/ clean
- make -C amalgamation/ USE_BLAS=openblas
-}
-
-build_ubuntu_amalgamation_min() {
- set -ex
- # Amalgamation can not be run with -j nproc
- make -C amalgamation/ clean
- make -C amalgamation/ USE_BLAS=openblas MIN=1
-}
-
-build_ubuntu_gpu_cmake_mkldnn() {
- set -ex
- cd /work/build
- cmake \
- -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
- -DCMAKE_C_COMPILER_LAUNCHER=ccache \
- -DUSE_CUDA=1 \
- -DUSE_CUDNN=1 \
- -DUSE_MKLML_MKL=1 \
- -DUSE_MKLDNN=1 \
- -DCMAKE_BUILD_TYPE=Release \
- -G Ninja \
- /work/mxnet
-
- ninja -v
- # libmkldnn.so.0 is a link file. We need an actual binary file named libmkldnn.so.0.
- cp 3rdparty/mkldnn/src/libmkldnn.so.0 3rdparty/mkldnn/src/libmkldnn.so.0.tmp
- mv 3rdparty/mkldnn/src/libmkldnn.so.0.tmp 3rdparty/mkldnn/src/libmkldnn.so.0
-}
-
build_ubuntu_gpu_cmake() {
set -ex
cd /work/build
cmake \
- -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
- -DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DUSE_CUDA=1 \
-DUSE_CUDNN=1 \
-DUSE_MKLML_MKL=0 \
@@ -523,75 +135,6 @@ sanity_check() {
nosetests-3.4 tests/tutorials/test_sanity_tutorials.py
}
-
-unittest_ubuntu_python2_cpu() {
- set -ex
- export PYTHONPATH=./python/
- # MXNET_MKLDNN_DEBUG is buggy and produces false positives
- # https://github.com/apache/incubator-mxnet/issues/10026
- #export MXNET_MKLDNN_DEBUG=1 # Ignored if not present
- export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
- nosetests-2.7 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_unittest.xml --verbose tests/python/unittest
- nosetests-2.7 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_train.xml --verbose tests/python/train
- nosetests-2.7 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_quantization.xml --verbose tests/python/quantization
-}
-
-unittest_ubuntu_python3_cpu() {
- set -ex
- export PYTHONPATH=./python/
- # MXNET_MKLDNN_DEBUG is buggy and produces false positives
- # https://github.com/apache/incubator-mxnet/issues/10026
- #export MXNET_MKLDNN_DEBUG=1 # Ignored if not present
- export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
- nosetests-3.4 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_unittest.xml --verbose tests/python/unittest
- nosetests-3.4 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_quantization.xml --verbose tests/python/quantization
-}
-
-unittest_ubuntu_python3_cpu_mkldnn() {
- set -ex
- export PYTHONPATH=./python/
- # MXNET_MKLDNN_DEBUG is buggy and produces false positives
- # https://github.com/apache/incubator-mxnet/issues/10026
- #export MXNET_MKLDNN_DEBUG=1 # Ignored if not present
- export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
- nosetests-3.4 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_unittest.xml --verbose tests/python/unittest
- nosetests-3.4 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_mkl.xml --verbose tests/python/mkl
-}
-
-unittest_ubuntu_python2_gpu() {
- set -ex
- export PYTHONPATH=./python/
- # MXNET_MKLDNN_DEBUG is buggy and produces false positives
- # https://github.com/apache/incubator-mxnet/issues/10026
- #export MXNET_MKLDNN_DEBUG=1 # Ignored if not present
- export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
- nosetests-2.7 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_gpu.xml --verbose tests/python/gpu
-}
-
-tutorialtest_ubuntu_python3_gpu() {
- set -ex
- cd /work/mxnet/docs
- export MXNET_DOCS_BUILD_MXNET=0
- make html
- export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
- export PYTHONPATH=/work/mxnet/python/
- export MXNET_TUTORIAL_TEST_KERNEL=python3
- cd /work/mxnet/tests/tutorials
- nosetests-3.4 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_tutorials.xml test_tutorials.py --nologcapture
-}
-
-tutorialtest_ubuntu_python2_gpu() {
- set -ex
- cd /work/mxnet/docs
- export MXNET_DOCS_BUILD_MXNET=0
- make html
- export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
- export PYTHONPATH=/work/mxnet/python/
- export MXNET_TUTORIAL_TEST_KERNEL=python2
- cd /work/mxnet/tests/tutorials
- nosetests-3.4 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_tutorials.xml test_tutorials.py --nologcapture
-}
-
unittest_ubuntu_python3_gpu() {
set -ex
export PYTHONPATH=./python/
@@ -602,187 +145,6 @@ unittest_ubuntu_python3_gpu() {
nosetests-3.4 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_gpu.xml --verbose tests/python/gpu
}
-unittest_ubuntu_python3_gpu_nocudnn() {
- set -ex
- export PYTHONPATH=./python/
- export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
- export CUDNN_OFF_TEST_ONLY=true
- nosetests-3.4 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_gpu.xml --verbose tests/python/gpu
-}
-
-# quantization gpu currently only runs on P3 instances
-# need to separte it from unittest_ubuntu_python2_gpu()
-unittest_ubuntu_python2_quantization_gpu() {
- set -ex
- export PYTHONPATH=./python/
- # MXNET_MKLDNN_DEBUG is buggy and produces false positives
- # https://github.com/apache/incubator-mxnet/issues/10026
- #export MXNET_MKLDNN_DEBUG=1 # Ignored if not present
- export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
- nosetests-2.7 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_quantization_gpu.xml --verbose tests/python/quantization_gpu
-}
-
-# quantization gpu currently only runs on P3 instances
-# need to separte it from unittest_ubuntu_python3_gpu()
-unittest_ubuntu_python3_quantization_gpu() {
- set -ex
- export PYTHONPATH=./python/
- # MXNET_MKLDNN_DEBUG is buggy and produces false positives
- # https://github.com/apache/incubator-mxnet/issues/10026
- #export MXNET_MKLDNN_DEBUG=1 # Ignored if not present
- export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
- nosetests-3.4 $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_quantization_gpu.xml --verbose tests/python/quantization_gpu
-}
-
-unittest_ubuntu_cpu_scala() {
- set -ex
- make scalapkg USE_BLAS=openblas USE_DIST_KVSTORE=1
- make scalaunittest USE_BLAS=openblas USE_DIST_KVSTORE=1
-}
-
-unittest_ubuntu_cpu_clojure() {
- set -ex
- make scalapkg USE_OPENCV=1 USE_BLAS=openblas USE_DIST_KVSTORE=1
- make scalainstall USE_OPENCV=1 USE_BLAS=openblas USE_DIST_KVSTORE=1
- ./contrib/clojure-package/ci-test.sh
-}
-
-unittest_ubuntu_cpugpu_perl() {
- set -ex
- ./perl-package/test.sh
-}
-
-unittest_ubuntu_gpu_cpp() {
- set -ex
- build/tests/mxnet_unit_tests
-}
-
-unittest_ubuntu_cpu_R() {
- set -ex
- mkdir -p /tmp/r-site-library
- # make -j not supported
- make rpkg USE_BLAS=openblas R_LIBS=/tmp/r-site-library
- R CMD INSTALL --library=/tmp/r-site-library R-package
- make rpkgtest R_LIBS=/tmp/r-site-library
-}
-
-unittest_ubuntu_gpu_R() {
- set -ex
- mkdir -p /tmp/r-site-library
- # make -j not supported
- make rpkg USE_BLAS=openblas R_LIBS=/tmp/r-site-library
- R CMD INSTALL --library=/tmp/r-site-library R-package
- make rpkgtest R_LIBS=/tmp/r-site-library R_GPU_ENABLE=1
-}
-
-unittest_centos7_cpu() {
- set -ex
- cd /work/mxnet
- python3.6 -m "nose" $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_unittest.xml --verbose tests/python/unittest
- python3.6 -m "nose" $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_train.xml --verbose tests/python/train
-}
-
-unittest_centos7_gpu() {
- set -ex
- cd /work/mxnet
- python3.6 -m "nose" $NOSE_COVERAGE_ARGUMENTS --with-xunit --xunit-file nosetests_gpu.xml --verbose tests/python/gpu
-}
-
-integrationtest_ubuntu_cpu_onnx() {
- set -ex
- export PYTHONPATH=./python/
- python example/onnx/super_resolution.py
- pytest tests/python-pytest/onnx/import/mxnet_backend_test.py
- pytest tests/python-pytest/onnx/import/onnx_import_test.py
- pytest tests/python-pytest/onnx/import/gluon_backend_test.py
- pytest tests/python-pytest/onnx/export/onnx_backend_test.py
- python tests/python-pytest/onnx/export/mxnet_export_test.py
-}
-
-integrationtest_ubuntu_gpu_python() {
- set -ex
- export PYTHONPATH=./python/
- export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
- python example/image-classification/test_score.py
-}
-
-integrationtest_ubuntu_gpu_caffe() {
- set -ex
- export PYTHONPATH=/work/deps/caffe/python:./python
- python tools/caffe_converter/test_converter.py
-}
-
-integrationtest_ubuntu_gpu_cpp_package() {
- set -ex
- cpp-package/tests/ci_test.sh
-}
-
-integrationtest_ubuntu_cpu_dist_kvstore() {
- set -ex
- export PYTHONPATH=./python/
- export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
- export MXNET_USE_OPERATOR_TUNING=0
- cd tests/nightly/
- ../../tools/launch.py -n 7 --launcher local python dist_sync_kvstore.py --type=gluon_step_cpu
- ../../tools/launch.py -n 7 --launcher local python dist_sync_kvstore.py --type=gluon_sparse_step_cpu
- ../../tools/launch.py -n 7 --launcher local python dist_sync_kvstore.py --type=invalid_cpu
- ../../tools/launch.py -n 7 --launcher local python dist_sync_kvstore.py --type=gluon_type_cpu
- ../../tools/launch.py -n 7 --launcher local python dist_sync_kvstore.py
- ../../tools/launch.py -n 7 --launcher local python dist_sync_kvstore.py --no-multiprecision
- ../../tools/launch.py -n 7 --launcher local python dist_sync_kvstore.py --type=compressed_cpu
- ../../tools/launch.py -n 7 --launcher local python dist_sync_kvstore.py --type=compressed_cpu --no-multiprecision
- ../../tools/launch.py -n 3 --launcher local python test_server_profiling.py
-}
-
-integrationtest_ubuntu_gpu_scala() {
- set -ex
- make scalapkg USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 USE_DIST_KVSTORE=1 SCALA_ON_GPU=1
- make scalaintegrationtest USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 SCALA_TEST_ON_GPU=1 USE_DIST_KVSTORE=1
-}
-
-integrationtest_ubuntu_gpu_dist_kvstore() {
- set -ex
- export PYTHONPATH=./python/
- export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
- cd tests/nightly/
- ../../tools/launch.py -n 7 --launcher local python dist_device_sync_kvstore.py
- ../../tools/launch.py -n 7 --launcher local python dist_sync_kvstore.py --type=init_gpu
-}
-
-test_ubuntu_cpu_python2() {
- set -ex
- pushd .
- export MXNET_LIBRARY_PATH=/work/build/libmxnet.so
-
- VENV=mxnet_py2_venv
- virtualenv -p `which python2` $VENV
- source $VENV/bin/activate
- pip install nose nose-timer
-
- cd /work/mxnet/python
- pip install -e .
- cd /work/mxnet
- python -m "nose" $NOSE_COVERAGE_ARGUMENTS --with-timer --verbose tests/python/unittest
- popd
-}
-
-test_ubuntu_cpu_python3() {
- set -ex
- pushd .
- export MXNET_LIBRARY_PATH=/work/build/libmxnet.so
- VENV=mxnet_py3_venv
- virtualenv -p `which python3` $VENV
- source $VENV/bin/activate
-
- cd /work/mxnet/python
- pip3 install nose nose-timer
- pip3 install -e .
- cd /work/mxnet
- python3 -m "nose" $NOSE_COVERAGE_ARGUMENTS --with-timer --verbose tests/python/unittest
-
- popd
-}
-
build_docs() {
set -ex
pushd .
@@ -799,8 +161,6 @@ build_docs() {
}
-# Functions that run the nightly Tests:
-
#Runs Apache RAT Check on MXNet Source for License Headers
nightly_test_rat_check() {
set -e
@@ -825,105 +185,6 @@ nightly_test_rat_check() {
popd
}
-#Checks MXNet for Compilation Warnings
-nightly_test_compilation_warning() {
- set -ex
- export PYTHONPATH=./python/
- ./tests/nightly/compilation_warnings/compilation_warnings.sh
-}
-
-#Checks the MXNet Installation Guide - currently checks pip, build from source and virtual env on cpu and gpu
-nightly_test_installation() {
- set -ex
- # The run_test_installation_docs.sh expects the path to index.md and the first and last line numbers of the index.md file
- # First execute the test script and then call the method specified by the Jenkinsfile - ${1}
- source ./tests/jenkins/run_test_installation_docs.sh docs/install/index.md 1 1686; ${1}
-}
-
-#Runs a simple MNIST training example
-nightly_test_image_classification() {
- set -ex
- ./tests/nightly/test_image_classification.sh
-}
-
-#Single Node KVStore Test
-nightly_test_KVStore_singleNode() {
- set -ex
- export PYTHONPATH=./python/
- python tests/nightly/test_kvstore.py
-}
-
-#Tests Amalgamation Build with 5 different sets of flags
-nightly_test_amalgamation() {
- set -ex
- # Amalgamation can not be run with -j nproc
- make -C amalgamation/ clean
- make -C amalgamation/ ${1} ${2}
-}
-
-#Tests Amalgamation Build for Javascript
-nightly_test_javascript() {
- set -ex
- export LLVM=/work/deps/emscripten-fastcomp/build/bin
- # This part is needed to run emcc correctly
- cd /work/deps/emscripten
- ./emcc
- touch ~/.emscripten
- make -C /work/mxnet/amalgamation libmxnet_predict.js MIN=1 EMCC=/work/deps/emscripten/emcc
-}
-
-#Tests Model backwards compatibility on MXNet
-nightly_model_backwards_compat_test() {
- set -ex
- export PYTHONPATH=/work/mxnet/python/
- ./tests/nightly/model_backwards_compatibility_check/model_backward_compat_checker.sh
-}
-
-#Backfills S3 bucket with models trained on earlier versions of mxnet
-nightly_model_backwards_compat_train() {
- set -ex
- export PYTHONPATH=./python/
- ./tests/nightly/model_backwards_compatibility_check/train_mxnet_legacy_models.sh
-}
-
-# Nightly 'MXNet: The Straight Dope' Single-GPU Tests
-nightly_straight_dope_python2_single_gpu_tests() {
- set -ex
- cd /work/mxnet/tests/nightly/straight_dope
- export PYTHONPATH=/work/mxnet/python/
- export MXNET_TEST_KERNEL=python2
- nosetests-2.7 --with-xunit --xunit-file nosetests_straight_dope_python2_single_gpu.xml \
- test_notebooks_single_gpu.py --nologcapture
-}
-
-nightly_straight_dope_python3_single_gpu_tests() {
- set -ex
- cd /work/mxnet/tests/nightly/straight_dope
- export PYTHONPATH=/work/mxnet/python/
- export MXNET_TEST_KERNEL=python3
- nosetests-3.4 --with-xunit --xunit-file nosetests_straight_dope_python3_single_gpu.xml \
- test_notebooks_single_gpu.py --nologcapture
-}
-
-# Nightly 'MXNet: The Straight Dope' Multi-GPU Tests
-nightly_straight_dope_python2_multi_gpu_tests() {
- set -ex
- cd /work/mxnet/tests/nightly/straight_dope
- export PYTHONPATH=/work/mxnet/python/
- export MXNET_TEST_KERNEL=python2
- nosetests-2.7 --with-xunit --xunit-file nosetests_straight_dope_python2_multi_gpu.xml \
- test_notebooks_multi_gpu.py --nologcapture
-}
-
-nightly_straight_dope_python3_multi_gpu_tests() {
- set -ex
- cd /work/mxnet/tests/nightly/straight_dope
- export PYTHONPATH=/work/mxnet/python/
- export MXNET_TEST_KERNEL=python3
- nosetests-3.4 --with-xunit --xunit-file nosetests_straight_dope_python3_multi_gpu.xml \
- test_notebooks_multi_gpu.py --nologcapture
-}
-
# Deploy
deploy_docs() {
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services