You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by GitBox <gi...@apache.org> on 2018/06/28 09:17:59 UTC
[GitHub] marcoabreu closed pull request #11457: Move Windows GPU tests to
nightly and move MKLDNN from GPU to CPU
marcoabreu closed pull request #11457: Move Windows GPU tests to nightly and move MKLDNN from GPU to CPU
URL: https://github.com/apache/incubator-mxnet/pull/11457
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/Jenkinsfile b/Jenkinsfile
index 20ee040217f..cf7bc65dbd9 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -378,50 +378,17 @@ try {
}
}
},
- //Todo: Set specific CUDA_ARCh for windows builds in cmake
- 'Build GPU windows':{
+ 'Build CPU MKLDNN windows':{
node('mxnetwindows-cpu') {
timeout(time: max_time, unit: 'MINUTES') {
- ws('workspace/build-gpu') {
- withEnv(['OpenBLAS_HOME=C:\\mxnet\\openblas', 'OpenCV_DIR=C:\\mxnet\\opencv_vc14', 'CUDA_PATH=C:\\CUDA\\v8.0']) {
- init_git_win()
- bat """mkdir build_vc14_gpu
- call "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\bin\\x86_amd64\\vcvarsx86_amd64.bat"
- cd build_vc14_gpu
- cmake -G \"NMake Makefiles JOM\" -DUSE_CUDA=1 -DUSE_CUDNN=1 -DUSE_NVRTC=1 -DUSE_OPENCV=1 -DUSE_OPENMP=1 -DUSE_PROFILER=1 -DUSE_BLAS=open -DUSE_LAPACK=1 -DUSE_DIST_KVSTORE=0 -DCUDA_ARCH_NAME=All -DCMAKE_CXX_FLAGS_RELEASE="/FS /MD /O2 /Ob2 /DNDEBUG" -DCMAKE_BUILD_TYPE=Release -DUSE_MKL_IF_AVAILABLE=0 ${env.WORKSPACE}"""
- bat 'C:\\mxnet\\build_vc14_gpu.bat'
- bat '''rmdir /s/q pkg_vc14_gpu
- mkdir pkg_vc14_gpu\\lib
- mkdir pkg_vc14_gpu\\python
- mkdir pkg_vc14_gpu\\include
- mkdir pkg_vc14_gpu\\build
- copy build_vc14_gpu\\libmxnet.lib pkg_vc14_gpu\\lib
- copy build_vc14_gpu\\libmxnet.dll pkg_vc14_gpu\\build
- xcopy python pkg_vc14_gpu\\python /E /I /Y
- xcopy include pkg_vc14_gpu\\include /E /I /Y
- xcopy 3rdparty\\dmlc-core\\include pkg_vc14_gpu\\include /E /I /Y
- xcopy 3rdparty\\mshadow\\mshadow pkg_vc14_gpu\\include\\mshadow /E /I /Y
- xcopy 3rdparty\\nnvm\\include pkg_vc14_gpu\\nnvm\\include /E /I /Y
- del /Q *.7z
- 7z.exe a vc14_gpu.7z pkg_vc14_gpu\\
- '''
- stash includes: 'vc14_gpu.7z', name: 'vc14_gpu'
- }
- }
- }
- }
- },
- 'Build GPU MKLDNN windows':{
- node('mxnetwindows-cpu') {
- timeout(time: max_time, unit: 'MINUTES') {
- ws('workspace/build-gpu') {
- withEnv(['OpenBLAS_HOME=C:\\mxnet\\openblas', 'OpenCV_DIR=C:\\mxnet\\opencv_vc14', 'CUDA_PATH=C:\\CUDA\\v8.0','BUILD_NAME=vc14_gpu_mkldnn']) {
+ ws('workspace/build-cpu-mkldnn') {
+ withEnv(['OpenBLAS_HOME=C:\\mxnet\\openblas', 'OpenCV_DIR=C:\\mxnet\\opencv_vc14', 'CUDA_PATH=C:\\CUDA\\v8.0','BUILD_NAME=vc14_cpu_mkldnn']) {
init_git_win()
bat """mkdir build_%BUILD_NAME%
call "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\bin\\x86_amd64\\vcvarsx86_amd64.bat"
cd build_%BUILD_NAME%
copy ${env.WORKSPACE}\\3rdparty\\mkldnn\\config_template.vcxproj.user ${env.WORKSPACE}\\config_template.vcxproj.user /y
- cmake -G \"NMake Makefiles JOM\" -DUSE_CUDA=1 -DUSE_CUDNN=1 -DUSE_NVRTC=1 -DUSE_OPENCV=1 -DUSE_OPENMP=1 -DUSE_PROFILER=1 -DUSE_BLAS=open -DUSE_LAPACK=1 -DUSE_DIST_KVSTORE=0 -DCUDA_ARCH_NAME=All -DUSE_MKLDNN=1 -DCMAKE_CXX_FLAGS_RELEASE="/FS /MD /O2 /Ob2 /DNDEBUG" -DCMAKE_BUILD_TYPE=Release ${env.WORKSPACE}"""
+ cmake -G \"NMake Makefiles JOM\" -DUSE_CUDA=0 -DUSE_CUDNN=0 -DUSE_NVRTC=0 -DUSE_OPENCV=1 -DUSE_OPENMP=1 -DUSE_PROFILER=1 -DUSE_BLAS=open -DUSE_LAPACK=1 -DUSE_DIST_KVSTORE=0 -DCUDA_ARCH_NAME=All -DUSE_MKLDNN=1 -DCMAKE_CXX_FLAGS_RELEASE="/FS /MD /O2 /Ob2 /DNDEBUG" -DCMAKE_BUILD_TYPE=Release ${env.WORKSPACE}"""
bat '''
call "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\bin\\x86_amd64\\vcvarsx86_amd64.bat"
cd build_%BUILD_NAME%
@@ -446,7 +413,7 @@ try {
del /Q *.7z
7z.exe a %BUILD_NAME%.7z pkg_%BUILD_NAME%\\
'''
- stash includes: 'vc14_gpu_mkldnn.7z', name: 'vc14_gpu_mkldnn'
+ stash includes: 'vc14_cpu_mkldnn.7z', name: 'vc14_cpu_mkldnn'
}
}
}
@@ -774,7 +741,6 @@ try {
}
}
},
-
'Python 2: CPU Win':{
node('mxnetwindows-cpu') {
timeout(time: max_time, unit: 'MINUTES') {
@@ -797,92 +763,23 @@ try {
}
}
},
- 'Python 3: CPU Win': {
+ 'Python 3: MKLDNN-CPU Win':{
node('mxnetwindows-cpu') {
timeout(time: max_time, unit: 'MINUTES') {
- ws('workspace/ut-python-cpu') {
+ ws('workspace/ut-python-cpu-mkldnn') {
try {
init_git_win()
- unstash 'vc14_cpu'
- bat '''rmdir /s/q pkg_vc14_cpu
- 7z x -y vc14_cpu.7z'''
+ unstash 'vc14_cpu_mkldnn'
+ bat '''rmdir /s/q pkg_vc14_cpu_mkldnn
+ 7z x -y vc14_cpu_mkldnn.7z'''
bat """xcopy C:\\mxnet\\data data /E /I /Y
xcopy C:\\mxnet\\model model /E /I /Y
call activate py3
- set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_cpu\\python
- del /S /Q ${env.WORKSPACE}\\pkg_vc14_cpu\\python\\*.pyc
+ set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_cpu_mkldnn\\python
+ del /S /Q ${env.WORKSPACE}\\pkg_vc14_cpu_mkldnn\\python\\*.pyc
C:\\mxnet\\test_cpu.bat"""
} finally {
- collect_test_results_windows('nosetests_unittest.xml', 'nosetests_unittest_windows_python3_cpu.xml')
- }
- }
- }
- }
- },
- 'Python 2: GPU Win':{
- node('mxnetwindows-gpu') {
- timeout(time: max_time, unit: 'MINUTES') {
- ws('workspace/ut-python-gpu') {
- try {
- init_git_win()
- unstash 'vc14_gpu'
- bat '''rmdir /s/q pkg_vc14_gpu
- 7z x -y vc14_gpu.7z'''
- bat """xcopy C:\\mxnet\\data data /E /I /Y
- xcopy C:\\mxnet\\model model /E /I /Y
- call activate py2
- set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_gpu\\python
- del /S /Q ${env.WORKSPACE}\\pkg_vc14_gpu\\python\\*.pyc
- C:\\mxnet\\test_gpu.bat"""
- } finally {
- collect_test_results_windows('nosetests_gpu_forward.xml', 'nosetests_gpu_forward_windows_python2_gpu.xml')
- collect_test_results_windows('nosetests_gpu_operator.xml', 'nosetests_gpu_operator_windows_python2_gpu.xml')
- }
- }
- }
- }
- },
- 'Python 3: GPU Win':{
- node('mxnetwindows-gpu') {
- timeout(time: max_time, unit: 'MINUTES') {
- ws('workspace/ut-python-gpu') {
- try {
- init_git_win()
- unstash 'vc14_gpu'
- bat '''rmdir /s/q pkg_vc14_gpu
- 7z x -y vc14_gpu.7z'''
- bat """xcopy C:\\mxnet\\data data /E /I /Y
- xcopy C:\\mxnet\\model model /E /I /Y
- call activate py3
- set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_gpu\\python
- del /S /Q ${env.WORKSPACE}\\pkg_vc14_gpu\\python\\*.pyc
- C:\\mxnet\\test_gpu.bat"""
- } finally {
- collect_test_results_windows('nosetests_gpu_forward.xml', 'nosetests_gpu_forward_windows_python3_gpu.xml')
- collect_test_results_windows('nosetests_gpu_operator.xml', 'nosetests_gpu_operator_windows_python3_gpu.xml')
- }
- }
- }
- }
- },
- 'Python 3: MKLDNN-GPU Win':{
- node('mxnetwindows-gpu') {
- timeout(time: max_time, unit: 'MINUTES') {
- ws('workspace/ut-python-gpu') {
- try {
- init_git_win()
- unstash 'vc14_gpu_mkldnn'
- bat '''rmdir /s/q pkg_vc14_gpu_mkldnn
- 7z x -y vc14_gpu_mkldnn.7z'''
- bat """xcopy C:\\mxnet\\data data /E /I /Y
- xcopy C:\\mxnet\\model model /E /I /Y
- call activate py3
- set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_gpu_mkldnn\\python
- del /S /Q ${env.WORKSPACE}\\pkg_vc14_gpu_mkldnn\\python\\*.pyc
- C:\\mxnet\\test_gpu.bat"""
- } finally {
- collect_test_results_windows('nosetests_gpu_forward.xml', 'nosetests_gpu_forward_windows_python3_gpu_mkldnn.xml')
- collect_test_results_windows('nosetests_gpu_operator.xml', 'nosetests_gpu_operator_windows_python3_gpu_mkldnn.xml')
+ collect_test_results_windows('nosetests_unittest.xml', 'nosetests_unittest_windows_python3_cpu_mkldnn.xml')
}
}
}
@@ -900,7 +797,7 @@ try {
}
}
},
- 'Python GPU': {
+ 'Python Integration GPU': {
node('mxnetlinux-gpu') {
ws('workspace/it-python-gpu') {
timeout(time: max_time, unit: 'MINUTES') {
@@ -912,7 +809,7 @@ try {
}
}
},
- 'Caffe GPU': {
+ 'Caffe Converter GPU': {
node('mxnetlinux-gpu') {
ws('workspace/it-caffe') {
timeout(time: max_time, unit: 'MINUTES') {
diff --git a/tests/nightly/JenkinsfileForBinaries b/tests/nightly/JenkinsfileForBinaries
index c0c14b26667..ee1641af187 100755
--- a/tests/nightly/JenkinsfileForBinaries
+++ b/tests/nightly/JenkinsfileForBinaries
@@ -77,7 +77,73 @@ try {
pack_lib('gpu', mx_lib)
}
}
- }
+ },
+ 'Build CPU windows':{
+ node('mxnetwindows-cpu') {
+ timeout(time: max_time, unit: 'MINUTES') {
+ ws('workspace/build-cpu') {
+ withEnv(['OpenBLAS_HOME=C:\\mxnet\\openblas', 'OpenCV_DIR=C:\\mxnet\\opencv_vc14', 'CUDA_PATH=C:\\CUDA\\v8.0']) {
+ init_git_win()
+ bat """mkdir build_vc14_cpu
+ call "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\bin\\x86_amd64\\vcvarsx86_amd64.bat"
+ cd build_vc14_cpu
+ cmake -G \"Visual Studio 14 2015 Win64\" -DUSE_CUDA=0 -DUSE_CUDNN=0 -DUSE_NVRTC=0 -DUSE_OPENCV=1 -DUSE_OPENMP=1 -DUSE_PROFILER=1 -DUSE_BLAS=open -DUSE_LAPACK=1 -DUSE_DIST_KVSTORE=0 -DUSE_MKL_IF_AVAILABLE=0 ${env.WORKSPACE}"""
+ bat 'C:\\mxnet\\build_vc14_cpu.bat'
+
+ bat '''rmdir /s/q pkg_vc14_cpu
+ mkdir pkg_vc14_cpu\\lib
+ mkdir pkg_vc14_cpu\\python
+ mkdir pkg_vc14_cpu\\include
+ mkdir pkg_vc14_cpu\\build
+ copy build_vc14_cpu\\Release\\libmxnet.lib pkg_vc14_cpu\\lib
+ copy build_vc14_cpu\\Release\\libmxnet.dll pkg_vc14_cpu\\build
+ xcopy python pkg_vc14_cpu\\python /E /I /Y
+ xcopy include pkg_vc14_cpu\\include /E /I /Y
+ xcopy 3rdparty\\dmlc-core\\include pkg_vc14_cpu\\include /E /I /Y
+ xcopy 3rdparty\\mshadow\\mshadow pkg_vc14_cpu\\include\\mshadow /E /I /Y
+ xcopy 3rdparty\\nnvm\\include pkg_vc14_cpu\\nnvm\\include /E /I /Y
+ del /Q *.7z
+ 7z.exe a vc14_cpu.7z pkg_vc14_cpu\\
+ '''
+ stash includes: 'vc14_cpu.7z', name: 'vc14_cpu'
+ }
+ }
+ }
+ }
+ },
+ //Todo: Set specific CUDA_ARCh for windows builds in cmake
+ 'Build GPU windows':{
+ node('mxnetwindows-cpu') {
+ timeout(time: max_time, unit: 'MINUTES') {
+ ws('workspace/build-gpu') {
+ withEnv(['OpenBLAS_HOME=C:\\mxnet\\openblas', 'OpenCV_DIR=C:\\mxnet\\opencv_vc14', 'CUDA_PATH=C:\\CUDA\\v8.0']) {
+ init_git_win()
+ bat """mkdir build_vc14_gpu
+ call "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\bin\\x86_amd64\\vcvarsx86_amd64.bat"
+ cd build_vc14_gpu
+ cmake -G \"NMake Makefiles JOM\" -DUSE_CUDA=1 -DUSE_CUDNN=1 -DUSE_NVRTC=1 -DUSE_OPENCV=1 -DUSE_OPENMP=1 -DUSE_PROFILER=1 -DUSE_BLAS=open -DUSE_LAPACK=1 -DUSE_DIST_KVSTORE=0 -DCUDA_ARCH_NAME=All -DCMAKE_CXX_FLAGS_RELEASE="/FS /MD /O2 /Ob2 /DNDEBUG" -DCMAKE_BUILD_TYPE=Release -DUSE_MKL_IF_AVAILABLE=0 ${env.WORKSPACE}"""
+ bat 'C:\\mxnet\\build_vc14_gpu.bat'
+ bat '''rmdir /s/q pkg_vc14_gpu
+ mkdir pkg_vc14_gpu\\lib
+ mkdir pkg_vc14_gpu\\python
+ mkdir pkg_vc14_gpu\\include
+ mkdir pkg_vc14_gpu\\build
+ copy build_vc14_gpu\\libmxnet.lib pkg_vc14_gpu\\lib
+ copy build_vc14_gpu\\libmxnet.dll pkg_vc14_gpu\\build
+ xcopy python pkg_vc14_gpu\\python /E /I /Y
+ xcopy include pkg_vc14_gpu\\include /E /I /Y
+ xcopy 3rdparty\\dmlc-core\\include pkg_vc14_gpu\\include /E /I /Y
+ xcopy 3rdparty\\mshadow\\mshadow pkg_vc14_gpu\\include\\mshadow /E /I /Y
+ xcopy 3rdparty\\nnvm\\include pkg_vc14_gpu\\nnvm\\include /E /I /Y
+ del /Q *.7z
+ 7z.exe a vc14_gpu.7z pkg_vc14_gpu\\
+ '''
+ stash includes: 'vc14_gpu.7z', name: 'vc14_gpu'
+ }
+ }
+ }
+ }
+ },
}
stage('NightlyTests'){
@@ -98,7 +164,52 @@ try {
docker_run('ubuntu_nightly_gpu', 'nightly_test_KVStore_singleNode', true)
}
}
- }
+ },
+ 'Python 3: CPU Win': {
+ node('mxnetwindows-cpu') {
+ timeout(time: max_time, unit: 'MINUTES') {
+ ws('workspace/ut-python-cpu') {
+ try {
+ init_git_win()
+ unstash 'vc14_cpu'
+ bat '''rmdir /s/q pkg_vc14_cpu
+ 7z x -y vc14_cpu.7z'''
+ bat """xcopy C:\\mxnet\\data data /E /I /Y
+ xcopy C:\\mxnet\\model model /E /I /Y
+ call activate py3
+ set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_cpu\\python
+ del /S /Q ${env.WORKSPACE}\\pkg_vc14_cpu\\python\\*.pyc
+ C:\\mxnet\\test_cpu.bat"""
+ } finally {
+ collect_test_results_windows('nosetests_unittest.xml', 'nosetests_unittest_windows_python3_cpu.xml')
+ }
+ }
+ }
+ }
+ },
+ 'Python 3: GPU Win':{
+ node('mxnetwindows-gpu') {
+ timeout(time: max_time, unit: 'MINUTES') {
+ ws('workspace/ut-python-gpu') {
+ try {
+ init_git_win()
+ unstash 'vc14_gpu'
+ bat '''rmdir /s/q pkg_vc14_gpu
+ 7z x -y vc14_gpu.7z'''
+ bat """xcopy C:\\mxnet\\data data /E /I /Y
+ xcopy C:\\mxnet\\model model /E /I /Y
+ call activate py3
+ set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_gpu\\python
+ del /S /Q ${env.WORKSPACE}\\pkg_vc14_gpu\\python\\*.pyc
+ C:\\mxnet\\test_gpu.bat"""
+ } finally {
+ collect_test_results_windows('nosetests_gpu_forward.xml', 'nosetests_gpu_forward_windows_python3_gpu.xml')
+ collect_test_results_windows('nosetests_gpu_operator.xml', 'nosetests_gpu_operator_windows_python3_gpu.xml')
+ }
+ }
+ }
+ }
+ },
}
} catch (caughtError) {
node("mxnetlinux-cpu") {
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services