You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by cj...@apache.org on 2018/04/04 21:55:44 UTC
[incubator-mxnet] branch master updated: [MXNET-136] Enabling
USE_DIST_KVSTORE flag for CI (#10232)
This is an automated email from the ASF dual-hosted git repository.
cjolivier01 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
The following commit(s) were added to refs/heads/master by this push:
new f950702 [MXNET-136] Enabling USE_DIST_KVSTORE flag for CI (#10232)
f950702 is described below
commit f95070239e3b2b1722cfc29f6a1807965b861cdf
Author: Rahul Huilgol <ra...@gmail.com>
AuthorDate: Wed Apr 4 14:55:39 2018 -0700
[MXNET-136] Enabling USE_DIST_KVSTORE flag for CI (#10232)
* test dist_kvstore compilation for make
* add cmake functions
* install zmq
* update ps-lite
* fix scala build failure
* updating ps-lite
* stash more libs for scala link
* update ps-lite executable path
---
3rdparty/ps-lite | 2 +-
Jenkinsfile | 12 +++++++-----
ci/docker/install/ubuntu_core.sh | 1 +
ci/docker/runtime_functions.sh | 16 ++++++++++++----
4 files changed, 21 insertions(+), 10 deletions(-)
diff --git a/3rdparty/ps-lite b/3rdparty/ps-lite
index aee3252..a6dda54 160000
--- a/3rdparty/ps-lite
+++ b/3rdparty/ps-lite
@@ -1 +1 @@
-Subproject commit aee325276bccb092f516df0bce30d3a8333f4038
+Subproject commit a6dda54604a07d1fb21b016ed1e3f4246b08222a
diff --git a/Jenkinsfile b/Jenkinsfile
index 45b86fb..3892906 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -22,6 +22,8 @@
// mxnet libraries
mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/nnvm/lib/libnnvm.a'
+// for scala build, need to pass extra libs when run with dist_kvstore
+mx_dist_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a'
// mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default.
mx_cmake_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so'
mx_cmake_mkldnn_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, build/3rdparty/mkldnn/src/libmkldnn.so, build/3rdparty/mkldnn/src/libmkldnn.so.0'
@@ -164,7 +166,7 @@ try {
ws('workspace/build-cpu-openblas') {
init_git()
sh "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh build_ubuntu_cpu_openblas"
- pack_lib('cpu')
+ pack_lib('cpu', mx_dist_lib)
}
}
},
@@ -224,8 +226,8 @@ try {
node('mxnetlinux-cpu') {
ws('workspace/build-gpu') {
init_git()
- sh "ci/build.py --platform ubuntu_build_cuda /work/runtime_functions.sh build_ubuntu_gpu_cuda91_cudnn7"
- pack_lib('gpu')
+ sh "ci/build.py --platform ubuntu_build_cuda /work/runtime_functions.sh build_ubuntu_gpu_cuda91_cudnn7"
+ pack_lib('gpu', mx_dist_lib)
stash includes: 'build/cpp-package/example/test_score', name: 'cpp_test_score'
stash includes: 'build/cpp-package/example/test_optimizer', name: 'cpp_test_optimizer'
}
@@ -475,7 +477,7 @@ try {
node('mxnetlinux-cpu') {
ws('workspace/ut-scala-cpu') {
init_git()
- unpack_lib('cpu')
+ unpack_lib('cpu', mx_dist_lib)
timeout(time: max_time, unit: 'MINUTES') {
sh "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh unittest_ubuntu_cpu_scala"
}
@@ -486,7 +488,7 @@ try {
node('mxnetlinux-gpu') {
ws('workspace/ut-scala-gpu') {
init_git()
- unpack_lib('gpu')
+ unpack_lib('gpu', mx_dist_lib)
timeout(time: max_time, unit: 'MINUTES') {
sh "ci/build.py --nvidiadocker --platform ubuntu_gpu /work/runtime_functions.sh unittest_ubuntu_gpu_scala"
}
diff --git a/ci/docker/install/ubuntu_core.sh b/ci/docker/install/ubuntu_core.sh
index dc9b091..f588d06 100755
--- a/ci/docker/install/ubuntu_core.sh
+++ b/ci/docker/install/ubuntu_core.sh
@@ -33,6 +33,7 @@ apt-get install -y \
wget \
unzip \
sudo \
+ libzmq3-dev \
software-properties-common \
ninja-build
diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index 3d58f34..a97f6b9 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -131,6 +131,7 @@ build_amzn_linux_cpu() {
-DCMAKE_BUILD_TYPE=RelWithDebInfo\
-DUSE_MKL_IF_AVAILABLE=OFF\
-DUSE_LAPACK=OFF\
+ -DUSE_DIST_KVSTORE=ON\
-G Ninja /work/mxnet
ninja
export MXNET_LIBRARY_PATH=`pwd`/libmxnet.so
@@ -180,6 +181,7 @@ build_centos7_cpu() {
USE_LAPACK=1 \
USE_LAPACK_PATH=/usr/lib64/liblapack.so \
USE_BLAS=openblas \
+ USE_DIST_KVSTORE=1 \
-j$(nproc)
}
@@ -206,6 +208,7 @@ build_centos7_gpu() {
USE_CUDA=1 \
USE_CUDA_PATH=/usr/local/cuda \
USE_CUDNN=1 \
+ USE_DIST_KVSTORE=1 \
-j$(nproc)
}
@@ -215,6 +218,7 @@ build_ubuntu_cpu_openblas() {
DEV=1 \
USE_CPP_PACKAGE=1 \
USE_BLAS=openblas \
+ USE_DIST_KVSTORE=1 \
-j$(nproc)
}
@@ -224,6 +228,7 @@ build_ubuntu_cpu_clang39() {
USE_CPP_PACKAGE=1 \
USE_BLAS=openblas \
USE_OPENMP=0 \
+ USE_DIST_KVSTORE=1 \
CXX=clang++-3.9 \
CC=clang-3.9 \
-j$(nproc)
@@ -235,6 +240,7 @@ build_ubuntu_cpu_clang50() {
USE_CPP_PACKAGE=1 \
USE_BLAS=openblas \
USE_OPENMP=1 \
+ USE_DIST_KVSTORE=1 \
CXX=clang++-5.0 \
CC=clang-5.0 \
-j$(nproc)
@@ -296,6 +302,7 @@ build_ubuntu_gpu_cuda91_cudnn7() {
USE_CUDA_PATH=/usr/local/cuda \
USE_CUDNN=1 \
USE_CPP_PACKAGE=1 \
+ USE_DIST_KVSTORE=1 \
-j$(nproc)
}
@@ -336,6 +343,7 @@ build_ubuntu_gpu_cmake() {
-DUSE_CUDNN=1 \
-DUSE_MKLML_MKL=0 \
-DUSE_MKLDNN=0 \
+ -DUSE_DIST_KVSTORE=1 \
-DCMAKE_BUILD_TYPE=Release \
-G Ninja \
/work/mxnet
@@ -423,14 +431,14 @@ unittest_ubuntu_python3_quantization_gpu() {
unittest_ubuntu_cpu_scala() {
set -ex
- make scalapkg USE_BLAS=openblas
- make scalatest USE_BLAS=openblas
+ make scalapkg USE_BLAS=openblas USE_DIST_KVSTORE=1
+ make scalatest USE_BLAS=openblas USE_DIST_KVSTORE=1
}
unittest_ubuntu_gpu_scala() {
set -ex
- make scalapkg USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1
- make scalatest USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 SCALA_TEST_ON_GPU=1
+ make scalapkg USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 USE_DIST_KVSTORE=1
+ make scalatest USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 SCALA_TEST_ON_GPU=1 USE_DIST_KVSTORE=1
}
unittest_ubuntu_cpugpu_perl() {
--
To stop receiving notification emails like this one, please contact
cjolivier01@apache.org.