You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by GitBox <gi...@apache.org> on 2018/12/29 00:30:51 UTC
[GitHub] azai91 closed pull request #13464: [WIP] Feature/mklnn default make
+ test on jenkins
azai91 closed pull request #13464: [WIP] Feature/mklnn default make + test on jenkins
URL: https://github.com/apache/incubator-mxnet/pull/13464
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 3b8bbd2e027..f09a9c29e9d 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -20,7 +20,7 @@ mxnet_option(USE_F16C "Build with x86 F16C instruction support" ON)
mxnet_option(USE_LAPACK "Build with lapack support" ON)
mxnet_option(USE_MKL_IF_AVAILABLE "Use MKL if found" ON)
mxnet_option(USE_MKLML_MKL "Use MKLDNN variant of MKL (if MKL found)" ON IF USE_MKL_IF_AVAILABLE AND (NOT APPLE))
-mxnet_option(USE_MKLDNN "Use MKLDNN variant of MKL (if MKL found)" ON IF USE_MKL_IF_AVAILABLE AND (NOT APPLE))
+mxnet_option(USE_MKLDNN "Use MKLDNN variant of MKL (if MKL found)" ON IF USE_MKL_IF_AVAILABLE)
mxnet_option(USE_OPERATOR_TUNING "Enable auto-tuning of operators" ON IF NOT MSVC)
mxnet_option(USE_GPERFTOOLS "Build with GPerfTools support (if found)" ON)
mxnet_option(USE_JEMALLOC "Build with Jemalloc support" ON)
diff --git a/Makefile b/Makefile
index f15968bfe52..dc17fcae7bd 100644
--- a/Makefile
+++ b/Makefile
@@ -60,6 +60,10 @@ endif
# use customized config file
include $(config)
+ifndef $(USE_MKLDNN)
+ USE_MKLDNN = 1
+endif
+
ifeq ($(USE_MKL2017), 1)
$(warning "USE_MKL2017 is deprecated. We will switch to USE_MKLDNN.")
USE_MKLDNN=1
@@ -132,7 +136,7 @@ ifeq ($(USE_MKLDNN), 1)
LDFLAGS += -L$(MKLROOT)/lib
endif
CFLAGS += -I$(MKLDNNROOT)/include
- LDFLAGS += -L$(MKLDNNROOT)/lib -lmkldnn -Wl,-rpath,'$${ORIGIN}'
+ LDFLAGS += -L$(MKLDNNROOT)/lib -lmkldnn -Wl,-rpath,$(ROOTDIR)/lib
endif
# setup opencv
diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index 6dd5bb6f239..5f60738ee5f 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -253,6 +253,12 @@ build_centos7_cpu() {
USE_BLAS=openblas \
USE_DIST_KVSTORE=1 \
-j$(nproc)
+ cp lib/libmkldnn.so.0 lib/libmkldnn.so.0.tmp
+ mv lib/libmkldnn.so.0.tmp lib/libmkldnn.so.0
+ cp lib/libmklml_intel.so lib/libmklml_intel.so.tmp
+ mv lib/libmklml_intel.so.tmp lib/libmklml_intel.so
+ cp lib/libiomp5.so lib/libiomp5.so.tmp
+ mv lib/libiomp5.so.tmp lib/libiomp5.so
}
build_amzn_linux_cpu() {
@@ -285,7 +291,6 @@ build_centos7_mkldnn() {
ENABLE_TESTCOVERAGE=1 \
USE_LAPACK=1 \
USE_LAPACK_PATH=/usr/lib64/liblapack.so \
- USE_MKLDNN=1 \
USE_BLAS=openblas \
-j$(nproc)
}
@@ -307,6 +312,12 @@ build_centos7_gpu() {
USE_DIST_KVSTORE=1 \
CUDA_ARCH="$CI_CUDA_COMPUTE_CAPABILITIES" \
-j$(nproc)
+ cp lib/libmkldnn.so.0 lib/libmkldnn.so.0.tmp
+ mv lib/libmkldnn.so.0.tmp lib/libmkldnn.so.0
+ cp lib/libmklml_intel.so lib/libmklml_intel.so.tmp
+ mv lib/libmklml_intel.so.tmp lib/libmklml_intel.so
+ cp lib/libiomp5.so lib/libiomp5.so.tmp
+ mv lib/libiomp5.so.tmp lib/libiomp5.so
}
build_ubuntu_cpu() {
@@ -458,7 +469,6 @@ build_ubuntu_cpu_clang39_mkldnn() {
ENABLE_TESTCOVERAGE=1 \
USE_CPP_PACKAGE=1 \
USE_BLAS=openblas \
- USE_MKLDNN=1 \
USE_OPENMP=0 \
-j$(nproc)
}
@@ -475,7 +485,6 @@ build_ubuntu_cpu_clang60_mkldnn() {
ENABLE_TESTCOVERAGE=1 \
USE_CPP_PACKAGE=1 \
USE_BLAS=openblas \
- USE_MKLDNN=1 \
USE_OPENMP=1 \
-j$(nproc)
}
@@ -490,7 +499,6 @@ build_ubuntu_cpu_mkldnn() {
ENABLE_TESTCOVERAGE=1 \
USE_CPP_PACKAGE=1 \
USE_BLAS=openblas \
- USE_MKLDNN=1 \
-j$(nproc)
}
@@ -578,7 +586,6 @@ build_ubuntu_gpu_mkldnn() {
ENABLE_TESTCOVERAGE=1 \
USE_CPP_PACKAGE=1 \
USE_BLAS=openblas \
- USE_MKLDNN=1 \
USE_CUDA=1 \
USE_CUDA_PATH=/usr/local/cuda \
USE_CUDNN=1 \
@@ -595,7 +602,6 @@ build_ubuntu_gpu_mkldnn_nocudnn() {
DEV=1 \
ENABLE_TESTCOVERAGE=1 \
USE_BLAS=openblas \
- USE_MKLDNN=1 \
USE_CUDA=1 \
USE_CUDA_PATH=/usr/local/cuda \
USE_CUDNN=0 \
@@ -618,6 +624,10 @@ build_ubuntu_gpu_cuda91_cudnn7() {
USE_DIST_KVSTORE=1 \
CUDA_ARCH="$CI_CUDA_COMPUTE_CAPABILITIES" \
-j$(nproc)
+ cp lib/libmkldnn.so.0 lib/libmkldnn.so.0.tmp
+ mv lib/libmkldnn.so.0.tmp lib/libmkldnn.so.0
+ cp lib/libiomp5.so lib/libiomp5.so.tmp
+ mv lib/libiomp5.so.tmp lib/libiomp5.so
}
build_ubuntu_amalgamation() {
@@ -672,7 +682,6 @@ build_ubuntu_gpu_cmake() {
-DUSE_CUDA=1 \
-DUSE_CUDNN=1 \
-DUSE_MKLML_MKL=0 \
- -DUSE_MKLDNN=0 \
-DUSE_DIST_KVSTORE=1 \
-DCMAKE_BUILD_TYPE=Release \
-DCUDA_ARCH_NAME=Manual \
@@ -681,6 +690,9 @@ build_ubuntu_gpu_cmake() {
/work/mxnet
ninja -v
+ # libmkldnn.so.0 is a link file. We need an actual binary file named libmkldnn.so.0.
+ cp 3rdparty/mkldnn/src/libmkldnn.so.0 3rdparty/mkldnn/src/libmkldnn.so.0.tmp
+ mv 3rdparty/mkldnn/src/libmkldnn.so.0.tmp 3rdparty/mkldnn/src/libmkldnn.so.0
}
build_ubuntu_blc() {
@@ -792,6 +804,9 @@ unittest_ubuntu_cpu_scala() {
unittest_centos7_cpu_scala() {
set -ex
+ mkdir -p /work/mxnet/3rdparty/mkldnn/build/install/lib/
+ cp lib/libmkldnn.so.0 /work/mxnet/3rdparty/mkldnn/build/install/lib/libmkldnn.so
+ cp lib/libmklml_intel.so /work/mxnet/3rdparty/mkldnn/build/install/lib/libmklml_intel.so
cd /work/mxnet
make scalapkg USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1
make scalaunittest USE_BLAS=openblas USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1
@@ -861,6 +876,7 @@ unittest_ubuntu_cpu_julia06() {
# FIXME
export LD_PRELOAD='/usr/lib/x86_64-linux-gnu/libjemalloc.so'
+ export LD_LIBRARY_PATH=/work/mxnet/lib:$LD_LIBRARY_PATH
# use the prebuilt binary from $MXNET_HOME/lib
julia -e 'Pkg.build("MXNet")'
@@ -944,6 +960,9 @@ integrationtest_ubuntu_cpu_dist_kvstore() {
integrationtest_ubuntu_gpu_scala() {
set -ex
+ mkdir -p /work/mxnet/3rdparty/mkldnn/build/install/lib/
+ cp lib/libmkldnn.so.0 /work/mxnet/3rdparty/mkldnn/build/install/lib/libmkldnn.so
+ cp lib/libmklml_intel.so /work/mxnet/3rdparty/mkldnn/build/install/lib/libmklml_intel.so
make scalapkg USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 USE_DIST_KVSTORE=1 SCALA_ON_GPU=1 ENABLE_TESTCOVERAGE=1
make scalaintegrationtest USE_OPENCV=1 USE_BLAS=openblas USE_CUDA=1 USE_CUDA_PATH=/usr/local/cuda USE_CUDNN=1 SCALA_TEST_ON_GPU=1 USE_DIST_KVSTORE=1 ENABLE_TESTCOVERAGE=1
}
@@ -1187,6 +1206,7 @@ deploy_jl_docs() {
# FIXME
export LD_PRELOAD='/usr/lib/x86_64-linux-gnu/libjemalloc.so'
+ export LD_LIBRARY_PATH=/work/mxnet/lib:$LD_LIBRARY_PATH
# use the prebuilt binary from $MXNET_HOME/lib
julia -e 'Pkg.build("MXNet")'
diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy
index 74bde1eee21..94cc81a444a 100644
--- a/ci/jenkins/Jenkins_steps.groovy
+++ b/ci/jenkins/Jenkins_steps.groovy
@@ -23,22 +23,22 @@
utils = load('ci/Jenkinsfile_utils.groovy')
// mxnet libraries
-mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a'
+mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, lib/libmklml_intel.so, lib/libmkldnn.so.0, lib/libiomp5.so'
// Python wheels
mx_pip = 'build/*.whl'
// for scala build, need to pass extra libs when run with dist_kvstore
-mx_dist_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a'
+mx_dist_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, lib/libmklml_intel.so, lib/libmkldnn.so.0, lib/libiomp5.so'
// mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default.
-mx_cmake_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so'
+mx_cmake_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, build/3rdparty/mkldnn/src/libmkldnn.so.0'
// mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default.
mx_cmake_lib_debug = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests'
mx_cmake_mkldnn_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, build/3rdparty/mkldnn/src/libmkldnn.so.0'
mx_mkldnn_lib = 'lib/libmxnet.so, lib/libmxnet.a, lib/libiomp5.so, lib/libmkldnn.so.0, lib/libmklml_intel.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a'
-mx_tensorrt_lib = 'lib/libmxnet.so, lib/libnvonnxparser_runtime.so.0, lib/libnvonnxparser.so.0, lib/libonnx_proto.so, lib/libonnx.so'
-mx_lib_cpp_examples = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, build/cpp-package/example/*'
-mx_lib_cpp_examples_cpu = 'build/libmxnet.so, build/cpp-package/example/*'
+mx_tensorrt_lib = 'lib/libmxnet.so, lib/libnvonnxparser_runtime.so.0, lib/libnvonnxparser.so.0, lib/libonnx_proto.so, lib/libonnx.so, lib/libmklml_intel.so, lib/libmkldnn.so.0, lib/libiomp5.so'
+mx_lib_cpp_examples = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, build/cpp-package/example/*, lib/libmklml_intel.so, lib/libmkldnn.so.0, lib/libiomp5.so'
+mx_lib_cpp_examples_cpu = 'build/libmxnet.so, build/cpp-package/example/*, lib/libmklml_intel.so, lib/libmkldnn.so.0, lib/libiomp5.so'
// Python unittest for CPU
// Python 2
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services