You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by an...@apache.org on 2018/12/13 07:54:35 UTC
[incubator-mxnet] branch master updated: Feature/mkldnn static
(#13628)
This is an automated email from the ASF dual-hosted git repository.
anirudh2290 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
The following commit(s) were added to refs/heads/master by this push:
new 5bcf2bd Feature/mkldnn static (#13628)
5bcf2bd is described below
commit 5bcf2bd6e8b48fa27bfcfdafd06401ec2d28978b
Author: Alexander Zai <az...@gmail.com>
AuthorDate: Wed Dec 12 23:54:18 2018 -0800
Feature/mkldnn static (#13628)
* Revert "Revert "Feature/mkldnn static 2 (#13503)" (#13540)"
This reverts commit a3eca5f5c96eed0bc29bd4e58e470997091a1fb3.
* include headers on mkldnn lib
* retrigger
* retrigger
---
CMakeLists.txt | 1 +
Makefile | 7 ++++-
ci/docker/runtime_functions.sh | 3 --
ci/jenkins/Jenkins_steps.groovy | 8 ++---
mkldnn.mk | 12 +++++--
tests/cpp/unittest.mk | 8 ++---
tests/python/mkl/test_mkldnn.py | 6 +---
tests/python/mkl/test_mkldnn_install.py | 56 ---------------------------------
8 files changed, 25 insertions(+), 76 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 3b8bbd2..1617056 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -227,6 +227,7 @@ if(USE_MKLDNN)
include(cmake/DownloadMKLML.cmake)
# CPU architecture (e.g., C5) can't run on another architecture (e.g., g3).
if(NOT MSVC)
+ set(MKLDNN_LIBRARY_TYPE "STATIC" CACHE INTERNAL "" FORCE)
set(ARCH_OPT_FLAGS "-mtune=generic")
else()
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /EHsc")
diff --git a/Makefile b/Makefile
index f15968b..31722e8 100644
--- a/Makefile
+++ b/Makefile
@@ -132,7 +132,12 @@ ifeq ($(USE_MKLDNN), 1)
LDFLAGS += -L$(MKLROOT)/lib
endif
CFLAGS += -I$(MKLDNNROOT)/include
- LDFLAGS += -L$(MKLDNNROOT)/lib -lmkldnn -Wl,-rpath,'$${ORIGIN}'
+ # MKLDNN but to needs to be dynamically linked for windows as not all VS compilers support static linking
+ ifneq ($(UNAME_S), Windows)
+ LIB_DEP += $(MKLDNNROOT)/lib/libmkldnn.a
+ else
+ LDFLAGS += -L$(MKLDNNROOT)/lib -lmkldnn -Wl,-rpath,'$${ORIGIN}'
+ endif
endif
# setup opencv
diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index 6dd5bb6..82e6feb 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -657,9 +657,6 @@ build_ubuntu_gpu_cmake_mkldnn() {
/work/mxnet
ninja -v
- # libmkldnn.so.0 is a link file. We need an actual binary file named libmkldnn.so.0.
- cp 3rdparty/mkldnn/src/libmkldnn.so.0 3rdparty/mkldnn/src/libmkldnn.so.0.tmp
- mv 3rdparty/mkldnn/src/libmkldnn.so.0.tmp 3rdparty/mkldnn/src/libmkldnn.so.0
}
build_ubuntu_gpu_cmake() {
diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy
index 74bde1e..d5cbd97 100644
--- a/ci/jenkins/Jenkins_steps.groovy
+++ b/ci/jenkins/Jenkins_steps.groovy
@@ -23,19 +23,19 @@
utils = load('ci/Jenkinsfile_utils.groovy')
// mxnet libraries
-mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a'
+mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, lib/libiomp5.so, lib/libmklml_intel.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a'
// Python wheels
mx_pip = 'build/*.whl'
// for scala build, need to pass extra libs when run with dist_kvstore
-mx_dist_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a'
+mx_dist_lib = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, lib/libmkldnn.a'
// mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default.
mx_cmake_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so'
// mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default.
mx_cmake_lib_debug = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests'
-mx_cmake_mkldnn_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so, build/3rdparty/mkldnn/src/libmkldnn.so.0'
-mx_mkldnn_lib = 'lib/libmxnet.so, lib/libmxnet.a, lib/libiomp5.so, lib/libmkldnn.so.0, lib/libmklml_intel.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a'
+mx_cmake_mkldnn_lib = 'build/libmxnet.so, build/libmxnet.a, build/3rdparty/dmlc-core/libdmlc.a, build/tests/mxnet_unit_tests, build/3rdparty/openmp/runtime/src/libomp.so'
+mx_mkldnn_lib = 'lib/libmxnet.so, lib/libmxnet.a, lib/libiomp5.so, lib/libmklml_intel.so, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a'
mx_tensorrt_lib = 'lib/libmxnet.so, lib/libnvonnxparser_runtime.so.0, lib/libnvonnxparser.so.0, lib/libonnx_proto.so, lib/libonnx.so'
mx_lib_cpp_examples = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, build/cpp-package/example/*'
mx_lib_cpp_examples_cpu = 'build/libmxnet.so, build/cpp-package/example/*'
diff --git a/mkldnn.mk b/mkldnn.mk
index d79bbe7..5af3e9b 100644
--- a/mkldnn.mk
+++ b/mkldnn.mk
@@ -19,14 +19,20 @@ ifeq ($(USE_MKLDNN), 1)
MKLDNN_SUBMODDIR = $(ROOTDIR)/3rdparty/mkldnn
MKLDNN_BUILDDIR = $(MKLDNN_SUBMODDIR)/build
MXNET_LIBDIR = $(ROOTDIR)/lib
+ MKLDNN_LIBRARY_TYPE=STATIC
ifeq ($(UNAME_S), Darwin)
OMP_LIBFILE = $(MKLDNNROOT)/lib/libiomp5.dylib
MKLML_LIBFILE = $(MKLDNNROOT)/lib/libmklml.dylib
- MKLDNN_LIBFILE = $(MKLDNNROOT)/lib/libmkldnn.0.dylib
+ MKLDNN_LIBFILE = $(MKLDNNROOT)/lib/libmkldnn.a
+else ifeq ($(UNAME_S), Windows)
+ OMP_LIBFILE = $(MKLDNNROOT)/lib/libiomp5.so
+ MKLML_LIBFILE = $(MKLDNNROOT)/lib/libmklml_intel.so
+ MKLDNN_LIBFILE = $(MKLDNNROOT)/lib/libmkldnn.so
+ MKLDNN_LIBRARY_TYPE=SHARED
else
OMP_LIBFILE = $(MKLDNNROOT)/lib/libiomp5.so
MKLML_LIBFILE = $(MKLDNNROOT)/lib/libmklml_intel.so
- MKLDNN_LIBFILE = $(MKLDNNROOT)/lib/libmkldnn.so.0
+ MKLDNN_LIBFILE = $(MKLDNNROOT)/lib/libmkldnn.a
endif
endif
@@ -37,7 +43,7 @@ mkldnn_build: $(MKLDNN_LIBFILE)
$(MKLDNN_LIBFILE):
mkdir -p $(MKLDNNROOT)
cd $(MKLDNN_SUBMODDIR) && rm -rf external && cd scripts && ./prepare_mkl.sh && cd .. && cp -a external/*/* $(MKLDNNROOT)/.
- cmake $(MKLDNN_SUBMODDIR) -DCMAKE_INSTALL_PREFIX=$(MKLDNNROOT) -B$(MKLDNN_BUILDDIR) -DARCH_OPT_FLAGS="-mtune=generic" -DWITH_TEST=OFF -DWITH_EXAMPLE=OFF
+ cmake $(MKLDNN_SUBMODDIR) -DCMAKE_INSTALL_PREFIX=$(MKLDNNROOT) -B$(MKLDNN_BUILDDIR) -DARCH_OPT_FLAGS="-mtune=generic" -DWITH_TEST=OFF -DWITH_EXAMPLE=OFF -DMKLDNN_LIBRARY_TYPE=$(MKLDNN_LIBRARY_TYPE)
$(MAKE) -C $(MKLDNN_BUILDDIR) VERBOSE=1
$(MAKE) -C $(MKLDNN_BUILDDIR) install
mkdir -p $(MXNET_LIBDIR)
diff --git a/tests/cpp/unittest.mk b/tests/cpp/unittest.mk
index 746ee2f..665ce69 100644
--- a/tests/cpp/unittest.mk
+++ b/tests/cpp/unittest.mk
@@ -41,22 +41,22 @@ gtest-all.o : $(GTEST_SRCS_)
gtest.a : gtest-all.o
$(AR) $(ARFLAGS) $@ $^
-build/tests/cpp/%.o : tests/cpp/%.cc | mkldnn
+build/tests/cpp/%.o : tests/cpp/%.cc
@mkdir -p $(@D)
$(CXX) -std=c++11 $(TEST_CFLAGS) -I$(GTEST_INC) -MM -MT tests/cpp/$* $< > build/tests/cpp/$*.d
$(CXX) -c -std=c++11 $(TEST_CFLAGS) -I$(GTEST_INC) -o build/tests/cpp/$*.o $(filter %.cc %.a, $^)
-build/tests/cpp/operator/%.o : tests/cpp/operator/%.cc | mkldnn
+build/tests/cpp/operator/%.o : tests/cpp/operator/%.cc
@mkdir -p $(@D)
$(CXX) -std=c++11 $(TEST_CFLAGS) -I$(GTEST_INC) -MM -MT tests/cpp/operator/$* $< > build/tests/cpp/operator/$*.d
$(CXX) -c -std=c++11 $(TEST_CFLAGS) -I$(GTEST_INC) -o build/tests/cpp/operator/$*.o $(filter %.cc %.a, $^)
-build/tests/cpp/storage/%.o : tests/cpp/storage/%.cc | mkldnn
+build/tests/cpp/storage/%.o : tests/cpp/storage/%.cc
@mkdir -p $(@D)
$(CXX) -std=c++11 $(TEST_CFLAGS) -I$(GTEST_INC) -MM -MT tests/cpp/storage/$* $< > build/tests/cpp/storage/$*.d
$(CXX) -c -std=c++11 $(TEST_CFLAGS) -I$(GTEST_INC) -o build/tests/cpp/storage/$*.o $(filter %.cc %.a, $^)
-build/tests/cpp/engine/%.o : tests/cpp/engine/%.cc | mkldnn
+build/tests/cpp/engine/%.o : tests/cpp/engine/%.cc
@mkdir -p $(@D)
$(CXX) -std=c++11 $(TEST_CFLAGS) -I$(GTEST_INC) -MM -MT tests/cpp/engine/$* $< > build/tests/cpp/engine/$*.d
$(CXX) -c -std=c++11 $(TEST_CFLAGS) -I$(GTEST_INC) -o build/tests/cpp/engine/$*.o $(filter %.cc %.a, $^)
diff --git a/tests/python/mkl/test_mkldnn.py b/tests/python/mkl/test_mkldnn.py
index c6c0a08..d9d3abf 100644
--- a/tests/python/mkl/test_mkldnn.py
+++ b/tests/python/mkl/test_mkldnn.py
@@ -27,7 +27,6 @@ from mxnet.test_utils import rand_ndarray, assert_almost_equal
from mxnet import gluon
from mxnet.gluon import nn
from mxnet.test_utils import *
-import test_mkldnn_install as install
curr_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__)))
sys.path.append(os.path.join(curr_path, '../unittest/'))
from common import with_seed
@@ -441,7 +440,4 @@ def test_non_mkldnn_fcomputeex():
custom = mx.symbol.Custom(name='custom', data=conv, op_type='custom')
exec1 = custom.bind(mx.cpu(), args={'data': mx.nd.ones([10,3,96,96]), 'conv_weight': mx.nd.ones([8,3,5,5])})
exec1.forward()[0].wait_to_read()
-
-
-if __name__ == '__main__':
- install.test_mkldnn_install()
+
diff --git a/tests/python/mkl/test_mkldnn_install.py b/tests/python/mkl/test_mkldnn_install.py
deleted file mode 100644
index c2f26df..0000000
--- a/tests/python/mkl/test_mkldnn_install.py
+++ /dev/null
@@ -1,56 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-"""
-MKL-DNN related test cases
-"""
-
-import sys
-import os
-import logging
-
-
-def test_mkldnn_install():
- """
- This test will verify that MXNet is built/installed correctly when
- compiled with Intel MKL-DNN library. The method will try to import
- the mxnet module and see if the mkldnn library is mapped to this
- process's address space.
- """
- logging.basicConfig(level=logging.INFO)
-
- if not sys.platform.startswith('linux'):
- logging.info("Bypass mkldnn install test for non-Linux OS")
- return
-
- try:
- #pylint: disable=unused-variable
- import mxnet as mx
- except (ImportError, OSError) as e:
- assert 0, "Import mxnet error: %s. Please double check your build/" \
- "install steps or environment variable settings" % str(e)
-
- pid = os.getpid()
- rc = os.system("cat /proc/" + str(pid) +
- "/maps | grep libmkldnn > /dev/null")
-
- if rc == 0:
- logging.info("MXNet is built/installed correctly with MKL-DNN")
- else:
- assert 0, "MXNet is built/installed incorrectly with MKL-DNN, please " \
- "double check your build/install steps or environment " \
- "variable settings"