You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by GitBox <gi...@apache.org> on 2018/06/15 00:32:03 UTC
[GitHub] marcoabreu closed pull request #11245: [MXNET-244][MXNET-523][ARM]
improvements to ARMv7 based builds.
marcoabreu closed pull request #11245: [MXNET-244][MXNET-523][ARM] improvements to ARMv7 based builds.
URL: https://github.com/apache/incubator-mxnet/pull/11245
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/CMakeLists.txt b/CMakeLists.txt
index e57c00b69e9..8a1765a0e67 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -321,14 +321,15 @@ endif()
# ---[ OpenCV
if(USE_OPENCV)
- find_package(OpenCV QUIET COMPONENTS core highgui imgproc imgcodecs)
+ find_package(OpenCV COMPONENTS core highgui imgproc imgcodecs)
if(NOT OpenCV_FOUND) # if not OpenCV 3.x, then imgcodecs are not found
+ message(STATUS "OpenCV imgcodecs missing")
find_package(OpenCV REQUIRED COMPONENTS core highgui imgproc)
endif()
include_directories(SYSTEM ${OpenCV_INCLUDE_DIRS})
list(APPEND mxnet_LINKER_LIBS ${OpenCV_LIBS})
message(STATUS " OpenCV_LIBS=${OpenCV_LIBS}")
- message(STATUS "OpenCV found (${OpenCV_CONFIG_PATH})")
+ message(STATUS "OpenCV ${OpenCV_VERSION} found (${OpenCV_CONFIG_PATH})")
add_definitions(-DMXNET_USE_OPENCV=1)
else(USE_OPENCV)
message(STATUS "OpenCV Disabled")
@@ -340,7 +341,11 @@ if(USE_OPENMP)
find_package(OpenMP REQUIRED)
# This should build on Windows, but there's some problem and I don't have a Windows box, so
# could a Windows user please fix?
- if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/openmp/CMakeLists.txt AND SYSTEM_ARCHITECTURE STREQUAL "x86_64" AND NOT MSVC)
+ if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/openmp/CMakeLists.txt
+ AND SYSTEM_ARCHITECTURE STREQUAL "x86_64"
+ AND NOT MSVC
+ AND NOT CMAKE_CROSSCOMPILING)
+
# Intel/llvm OpenMP: https://github.com/llvm-mirror/openmp
set(OPENMP_STANDALONE_BUILD TRUE)
set(LIBOMP_ENABLE_SHARED TRUE)
@@ -648,7 +653,7 @@ if(USE_PLUGINS_WARPCTC)
endif()
-if(USE_OPENCV)
+if(USE_OPENCV AND OpenCV_VERSION_MAJOR GREATER 2)
add_executable(im2rec "tools/im2rec.cc")
if(MSVC)
target_link_libraries(im2rec mxnet)
@@ -662,6 +667,9 @@ if(USE_OPENCV)
${nnvm_LINKER_LIBS}
${pslite_LINKER_LIBS}
)
+else()
+ message(WARNING "OpenCV_VERSION_MAJOR: ${OpenCV_VERSION_MAJOR}, version 3 with imgcodecs \
+ is required for im2rec, im2rec will not be available")
endif()
target_link_libraries(mxnet PUBLIC dmlc)
diff --git a/ci/docker/Dockerfile.build.android_arm64 b/ci/docker/Dockerfile.build.android_arm64
index 4bd4fd30922..04ba5c90ea3 100755
--- a/ci/docker/Dockerfile.build.android_arm64
+++ b/ci/docker/Dockerfile.build.android_arm64
@@ -31,10 +31,7 @@ MAINTAINER Pedro Larroy "pllarroy@amazon.com"
# extract ccache binary into latest context
COPY --from=ccachebuilder /usr/local/bin/ccache /usr/local/bin/ccache
-# The cross-compiling emulator
RUN apt-get update && apt-get install -y \
- qemu-user \
- qemu-user-static \
unzip
ENV CROSS_TRIPLE=aarch64-linux-android
diff --git a/ci/docker/Dockerfile.build.arm64 b/ci/docker/Dockerfile.build.arm64
index a1f752bbf64..2a950078b42 100755
--- a/ci/docker/Dockerfile.build.arm64
+++ b/ci/docker/Dockerfile.build.arm64
@@ -33,17 +33,19 @@ FROM mxnetci/dockcross-linux-arm64:05082018
COPY --from=ccachebuilder /usr/local/bin/ccache /usr/local/bin/ccache
ENV ARCH aarch64
-ENV FC /usr/bin/${CROSS_TRIPLE}-gfortran
ENV HOSTCC gcc
ENV TARGET ARMV8
-WORKDIR /work
+WORKDIR /work/deps
-# Build OpenBLAS
-RUN git clone --recursive -b v0.2.20 https://github.com/xianyi/OpenBLAS.git && \
- cd OpenBLAS && \
- make -j$(nproc) && \
- PREFIX=${CROSS_ROOT} make install
+COPY install/ubuntu_arm.sh /work/
+RUN /work/ubuntu_arm.sh
+
+COPY install/arm_openblas.sh /work/
+RUN /work/arm_openblas.sh
+
+ENV OpenBLAS_HOME=${CROSS_ROOT}
+ENV OpenBLAS_DIR=${CROSS_ROOT}
COPY runtime_functions.sh /work/
-WORKDIR /work/mxnet
+WORKDIR /work/build
diff --git a/ci/docker/Dockerfile.build.armv6 b/ci/docker/Dockerfile.build.armv6
index c073992406f..93be54025f4 100755
--- a/ci/docker/Dockerfile.build.armv6
+++ b/ci/docker/Dockerfile.build.armv6
@@ -36,11 +36,14 @@ ENV TARGET ARMV6
WORKDIR /work/deps
-# Build OpenBLAS
-RUN git clone --recursive -b v0.2.20 https://github.com/xianyi/OpenBLAS.git && \
- cd OpenBLAS && \
- make -j$(nproc) && \
- make PREFIX=$CROSS_ROOT install
+COPY install/ubuntu_arm.sh /work/
+RUN /work/ubuntu_arm.sh
+
+COPY install/arm_openblas.sh /work/
+RUN /work/arm_openblas.sh
+
+ENV OpenBLAS_HOME=${CROSS_ROOT}
+ENV OpenBLAS_DIR=${CROSS_ROOT}
COPY runtime_functions.sh /work/
WORKDIR /work/mxnet
diff --git a/ci/docker/Dockerfile.build.armv7 b/ci/docker/Dockerfile.build.armv7
index 627486c0537..95e05e7cb51 100755
--- a/ci/docker/Dockerfile.build.armv7
+++ b/ci/docker/Dockerfile.build.armv7
@@ -30,13 +30,20 @@ FROM dockcross/linux-armv7
# extract ccache binary into latest context
COPY --from=ccachebuilder /usr/local/bin/ccache /usr/local/bin/ccache
-ENV ARCH armv71
-ENV CC /usr/bin/arm-linux-gnueabihf-gcc
-ENV CXX /usr/bin/arm-linux-gnueabihf-g++
+ENV ARCH armv7l
+ENV HOSTCC gcc
+ENV TARGET ARMV7
-RUN apt-get update && \
- apt-get install -y libopenblas-dev:armhf && \
- rm -rf /var/lib/apt/lists/*
+WORKDIR /work/deps
+
+COPY install/ubuntu_arm.sh /work/
+RUN /work/ubuntu_arm.sh
+
+COPY install/arm_openblas.sh /work/
+RUN /work/arm_openblas.sh
+
+ENV OpenBLAS_HOME=${CROSS_ROOT}
+ENV OpenBLAS_DIR=${CROSS_ROOT}
COPY runtime_functions.sh /work/
-WORKDIR /work/build
+WORKDIR /work/mxnet
diff --git a/ci/docker/Dockerfile.build.jetson b/ci/docker/Dockerfile.build.jetson
index 8a8bb97aa15..098225e5af5 100755
--- a/ci/docker/Dockerfile.build.jetson
+++ b/ci/docker/Dockerfile.build.jetson
@@ -37,17 +37,16 @@ FROM mxnetci/dockcross-linux-arm64:05082018
COPY --from=ccachebuilder /usr/local/bin/ccache /usr/local/bin/ccache
ENV ARCH aarch64
-ENV FC /usr/bin/${CROSS_TRIPLE}-gfortran
ENV HOSTCC gcc
ENV TARGET ARMV8
-WORKDIR /work
+WORKDIR /work/deps
-# Build OpenBLAS
-RUN git clone --recursive -b v0.2.20 https://github.com/xianyi/OpenBLAS.git && \
- cd OpenBLAS && \
- make -j$(nproc) && \
- PREFIX=${CROSS_ROOT} make install
+COPY install/ubuntu_arm.sh /work/
+RUN /work/ubuntu_arm.sh
+
+COPY install/arm_openblas.sh /work/
+RUN /work/arm_openblas.sh
ENV OpenBLAS_HOME=${CROSS_ROOT}
ENV OpenBLAS_DIR=${CROSS_ROOT}
diff --git a/ci/docker/install/arm_openblas.sh b/ci/docker/install/arm_openblas.sh
new file mode 100755
index 00000000000..fa2e5cae9cb
--- /dev/null
+++ b/ci/docker/install/arm_openblas.sh
@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -ex
+
+git clone --recursive -b v0.2.20 https://github.com/xianyi/OpenBLAS.git
+
+cd OpenBLAS
+make -j$(nproc)
+PREFIX=${CROSS_ROOT} make install
+
+cd ..
+
+rm -rf OpenBLAS
diff --git a/ci/docker/install/ubuntu_arm.sh b/ci/docker/install/ubuntu_arm.sh
new file mode 100755
index 00000000000..becb012bd18
--- /dev/null
+++ b/ci/docker/install/ubuntu_arm.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -ex
+
+apt update
+apt install -y \
+ unzip
diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index 36e23879705..a008d1b2052 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -31,7 +31,6 @@ clean_repo() {
git submodule update --init --recursive
}
-# wrap compiler calls with ccache
build_ccache_wrappers() {
set -ex
@@ -63,34 +62,49 @@ build_ccache_wrappers() {
export CXX=`pwd`/cxx
}
-# Build commands: Every platform in docker/Dockerfile.build.<platform> should have a corresponding
-# function here with the same suffix:
+build_wheel() {
-build_jetson() {
set -ex
pushd .
- build_ccache_wrappers
+ PYTHON_DIR=${1:-/work/mxnet/python}
+ BUILD_DIR=${2:-/work/build}
- cp -f make/crosscompile.jetson.mk ./config.mk
+ # build
- make -j$(nproc)
+ export MXNET_LIBRARY_PATH=${BUILD_DIR}/libmxnet.so
- export MXNET_LIBRARY_PATH=`pwd`/libmxnet.so
- cd /work/mxnet/python
+ cd ${PYTHON_DIR}
python setup.py bdist_wheel --universal
+ # repackage
+
# Fix pathing issues in the wheel. We need to move libmxnet.so from the data folder to the
# mxnet folder, then repackage the wheel.
WHEEL=`readlink -f dist/*.whl`
TMPDIR=`mktemp -d`
- unzip -d $TMPDIR $WHEEL
- rm $WHEEL
- cd $TMPDIR
+ unzip -d ${TMPDIR} ${WHEEL}
+ rm ${WHEEL}
+ cd ${TMPDIR}
mv *.data/data/mxnet/libmxnet.so mxnet
- zip -r $WHEEL .
- cp $WHEEL /work/build
- rm -rf $TMPDIR
+ zip -r ${WHEEL} .
+ cp ${WHEEL} ${BUILD_DIR}
+ rm -rf ${TMPDIR}
+
+ popd
+}
+
+# Build commands: Every platform in docker/Dockerfile.build.<platform> should have a corresponding
+# function here with the same suffix:
+
+build_jetson() {
+ set -ex
+ pushd .
+
+ cp make/crosscompile.jetson.mk ./config.mk
+ make -j$(nproc)
+
+ build_wheel /work/mxnet/python /work/mxnet/lib
popd
}
@@ -107,7 +121,7 @@ build_armv6() {
# We do not need OpenMP, since most armv6 systems have only 1 core
cmake \
- -DCMAKE_TOOLCHAIN_FILE=$CROSS_ROOT/Toolchain.cmake \
+ -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DUSE_CUDA=OFF \
@@ -120,11 +134,9 @@ build_armv6() {
-DBUILD_CPP_EXAMPLES=OFF \
-Dmxnet_LINKER_LIBS=-lgfortran \
-G Ninja /work/mxnet
- ninja
- export MXNET_LIBRARY_PATH=`pwd`/libmxnet.so
- cd /work/mxnet/python
- python setup.py bdist_wheel --universal
- cp dist/*.whl /work/build
+
+ ninja -v
+ build_wheel
popd
}
@@ -132,21 +144,30 @@ build_armv7() {
set -ex
pushd .
cd /work/build
+
+ # Lapack functionality will be included and statically linked to openblas.
+ # But USE_LAPACK needs to be set to OFF, otherwise the main CMakeLists.txt
+ # file tries to add -llapack. Lapack functionality though, requires -lgfortran
+ # to be linked additionally.
+
cmake \
+ -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} \
+ -DCMAKE_CROSSCOMPILING=ON \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
- -DUSE_CUDA=OFF\
- -DUSE_OPENCV=OFF\
- -DUSE_OPENMP=OFF\
- -DUSE_SIGNAL_HANDLER=ON\
- -DCMAKE_BUILD_TYPE=RelWithDebInfo\
- -DUSE_MKL_IF_AVAILABLE=OFF\
+ -DUSE_CUDA=OFF \
+ -DUSE_OPENCV=OFF \
+ -DUSE_OPENMP=ON \
+ -DUSE_SIGNAL_HANDLER=ON \
+ -DCMAKE_BUILD_TYPE=Release \
+ -DUSE_MKL_IF_AVAILABLE=OFF \
+ -DUSE_LAPACK=OFF \
+ -DBUILD_CPP_EXAMPLES=OFF \
+ -Dmxnet_LINKER_LIBS=-lgfortran \
-G Ninja /work/mxnet
- ninja
- export MXNET_LIBRARY_PATH=`pwd`/libmxnet.so
- cd /work/mxnet/python
- python setup.py bdist_wheel --universal
- cp dist/*.whl /work/build
+
+ ninja -v
+ build_wheel
popd
}
@@ -164,7 +185,7 @@ build_amzn_linux_cpu() {
-DUSE_LAPACK=OFF\
-DUSE_DIST_KVSTORE=ON\
-G Ninja /work/mxnet
- ninja
+ ninja -v
export MXNET_LIBRARY_PATH=`pwd`/libmxnet.so
}
@@ -173,13 +194,14 @@ build_arm64() {
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DUSE_CUDA=OFF\
+ -DSUPPORT_F16C=OFF\
-DUSE_OPENCV=OFF\
-DUSE_OPENMP=OFF\
-DUSE_SIGNAL_HANDLER=ON\
- -DCMAKE_BUILD_TYPE=RelWithDebInfo\
+ -DCMAKE_BUILD_TYPE=Release\
-DUSE_MKL_IF_AVAILABLE=OFF\
-G Ninja /work/mxnet
- ninja
+ ninja -v
export MXNET_LIBRARY_PATH=`pwd`/libmxnet.so
cd /work/mxnet/python
python setup.py bdist_wheel --universal
@@ -194,6 +216,7 @@ build_android_arm64() {
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DUSE_CUDA=OFF\
-DUSE_SSE=OFF\
+ -DSUPPORT_F16C=OFF\
-DUSE_LAPACK=OFF\
-DUSE_OPENCV=OFF\
-DUSE_OPENMP=OFF\
@@ -201,7 +224,7 @@ build_android_arm64() {
-DCMAKE_BUILD_TYPE=RelWithDebInfo\
-DUSE_MKL_IF_AVAILABLE=OFF\
-G Ninja /work/mxnet
- ninja
+ ninja -v
export MXNET_LIBRARY_PATH=`pwd`/libmxnet.so
cd /work/mxnet/python
python setup.py bdist_wheel --universal
@@ -466,7 +489,7 @@ unittest_ubuntu_python3_cpu() {
unittest_ubuntu_python3_cpu_mkldnn() {
set -ex
- export PYTHONPATH=./python/
+ export PYTHONPATH=./python/
# MXNET_MKLDNN_DEBUG is buggy and produces false positives
# https://github.com/apache/incubator-mxnet/issues/10026
#export MXNET_MKLDNN_DEBUG=1 # Ignored if not present
diff --git a/example/image-classification/predict-cpp/CMakeLists.txt b/example/image-classification/predict-cpp/CMakeLists.txt
index a2f52b9df3a..c42d1917b76 100644
--- a/example/image-classification/predict-cpp/CMakeLists.txt
+++ b/example/image-classification/predict-cpp/CMakeLists.txt
@@ -1,7 +1,7 @@
# Check OpenCV
-if(NOT USE_OPENCV OR NOT OpenCV_FOUND)
+if(NOT USE_OPENCV OR NOT OpenCV_FOUND OR OpenCV_VERSION_MAJOR LESS 3)
message(WARNING "\
-OpenCV should be enabled and found to build image classification example, skipping...")
+OpenCV version >= 3 should be enabled and found to build image classification example, skipping...")
return()
endif()
diff --git a/tests/python/unittest/test_ndarray.py b/tests/python/unittest/test_ndarray.py
index 92cdb2ced9d..be14822384b 100644
--- a/tests/python/unittest/test_ndarray.py
+++ b/tests/python/unittest/test_ndarray.py
@@ -711,9 +711,8 @@ def get_values(ensure_unique):
k=dat_size*dat_size*dat_size*dat_size, is_ascend=False)
assert_almost_equal(nd_ret_argsort, gt)
- # test topk with a big shape
- a = mx.nd.arange(0, 54686454, step=1, repeat=1)
- assert_almost_equal(a.topk(k=54686454).asnumpy(), a.asnumpy()[::-1])
+ a = mx.nd.arange(0, 1024, step=1, repeat=1)
+ assert_almost_equal(a.topk(k=1024).asnumpy(), a.asnumpy()[::-1])
# Repeat those tests that don't involve indices. These should pass even with
# duplicated input data values (over many repeated runs with different random seeds,
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services