You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by GitBox <gi...@apache.org> on 2018/06/07 16:32:59 UTC
[GitHub] marcoabreu closed pull request #11187: Devel arm rebased,
includes anton PR #11053
marcoabreu closed pull request #11187: Devel arm rebased, includes anton PR #11053
URL: https://github.com/apache/incubator-mxnet/pull/11187
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/CMakeLists.txt b/CMakeLists.txt
index e57c00b69e9..8a1765a0e67 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -321,14 +321,15 @@ endif()
# ---[ OpenCV
if(USE_OPENCV)
- find_package(OpenCV QUIET COMPONENTS core highgui imgproc imgcodecs)
+ find_package(OpenCV COMPONENTS core highgui imgproc imgcodecs)
if(NOT OpenCV_FOUND) # if not OpenCV 3.x, then imgcodecs are not found
+ message(STATUS "OpenCV imgcodecs missing")
find_package(OpenCV REQUIRED COMPONENTS core highgui imgproc)
endif()
include_directories(SYSTEM ${OpenCV_INCLUDE_DIRS})
list(APPEND mxnet_LINKER_LIBS ${OpenCV_LIBS})
message(STATUS " OpenCV_LIBS=${OpenCV_LIBS}")
- message(STATUS "OpenCV found (${OpenCV_CONFIG_PATH})")
+ message(STATUS "OpenCV ${OpenCV_VERSION} found (${OpenCV_CONFIG_PATH})")
add_definitions(-DMXNET_USE_OPENCV=1)
else(USE_OPENCV)
message(STATUS "OpenCV Disabled")
@@ -340,7 +341,11 @@ if(USE_OPENMP)
find_package(OpenMP REQUIRED)
# This should build on Windows, but there's some problem and I don't have a Windows box, so
# could a Windows user please fix?
- if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/openmp/CMakeLists.txt AND SYSTEM_ARCHITECTURE STREQUAL "x86_64" AND NOT MSVC)
+ if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/openmp/CMakeLists.txt
+ AND SYSTEM_ARCHITECTURE STREQUAL "x86_64"
+ AND NOT MSVC
+ AND NOT CMAKE_CROSSCOMPILING)
+
# Intel/llvm OpenMP: https://github.com/llvm-mirror/openmp
set(OPENMP_STANDALONE_BUILD TRUE)
set(LIBOMP_ENABLE_SHARED TRUE)
@@ -648,7 +653,7 @@ if(USE_PLUGINS_WARPCTC)
endif()
-if(USE_OPENCV)
+if(USE_OPENCV AND OpenCV_VERSION_MAJOR GREATER 2)
add_executable(im2rec "tools/im2rec.cc")
if(MSVC)
target_link_libraries(im2rec mxnet)
@@ -662,6 +667,9 @@ if(USE_OPENCV)
${nnvm_LINKER_LIBS}
${pslite_LINKER_LIBS}
)
+else()
+ message(WARNING "OpenCV_VERSION_MAJOR: ${OpenCV_VERSION_MAJOR}, version 3 with imgcodecs \
+ is required for im2rec, im2rec will not be available")
endif()
target_link_libraries(mxnet PUBLIC dmlc)
diff --git a/ci/docker/Dockerfile.build.arm64 b/ci/docker/Dockerfile.build.arm64
index ec949600f73..7a2e1723360 100755
--- a/ci/docker/Dockerfile.build.arm64
+++ b/ci/docker/Dockerfile.build.arm64
@@ -27,13 +27,16 @@ ENV FC /usr/bin/${CROSS_TRIPLE}-gfortran
ENV HOSTCC gcc
ENV TARGET ARMV8
-WORKDIR /work
+WORKDIR /work/deps
-# Build OpenBLAS
-RUN git clone --recursive -b v0.2.20 https://github.com/xianyi/OpenBLAS.git && \
- cd OpenBLAS && \
- make -j$(nproc) && \
- PREFIX=${CROSS_ROOT} make install
+COPY install/ubuntu_arm.sh /work/
+RUN /work/ubuntu_arm.sh
+
+COPY install/arm_openblas.sh /work/
+RUN /work/arm_openblas.sh
+
+ENV OpenBLAS_HOME=${CROSS_ROOT}
+ENV OpenBLAS_DIR=${CROSS_ROOT}
COPY runtime_functions.sh /work/
WORKDIR /work/mxnet
diff --git a/ci/docker/Dockerfile.build.armv6 b/ci/docker/Dockerfile.build.armv6
index 20739dabe2e..79fb4c61c7b 100755
--- a/ci/docker/Dockerfile.build.armv6
+++ b/ci/docker/Dockerfile.build.armv6
@@ -20,6 +20,9 @@
FROM dockcross/linux-armv6
+RUN apt-get update
+RUN apt-get install -y unzip
+
ENV ARCH armv6l
ENV FC=/usr/bin/${CROSS_TRIPLE}-gfortran
ENV HOSTCC gcc
@@ -27,11 +30,14 @@ ENV TARGET ARMV6
WORKDIR /work/deps
-# Build OpenBLAS
-RUN git clone --recursive -b v0.2.20 https://github.com/xianyi/OpenBLAS.git && \
- cd OpenBLAS && \
- make -j$(nproc) && \
- make PREFIX=$CROSS_ROOT install
+COPY install/ubuntu_arm.sh /work/
+RUN /work/ubuntu_arm.sh
+
+COPY install/arm_openblas.sh /work/
+RUN /work/arm_openblas.sh
+
+ENV OpenBLAS_HOME=${CROSS_ROOT}
+ENV OpenBLAS_DIR=${CROSS_ROOT}
COPY runtime_functions.sh /work/
WORKDIR /work/mxnet
diff --git a/ci/docker/Dockerfile.build.armv7 b/ci/docker/Dockerfile.build.armv7
index c2493063518..04002dcb292 100755
--- a/ci/docker/Dockerfile.build.armv7
+++ b/ci/docker/Dockerfile.build.armv7
@@ -16,17 +16,25 @@
# specific language governing permissions and limitations
# under the License.
#
-# Dockerfile to build MXNet for Android ARMv7
+# Dockerfile to build MXNet for ARMv7 (Android & RPi)
FROM dockcross/linux-armv7
-ENV ARCH armv71
-ENV CC /usr/bin/arm-linux-gnueabihf-gcc
-ENV CXX /usr/bin/arm-linux-gnueabihf-g++
+ENV ARCH armv7l
+ENV HOSTCC gcc
+ENV TARGET ARMV7
+ENV FC /usr/bin/${CROSS_TRIPLE}-gfortran
-RUN apt-get update && \
- apt-get install -y libopenblas-dev:armhf && \
- rm -rf /var/lib/apt/lists/*
+WORKDIR /work/deps
+
+COPY install/ubuntu_arm.sh /work/
+RUN /work/ubuntu_arm.sh
+
+COPY install/arm_openblas.sh /work/
+RUN /work/arm_openblas.sh
+
+ENV OpenBLAS_HOME=${CROSS_ROOT}
+ENV OpenBLAS_DIR=${CROSS_ROOT}
COPY runtime_functions.sh /work/
-WORKDIR /work/build
+WORKDIR /work/mxnet
diff --git a/ci/docker/Dockerfile.build.jetson b/ci/docker/Dockerfile.build.jetson
index c358edb1fb0..5bbc5d4f4be 100755
--- a/ci/docker/Dockerfile.build.jetson
+++ b/ci/docker/Dockerfile.build.jetson
@@ -31,13 +31,16 @@ ENV FC /usr/bin/${CROSS_TRIPLE}-gfortran
ENV HOSTCC gcc
ENV TARGET ARMV8
-WORKDIR /work
+WORKDIR /work/deps
-# Build OpenBLAS
-RUN git clone --recursive -b v0.2.20 https://github.com/xianyi/OpenBLAS.git && \
- cd OpenBLAS && \
- make -j$(nproc) && \
- PREFIX=${CROSS_ROOT} make install
+COPY install/ubuntu_arm.sh /work/
+RUN /work/ubuntu_arm.sh
+
+COPY install/arm_openblas.sh /work/
+RUN /work/arm_openblas.sh
+
+ENV OpenBLAS_HOME=${CROSS_ROOT}
+ENV OpenBLAS_DIR=${CROSS_ROOT}
# Setup CUDA build env (including configuring and copying nvcc)
COPY --from=cudabuilder /usr/local/cuda /usr/local/cuda
diff --git a/ci/docker/install/arm_openblas.sh b/ci/docker/install/arm_openblas.sh
new file mode 100755
index 00000000000..fa2e5cae9cb
--- /dev/null
+++ b/ci/docker/install/arm_openblas.sh
@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -ex
+
+git clone --recursive -b v0.2.20 https://github.com/xianyi/OpenBLAS.git
+
+cd OpenBLAS
+make -j$(nproc)
+PREFIX=${CROSS_ROOT} make install
+
+cd ..
+
+rm -rf OpenBLAS
diff --git a/ci/docker/install/ubuntu_arm.sh b/ci/docker/install/ubuntu_arm.sh
new file mode 100755
index 00000000000..becb012bd18
--- /dev/null
+++ b/ci/docker/install/ubuntu_arm.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -ex
+
+apt update
+apt install -y \
+ unzip
diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index fa9de6112ff..24836d5e5b9 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -31,31 +31,49 @@ clean_repo() {
git submodule update --init --recursive
}
+build_wheel() {
-# Build commands: Every platform in docker/Dockerfile.build.<platform> should have a corresponding
-# function here with the same suffix:
-
-build_jetson() {
set -ex
pushd .
- mv make/crosscompile.jetson.mk make/config.mk
- make -j$(nproc)
- export MXNET_LIBRARY_PATH=`pwd`/libmxnet.so
- cd /work/mxnet/python
+ PYTHON_DIR=${1:-/work/mxnet/python}
+ BUILD_DIR=${2:-/work/build}
+
+ # build
+
+ export MXNET_LIBRARY_PATH=${BUILD_DIR}/libmxnet.so
+
+ cd ${PYTHON_DIR}
python setup.py bdist_wheel --universal
+ # repackage
+
# Fix pathing issues in the wheel. We need to move libmxnet.so from the data folder to the
# mxnet folder, then repackage the wheel.
WHEEL=`readlink -f dist/*.whl`
TMPDIR=`mktemp -d`
- unzip -d $TMPDIR $WHEEL
- rm $WHEEL
- cd $TMPDIR
+ unzip -d ${TMPDIR} ${WHEEL}
+ rm ${WHEEL}
+ cd ${TMPDIR}
mv *.data/data/mxnet/libmxnet.so mxnet
- zip -r $WHEEL .
- cp $WHEEL /work/build
- rm -rf $TMPDIR
+ zip -r ${WHEEL} .
+ cp ${WHEEL} ${BUILD_DIR}
+ rm -rf ${TMPDIR}
+
+ popd
+}
+
+# Build commands: Every platform in docker/Dockerfile.build.<platform> should have a corresponding
+# function here with the same suffix:
+
+build_jetson() {
+ set -ex
+ pushd .
+
+ cp make/crosscompile.jetson.mk ./config.mk
+ make -j$(nproc)
+
+ build_wheel /work/mxnet/python /work/mxnet/lib
popd
}
@@ -72,42 +90,56 @@ build_armv6() {
# We do not need OpenMP, since most armv6 systems have only 1 core
cmake \
- -DCMAKE_TOOLCHAIN_FILE=$CROSS_ROOT/Toolchain.cmake \
+ -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} \
-DUSE_CUDA=OFF \
-DUSE_OPENCV=OFF \
-DUSE_OPENMP=OFF \
-DUSE_SIGNAL_HANDLER=ON \
- -DCMAKE_BUILD_TYPE=Release \
+ -DCMAKE_BUILD_TYPE=Release\
-DUSE_MKL_IF_AVAILABLE=OFF \
-DUSE_LAPACK=OFF \
-DBUILD_CPP_EXAMPLES=OFF \
-Dmxnet_LINKER_LIBS=-lgfortran \
-G Ninja /work/mxnet
+
ninja
- export MXNET_LIBRARY_PATH=`pwd`/libmxnet.so
- cd /work/mxnet/python
- python setup.py bdist_wheel --universal
- cp dist/*.whl /work/build
+ build_wheel
popd
}
build_armv7() {
set -ex
pushd .
+
+ # uncomment for make based build
+ # cp make/crosscompile.armv7.mk ./config.mk
+ # make -j$(nproc)
+
+ # build_wheel /work/mxnet/python /work/mxnet/lib
+
cd /work/build
- cmake\
- -DUSE_CUDA=OFF\
- -DUSE_OPENCV=OFF\
- -DUSE_OPENMP=OFF\
- -DUSE_SIGNAL_HANDLER=ON\
- -DCMAKE_BUILD_TYPE=RelWithDebInfo\
- -DUSE_MKL_IF_AVAILABLE=OFF\
+
+ # Lapack functionality will be included and statically linked to openblas.
+ # But USE_LAPACK needs to be set to OFF, otherwise the main CMakeLists.txt
+ # file tries to add -llapack. Lapack functionality though, requires -lgfortran
+ # to be linked additionally.
+
+ cmake \
+ -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} \
+ -DCMAKE_CROSSCOMPILING=ON \
+ -DUSE_CUDA=OFF \
+ -DUSE_OPENCV=OFF \
+ -DUSE_OPENMP=ON \
+ -DUSE_SIGNAL_HANDLER=ON \
+ -DCMAKE_BUILD_TYPE=Release \
+ -DUSE_MKL_IF_AVAILABLE=OFF \
+ -DUSE_LAPACK=OFF \
+ -DBUILD_CPP_EXAMPLES=OFF \
+ -Dmxnet_LINKER_LIBS=-lgfortran \
-G Ninja /work/mxnet
+
ninja
- export MXNET_LIBRARY_PATH=`pwd`/libmxnet.so
- cd /work/mxnet/python
- python setup.py bdist_wheel --universal
- cp dist/*.whl /work/build
+ build_wheel
popd
}
@@ -133,7 +165,7 @@ build_arm64() {
-DUSE_OPENCV=OFF\
-DUSE_OPENMP=OFF\
-DUSE_SIGNAL_HANDLER=ON\
- -DCMAKE_BUILD_TYPE=RelWithDebInfo\
+ -DCMAKE_BUILD_TYPE=Release\
-DUSE_MKL_IF_AVAILABLE=OFF\
-G Ninja /work/mxnet
ninja
diff --git a/example/image-classification/predict-cpp/CMakeLists.txt b/example/image-classification/predict-cpp/CMakeLists.txt
index a2f52b9df3a..c42d1917b76 100644
--- a/example/image-classification/predict-cpp/CMakeLists.txt
+++ b/example/image-classification/predict-cpp/CMakeLists.txt
@@ -1,7 +1,7 @@
# Check OpenCV
-if(NOT USE_OPENCV OR NOT OpenCV_FOUND)
+if(NOT USE_OPENCV OR NOT OpenCV_FOUND OR OpenCV_VERSION_MAJOR LESS 3)
message(WARNING "\
-OpenCV should be enabled and found to build image classification example, skipping...")
+OpenCV version >= 3 should be enabled and found to build image classification example, skipping...")
return()
endif()
diff --git a/make/crosscompile.armv7.mk b/make/crosscompile.armv7.mk
new file mode 100644
index 00000000000..47402451d8d
--- /dev/null
+++ b/make/crosscompile.armv7.mk
@@ -0,0 +1,198 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+#-------------------------------------------------------------------------------
+# Template configuration for compiling mxnet
+#
+# If you want to change the configuration, please use the following
+# steps. Assume you are on the root directory of mxnet. First copy the this
+# file so that any local changes will be ignored by git
+#
+# $ cp make/config.mk .
+#
+# Next modify the according entries, and then compile by
+#
+# $ make
+#
+# or build in parallel with 8 threads
+#
+# $ make -j8
+#-------------------------------------------------------------------------------
+
+#---------------------
+# For cross compilation we only explictily set a compiler when one is not already present.
+#--------------------
+
+ifndef CC
+export CC = gcc
+endif
+ifndef CXX
+export CXX = g++
+endif
+ifndef NVCC
+export NVCC = nvcc
+endif
+
+# whether compile with options for MXNet developer
+DEV = 0
+
+# whether compile with debug
+DEBUG = 0
+
+# whether to turn on segfault signal handler to log the stack trace
+USE_SIGNAL_HANDLER = 1
+
+# the additional link flags you want to add
+ADD_LDFLAGS =
+
+# the additional compile flags you want to add
+ADD_CFLAGS =
+
+#---------------------------------------------
+# matrix computation libraries for CPU/GPU
+#---------------------------------------------
+
+# whether use CUDA during compile
+USE_CUDA = 0
+
+# add the path to CUDA library to link and compile flag
+# if you have already add them to environment variable, leave it as NONE
+USE_CUDA_PATH = NONE
+
+# whether to enable CUDA runtime compilation
+ENABLE_CUDA_RTC = 0
+
+# whether use CuDNN R3 library
+USE_CUDNN = 0
+
+#whether to use NCCL library
+USE_NCCL = 0
+#add the path to NCCL library
+USE_NCCL_PATH = NONE
+
+# whether use opencv during compilation
+# you can disable it, however, you will not able to use
+# imbin iterator
+USE_OPENCV = 0
+
+#whether use libjpeg-turbo for image decode without OpenCV wrapper
+USE_LIBJPEG_TURBO = 0
+#add the path to libjpeg-turbo library
+USE_LIBJPEG_TURBO_PATH = NONE
+
+# use openmp for parallelization
+USE_OPENMP = 1
+
+# whether use MKL-DNN library
+USE_MKLDNN = 0
+
+# whether use NNPACK library
+USE_NNPACK = 0
+
+# choose the version of blas you want to use
+# can be: mkl, blas, atlas, openblas
+# in default use atlas for linux while apple for osx
+UNAME_S := $(shell uname -s)
+USE_BLAS = openblas
+
+# whether use lapack during compilation
+# only effective when compiled with blas versions openblas/apple/atlas/mkl
+USE_LAPACK = 1
+
+# path to lapack library in case of a non-standard installation
+USE_LAPACK_PATH =
+
+# add path to intel library, you may need it for MKL, if you did not add the path
+# to environment variable
+USE_INTEL_PATH = NONE
+
+# If use MKL only for BLAS, choose static link automatically to allow python wrapper
+ifeq ($(USE_BLAS), mkl)
+USE_STATIC_MKL = 1
+else
+USE_STATIC_MKL = NONE
+endif
+
+#----------------------------
+# Settings for power and arm arch
+#----------------------------
+USE_SSE = 0
+
+# Turn off F16C instruction set support
+USE_F16C = 0
+
+#----------------------------
+# distributed computing
+#----------------------------
+
+# whether or not to enable multi-machine supporting
+USE_DIST_KVSTORE = 0
+
+# whether or not allow to read and write HDFS directly. If yes, then hadoop is
+# required
+USE_HDFS = 0
+
+# path to libjvm.so. required if USE_HDFS=1
+LIBJVM = NONE
+
+# whether or not allow to read and write AWS S3 directly. If yes, then
+# libcurl4-openssl-dev is required, it can be installed on Ubuntu by
+# sudo apt-get install -y libcurl4-openssl-dev
+USE_S3 = 0
+
+#----------------------------
+# performance settings
+#----------------------------
+# Use operator tuning
+USE_OPERATOR_TUNING = 1
+
+# Use gperftools if found
+USE_GPERFTOOLS = 1
+
+# Use JEMalloc if found, and not using gperftools
+USE_JEMALLOC = 1
+
+#----------------------------
+# additional operators
+#----------------------------
+
+# path to folders containing projects specific operators that you don't want to put in src/operators
+EXTRA_OPERATORS =
+
+#----------------------------
+# other features
+#----------------------------
+
+# Create C++ interface package
+USE_CPP_PACKAGE = 0
+
+#----------------------------
+# plugins
+#----------------------------
+
+# whether to use caffe integration. This requires installing caffe.
+# You also need to add CAFFE_PATH/build/lib to your LD_LIBRARY_PATH
+# CAFFE_PATH = $(HOME)/caffe
+# MXNET_PLUGINS += plugin/caffe/caffe.mk
+
+# WARPCTC_PATH = $(HOME)/warp-ctc
+# MXNET_PLUGINS += plugin/warpctc/warpctc.mk
+
+# whether to use sframe integration. This requires build sframe
+# git@github.com:dato-code/SFrame.git
+# SFRAME_PATH = $(HOME)/SFrame
+# MXNET_PLUGINS += plugin/sframe/plugin.mk
diff --git a/tests/python/unittest/test_ndarray.py b/tests/python/unittest/test_ndarray.py
index a0604658ee1..7c50a424b40 100644
--- a/tests/python/unittest/test_ndarray.py
+++ b/tests/python/unittest/test_ndarray.py
@@ -711,9 +711,8 @@ def get_values(ensure_unique):
k=dat_size*dat_size*dat_size*dat_size, is_ascend=False)
assert_almost_equal(nd_ret_argsort, gt)
- # test topk with a big shape
- a = mx.nd.arange(0, 54686454, step=1, repeat=1)
- assert_almost_equal(a.topk(k=54686454).asnumpy(), a.asnumpy()[::-1])
+ a = mx.nd.arange(0, 1024, step=1, repeat=1)
+ assert_almost_equal(a.topk(k=1024).asnumpy(), a.asnumpy()[::-1])
# Repeat those tests that don't involve indices. These should pass even with
# duplicated input data values (over many repeated runs with different random seeds,
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services