You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by ib...@apache.org on 2019/09/09 05:04:36 UTC
[incubator-mxnet] branch ib/jl-cmake-build created (now 82a8b6e)
This is an automated email from the ASF dual-hosted git repository.
iblis pushed a change to branch ib/jl-cmake-build
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git.
at 82a8b6e julia: migrate build process to cmake
This branch includes the following new commits:
new 82a8b6e julia: migrate build process to cmake
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[incubator-mxnet] 01/01: julia: migrate build process to cmake
Posted by ib...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
iblis pushed a commit to branch ib/jl-cmake-build
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
commit 82a8b6e9796d352beabc19351d729edcf9219990
Author: Iblis Lin <ib...@hs.ntnu.edu.tw>
AuthorDate: Mon Sep 9 05:01:02 2019 +0000
julia: migrate build process to cmake
The original build process triggered by `Pkg.add` is
done by GNU make.
---
julia/Project.toml | 1 +
julia/deps/build.jl | 141 ++++++++++++++++------------------------------------
julia/src/base.jl | 19 +++----
3 files changed, 53 insertions(+), 108 deletions(-)
diff --git a/julia/Project.toml b/julia/Project.toml
index 994a696..ca41bb7 100644
--- a/julia/Project.toml
+++ b/julia/Project.toml
@@ -5,6 +5,7 @@ version = "1.6.0"
[deps]
BinDeps = "9e28174c-4ba2-5203-b857-d8d62c4213ee"
+CMake = "631607c0-34d2-5d66-819e-eb0f9aa2061a"
Formatting = "59287772-0a20-5a39-b81b-1366585eb4c0"
JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
diff --git a/julia/deps/build.jl b/julia/deps/build.jl
index dfd3f78..96b1634 100644
--- a/julia/deps/build.jl
+++ b/julia/deps/build.jl
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+using CMake
using JSON
using Libdl
using LinearAlgebra
@@ -75,7 +76,7 @@ if Sys.isunix()
nvcc_path = Sys.which("nvcc")
if nvcc_path ≢ nothing
@info "Found nvcc: $nvcc_path"
- push!(CUDAPATHS, replace(nvcc_path, "bin/nvcc", "lib64"))
+ push!(CUDAPATHS, replace(nvcc_path, "bin/nvcc" => "lib64"))
end
end
@@ -114,19 +115,17 @@ else
end
# propagate more build flags from ENV
-const CC = get(ENV, "CC", nothing)
-const CXX = get(ENV, "CXX", nothing)
-const ADD_CFLAGS = get(ENV, "ADD_CFLAGS", nothing)
-const ADD_LDFLAGS = get(ENV, "ADD_LDFLAGS", nothing)
-const USE_JEMALLOC = get(ENV, "USE_JEMALLOC", nothing) # "0" or "1"
-
-function get_cpucore()
- if haskey(ENV, "TRAVIS") # on travis-ci
- 2
- else
- min(Sys.CPU_THREADS, 32)
- end
-end
+const USE_JEMALLOC = get(ENV, "USE_JEMALLOC", nothing) # "ON" or "OFF"
+
+get_cpucore() = min(Sys.CPU_THREADS, 32)
+
+cmake_bool(x::Bool) = ifelse(x ≡ true, "ON", "OFF")
+
+cmake_jemalloc(::Nothing) = ""
+cmake_jemalloc(x::Bool) = "-DUSE_JEMALLOC=" * cmake_bool(x)
+
+cmake_cuda_path(::Nothing) = ""
+cmake_cuda_path(x::String) = "-DUSE_CUDA_PATH=" * x
using BinDeps
@BinDeps.setup
@@ -167,7 +166,7 @@ if !libmxnet_detected
# testing
run(`cmd /c dir "usr\\lib"`)
return
- end
+ end # if Sys.iswindows()
################################################################################
# If not found, try to build automatically using BinDeps
@@ -181,7 +180,6 @@ if !libmxnet_detected
ilp64 = "-DINTERFACE64"
end
- FORCE_LAPACK = false
if blas_vendor == :unknown
@info("Julia is built with an unkown blas library ($blas_path).")
@info("Attempting build without reusing the blas library")
@@ -191,33 +189,34 @@ if !libmxnet_detected
@info("Attempting build anyway.")
USE_JULIA_BLAS = true
else
- USE_JULIA_BLAS = true
- FORCE_LAPACK = true
+ USE_JULIA_BLAS = false
end
@info("USE_JULIA_BLAS -> $USE_JULIA_BLAS")
- blas_name = blas_vendor == :openblas64 ? "openblas" : string(blas_vendor)
- MSHADOW_LDFLAGS = "MSHADOW_LDFLAGS=-lm $blas_path"
+ blas_name = occursin("openblas", string(blas_vendor)) ? "open" : string(blas_vendor)
#--------------------------------------------------------------------------------
# Build libmxnet
mxnet = library_dependency("mxnet", aliases=["mxnet", "libmxnet", "libmxnet.so"])
- _prefix = joinpath(BinDeps.depsdir(mxnet), "usr")
+ _blddir = joinpath(BinDeps.depsdir(mxnet), "build")
_srcdir = joinpath(BinDeps.depsdir(mxnet), "src")
_mxdir = joinpath(_srcdir, "mxnet")
- _libdir = joinpath(_prefix, "lib")
- # We have do eagerly delete the installed libmxnet.so
+
+ # We have do eagerly delete the build stuffs.
# Otherwise we won't rebuild on an update.
- run(`rm -f $_libdir/libmxnet.$(Libdl.dlext)`)
+ rm(_blddir, recursive=true, force=true)
+
+ @debug "build dir -> $_blddir"
+
provides(BuildProcess,
(@build_steps begin
+ CreateDirectory(_blddir)
CreateDirectory(_srcdir)
- CreateDirectory(_libdir)
@build_steps begin
BinDeps.DirectoryRule(_mxdir, @build_steps begin
ChangeDirectory(_srcdir)
- `git clone https://github.com/apache/incubator-mxnet mxnet`
+ `git clone --recursive https://github.com/apache/incubator-mxnet mxnet`
end)
@build_steps begin
ChangeDirectory(_mxdir)
@@ -227,80 +226,28 @@ if !libmxnet_detected
else
`git checkout origin/$libmxnet_curr_ver`
end
- `git submodule update --init --recursive`
- `git -C 3rdparty/mshadow checkout -- make/mshadow.mk`
- `cp -v ../../cblas.h include/cblas.h`
- `sed -i -s "s/MSHADOW_CFLAGS = \(.*\)/MSHADOW_CFLAGS = \1 $ilp64/" 3rdparty/mshadow/make/mshadow.mk`
-
- # Copy config.mk, always override the file
- if Sys.isapple()
- `cp make/osx.mk config.mk`
- else
- `cp make/config.mk config.mk`
- end
-
- # Configure OpenCV
- `sed -i -s 's/USE_OPENCV = 1/USE_OPENCV = 0/' config.mk`
-
- # Configure CUDA
- if HAS_CUDA
- @build_steps begin
- `sed -i -s 's/USE_CUDA = 0/USE_CUDA = 1/' config.mk`
- # address https://github.com/apache/incubator-mxnet/pull/7856
- `sed -i -s "s/ADD_LDFLAGS =\(.*\)/ADD_LDFLAGS =\1 -lcublas -lcusolver -lcurand -lcudart/" config.mk`
- if haskey(ENV, "CUDA_HOME")
- `sed -i -s "s@USE_CUDA_PATH = NONE@USE_CUDA_PATH = $(ENV["CUDA_HOME"])@" config.mk`
- end
- if haskey(ENV, "CUDA_HOME")
- # address https://github.com/apache/incubator-mxnet/pull/7838
- flag = "-L$(ENV["CUDA_HOME"])/lib64 -L$(ENV["CUDA_HOME"])/lib"
- `sed -i -s "s@ADD_LDFLAGS =\(.*\)@ADD_LDFLAGS =\1 $flag@" config.mk`
- end
- if HAS_CUDNN
- `sed -i -s 's/USE_CUDNN = 0/USE_CUDNN = 1/' config.mk`
- end
- end
- end
-
- # Force enable LAPACK build
- # Julia's OpenBLAS has LAPACK functionality already
- if FORCE_LAPACK
- if Sys.isapple()
- MSHADOW_LDFLAGS *= " -framework Accelerate"
- end
- `sed -i -s 's/ADD_CFLAGS =\(.*\)/ADD_CFLAGS =\1 -DMXNET_USE_LAPACK/' config.mk`
- end
-
- # propagate more build flags from ENV
- if CC != nothing
- `sed -i -s "s@^export CC =\(.*\)@export CC = $CC@" config.mk`
- end
- if CXX != nothing
- `sed -i -s "s@^export CXX =\(.*\)@export CXX = $CXX@" config.mk`
- end
- if ADD_CFLAGS != nothing
- `sed -i -s "s@ADD_CFLAGS =\(.*\)@ADD_CFLAGS =\1 $ADD_CFLAGS@" config.mk`
- end
- if ADD_LDFLAGS != nothing
- `sed -i -s "s@ADD_LDFLAGS =\(.*\)@ADD_LDFLAGS =\1 $ADD_LDFLAGS@" config.mk`
- end
- if USE_JEMALLOC != nothing
- `sed -i -s "s@USE_JEMALLOC =\(.*\)@USE_JEMALLOC = $USE_JEMALLOC@" config.mk`
- end
-
- if USE_JULIA_BLAS
- `make -j$(get_cpucore()) USE_BLAS=$blas_name $MSHADOW_LDFLAGS`
- else
- `make -j$(get_cpucore())`
- end
+ `cp -f -v ../../cblas.h include/cblas.h`
+ end
+ @build_steps begin
+ ChangeDirectory(_blddir)
+ `$cmake
+ -DCMAKE_BUILD_TYPE=$(libmxnet_curr_ver == "master" ? "Debug" : "Release")
+ -DUSE_BLAS=$blas_name
+ -DUSE_OPENCV=$(cmake_bool(false))
+ -DUSE_CUDA=$(cmake_bool(HAS_CUDA))
+ -DUSE_CUDNN=$(cmake_bool(HAS_CUDNN))
+ $(cmake_jemalloc(USE_JEMALLOC))
+ $(cmake_cuda_path(get(ENV, "CUDA_HOME", nothing)))
+ $_mxdir`
+ `make -j$(get_cpucore()) VERBOSE=$(Int(libmxnet_curr_ver == "master"))`
end
- FileRule(joinpath(_libdir, "libmxnet.$(Libdl.dlext)"), @build_steps begin
- # the output file on macos is still in `.so` suffix
- # so we rename it
- `cp $_mxdir/lib/libmxnet.so $_libdir/libmxnet.$(Libdl.dlext)`
+ FileRule(joinpath(_blddir, "libmxnet.$(Libdl.dlext)"), @build_steps begin
+ # the output file on macos is still in `.so` suffix,
+ # so we create a soft link for it.
+ `ln -s libmxnet.so $_blddir/libmxnet.$(Libdl.dlext)`
end)
end
- end), mxnet, installed_libpath=_libdir)
+ end), mxnet, installed_libpath=_blddir)
@BinDeps.install Dict(:mxnet => :mxnet)
end
diff --git a/julia/src/base.jl b/julia/src/base.jl
index 7a2bde0..3460c9d 100644
--- a/julia/src/base.jl
+++ b/julia/src/base.jl
@@ -58,11 +58,11 @@ function _get_search_names()
end
function _get_search_dirs()
- # TODO: remove MXNET_HOME backward compatibility in v1.7
+ # TODO: remove MXNET_HOME backward compatibility in v2.0
if haskey(ENV, "MXNET_HOME")
@warn "The environment variable `MXNET_HOME` has been renamed, please use `MXNET_ROOT` instead."
end
- A = [joinpath(@__DIR__, "..", "deps", "usr", "lib")]
+ A = [joinpath(@__DIR__, "..", "deps", "build")]
MXNET_ROOT = get(ENV, "MXNET_ROOT", get(ENV, "MXNET_HOME", ""))
if !isempty(MXNET_ROOT)
!isabspath(MXNET_ROOT) && error("MXNET_ROOT should be a absolute path")
@@ -71,19 +71,16 @@ function _get_search_dirs()
A
end
-const MXNET_LIB = Libdl.find_library(_get_search_names(), _get_search_dirs())
-const LIB_VERSION = Ref{Cint}(0)
+const MXNET_LIB = Libdl.find_library(_get_search_names(), _get_search_dirs())
+const LIB_VERSION = Ref{Cint}(C_NULL)
if isempty(MXNET_LIB)
- # touch this file, so that after the user properly build libmxnet, the precompiled
- # MXNet.ji will be re-compiled to get MXNET_LIB properly.
- touch(@__FILE__)
error("Cannot find or load libmxnet.$(Libdl.dlext). " *
"Please see the document on how to build it.")
-else
- include_dependency(MXNET_LIB)
end
+include_dependency(MXNET_LIB)
+
function __init__()
# TODO: bug in nnvm, if do not call this, call get handle "_copyto" will fail
_get_libmx_op_names()
@@ -92,12 +89,12 @@ function __init__()
atexit() do
# notify libmxnet we are shutting down
- ccall( ("MXNotifyShutdown", MXNET_LIB), Cint, () )
+ ccall(("MXNotifyShutdown", MXNET_LIB), Cint, ())
end
end
function mx_get_last_error()
- msg = ccall( ("MXGetLastError", MXNET_LIB), char_p, () )
+ msg = ccall(("MXGetLastError", MXNET_LIB), char_p, ())
if msg == C_NULL
throw(MXError("Failed to get last error message"))
end