You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by cj...@apache.org on 2018/01/17 17:19:40 UTC
[incubator-mxnet] branch master updated: Add CMake CUDA build job
to the CI. (#9329)
This is an automated email from the ASF dual-hosted git repository.
cjolivier01 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
The following commit(s) were added to refs/heads/master by this push:
new b023bd5 Add CMake CUDA build job to the CI. (#9329)
b023bd5 is described below
commit b023bd5855c91cf7c05a06c7ca4bb58799e206d9
Author: Kellen Sunderland <ke...@gmail.com>
AuthorDate: Wed Jan 17 18:19:28 2018 +0100
Add CMake CUDA build job to the CI. (#9329)
---
Jenkinsfile | 42 ++++++++++++++++++++++++++++++++++++++++--
tests/ci_build/ci_build.sh | 21 ++++++++++++++++++---
2 files changed, 58 insertions(+), 5 deletions(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index b7a8f60..2bffdd4 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -4,6 +4,8 @@
// mxnet libraries
mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, dmlc-core/libdmlc.a, nnvm/lib/libnnvm.a'
+// mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default.
+mx_cmake_lib = 'build/libmxnet.so, build/libmxnet.a, build/dmlc-core/libdmlc.a'
// command to start a docker container
docker_run = 'tests/ci_build/ci_build.sh'
// timeout in minutes
@@ -50,7 +52,7 @@ def init_git_win() {
}
}
-// Run make. First try to do an incremental make from a previous workspace in hope to
+// Run make. First try to do an incremental build from a previous workspace in hope to
// accelerate the compilation. If something wrong, clean the workspace and then
// build from scratch.
def make(docker_type, make_flag) {
@@ -66,6 +68,26 @@ def make(docker_type, make_flag) {
}
}
+// Run cmake. First try to do an incremental build from a previous workspace in hope to
+// accelerate the compilation. If something wrong, clean the workspace and then
+// build from scratch.
+def cmake(docker_type, cmake_defines, make_flags) {
+ timeout(time: max_time, unit: 'MINUTES') {
+ try {
+ sh "${docker_run} ${docker_type} --dockerbinary docker mkdir build"
+ sh "WORKDIR=/workspace/build ${docker_run} ${docker_type} --dockerbinary docker cmake ${cmake_defines} .."
+ sh "WORKDIR=/workspace/build ${docker_run} ${docker_type} --dockerbinary docker make ${make_flags}"
+ } catch (exc) {
+ echo 'Incremental compilation failed with ${exc}. Fall back to build from scratch'
+ sh "${docker_run} ${docker_type} --dockerbinary docker sudo make clean"
+ sh "${docker_run} ${docker_type} --dockerbinary docker sudo make -C amalgamation/ clean"
+ sh "${docker_run} ${docker_type} --dockerbinary docker mkdir build"
+ sh "WORKDIR=/workspace/build ${docker_run} ${docker_type} --dockerbinary docker cmake ${cmake_defines} .."
+ sh "WORKDIR=/workspace/build ${docker_run} ${docker_type} --dockerbinary docker make ${make_flags}"
+ }
+ }
+}
+
// pack libraries for later use
def pack_lib(name, libs=mx_lib) {
sh """
@@ -75,7 +97,6 @@ echo ${libs} | sed -e 's/,/ /g' | xargs md5sum
stash includes: libs, name: name
}
-
// unpack libraries saved before
def unpack_lib(name, libs=mx_lib) {
unstash name
@@ -222,6 +243,23 @@ try {
}
}
},
+ 'GPU: CMake': {
+ node('mxnetlinux-cpu') {
+ ws('workspace/build-cmake-gpu') {
+ init_git()
+ def defines = """ \
+ -DUSE_CUDA=1 \
+ -DUSE_CUDNN=1 \
+ -DCMAKE_BUILD_TYPE=Release \
+ """
+ def flag = """ \
+ -j\$(nproc)
+ """
+ cmake("build_cuda", defines, flag)
+ pack_lib('cmake_gpu', mx_cmake_lib)
+ }
+ }
+ },
'GPU: MKLML': {
node('mxnetlinux-cpu') {
ws('workspace/build-mklml-gpu') {
diff --git a/tests/ci_build/ci_build.sh b/tests/ci_build/ci_build.sh
index 118f618..794a4c5 100755
--- a/tests/ci_build/ci_build.sh
+++ b/tests/ci_build/ci_build.sh
@@ -101,6 +101,11 @@ function upsearch () {
# reasonable defaults if you run it outside of Jenkins.
WORKSPACE="${WORKSPACE:-${SCRIPT_DIR}/../../}"
+# Set up a default WORKDIR, unless otherwise specified.
+if [[ -z "${WORKDIR}" ]]; then
+ WORKDIR="/workspace"
+fi
+
# Determine the docker image name
DOCKER_IMG_NAME="mx-ci.${CONTAINER_TYPE}"
@@ -111,16 +116,21 @@ DOCKER_IMG_NAME=$(echo "${DOCKER_IMG_NAME}" | sed -e 's/=/_/g' -e 's/,/-/g')
# Convert to all lower-case, as per requirement of Docker image names
DOCKER_IMG_NAME=$(echo "${DOCKER_IMG_NAME}" | tr '[:upper:]' '[:lower:]')
-# skip with_the_same_user for non-linux
+# Skip with_the_same_user for non-linux
uname=`uname`
if [[ "$uname" == "Linux" ]]; then
- PRE_COMMAND="tests/ci_build/with_the_same_user"
+ # By convention the root of our source dir is always mapped to the /workspace folder
+ # inside the docker container. Our working directory when we start the container
+ # is variable, so we should ensure we call commands such as with_the_same_user
+ # with their absolute paths.
+ PRE_COMMAND="/workspace/tests/ci_build/with_the_same_user"
else
PRE_COMMAND=""
fi
# Print arguments.
echo "WORKSPACE: ${WORKSPACE}"
+echo "WORKDIR: ${WORKDIR}"
echo "CI_DOCKER_EXTRA_PARAMS: ${CI_DOCKER_EXTRA_PARAMS[@]}"
echo "COMMAND: ${COMMAND[@]}"
echo "CONTAINER_TYPE: ${CONTAINER_TYPE}"
@@ -153,9 +163,14 @@ echo "Running '${COMMAND[@]}' inside ${DOCKER_IMG_NAME}..."
# Turning off MXNET_STORAGE_FALLBACK_LOG_WARNING temporarily per this issue:
# https://github.com/apache/incubator-mxnet/issues/8980
+# By convention always map the root of the MXNet source directory into /workspace.
+# ${WORKSPACE} represents the path to the source folder on the host system.
+# ${WORKDIR} is the working directory we start the container in. By default this
+# is /workspace, but for example when running cmake it is sometimes /workspace/build.
+
${DOCKER_BINARY} run --rm --pid=host \
-v ${WORKSPACE}:/workspace \
- -w /workspace \
+ -w ${WORKDIR} \
-e "CI_BUILD_HOME=${WORKSPACE}" \
-e "CI_BUILD_USER=$(id -u -n)" \
-e "CI_BUILD_UID=$(id -u)" \
--
To stop receiving notification emails like this one, please contact
['"commits@mxnet.apache.org" <co...@mxnet.apache.org>'].