[GitHub] cjolivier01 closed pull request #9329: Add CMake CI build job

[GitBox <gi...@apache.org>]

cjolivier01 closed pull request #9329: Add CMake CI build job
URL: https://github.com/apache/incubator-mxnet/pull/9329
 
 
   

This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:

As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):

diff --git a/Jenkinsfile b/Jenkinsfile
index b7a8f60cb9..2bffdd4e5a 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -4,6 +4,8 @@
 
 // mxnet libraries
 mx_lib = 'lib/libmxnet.so, lib/libmxnet.a, dmlc-core/libdmlc.a, nnvm/lib/libnnvm.a'
+// mxnet cmake libraries, in cmake builds we do not produce a libnvvm static library by default.
+mx_cmake_lib = 'build/libmxnet.so, build/libmxnet.a, build/dmlc-core/libdmlc.a'
 // command to start a docker container
 docker_run = 'tests/ci_build/ci_build.sh'
 // timeout in minutes
@@ -50,7 +52,7 @@ def init_git_win() {
   }
 }
 
-// Run make. First try to do an incremental make from a previous workspace in hope to
+// Run make. First try to do an incremental build from a previous workspace in hope to
 // accelerate the compilation. If something wrong, clean the workspace and then
 // build from scratch.
 def make(docker_type, make_flag) {
@@ -66,6 +68,26 @@ def make(docker_type, make_flag) {
   }
 }
 
+// Run cmake. First try to do an incremental build from a previous workspace in hope to
+// accelerate the compilation. If something wrong, clean the workspace and then
+// build from scratch.
+def cmake(docker_type, cmake_defines, make_flags) {
+  timeout(time: max_time, unit: 'MINUTES') {
+    try {
+      sh "${docker_run} ${docker_type} --dockerbinary docker mkdir build"
+      sh "WORKDIR=/workspace/build ${docker_run} ${docker_type} --dockerbinary docker cmake ${cmake_defines} .."
+      sh "WORKDIR=/workspace/build ${docker_run} ${docker_type} --dockerbinary docker make ${make_flags}"
+    } catch (exc) {
+      echo 'Incremental compilation failed with ${exc}. Fall back to build from scratch'
+      sh "${docker_run} ${docker_type} --dockerbinary docker sudo make clean"
+      sh "${docker_run} ${docker_type} --dockerbinary docker sudo make -C amalgamation/ clean"
+      sh "${docker_run} ${docker_type} --dockerbinary docker mkdir build"
+      sh "WORKDIR=/workspace/build ${docker_run} ${docker_type} --dockerbinary docker cmake ${cmake_defines} .."
+      sh "WORKDIR=/workspace/build ${docker_run} ${docker_type} --dockerbinary docker make ${make_flags}"
+    }
+  }
+}
+
 // pack libraries for later use
 def pack_lib(name, libs=mx_lib) {
   sh """
@@ -75,7 +97,6 @@ echo ${libs} | sed -e 's/,/ /g' | xargs md5sum
   stash includes: libs, name: name
 }
 
-
 // unpack libraries saved before
 def unpack_lib(name, libs=mx_lib) {
   unstash name
@@ -222,6 +243,23 @@ try {
         }
       }
     },
+    'GPU: CMake': {
+      node('mxnetlinux-cpu') {
+        ws('workspace/build-cmake-gpu') {
+          init_git()
+          def defines = """            \
+            -DUSE_CUDA=1               \
+            -DUSE_CUDNN=1              \
+            -DCMAKE_BUILD_TYPE=Release \
+            """
+            def flag = """             \
+            -j\$(nproc)
+            """
+          cmake("build_cuda", defines, flag)
+          pack_lib('cmake_gpu', mx_cmake_lib)
+        }
+      }
+    },
     'GPU: MKLML': {
       node('mxnetlinux-cpu') {
         ws('workspace/build-mklml-gpu') {
diff --git a/tests/ci_build/ci_build.sh b/tests/ci_build/ci_build.sh
index 118f618fa7..794a4c55ee 100755
--- a/tests/ci_build/ci_build.sh
+++ b/tests/ci_build/ci_build.sh
@@ -101,6 +101,11 @@ function upsearch () {
 # reasonable defaults if you run it outside of Jenkins.
 WORKSPACE="${WORKSPACE:-${SCRIPT_DIR}/../../}"
 
+# Set up a default WORKDIR, unless otherwise specified.
+if [[ -z "${WORKDIR}" ]]; then
+    WORKDIR="/workspace"
+fi
+
 # Determine the docker image name
 DOCKER_IMG_NAME="mx-ci.${CONTAINER_TYPE}"
 
@@ -111,16 +116,21 @@ DOCKER_IMG_NAME=$(echo "${DOCKER_IMG_NAME}" | sed -e 's/=/_/g' -e 's/,/-/g')
 # Convert to all lower-case, as per requirement of Docker image names
 DOCKER_IMG_NAME=$(echo "${DOCKER_IMG_NAME}" | tr '[:upper:]' '[:lower:]')
 
-# skip with_the_same_user for non-linux
+# Skip with_the_same_user for non-linux
 uname=`uname`
 if [[ "$uname" == "Linux" ]]; then
-    PRE_COMMAND="tests/ci_build/with_the_same_user"
+    # By convention the root of our source dir is always mapped to the /workspace folder
+    # inside the docker container.  Our working directory when we start the container
+    # is variable, so we should ensure we call commands such as with_the_same_user
+    # with their absolute paths.
+    PRE_COMMAND="/workspace/tests/ci_build/with_the_same_user"
 else
     PRE_COMMAND=""
 fi
 
 # Print arguments.
 echo "WORKSPACE: ${WORKSPACE}"
+echo "WORKDIR: ${WORKDIR}"
 echo "CI_DOCKER_EXTRA_PARAMS: ${CI_DOCKER_EXTRA_PARAMS[@]}"
 echo "COMMAND: ${COMMAND[@]}"
 echo "CONTAINER_TYPE: ${CONTAINER_TYPE}"
@@ -153,9 +163,14 @@ echo "Running '${COMMAND[@]}' inside ${DOCKER_IMG_NAME}..."
 # Turning off MXNET_STORAGE_FALLBACK_LOG_WARNING temporarily per this issue:
 # https://github.com/apache/incubator-mxnet/issues/8980
 
+# By convention always map the root of the MXNet source directory into /workspace.
+# ${WORKSPACE} represents the path to the source folder on the host system.
+# ${WORKDIR} is the working directory we start the container in.  By default this
+# is /workspace, but for example when running cmake it is sometimes /workspace/build.
+
 ${DOCKER_BINARY} run --rm --pid=host \
     -v ${WORKSPACE}:/workspace \
-    -w /workspace \
+    -w ${WORKDIR} \
     -e "CI_BUILD_HOME=${WORKSPACE}" \
     -e "CI_BUILD_USER=$(id -u -n)" \
     -e "CI_BUILD_UID=$(id -u)" \


 

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services