You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by ma...@apache.org on 2018/02/09 01:03:23 UTC

[incubator-mxnet] branch master updated: CI environment refinements (#9741)

This is an automated email from the ASF dual-hosted git repository.

marcoabreu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git


The following commit(s) were added to refs/heads/master by this push:
     new 3f42582  CI environment refinements (#9741)
3f42582 is described below

commit 3f42582405662b903680dc4d26f33b69ddf0e98d
Author: Pedro Larroy <92...@users.noreply.github.com>
AuthorDate: Fri Feb 9 02:03:18 2018 +0100

    CI environment refinements (#9741)
    
    * Remove uneeded gtest dependency, build with verbosely with CMake & ninja.
    
    * Add set -x and set -e to install scripts to fail on exec and show commands
---
 Jenkinsfile                                     | 23 ++++++++++-------------
 tests/ci_build/install/install_julia.sh         |  3 +--
 tests/ci_build/install/install_library.sh       |  2 ++
 tests/ci_build/install/install_maven.sh         |  3 +--
 tests/ci_build/install/install_openblas.sh      |  3 +--
 tests/ci_build/install/install_opencv.sh        |  3 +--
 tests/ci_build/install/install_python2.sh       |  3 +--
 tests/ci_build/install/install_python3.sh       |  3 +--
 tests/ci_build/install/install_testdeps.sh      | 16 +---------------
 tests/ci_build/install/ubuntu_install_core.sh   |  6 +++---
 tests/ci_build/install/ubuntu_install_nvidia.sh |  2 ++
 tests/ci_build/install/ubuntu_install_perl.sh   |  2 ++
 tests/ci_build/install/ubuntu_install_python.sh |  3 ++-
 tests/ci_build/install/ubuntu_install_r.sh      |  2 ++
 tests/ci_build/install/ubuntu_install_scala.sh  |  2 ++
 15 files changed, 32 insertions(+), 44 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 8e5ee2c..b741d9a 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -89,19 +89,18 @@ def make(docker_type, make_flag) {
 // Run cmake. First try to do an incremental build from a previous workspace in hope to
 // accelerate the compilation. If something wrong, clean the workspace and then
 // build from scratch.
-def cmake(docker_type, cmake_defines, make_flags) {
+def cmake(docker_type, cmake_defines, flags) {
   timeout(time: max_time, unit: 'MINUTES') {
     try {
-      sh "${docker_run} ${docker_type} --dockerbinary docker mkdir build"
-      sh "WORKDIR=/workspace/build ${docker_run} ${docker_type} --dockerbinary docker cmake ${cmake_defines} .."
-      sh "WORKDIR=/workspace/build ${docker_run} ${docker_type} --dockerbinary docker make ${make_flags}"
+      sh "${docker_run} ${docker_type} --dockerbinary docker mkdir -p build"
+      sh "WORKDIR=/workspace/build ${docker_run} ${docker_type} --dockerbinary docker cmake ${cmake_defines} -G Ninja .."
+      sh "WORKDIR=/workspace/build ${docker_run} ${docker_type} --dockerbinary docker ninja ${flags}"
     } catch (exc) {
       echo 'Incremental compilation failed with ${exc}. Fall back to build from scratch'
-      sh "${docker_run} ${docker_type} --dockerbinary docker sudo make clean"
-      sh "${docker_run} ${docker_type} --dockerbinary docker sudo make -C amalgamation/ clean"
-      sh "${docker_run} ${docker_type} --dockerbinary docker mkdir build"
-      sh "WORKDIR=/workspace/build ${docker_run} ${docker_type} --dockerbinary docker cmake ${cmake_defines} .."
-      sh "WORKDIR=/workspace/build ${docker_run} ${docker_type} --dockerbinary docker make ${make_flags}"
+      sh "${docker_run} ${docker_type} --dockerbinary docker git clean -fdx"
+      sh "${docker_run} ${docker_type} --dockerbinary docker mkdir -p build"
+      sh "WORKDIR=/workspace/build ${docker_run} ${docker_type} --dockerbinary docker cmake ${cmake_defines} -G Ninja .."
+      sh "WORKDIR=/workspace/build ${docker_run} ${docker_type} --dockerbinary docker ninja ${flags}"
     }
   }
 }
@@ -270,10 +269,8 @@ try {
             -DUSE_CUDNN=1              \
             -DCMAKE_BUILD_TYPE=Release \
             """
-            def flag = """             \
-            -j\$(nproc)
-            """
-          cmake("build_cuda", defines, flag)
+            def flag = "-v"
+            cmake("build_cuda", defines, flag)
           pack_lib('cmake_gpu', mx_cmake_lib)
         }
       }
diff --git a/tests/ci_build/install/install_julia.sh b/tests/ci_build/install/install_julia.sh
index 5007c94..e36dfb5 100644
--- a/tests/ci_build/install/install_julia.sh
+++ b/tests/ci_build/install/install_julia.sh
@@ -16,9 +16,8 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-
-
 set -e
+set -x
 
 wget https://julialang.s3.amazonaws.com/bin/linux/x64/0.5/julia-0.5.0-linux-x86_64.tar.gz
 mv julia-0.5.0-linux-x86_64.tar.gz /tmp/
diff --git a/tests/ci_build/install/install_library.sh b/tests/ci_build/install/install_library.sh
index 399f6a0..f3d7868 100644
--- a/tests/ci_build/install/install_library.sh
+++ b/tests/ci_build/install/install_library.sh
@@ -16,6 +16,8 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
+set -e
+set -x
 
 
 yum install graphviz
diff --git a/tests/ci_build/install/install_maven.sh b/tests/ci_build/install/install_maven.sh
index 666ebde..f124186 100644
--- a/tests/ci_build/install/install_maven.sh
+++ b/tests/ci_build/install/install_maven.sh
@@ -16,9 +16,8 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-
-
 set -e
+set -x
 
 wget http://mirrors.ocf.berkeley.edu/apache/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz
 mv apache-maven-3.3.9-bin.tar.gz /tmp/
diff --git a/tests/ci_build/install/install_openblas.sh b/tests/ci_build/install/install_openblas.sh
index 2ec5eeb..5c92823 100644
--- a/tests/ci_build/install/install_openblas.sh
+++ b/tests/ci_build/install/install_openblas.sh
@@ -16,9 +16,8 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-
-
 set -e
+set -x
 
 git clone https://github.com/xianyi/OpenBLAS
 cd OpenBLAS
diff --git a/tests/ci_build/install/install_opencv.sh b/tests/ci_build/install/install_opencv.sh
index 08a4d9b..9364a56 100644
--- a/tests/ci_build/install/install_opencv.sh
+++ b/tests/ci_build/install/install_opencv.sh
@@ -16,9 +16,8 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-
-
 set -e
+set -x
 
 yum groupinstall -y "Development Tools"
 yum install -y cmake python27 python27-setuptools
diff --git a/tests/ci_build/install/install_python2.sh b/tests/ci_build/install/install_python2.sh
index ec4bbb9..5c7c453 100644
--- a/tests/ci_build/install/install_python2.sh
+++ b/tests/ci_build/install/install_python2.sh
@@ -16,9 +16,8 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-
-
 set -e
+set -x
 
 yum groupinstall -y "Development Tools"
 yum install -y mlocate python27 python27-setuptools python27-tools python27-numpy python27-scipy python27-nose python27-matplotlib unzip
diff --git a/tests/ci_build/install/install_python3.sh b/tests/ci_build/install/install_python3.sh
index ee89161..0df12eb 100644
--- a/tests/ci_build/install/install_python3.sh
+++ b/tests/ci_build/install/install_python3.sh
@@ -16,9 +16,8 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-
-
 set -e
+set -x
 
 wget https://bootstrap.pypa.io/get-pip.py || exit 1
 mkdir py3
diff --git a/tests/ci_build/install/install_testdeps.sh b/tests/ci_build/install/install_testdeps.sh
index c777348..26da186 100644
--- a/tests/ci_build/install/install_testdeps.sh
+++ b/tests/ci_build/install/install_testdeps.sh
@@ -16,25 +16,11 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-
-
 set -e
+set -x
 
 pip install cpplint 'pylint==1.4.4' 'astroid==1.3.6'
 
-git clone https://github.com/google/googletest.git
-cd googletest/googletest/make
-make PREFIX=/usr/local
-cd ..
-export GTEST_DIR=${PWD}
-g++ -isystem ${GTEST_DIR}/include -I${GTEST_DIR} -pthread -c ${GTEST_DIR}/src/gtest-all.cc
-ar -rv libgtest.a gtest-all.o
-mkdir -p /usr/local/gtest/include
-mkdir -p /usr/local/gtest/lib
-cp libgtest.a /usr/local/gtest/lib
-cp -r include/ /usr/local/gtest/
-export LD_LIBRARY_PATH=/usr/local/gtest/lib:$LD_LIBRARY_PATH
-
 pip3 install nose
 ln -s -f /opt/bin/nosetests /usr/local/bin/nosetests3
 ln -s -f /opt/bin/nosetests-3.4 /usr/local/bin/nosetests-3.4
diff --git a/tests/ci_build/install/ubuntu_install_core.sh b/tests/ci_build/install/ubuntu_install_core.sh
index eefd759..74fb05a 100755
--- a/tests/ci_build/install/ubuntu_install_core.sh
+++ b/tests/ci_build/install/ubuntu_install_core.sh
@@ -16,14 +16,14 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
+set -e
+set -x
 
 # install libraries for building mxnet c++ core on ubuntu
 
 apt-get update && apt-get install -y \
     build-essential git libopenblas-dev liblapack-dev libopencv-dev \
-    libcurl4-openssl-dev libgtest-dev cmake wget unzip sudo  
+    libcurl4-openssl-dev cmake wget unzip sudo ninja-build
 
 # Link Openblas to Cblas as this link does not exist on ubuntu16.04
 ln -s /usr/lib/libopenblas.so /usr/lib/libcblas.so
-
-cd /usr/src/gtest && cmake CMakeLists.txt && make && cp *.a /usr/lib
diff --git a/tests/ci_build/install/ubuntu_install_nvidia.sh b/tests/ci_build/install/ubuntu_install_nvidia.sh
index e1a3e4f..d349809 100755
--- a/tests/ci_build/install/ubuntu_install_nvidia.sh
+++ b/tests/ci_build/install/ubuntu_install_nvidia.sh
@@ -19,6 +19,8 @@
 
 # install nvidia libraries to compile and run CUDA without
 # the necessity of nvidia-docker and a GPU
+set -e
+set -x
 
 # Needed to run add-apt-repository
 apt update && apt install -y software-properties-common
diff --git a/tests/ci_build/install/ubuntu_install_perl.sh b/tests/ci_build/install/ubuntu_install_perl.sh
index af49952..217f322 100755
--- a/tests/ci_build/install/ubuntu_install_perl.sh
+++ b/tests/ci_build/install/ubuntu_install_perl.sh
@@ -16,6 +16,8 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
+set -e
+set -x
 
 # install libraries for mxnet's perl package on ubuntu
 apt-get update && apt-get install -y libmouse-perl pdl cpanminus swig libgraphviz-perl
diff --git a/tests/ci_build/install/ubuntu_install_python.sh b/tests/ci_build/install/ubuntu_install_python.sh
index db4e9c4..0ba9800 100755
--- a/tests/ci_build/install/ubuntu_install_python.sh
+++ b/tests/ci_build/install/ubuntu_install_python.sh
@@ -16,9 +16,10 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
+set -e
+set -x
 
 # install libraries for mxnet's python package on ubuntu
-
 apt-get update && apt-get install -y python-dev python3-dev
 
 # the version of the pip shipped with ubuntu may be too lower, install a recent version here
diff --git a/tests/ci_build/install/ubuntu_install_r.sh b/tests/ci_build/install/ubuntu_install_r.sh
index 6d34441..6855b33 100755
--- a/tests/ci_build/install/ubuntu_install_r.sh
+++ b/tests/ci_build/install/ubuntu_install_r.sh
@@ -16,6 +16,8 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
+set -e
+set -x
 
 # install libraries for mxnet's r package on ubuntu
 
diff --git a/tests/ci_build/install/ubuntu_install_scala.sh b/tests/ci_build/install/ubuntu_install_scala.sh
index 169ece0..7dd343a 100755
--- a/tests/ci_build/install/ubuntu_install_scala.sh
+++ b/tests/ci_build/install/ubuntu_install_scala.sh
@@ -16,6 +16,8 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
+set -e
+set -x
 
 # install libraries for mxnet's scala package on ubuntu
 

-- 
To stop receiving notification emails like this one, please contact
marcoabreu@apache.org.