You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by ma...@apache.org on 2018/02/09 01:03:23 UTC
[incubator-mxnet] branch master updated: CI environment refinements
(#9741)
This is an automated email from the ASF dual-hosted git repository.
marcoabreu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
The following commit(s) were added to refs/heads/master by this push:
new 3f42582 CI environment refinements (#9741)
3f42582 is described below
commit 3f42582405662b903680dc4d26f33b69ddf0e98d
Author: Pedro Larroy <92...@users.noreply.github.com>
AuthorDate: Fri Feb 9 02:03:18 2018 +0100
CI environment refinements (#9741)
* Remove uneeded gtest dependency, build with verbosely with CMake & ninja.
* Add set -x and set -e to install scripts to fail on exec and show commands
---
Jenkinsfile | 23 ++++++++++-------------
tests/ci_build/install/install_julia.sh | 3 +--
tests/ci_build/install/install_library.sh | 2 ++
tests/ci_build/install/install_maven.sh | 3 +--
tests/ci_build/install/install_openblas.sh | 3 +--
tests/ci_build/install/install_opencv.sh | 3 +--
tests/ci_build/install/install_python2.sh | 3 +--
tests/ci_build/install/install_python3.sh | 3 +--
tests/ci_build/install/install_testdeps.sh | 16 +---------------
tests/ci_build/install/ubuntu_install_core.sh | 6 +++---
tests/ci_build/install/ubuntu_install_nvidia.sh | 2 ++
tests/ci_build/install/ubuntu_install_perl.sh | 2 ++
tests/ci_build/install/ubuntu_install_python.sh | 3 ++-
tests/ci_build/install/ubuntu_install_r.sh | 2 ++
tests/ci_build/install/ubuntu_install_scala.sh | 2 ++
15 files changed, 32 insertions(+), 44 deletions(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index 8e5ee2c..b741d9a 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -89,19 +89,18 @@ def make(docker_type, make_flag) {
// Run cmake. First try to do an incremental build from a previous workspace in hope to
// accelerate the compilation. If something wrong, clean the workspace and then
// build from scratch.
-def cmake(docker_type, cmake_defines, make_flags) {
+def cmake(docker_type, cmake_defines, flags) {
timeout(time: max_time, unit: 'MINUTES') {
try {
- sh "${docker_run} ${docker_type} --dockerbinary docker mkdir build"
- sh "WORKDIR=/workspace/build ${docker_run} ${docker_type} --dockerbinary docker cmake ${cmake_defines} .."
- sh "WORKDIR=/workspace/build ${docker_run} ${docker_type} --dockerbinary docker make ${make_flags}"
+ sh "${docker_run} ${docker_type} --dockerbinary docker mkdir -p build"
+ sh "WORKDIR=/workspace/build ${docker_run} ${docker_type} --dockerbinary docker cmake ${cmake_defines} -G Ninja .."
+ sh "WORKDIR=/workspace/build ${docker_run} ${docker_type} --dockerbinary docker ninja ${flags}"
} catch (exc) {
echo 'Incremental compilation failed with ${exc}. Fall back to build from scratch'
- sh "${docker_run} ${docker_type} --dockerbinary docker sudo make clean"
- sh "${docker_run} ${docker_type} --dockerbinary docker sudo make -C amalgamation/ clean"
- sh "${docker_run} ${docker_type} --dockerbinary docker mkdir build"
- sh "WORKDIR=/workspace/build ${docker_run} ${docker_type} --dockerbinary docker cmake ${cmake_defines} .."
- sh "WORKDIR=/workspace/build ${docker_run} ${docker_type} --dockerbinary docker make ${make_flags}"
+ sh "${docker_run} ${docker_type} --dockerbinary docker git clean -fdx"
+ sh "${docker_run} ${docker_type} --dockerbinary docker mkdir -p build"
+ sh "WORKDIR=/workspace/build ${docker_run} ${docker_type} --dockerbinary docker cmake ${cmake_defines} -G Ninja .."
+ sh "WORKDIR=/workspace/build ${docker_run} ${docker_type} --dockerbinary docker ninja ${flags}"
}
}
}
@@ -270,10 +269,8 @@ try {
-DUSE_CUDNN=1 \
-DCMAKE_BUILD_TYPE=Release \
"""
- def flag = """ \
- -j\$(nproc)
- """
- cmake("build_cuda", defines, flag)
+ def flag = "-v"
+ cmake("build_cuda", defines, flag)
pack_lib('cmake_gpu', mx_cmake_lib)
}
}
diff --git a/tests/ci_build/install/install_julia.sh b/tests/ci_build/install/install_julia.sh
index 5007c94..e36dfb5 100644
--- a/tests/ci_build/install/install_julia.sh
+++ b/tests/ci_build/install/install_julia.sh
@@ -16,9 +16,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
-
set -e
+set -x
wget https://julialang.s3.amazonaws.com/bin/linux/x64/0.5/julia-0.5.0-linux-x86_64.tar.gz
mv julia-0.5.0-linux-x86_64.tar.gz /tmp/
diff --git a/tests/ci_build/install/install_library.sh b/tests/ci_build/install/install_library.sh
index 399f6a0..f3d7868 100644
--- a/tests/ci_build/install/install_library.sh
+++ b/tests/ci_build/install/install_library.sh
@@ -16,6 +16,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+set -e
+set -x
yum install graphviz
diff --git a/tests/ci_build/install/install_maven.sh b/tests/ci_build/install/install_maven.sh
index 666ebde..f124186 100644
--- a/tests/ci_build/install/install_maven.sh
+++ b/tests/ci_build/install/install_maven.sh
@@ -16,9 +16,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
-
set -e
+set -x
wget http://mirrors.ocf.berkeley.edu/apache/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz
mv apache-maven-3.3.9-bin.tar.gz /tmp/
diff --git a/tests/ci_build/install/install_openblas.sh b/tests/ci_build/install/install_openblas.sh
index 2ec5eeb..5c92823 100644
--- a/tests/ci_build/install/install_openblas.sh
+++ b/tests/ci_build/install/install_openblas.sh
@@ -16,9 +16,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
-
set -e
+set -x
git clone https://github.com/xianyi/OpenBLAS
cd OpenBLAS
diff --git a/tests/ci_build/install/install_opencv.sh b/tests/ci_build/install/install_opencv.sh
index 08a4d9b..9364a56 100644
--- a/tests/ci_build/install/install_opencv.sh
+++ b/tests/ci_build/install/install_opencv.sh
@@ -16,9 +16,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
-
set -e
+set -x
yum groupinstall -y "Development Tools"
yum install -y cmake python27 python27-setuptools
diff --git a/tests/ci_build/install/install_python2.sh b/tests/ci_build/install/install_python2.sh
index ec4bbb9..5c7c453 100644
--- a/tests/ci_build/install/install_python2.sh
+++ b/tests/ci_build/install/install_python2.sh
@@ -16,9 +16,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
-
set -e
+set -x
yum groupinstall -y "Development Tools"
yum install -y mlocate python27 python27-setuptools python27-tools python27-numpy python27-scipy python27-nose python27-matplotlib unzip
diff --git a/tests/ci_build/install/install_python3.sh b/tests/ci_build/install/install_python3.sh
index ee89161..0df12eb 100644
--- a/tests/ci_build/install/install_python3.sh
+++ b/tests/ci_build/install/install_python3.sh
@@ -16,9 +16,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
-
set -e
+set -x
wget https://bootstrap.pypa.io/get-pip.py || exit 1
mkdir py3
diff --git a/tests/ci_build/install/install_testdeps.sh b/tests/ci_build/install/install_testdeps.sh
index c777348..26da186 100644
--- a/tests/ci_build/install/install_testdeps.sh
+++ b/tests/ci_build/install/install_testdeps.sh
@@ -16,25 +16,11 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
-
set -e
+set -x
pip install cpplint 'pylint==1.4.4' 'astroid==1.3.6'
-git clone https://github.com/google/googletest.git
-cd googletest/googletest/make
-make PREFIX=/usr/local
-cd ..
-export GTEST_DIR=${PWD}
-g++ -isystem ${GTEST_DIR}/include -I${GTEST_DIR} -pthread -c ${GTEST_DIR}/src/gtest-all.cc
-ar -rv libgtest.a gtest-all.o
-mkdir -p /usr/local/gtest/include
-mkdir -p /usr/local/gtest/lib
-cp libgtest.a /usr/local/gtest/lib
-cp -r include/ /usr/local/gtest/
-export LD_LIBRARY_PATH=/usr/local/gtest/lib:$LD_LIBRARY_PATH
-
pip3 install nose
ln -s -f /opt/bin/nosetests /usr/local/bin/nosetests3
ln -s -f /opt/bin/nosetests-3.4 /usr/local/bin/nosetests-3.4
diff --git a/tests/ci_build/install/ubuntu_install_core.sh b/tests/ci_build/install/ubuntu_install_core.sh
index eefd759..74fb05a 100755
--- a/tests/ci_build/install/ubuntu_install_core.sh
+++ b/tests/ci_build/install/ubuntu_install_core.sh
@@ -16,14 +16,14 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+set -e
+set -x
# install libraries for building mxnet c++ core on ubuntu
apt-get update && apt-get install -y \
build-essential git libopenblas-dev liblapack-dev libopencv-dev \
- libcurl4-openssl-dev libgtest-dev cmake wget unzip sudo
+ libcurl4-openssl-dev cmake wget unzip sudo ninja-build
# Link Openblas to Cblas as this link does not exist on ubuntu16.04
ln -s /usr/lib/libopenblas.so /usr/lib/libcblas.so
-
-cd /usr/src/gtest && cmake CMakeLists.txt && make && cp *.a /usr/lib
diff --git a/tests/ci_build/install/ubuntu_install_nvidia.sh b/tests/ci_build/install/ubuntu_install_nvidia.sh
index e1a3e4f..d349809 100755
--- a/tests/ci_build/install/ubuntu_install_nvidia.sh
+++ b/tests/ci_build/install/ubuntu_install_nvidia.sh
@@ -19,6 +19,8 @@
# install nvidia libraries to compile and run CUDA without
# the necessity of nvidia-docker and a GPU
+set -e
+set -x
# Needed to run add-apt-repository
apt update && apt install -y software-properties-common
diff --git a/tests/ci_build/install/ubuntu_install_perl.sh b/tests/ci_build/install/ubuntu_install_perl.sh
index af49952..217f322 100755
--- a/tests/ci_build/install/ubuntu_install_perl.sh
+++ b/tests/ci_build/install/ubuntu_install_perl.sh
@@ -16,6 +16,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+set -e
+set -x
# install libraries for mxnet's perl package on ubuntu
apt-get update && apt-get install -y libmouse-perl pdl cpanminus swig libgraphviz-perl
diff --git a/tests/ci_build/install/ubuntu_install_python.sh b/tests/ci_build/install/ubuntu_install_python.sh
index db4e9c4..0ba9800 100755
--- a/tests/ci_build/install/ubuntu_install_python.sh
+++ b/tests/ci_build/install/ubuntu_install_python.sh
@@ -16,9 +16,10 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+set -e
+set -x
# install libraries for mxnet's python package on ubuntu
-
apt-get update && apt-get install -y python-dev python3-dev
# the version of the pip shipped with ubuntu may be too lower, install a recent version here
diff --git a/tests/ci_build/install/ubuntu_install_r.sh b/tests/ci_build/install/ubuntu_install_r.sh
index 6d34441..6855b33 100755
--- a/tests/ci_build/install/ubuntu_install_r.sh
+++ b/tests/ci_build/install/ubuntu_install_r.sh
@@ -16,6 +16,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+set -e
+set -x
# install libraries for mxnet's r package on ubuntu
diff --git a/tests/ci_build/install/ubuntu_install_scala.sh b/tests/ci_build/install/ubuntu_install_scala.sh
index 169ece0..7dd343a 100755
--- a/tests/ci_build/install/ubuntu_install_scala.sh
+++ b/tests/ci_build/install/ubuntu_install_scala.sh
@@ -16,6 +16,8 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+set -e
+set -x
# install libraries for mxnet's scala package on ubuntu
--
To stop receiving notification emails like this one, please contact
marcoabreu@apache.org.