You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by ma...@apache.org on 2018/09/05 22:46:41 UTC
[incubator-mxnet] branch master updated: [MXNET-908] Enable minimal
OSX Travis build (#12462)
This is an automated email from the ASF dual-hosted git repository.
marcoabreu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
The following commit(s) were added to refs/heads/master by this push:
new 28cb75a [MXNET-908] Enable minimal OSX Travis build (#12462)
28cb75a is described below
commit 28cb75af38c9b6ef4d34b283535eea04714525d3
Author: Kellen Sunderland <ke...@gmail.com>
AuthorDate: Thu Sep 6 00:46:31 2018 +0200
[MXNET-908] Enable minimal OSX Travis build (#12462)
---
.travis.yml | 107 ++---------
.../travis/install.sh | 16 +-
tests/travis/is_core_changed.sh | 60 ------
tests/travis/r_vignettes.R | 21 ---
tests/travis/run_test.sh | 210 ---------------------
tests/travis/setup.sh | 59 ------
6 files changed, 25 insertions(+), 448 deletions(-)
diff --git a/.travis.yml b/.travis.yml
index ca5d03b..f61bd86 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,108 +1,31 @@
-sudo: false
+sudo: true
language: cpp
+cache: ccache
+
os:
- # - linux
- osx
-osx_image: xcode8
-
-env:
- # code analysis
- # - TASK=lint
- # build mxnet.so with CUDA
- # - TASK=build
- # run tests/cpp
- - TASK=cpp_test
- # run tests/python
- - TASK=python_test
- - TASK=r_test
- # - TASK=julia JULIA_VER=0.4
- # - TASK=scala_test
-
- # TODO, R test, distributed test, clang, more g++ versions
+osx_image: xcode9.4
matrix:
include:
- - # os: linux
- # dist: trusty
- # env: TASK=perl_test
- os: osx
- ## sudo is required because
- ## prexexisting packages conflict
- ## with new ones.
- ## would be nice to have macports
- ## on travis osx, it has all needed perl packages
- sudo: required
- env: TASK=perl_test
-# env: TASK=julia JULIA_VER=0.4
-# - os: linux
-# env: TASK=build
-# - os: linux
-# env: TASK=cpp_test
-# - os: linux
-# env: TASK=python_test
-# - os: linux
-# env: TASK=r_test
-# - os: linux
-# env: TASK=scala_test
-
-# dependent apt packages
-addons:
- apt:
- sources:
- - ubuntu-toolchain-r-test
- packages:
- - doxygen
- - wget
- - git
- - libcurl4-openssl-dev
- - unzip
- - libatlas-dev
- - libopencv-dev
- - gcc-4.8
- - g++-4.8
- - python-numpy
- - python-nose
- - python3-numpy
- - python3-dev
- - python3-nose
- - python-h5py
- - python3-h5py
- - graphviz
- - libmouse-perl
- - pdl
- - cpanminus
- - swig
- - libgraphviz-perl
before_install:
- - export NVCC_PREFIX=${HOME}
- - source dmlc-core/scripts/travis/travis_setup_env.sh
- export PYTHONPATH=${PYTHONPATH}:${PWD}/python
- - export MAVEN_SKIP_RC=true
- - export MAVEN_OPTS="-Xmx512m -XX:MaxPermSize=256m -XX:-UseGCOverheadLimit -XX:+CMSClassUnloadingEnabled -XX:+UseConcMarkSweepGC"
install:
- - source tests/travis/setup.sh
-
+ - brew install ccache
+ - export PATH="/usr/local/opt/ccache/libexec:$PATH"
+ - source ci/travis/install.sh
+
+# We build with 2 concurrent jobs to match the number of cores present on MacOS virutal machines.
+# nproc does not report the correct number of cores reliably in Travis, so using nproc is not
+# recommended.
+# https://docs.travis-ci.com/user/reference/overview/
script:
- - tests/travis/run_test.sh
-
-cache:
- directories:
- - ${HOME}/.cache/usr
-
-before_cache:
- - dmlc-core/scripts/travis/travis_before_cache.sh
-
-after_failure:
- - tests/travis/travis_after_failure.sh
-
-notifications:
-# Emails are sent to the committer's git-configured email address by default,
- email:
- on_success: change
- on_failure: always
- #slack: dmlc:NmroCzntCiWOuxUZpii40USd
+ - export MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
+ - mv make/osx.mk config.mk
+ - make -j 2
diff --git a/tests/travis/travis_after_failure.sh b/ci/travis/install.sh
old mode 100755
new mode 100644
similarity index 70%
rename from tests/travis/travis_after_failure.sh
rename to ci/travis/install.sh
index 50754c9..d04dda7
--- a/tests/travis/travis_after_failure.sh
+++ b/ci/travis/install.sh
@@ -17,10 +17,14 @@
# specific language governing permissions and limitations
# under the License.
-
-if [ ${TASK} == "r_test" ]; then
- echo "Print the install log..."
- cat mxnet.Rcheck/*.out
- echo "Print the check log..."
- cat mxnet.Rcheck/*.log
+if [ ${TRAVIS_OS_NAME} == "osx" ]; then
+ brew update
+ brew install opencv
+ brew install python3
+ brew install fftw
+ brew install libpng
+ brew install ImageMagick
+ brew install swig
+ python -m pip install --user nose numpy cython scipy requests
+ python3 -m pip install --user nose numpy cython scipy requests
fi
diff --git a/tests/travis/is_core_changed.sh b/tests/travis/is_core_changed.sh
deleted file mode 100755
index 7b9eb61..0000000
--- a/tests/travis/is_core_changed.sh
+++ /dev/null
@@ -1,60 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-
-# this is a util script to test whether the "core" of
-# mxnet has changed. Please modify the regex patterns here
-# to ensure the components are covered if you add new "core"
-# components to mxnet
-
-# temporarily disable this b/c the OS X tests are failing mysteriously
-exit 0
-
-# DEBUG
-echo "Files changed in this PR includes:"
-echo "**********************************"
-git diff --name-only HEAD^
-echo "**********************************"
-
-# we ignore examples, and docs
-core_patterns=(
- '^dmlc-core'
- '^matlab'
- '^plugin'
- '^python'
- '^src'
- '^tools'
- '^R-package'
- '^amalgamation'
- '^include'
- '^mshadow'
- '^ps-lite'
- '^scala-package'
- '^tests'
-)
-
-for pat in ${core_patterns[@]}; do
- if git diff --name-only HEAD^ | grep "$pat"
- then
- exit
- fi
-done
-
-echo "I think we are good to skip this travis ci run now"
-exit 1 # means nothing has changed
diff --git a/tests/travis/r_vignettes.R b/tests/travis/r_vignettes.R
deleted file mode 100644
index 1b03b8b..0000000
--- a/tests/travis/r_vignettes.R
+++ /dev/null
@@ -1,21 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-fnames <- list.files("R-package/vignettes/", pattern="*.Rmd")
-sapply(fnames, function(x){
- knitr::purl(paste0("R-package/vignettes/", x))
- })
\ No newline at end of file
diff --git a/tests/travis/run_test.sh b/tests/travis/run_test.sh
deleted file mode 100755
index fd23f0e..0000000
--- a/tests/travis/run_test.sh
+++ /dev/null
@@ -1,210 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-
-if ! tests/travis/is_core_changed.sh
-then
- exit 0
-fi
-
-if [ ${TASK} == "lint" ]; then
- make lint || exit -1
- echo "Check documentations of c++ code..."
- make doc 2>log.txt
- (cat log.txt| grep -v ENABLE_PREPROCESSING |grep -v "unsupported tag") > logclean.txt
- echo "---------Error Log----------"
- cat logclean.txt
- echo "----------------------------"
- (cat logclean.txt|grep warning) && exit -1
- (cat logclean.txt|grep error) && exit -1
- exit 0
-fi
-
-cp make/config.mk config.mk
-
-if [[ ${TRAVIS_OS_NAME} == "osx" ]]; then
- echo "USE_BLAS=apple" >> config.mk
- echo "USE_OPENMP=0" >> config.mk
-else
- # use g++-4.8 for linux
- if [[ ${CXX} == "g++" ]]; then
- export CXX=g++-4.8
- fi
- echo "USE_BLAS=blas" >> config.mk
-fi
-echo "CXX=${CXX}" >>config.mk
-echo "USE_PROFILER=1" >> config.mk
-
-if [ ${TASK} == "build" ]; then
- if [ ${TRAVIS_OS_NAME} == "linux" ]; then
- echo "USE_CUDA=1" >> config.mk
- ./dmlc-core/scripts/setup_nvcc.sh $NVCC_PREFIX
- fi
- make all
- exit $?
-fi
-
-if [ ${TASK} == "cpp_test" ]; then
- make -f dmlc-core/scripts/packages.mk gtest
- echo "GTEST_PATH="${CACHE_PREFIX} >> config.mk
- make test || exit -1
- export MXNET_ENGINE_INFO=true
- ./build/tests/cpp/mxnet_test
- exit 0
-fi
-
-if [ ${TASK} == "r_test" ]; then
- make all || exit -1
- # use cached dir for storing data
- rm -rf ${PWD}/data
- mkdir -p ${CACHE_PREFIX}/data
- ln -s ${CACHE_PREFIX}/data ${PWD}/data
-
- set -e
- export _R_CHECK_TIMINGS_=0
-
- if [[ ${TRAVIS_OS_NAME} == "osx" ]]; then
- wget https://cran.rstudio.com/bin/macosx/R-latest.pkg -O /tmp/R-latest.pkg
- sudo installer -pkg "/tmp/R-latest.pkg" -target /
- Rscript -e "install.packages('devtools', repo = 'https://cran.rstudio.com')"
- fi
-
- cd R-package
- Rscript -e "library(devtools); library(methods); options(repos=c(CRAN='https://cran.rstudio.com')); install_deps(dependencies = TRUE)"
- cd ..
-
- make rpkg
-# R CMD check --no-examples --no-manual --no-vignettes --no-build-vignettes mxnet_*.tar.gz
- R CMD INSTALL mxnet_*.tar.gz
-
- Rscript tests/travis/r_vignettes.R
-
- wget http://data.mxnet.io/mxnet/data/Inception.zip
- unzip Inception.zip && rm -rf Inception.zip
- wget http://data.mxnet.io/mxnet/data/mnist.zip
- unzip mnist.zip && rm -rf mnist.zip
-
- cat CallbackFunctionTutorial.R \
- fiveMinutesNeuralNetwork.R \
- mnistCompetition.R \
- ndarrayAndSymbolTutorial.R > r_test.R
-
- Rscript r_test.R || exit -1
-
- exit 0
-fi
-
-if [ ${TASK} == "python_test" ]; then
- make all || exit -1
- # use cached dir for storing data
- rm -rf ${PWD}/data
- mkdir -p ${PWD}/data
-
- if [ ${TRAVIS_OS_NAME} == "osx" ]; then
- python -m nose -v tests/python/unittest || exit -1
- python3 -m nose -v tests/python/unittest || exit -1
- # make cython3
- # cython tests
- # export MXNET_ENFORCE_CYTHON=1
- # python3 -m nose tests/python/unittest || exit -1
- python3 -m nose -v tests/python/train || exit -1
- python -m nose -v tests/python/doctest || exit -1
- python3 -m nose -v tests/python/doctest || exit -1
- else
- nosetests -v tests/python/unittest || exit -1
- nosetests3 -v tests/python/unittest || exit -1
- nosetests3 -v tests/python/train || exit -1
- nosetests -v tests/python/doctest || exit -1
- nosetests3 -v tests/python/doctest || exit -1
- fi
- exit 0
-fi
-
-if [ ${TASK} == "julia" ]; then
- make all || exit -1
- # use cached dir for storing data
- rm -rf ${PWD}/data
- mkdir -p ${PWD}/data
-
- export MXNET_HOME="${PWD}"
- julia -e 'Pkg.clone("MXNet"); Pkg.checkout("MXNet"); Pkg.build("MXNet"); Pkg.test("MXNet")' || exit -1
- exit 0
-fi
-
-if [ ${TASK} == "scala_test" ]; then
- if [ ${TRAVIS_OS_NAME} == "osx" ]; then
- LIB_GOMP_PATH=`find /usr/local/lib -name libgomp.dylib | grep -v i386 | head -n1`
- ln -sf $LIB_GOMP_PATH /usr/local/lib/libgomp.dylib
- fi
- make all || exit -1
- # use cached dir for storing data
- rm -rf ${PWD}/data
- mkdir -p ${PWD}/data
-
- export JAVA_HOME=$(/usr/libexec/java_home)
-
- make scalapkg || exit -1
- make scalatest || exit -1
-
- exit 0
-fi
-
-if [ ${TASK} == "perl_test" ]; then
- make all || exit -1
-
- # use cached dir for storing data
- MXNET_HOME=${PWD}
- rm -rf ${MXNET_HOME}/perl-package/AI-MXNet/data
- mkdir -p ${CACHE_PREFIX}/data
- ln -s ${CACHE_PREFIX}/data ${MXNET_HOME}/perl-package/AI-MXNet/data
-
- export LD_LIBRARY_PATH=${MXNET_HOME}/lib
- export PERL5LIB=${HOME}/perl5/lib/perl5
-
- cd ${MXNET_HOME}/perl-package/AI-MXNetCAPI/
- perl Makefile.PL INSTALL_BASE=${HOME}/perl5
- make || exit -1
- if [ ${TRAVIS_OS_NAME} == "osx" ]; then
- install_name_tool -change lib/libmxnet.so \
- ${MXNET_HOME}/lib/libmxnet.so \
- blib/arch/auto/AI/MXNetCAPI/MXNetCAPI.bundle
- fi
- make install || exit -1
-
- cd ${MXNET_HOME}/perl-package/AI-NNVMCAPI/
- perl Makefile.PL INSTALL_BASE=${HOME}/perl5
- make || exit -1
- if [ ${TRAVIS_OS_NAME} == "osx" ]; then
- install_name_tool -change lib/libmxnet.so \
- ${MXNET_HOME}/lib/libmxnet.so \
- blib/arch/auto/AI/NNVMCAPI/NNVMCAPI.bundle
- fi
- make install || exit -1
-
- cd ${MXNET_HOME}/perl-package/AI-MXNet/
- perl Makefile.PL
- make test || exit -1
- exit 0
-fi
-
-if [ ${TASK} == "cpp_package_test" ]; then
- MXNET_HOME=${PWD}
- make travis -C ${MXNET_HOME}/cpp-package/example
- exit 0
-fi
diff --git a/tests/travis/setup.sh b/tests/travis/setup.sh
deleted file mode 100755
index eec6c23..0000000
--- a/tests/travis/setup.sh
+++ /dev/null
@@ -1,59 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-
-if ! tests/travis/is_core_changed.sh
-then
- exit 0
-fi
-
-if [ ${TRAVIS_OS_NAME} == "osx" ]; then
- brew update
- brew tap homebrew/science
- brew install opencv
- brew install python3
- brew install fftw
- brew install libpng
- brew install ImageMagick
- brew install swig
- if [ ${TASK} == "python_test" ]; then
- python -m pip install --user nose numpy cython scipy
- python3 -m pip install --user nose numpy cython scipy
- fi
-fi
-
-if [ ${TASK} == "lint" ]; then
- pip install --user cpplint 'pylint==1.4.4' 'astroid==1.3.6'
-fi
-
-if [ ${TASK} == "julia" ]; then
- mkdir -p ~/julia
- curl -s -L --retry 7 "https://s3.amazonaws.com/julialang/bin/linux/x64/${JULIA_VER}/julia-${JULIA_VER}-latest-linux-x86_64.tar.gz" | tar -C ~/julia -x -z --strip-components=1 -f -
- export PATH="${PATH}:${HOME}/julia/bin"
- julia -e 'versioninfo()'
-fi
-
-if [ ${TASK} == "perl_test" ]; then
- if [ ${TRAVIS_OS_NAME} == "linux" ]; then
- cpanm -q -L "${HOME}/perl5" Function::Parameters Hash::Ordered PDL::CCS
- else
- sudo sh -c 'curl -L https://cpanmin.us | perl - App::cpanminus'
- sudo cpanm -q -n PDL Mouse Function::Parameters Hash::Ordered PDL::CCS
- fi
-fi