You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@singa.apache.org by zh...@apache.org on 2016/12/31 03:59:46 UTC
[2/3] incubator-singa git commit: SINGA-284 - Add python unittest
into Jenkins and link static libs into whl file
SINGA-284 - Add python unittest into Jenkins and link static libs into whl file
Update the script to package the whl file into a folder, whose name indicates the version of OS, cuda and cudnn.
Update the jenkins/docker files to avoid installing libprotobuf and openblas
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/761c463d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/761c463d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/761c463d
Branch: refs/heads/master
Commit: 761c463dd8a2aedd59af7f3a936e359080e6828d
Parents: e44658a
Author: wangwei <wa...@comp.nus.edu.sg>
Authored: Fri Dec 23 15:16:19 2016 +0800
Committer: wangwei <wa...@comp.nus.edu.sg>
Committed: Sat Dec 24 09:33:27 2016 +0800
----------------------------------------------------------------------
tool/jenkins/README.md | 8 ++++----
tool/jenkins/docker/ubuntu14.04/Dockerfile | 11 +++++++----
tool/jenkins/docker/ubuntu16.04/Dockerfile | 17 +++++++++++------
tool/jenkins/jenkins_wheel.sh | 21 +++++++++++++++++----
4 files changed, 39 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/761c463d/tool/jenkins/README.md
----------------------------------------------------------------------
diff --git a/tool/jenkins/README.md b/tool/jenkins/README.md
index d20470b..1b5fb7c 100644
--- a/tool/jenkins/README.md
+++ b/tool/jenkins/README.md
@@ -30,19 +30,19 @@ Create a multi-configuration project and configure project as follows:
* Poll SCM - Schedule - ``H/30 * * * *`` (pull every 30 minutes)
### Configuration Matrix
- * User-defined Axis - name ``support`` values ``BLAS CUDNN``
+ * User-defined Axis - name ``lang`` values ``CPP CUDA``
* Slave - name ``env`` Node/label ``tick all nodes``
### Build
- * Execute shell - command - ``bash -ex tool/jenkins/jenkins_wheel.sh $support``
+ * Execute shell - command - ``bash -ex tool/jenkins/jenkins_wheel.sh $lang``
### Post-build Actions
* Archive the artifacts - ``build/python/dist/**.whl``
+ * Send build artifacts over SSH - source files ``build/python/dist/*.tar.gz``, Remove prefix `build/python/dist`, Exec command `tar xf *.tar.gz && rm *.tar.gz`
* Publish JUnit test result report - Test report XMLs - ``**/gtest.xml, **/unittest.xml``
## Docker Images
-We provide in `docker` a number of singa docker images for Jenkins to use as
-slaves.
+We provide in `docker` a number of singa docker images for Jenkins to use as slaves.
## Access Control
Use `Role Strategy Plugin` to give read access for anonymous users.
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/761c463d/tool/jenkins/docker/ubuntu14.04/Dockerfile
----------------------------------------------------------------------
diff --git a/tool/jenkins/docker/ubuntu14.04/Dockerfile b/tool/jenkins/docker/ubuntu14.04/Dockerfile
index 9b974d2..6d96983 100644
--- a/tool/jenkins/docker/ubuntu14.04/Dockerfile
+++ b/tool/jenkins/docker/ubuntu14.04/Dockerfile
@@ -7,10 +7,12 @@
FROM nvidia/cuda:8.0-cudnn5-devel
# install dependencies
-RUN apt-get update && apt-get install -y \
- git vim wget build-essential autoconf libtool cmake \
- libpcre3-dev python-dev python-pip && \
- pip install numpy protobuf unittest-xml-reporting
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends git g++ cmake libprotobuf-dev libopenblas-dev protobuf-compiler python-dev python-pip wget\
+ && apt-get clean && apt-get autoremove && apt-get autoclean \
+ && rm -rf /var/lib/apt/lists/* \
+ && pip --no-cache-dir install -U pip wheel numpy setuptools unittest-xml-reporting protobuf>=2.5.0,<3
+
# install swig 3.0
RUN wget http://prdownloads.sourceforge.net/swig/swig-3.0.10.tar.gz && \
@@ -21,6 +23,7 @@ RUN wget http://prdownloads.sourceforge.net/swig/swig-3.0.10.tar.gz && \
ENV CPLUS_INCLUDE_PATH /usr/local/lib/python2.7/dist-packages/numpy/core/include:${CPLUS_INCLUDE_PATH}
ENV CMAKE_INCLUDE_PATH /usr/local/cuda/include:${CMAKE_INCLUDE_PATH}
ENV CMAKE_LIBRARY_PATH /usr/local/cuda/lib64:${CMAKE_LIBRARY_PATH}
+ENV OS_VERSION ubuntu14.04
# download singa source
RUN git clone https://github.com/apache/incubator-singa.git
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/761c463d/tool/jenkins/docker/ubuntu16.04/Dockerfile
----------------------------------------------------------------------
diff --git a/tool/jenkins/docker/ubuntu16.04/Dockerfile b/tool/jenkins/docker/ubuntu16.04/Dockerfile
index 612dd8b..1a9434c 100644
--- a/tool/jenkins/docker/ubuntu16.04/Dockerfile
+++ b/tool/jenkins/docker/ubuntu16.04/Dockerfile
@@ -4,21 +4,26 @@
FROM nvidia/cuda:8.0-cudnn5-devel-ubuntu16.04
# install dependencies
-RUN apt-get update && apt-get install -y \
- git wget build-essential autoconf libtool cmake \
- libpcre3-dev python-dev python-pip swig && \
- pip install numpy protobuf unittest-xml-reporting
+
+MAINTAINER incubator-singa dev@singa.incubator.apache.org
+
+# install dependencies
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends git g++ cmake libprotobuf-dev libopenblas-dev protobuf-compiler python-dev python-pip swig wget openssh-server\
+ && apt-get clean && apt-get autoremove && apt-get autoclean \
+ && rm -rf /var/lib/apt/lists/* \
+ && pip --no-cache-dir install -U pip wheel numpy setuptools unittest-xml-reporting protobuf>=2.5.0,<3
# set environment
ENV CPLUS_INCLUDE_PATH /usr/local/lib/python2.7/dist-packages/numpy/core/include:${CPLUS_INCLUDE_PATH}
ENV CMAKE_INCLUDE_PATH /usr/local/cuda/include:${CMAKE_INCLUDE_PATH}
ENV CMAKE_LIBRARY_PATH /usr/local/cuda/lib64:${CMAKE_LIBRARY_PATH}
+ENV OS_VERSION ubuntu16.04
# download singa source
RUN git clone https://github.com/apache/incubator-singa.git
-# install ssh service
-RUN apt-get install -y openssh-server
+# config ssh service
RUN mkdir /var/run/sshd
RUN echo 'root:singa' | chpasswd
RUN sed -i 's/PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/761c463d/tool/jenkins/jenkins_wheel.sh
----------------------------------------------------------------------
diff --git a/tool/jenkins/jenkins_wheel.sh b/tool/jenkins/jenkins_wheel.sh
index c3d075f..f131f68 100644
--- a/tool/jenkins/jenkins_wheel.sh
+++ b/tool/jenkins/jenkins_wheel.sh
@@ -27,26 +27,39 @@ echo OS version: `cat /etc/issue`
echo kernal version: `uname -a`
echo CUDA version: $CUDA_VERSION
echo CUDNN version: $CUDNN_VERSION
+echo OS name: $OS_VERSION
+COMMIT=`git rev-parse HEAD`
+echo COMMIT HASH: $COMMIT
# set parameters
+CUDA="OFF"
CUDNN="OFF"
-if [ $1 = "CUDNN" ]; then
+FOLDER=$COMMIT/$OS_VERSION-cpp/
+if [ $1 = "CUDA" ]; then
+ CUDA="ON"
CUDNN="ON"
+ FOLDER=$COMMIT/$OS_VERSION-cuda$CUDA_VERSION-cudnn$CUDNN_VERSION/
fi
+echo wheel file folder: build/python/dist/whl/$FOLDER
+
# setup env
rm -rf build
mkdir build
+
# compile singa c++
cd build
-cmake -DUSE_CUDNN=$CUDNN -DUSE_CUDA=$CUDNN -DUSE_MODULES=ON ../
+cmake -DUSE_CUDNN=$CUDNN -DUSE_CUDA=$CUDA -DUSE_MODULES=ON -DUSE_MODULES=ON ../
make
# unit test cpp code
./bin/test_singa --gtest_output=xml:./gtest.xml
# compile pysinga
cd python
python setup.py bdist_wheel
-# rename dist
+# mv whl file to a folder whose name identifies the OS, CUDA, CUDNN etc.
cd dist
-mv singa-1.0.1-py2-none-any.whl singa-1.0.0-cp27-none-linux_x86_64.whl
+mkdir -p $FOLDER
+mv *.whl $FOLDER/
+tar czf $FOLDER.tar.gz $FOLDER/*
+
# unit test python code
cd ../../../test/python
PYTHONPATH=../../build/python/ python run.py