You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@singa.apache.org by wa...@apache.org on 2016/12/14 06:25:38 UTC
[1/2] incubator-singa git commit: SINGA-280 Jenkins CI support
Repository: incubator-singa
Updated Branches:
refs/heads/master ee93b7dda -> 33cf5461b
SINGA-280 Jenkins CI support
Add Jenkins scripts
Add dockerfile for singa env
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/3b8cb947
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/3b8cb947
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/3b8cb947
Branch: refs/heads/master
Commit: 3b8cb94729e6dd04e34eb6792d8d691f62c1dfc3
Parents: ee93b7d
Author: WANG Sheng <wa...@gmail.com>
Authored: Thu Dec 1 16:35:46 2016 +0800
Committer: WANG Sheng <wa...@gmail.com>
Committed: Tue Dec 13 15:17:04 2016 +0800
----------------------------------------------------------------------
jenkins.sh | 51 --------------------
tool/jenkins/README.md | 42 ++++++++++++++++
.../docker/ubuntu14.04-cuda8-cudnn5/Dockerfile | 33 +++++++++++++
tool/jenkins/jenkins_pysinga.sh | 46 ++++++++++++++++++
4 files changed, 121 insertions(+), 51 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3b8cb947/jenkins.sh
----------------------------------------------------------------------
diff --git a/jenkins.sh b/jenkins.sh
deleted file mode 100644
index 347a55e..0000000
--- a/jenkins.sh
+++ /dev/null
@@ -1,51 +0,0 @@
-#!/usr/bin/env sh
-#/**
-# *
-# * Licensed to the Apache Software Foundation (ASF) under one
-# * or more contributor license agreements. See the NOTICE file
-# * distributed with this work for additional information
-# * regarding copyright ownership. The ASF licenses this file
-# * to you under the Apache License, Version 2.0 (the
-# * "License"); you may not use this file except in compliance
-# * with the License. You may obtain a copy of the License at
-# *
-# * http://www.apache.org/licenses/LICENSE-2.0
-# *
-# * Unless required by applicable law or agreed to in writing, software
-# * distributed under the License is distributed on an "AS IS" BASIS,
-# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# * See the License for the specific language governing permissions and
-# * limitations under the License.
-# */
-
-
-
-if [ $1 = "CPP" ]; then
- echo "CPP test"
- mkdir build
- rm -f gtest.xml
- cd build
- cmake -DUSE_CUDNN=OFF -DUSE_CUDA=OFF -DUSE_PYTHON=OFF ../
- make
- ./bin/test_singa --gtest_output=xml:./../gtest.xml
-fi
-
-if [ $1 = "CUDNN" ]; then
- echo "CUDNN test"
- git submodule init
- git submodule update
- mkdir build
- rm -f gtest.xml
- cd build
- cmake -DUSE_CUDNN=ON -DUSE_CUDA=ON -DUSE_PYTHON=OFF ../
- make
- ./bin/test_singa --gtest_output=xml:./../gtest.xml
-fi
-
-cd ..
-rm -rf build
-
-
-
-
-
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3b8cb947/tool/jenkins/README.md
----------------------------------------------------------------------
diff --git a/tool/jenkins/README.md b/tool/jenkins/README.md
new file mode 100644
index 0000000..f62fa35
--- /dev/null
+++ b/tool/jenkins/README.md
@@ -0,0 +1,42 @@
+# Jenkins CI Support
+
+## Introduction
+
+This documentation is to guide SINGA developers to setup Jenkins service.
+
+We use jenkins to support continuous integration.
+After each commit, we want to automatically compile and test SINGA
+under different OS and settings.
+Those built binaries need to be archived for users to download.
+
+
+## Install Jenkins
+[Jenkins Official Wiki](https://wiki.jenkins-ci.org/display/JENKINS/Installing+Jenkins)
+
+## Configure Jenkins Multi-configuration Project
+Create a multi-configuration project and configure project as follows:
+
+### Description
+ This job automatically pulls latest commits from apache SINGA github repository.
+ It compiles and tests SINGA in different environments and creates PySINGA wheel distribution accordingly.
+
+### General
+ * Discard old builds - Max # of builds to keep - 50
+ * GitHub project - ``https://github.com/apache/incubator-singa``
+
+### Source Code Management
+ * Git - Repository URL - ``https://github.com/apache/incubator-singa``
+ * Git - Branch Specifier - ``*/master``
+
+### Build Triggers
+ * Poll SCM - Schedule - ``H/30 * * * *`` (pull every 30 minutes)
+
+### Configuration Matrix
+ * User-defined Axis - name ``support`` values ``blas cudnn``
+
+### Build
+ * Execute shell - command - ``bash -ex tool/jenkins/jenkins_pysinga.sh $support``
+
+### Post-build Actions
+ * Archive the artifacts - ``build/python/dist/**.whl``
+ * Publish JUnit test result report - Test report XMLs - ``**/gtest.xml``
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3b8cb947/tool/jenkins/docker/ubuntu14.04-cuda8-cudnn5/Dockerfile
----------------------------------------------------------------------
diff --git a/tool/jenkins/docker/ubuntu14.04-cuda8-cudnn5/Dockerfile b/tool/jenkins/docker/ubuntu14.04-cuda8-cudnn5/Dockerfile
new file mode 100644
index 0000000..2feccd3
--- /dev/null
+++ b/tool/jenkins/docker/ubuntu14.04-cuda8-cudnn5/Dockerfile
@@ -0,0 +1,33 @@
+FROM nvidia/cuda:8.0-cudnn5-devel
+
+# install dependencies
+RUN apt-get update && apt-get install -y \
+ git vim wget build-essential autoconf libtool cmake \
+ libprotobuf-dev libopenblas-dev protobuf-compiler \
+ libpcre3-dev python-dev python-pip && \
+ pip install numpy
+
+# install swig 3.0
+RUN wget http://prdownloads.sourceforge.net/swig/swig-3.0.10.tar.gz && \
+ tar zxf swig-3.0.10.tar.gz && cd swig-3.0.10 && \
+ ./configure && make && make install
+
+# set environment
+ENV CPLUS_INCLUDE_PATH /usr/local/lib/python2.7/dist-packages/numpy/core/include:${CPLUS_INCLUDE_PATH}
+ENV CMAKE_INCLUDE_PATH /usr/local/cuda/include:${CMAKE_INCLUDE_PATH}
+ENV CMAKE_LIBRARY_PATH /usr/local/cuda/lib64:${CMAKE_LIBRARY_PATH}
+
+# download singa source
+RUN git clone https://github.com/apache/incubator-singa.git
+
+# install ssh service
+RUN apt-get install -y openssh-server
+RUN mkdir /var/run/sshd
+RUN echo 'root:singa' | chpasswd
+RUN sed -i 's/PermitRootLogin without-password/PermitRootLogin yes/' /etc/ssh/sshd_config
+# SSH login fix. Otherwise user is kicked off after login
+RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd
+
+EXPOSE 22
+
+CMD ["/usr/sbin/sshd", "-D"]
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3b8cb947/tool/jenkins/jenkins_pysinga.sh
----------------------------------------------------------------------
diff --git a/tool/jenkins/jenkins_pysinga.sh b/tool/jenkins/jenkins_pysinga.sh
new file mode 100644
index 0000000..b2e920a
--- /dev/null
+++ b/tool/jenkins/jenkins_pysinga.sh
@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+#/**
+# *
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+echo Compile, test and distribute PySINGA...
+echo Parameters: $1
+echo Workspace: `pwd`
+echo OS env: `uname -a`
+echo Cuda env: `nvcc --version`
+# set parameters
+CUDNN="OFF"
+if [ $1 = "cudnn" ]; then
+ CUDNN="ON"
+fi
+# setup env
+rm -rf build
+mkdir build
+# compile singa c++
+cd build
+cmake -DUSE_CUDNN=$CUDNN -DUSE_CUDA=$CUDNN ../
+make
+# unit test
+./bin/test_singa --gtest_output=xml:./../gtest.xml
+# compile pysinga
+cd python
+python setup.py bdist_wheel
+# rename dist
+cd dist
+mv singa-1.0.1-py2-none-any.whl singa-1.0.0-cp27-none-linux_x86_64.whl
+echo Job finished...
[2/2] incubator-singa git commit: SINGA 280 Jenkins CI support
Posted by wa...@apache.org.
SINGA 280 Jenkins CI support
Support Jenkins with multiple slaves to compile in different settings
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/33cf5461
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/33cf5461
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/33cf5461
Branch: refs/heads/master
Commit: 33cf5461b8937ff3a9a792ad4a32b27043cd46c3
Parents: 3b8cb94
Author: WANG Sheng <wa...@gmail.com>
Authored: Fri Dec 9 15:54:20 2016 +0800
Committer: WANG Sheng <wa...@gmail.com>
Committed: Tue Dec 13 15:51:03 2016 +0800
----------------------------------------------------------------------
tool/jenkins/README.md | 14 ++++--
.../docker/ubuntu14.04-cuda8-cudnn5/Dockerfile | 33 -------------
tool/jenkins/docker/ubuntu14.04/Dockerfile | 42 ++++++++++++++++
tool/jenkins/docker/ubuntu16.04/Dockerfile | 34 +++++++++++++
tool/jenkins/jenkins_pysinga.sh | 46 ------------------
tool/jenkins/jenkins_wheel.sh | 50 ++++++++++++++++++++
6 files changed, 136 insertions(+), 83 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/33cf5461/tool/jenkins/README.md
----------------------------------------------------------------------
diff --git a/tool/jenkins/README.md b/tool/jenkins/README.md
index f62fa35..6bd90df 100644
--- a/tool/jenkins/README.md
+++ b/tool/jenkins/README.md
@@ -1,7 +1,6 @@
# Jenkins CI Support
## Introduction
-
This documentation is to guide SINGA developers to setup Jenkins service.
We use jenkins to support continuous integration.
@@ -9,7 +8,6 @@ After each commit, we want to automatically compile and test SINGA
under different OS and settings.
Those built binaries need to be archived for users to download.
-
## Install Jenkins
[Jenkins Official Wiki](https://wiki.jenkins-ci.org/display/JENKINS/Installing+Jenkins)
@@ -32,11 +30,19 @@ Create a multi-configuration project and configure project as follows:
* Poll SCM - Schedule - ``H/30 * * * *`` (pull every 30 minutes)
### Configuration Matrix
- * User-defined Axis - name ``support`` values ``blas cudnn``
+ * User-defined Axis - name ``support`` values ``BLAS CUDNN``
+ * Slave - name ``env`` Node/label ``tick all nodes``
### Build
- * Execute shell - command - ``bash -ex tool/jenkins/jenkins_pysinga.sh $support``
+ * Execute shell - command - ``bash -ex tool/jenkins/jenkins_wheel.sh $support``
### Post-build Actions
* Archive the artifacts - ``build/python/dist/**.whl``
* Publish JUnit test result report - Test report XMLs - ``**/gtest.xml``
+
+## Docker Images
+We provide in `docker` a number of singa docker images for Jenkins to use as
+slaves.
+
+## Access Control
+Use `Role Strategy Plugin` to give read access for anonymous users.
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/33cf5461/tool/jenkins/docker/ubuntu14.04-cuda8-cudnn5/Dockerfile
----------------------------------------------------------------------
diff --git a/tool/jenkins/docker/ubuntu14.04-cuda8-cudnn5/Dockerfile b/tool/jenkins/docker/ubuntu14.04-cuda8-cudnn5/Dockerfile
deleted file mode 100644
index 2feccd3..0000000
--- a/tool/jenkins/docker/ubuntu14.04-cuda8-cudnn5/Dockerfile
+++ /dev/null
@@ -1,33 +0,0 @@
-FROM nvidia/cuda:8.0-cudnn5-devel
-
-# install dependencies
-RUN apt-get update && apt-get install -y \
- git vim wget build-essential autoconf libtool cmake \
- libprotobuf-dev libopenblas-dev protobuf-compiler \
- libpcre3-dev python-dev python-pip && \
- pip install numpy
-
-# install swig 3.0
-RUN wget http://prdownloads.sourceforge.net/swig/swig-3.0.10.tar.gz && \
- tar zxf swig-3.0.10.tar.gz && cd swig-3.0.10 && \
- ./configure && make && make install
-
-# set environment
-ENV CPLUS_INCLUDE_PATH /usr/local/lib/python2.7/dist-packages/numpy/core/include:${CPLUS_INCLUDE_PATH}
-ENV CMAKE_INCLUDE_PATH /usr/local/cuda/include:${CMAKE_INCLUDE_PATH}
-ENV CMAKE_LIBRARY_PATH /usr/local/cuda/lib64:${CMAKE_LIBRARY_PATH}
-
-# download singa source
-RUN git clone https://github.com/apache/incubator-singa.git
-
-# install ssh service
-RUN apt-get install -y openssh-server
-RUN mkdir /var/run/sshd
-RUN echo 'root:singa' | chpasswd
-RUN sed -i 's/PermitRootLogin without-password/PermitRootLogin yes/' /etc/ssh/sshd_config
-# SSH login fix. Otherwise user is kicked off after login
-RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd
-
-EXPOSE 22
-
-CMD ["/usr/sbin/sshd", "-D"]
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/33cf5461/tool/jenkins/docker/ubuntu14.04/Dockerfile
----------------------------------------------------------------------
diff --git a/tool/jenkins/docker/ubuntu14.04/Dockerfile b/tool/jenkins/docker/ubuntu14.04/Dockerfile
new file mode 100644
index 0000000..b664689
--- /dev/null
+++ b/tool/jenkins/docker/ubuntu14.04/Dockerfile
@@ -0,0 +1,42 @@
+# Base unbuntu 14.04 image from nvidia/cuda
+# Change tags to build with different cuda/cudnn versions:
+# FROM nvidia/cuda:8.0-cudnn5-devel
+# FROM nvidia/cuda:7.5-cudnn5-devel
+# FROM nvidia/cuda:7.5-cudnn4-devel
+# FROM nvidia/cuda:7.0-cudnn4-devel
+FROM nvidia/cuda:8.0-cudnn5-devel
+
+# install dependencies
+RUN apt-get update && apt-get install -y \
+ git vim wget build-essential autoconf libtool cmake \
+ libprotobuf-dev libopenblas-dev protobuf-compiler \
+ libpcre3-dev python-dev python-pip && \
+ pip install numpy
+
+# install swig 3.0
+RUN wget http://prdownloads.sourceforge.net/swig/swig-3.0.10.tar.gz && \
+ tar zxf swig-3.0.10.tar.gz && cd swig-3.0.10 && \
+ ./configure && make && make install
+
+# set environment
+ENV CPLUS_INCLUDE_PATH /usr/local/lib/python2.7/dist-packages/numpy/core/include:${CPLUS_INCLUDE_PATH}
+ENV CMAKE_INCLUDE_PATH /usr/local/cuda/include:${CMAKE_INCLUDE_PATH}
+ENV CMAKE_LIBRARY_PATH /usr/local/cuda/lib64:${CMAKE_LIBRARY_PATH}
+
+# download singa source
+RUN git clone https://github.com/apache/incubator-singa.git
+
+# install ssh service
+RUN apt-get install -y openssh-server
+RUN mkdir /var/run/sshd
+RUN echo 'root:singa' | chpasswd
+RUN sed -i 's/PermitRootLogin without-password/PermitRootLogin yes/' /etc/ssh/sshd_config
+# SSH login fix. Otherwise user is kicked off after login
+RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd
+
+# dump environment variables into files, so that ssh can see also
+RUN env | grep _ >> /etc/environment
+
+EXPOSE 22
+
+CMD ["/usr/sbin/sshd", "-D"]
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/33cf5461/tool/jenkins/docker/ubuntu16.04/Dockerfile
----------------------------------------------------------------------
diff --git a/tool/jenkins/docker/ubuntu16.04/Dockerfile b/tool/jenkins/docker/ubuntu16.04/Dockerfile
new file mode 100644
index 0000000..e7835f4
--- /dev/null
+++ b/tool/jenkins/docker/ubuntu16.04/Dockerfile
@@ -0,0 +1,34 @@
+# Base unbuntu 16.04 image from nvidia/cuda
+# Change tags to build with different cuda/cudnn versions:
+# FROM nvidia/cuda:8.0-cudnn5-devel-ubuntu16.04
+FROM nvidia/cuda:8.0-cudnn5-devel-ubuntu16.04
+
+# install dependencies
+RUN apt-get update && apt-get install -y \
+ git vim wget build-essential autoconf libtool cmake \
+ libprotobuf-dev libopenblas-dev protobuf-compiler \
+ libpcre3-dev python-dev python-pip swig && \
+ pip install numpy
+
+# set environment
+ENV CPLUS_INCLUDE_PATH /usr/local/lib/python2.7/dist-packages/numpy/core/include:${CPLUS_INCLUDE_PATH}
+ENV CMAKE_INCLUDE_PATH /usr/local/cuda/include:${CMAKE_INCLUDE_PATH}
+ENV CMAKE_LIBRARY_PATH /usr/local/cuda/lib64:${CMAKE_LIBRARY_PATH}
+
+# download singa source
+RUN git clone https://github.com/apache/incubator-singa.git
+
+# install ssh service
+RUN apt-get install -y openssh-server
+RUN mkdir /var/run/sshd
+RUN echo 'root:singa' | chpasswd
+RUN sed -i 's/PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config
+# SSH login fix. Otherwise user is kicked off after login
+RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd
+
+# dump environment variables into files, so that ssh can see also
+RUN env | grep _ >> /etc/environment
+
+EXPOSE 22
+
+CMD ["/usr/sbin/sshd", "-D"]
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/33cf5461/tool/jenkins/jenkins_pysinga.sh
----------------------------------------------------------------------
diff --git a/tool/jenkins/jenkins_pysinga.sh b/tool/jenkins/jenkins_pysinga.sh
deleted file mode 100644
index b2e920a..0000000
--- a/tool/jenkins/jenkins_pysinga.sh
+++ /dev/null
@@ -1,46 +0,0 @@
-#!/usr/bin/env bash
-#/**
-# *
-# * Licensed to the Apache Software Foundation (ASF) under one
-# * or more contributor license agreements. See the NOTICE file
-# * distributed with this work for additional information
-# * regarding copyright ownership. The ASF licenses this file
-# * to you under the Apache License, Version 2.0 (the
-# * "License"); you may not use this file except in compliance
-# * with the License. You may obtain a copy of the License at
-# *
-# * http://www.apache.org/licenses/LICENSE-2.0
-# *
-# * Unless required by applicable law or agreed to in writing, software
-# * distributed under the License is distributed on an "AS IS" BASIS,
-# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# * See the License for the specific language governing permissions and
-# * limitations under the License.
-# */
-
-echo Compile, test and distribute PySINGA...
-echo Parameters: $1
-echo Workspace: `pwd`
-echo OS env: `uname -a`
-echo Cuda env: `nvcc --version`
-# set parameters
-CUDNN="OFF"
-if [ $1 = "cudnn" ]; then
- CUDNN="ON"
-fi
-# setup env
-rm -rf build
-mkdir build
-# compile singa c++
-cd build
-cmake -DUSE_CUDNN=$CUDNN -DUSE_CUDA=$CUDNN ../
-make
-# unit test
-./bin/test_singa --gtest_output=xml:./../gtest.xml
-# compile pysinga
-cd python
-python setup.py bdist_wheel
-# rename dist
-cd dist
-mv singa-1.0.1-py2-none-any.whl singa-1.0.0-cp27-none-linux_x86_64.whl
-echo Job finished...
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/33cf5461/tool/jenkins/jenkins_wheel.sh
----------------------------------------------------------------------
diff --git a/tool/jenkins/jenkins_wheel.sh b/tool/jenkins/jenkins_wheel.sh
new file mode 100644
index 0000000..f82d070
--- /dev/null
+++ b/tool/jenkins/jenkins_wheel.sh
@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+#/**
+# *
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+# This script is used by Jenkins to compile and distribute SINGA wheel file.
+
+echo Compile, test and distribute PySINGA...
+echo parameters: $1
+echo workspace: `pwd`
+echo OS version: `cat /etc/issue`
+echo kernal version: `uname -a`
+echo CUDA version: $CUDA_VERSION
+echo CUDNN version: $CUDNN_VERSION
+# set parameters
+CUDNN="OFF"
+if [ $1 = "CUDNN" ]; then
+ CUDNN="ON"
+fi
+# setup env
+rm -rf build
+mkdir build
+# compile singa c++
+cd build
+cmake -DUSE_CUDNN=$CUDNN -DUSE_CUDA=$CUDNN ../
+make
+# unit test
+./bin/test_singa --gtest_output=xml:./../gtest.xml
+# compile pysinga
+cd python
+python setup.py bdist_wheel
+# rename dist
+cd dist
+mv singa-1.0.1-py2-none-any.whl singa-1.0.0-cp27-none-linux_x86_64.whl
+echo Job finished...