You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@singa.apache.org by zh...@apache.org on 2016/12/31 03:59:45 UTC
[1/3] incubator-singa git commit: SINGA-288 - Publish wheel of
PySINGA generated by Jenkins to public servers
Repository: incubator-singa
Updated Branches:
refs/heads/master 3f23c0d42 -> 330c87920
SINGA-288 - Publish wheel of PySINGA generated by Jenkins to public servers
Update dockefiles to pass the OS_VERSION as an environment variable, which would be used to create the wheel file path.
Update the jenkins_wheel.sh to move the whl file into $build_id/$commit_hash/$os_lang/ folder, and package this folder into $build_id.tar.gz.
Update the README.md file with instructions of configuring the Jenkins to use Publish over SSH to copy $build_id.tar.gz to the public server.
add build id to the prefix of artifact
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/330c8792
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/330c8792
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/330c8792
Branch: refs/heads/master
Commit: 330c87920057082fda2c35f664578bea97dbca91
Parents: 761c463
Author: wangwei <wa...@comp.nus.edu.sg>
Authored: Fri Dec 23 15:44:18 2016 +0800
Committer: wangwei <wa...@comp.nus.edu.sg>
Committed: Sat Dec 24 09:33:27 2016 +0800
----------------------------------------------------------------------
tool/jenkins/README.md | 24 +++++++++++++++++++++---
tool/jenkins/docker/ubuntu14.04/Dockerfile | 9 ++++-----
tool/jenkins/docker/ubuntu16.04/Dockerfile | 2 +-
tool/jenkins/jenkins_wheel.sh | 8 ++++----
4 files changed, 30 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/330c8792/tool/jenkins/README.md
----------------------------------------------------------------------
diff --git a/tool/jenkins/README.md b/tool/jenkins/README.md
index 1b5fb7c..b93b221 100644
--- a/tool/jenkins/README.md
+++ b/tool/jenkins/README.md
@@ -10,12 +10,13 @@ Those built binaries need to be archived for users to download.
## Install Jenkins
[Jenkins Official Wiki](https://wiki.jenkins-ci.org/display/JENKINS/Installing+Jenkins)
+The slave nodes for running different building environments are configured under 'Manage Jenkins'->'Manage nodes'.
## Configure Jenkins Multi-configuration Project
Create a multi-configuration project and configure project as follows:
### Description
- This job automatically pulls latest commits from apache SINGA github repository.
+ This job automatically pulls latest commits from SINGA github repository.
It compiles and tests SINGA in different environments and creates PySINGA wheel distribution accordingly.
### General
@@ -37,12 +38,29 @@ Create a multi-configuration project and configure project as follows:
* Execute shell - command - ``bash -ex tool/jenkins/jenkins_wheel.sh $lang``
### Post-build Actions
- * Archive the artifacts - ``build/python/dist/**.whl``
- * Send build artifacts over SSH - source files ``build/python/dist/*.tar.gz``, Remove prefix `build/python/dist`, Exec command `tar xf *.tar.gz && rm *.tar.gz`
* Publish JUnit test result report - Test report XMLs - ``**/gtest.xml, **/unittest.xml``
+ * Archive the artifacts - ``build/python/dist/**.whl``
+ * Send build artifacts over SSH
+ * In jenkins_wheel.sh, package the .whl file into $BUILD_ID.tar.gz. Inside the tar file,
+ the folder layout is `build_id/commit_hash/os_lang/*.whl`, where `os_lang` is the combination of os version, device programming language (cuda/cpp/opencl) and cudnn version.
+ * In `Manage Jenkins`-`Configure System`, configure the SSH for connecting to the remote public server and set the target folder location
+ * Source files - `build/python/dist/*.tar.gz`
+ * Remove prefix - `build/python/dist`
+ * Exec a command on the remote server to decompress the package and add a symlink to the latest build. E.g., on a Solaris server the command is
+
+ cd <target_folder> && gunzip $BUILD_ID.tar.gz && tar xf $BUILD_ID.tar && chmod -R 755 $BUILD_ID && /bin/rm -f $BUILD_ID.tar && /bin/rm -f latest && ln -s $BUILD_ID/* latest
+
+ * The file links on the remote public server would be like
+
+ 32/84d56b7/ubuntu16.04-cpp/singa-1.0.1-py2-none-any.whl
+ 32/84d56b7/ubuntu16.04-cuda8.0-cudnn5/singa-1.0.1-py2-none-any.whl
+
## Docker Images
We provide in `docker` a number of singa docker images for Jenkins to use as slaves.
+To run the docker images,
+
+ nvidia-docker run --name <jenkins-slaveXX> -d <Image ID>
## Access Control
Use `Role Strategy Plugin` to give read access for anonymous users.
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/330c8792/tool/jenkins/docker/ubuntu14.04/Dockerfile
----------------------------------------------------------------------
diff --git a/tool/jenkins/docker/ubuntu14.04/Dockerfile b/tool/jenkins/docker/ubuntu14.04/Dockerfile
index 6d96983..838be4a 100644
--- a/tool/jenkins/docker/ubuntu14.04/Dockerfile
+++ b/tool/jenkins/docker/ubuntu14.04/Dockerfile
@@ -4,14 +4,14 @@
# FROM nvidia/cuda:7.5-cudnn5-devel
# FROM nvidia/cuda:7.5-cudnn4-devel
# FROM nvidia/cuda:7.0-cudnn4-devel
-FROM nvidia/cuda:8.0-cudnn5-devel
+FROM nvidia/cuda:7.5-cudnn5-devel
# install dependencies
RUN apt-get update \
- && apt-get install -y --no-install-recommends git g++ cmake libprotobuf-dev libopenblas-dev protobuf-compiler python-dev python-pip wget\
+ && apt-get install -y --no-install-recommends git build-essential autoconf libtool cmake libpcre3-dev libprotobuf-dev libopenblas-dev protobuf-compiler python-dev python-pip wget openssh-server\
&& apt-get clean && apt-get autoremove && apt-get autoclean \
&& rm -rf /var/lib/apt/lists/* \
- && pip --no-cache-dir install -U pip wheel numpy setuptools unittest-xml-reporting protobuf>=2.5.0,<3
+ && pip install -U pip wheel numpy setuptools unittest-xml-reporting protobuf
# install swig 3.0
@@ -28,8 +28,7 @@ ENV OS_VERSION ubuntu14.04
# download singa source
RUN git clone https://github.com/apache/incubator-singa.git
-# install ssh service
-RUN apt-get install -y openssh-server
+# config ssh service
RUN mkdir /var/run/sshd
RUN echo 'root:singa' | chpasswd
RUN sed -i 's/PermitRootLogin without-password/PermitRootLogin yes/' /etc/ssh/sshd_config
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/330c8792/tool/jenkins/docker/ubuntu16.04/Dockerfile
----------------------------------------------------------------------
diff --git a/tool/jenkins/docker/ubuntu16.04/Dockerfile b/tool/jenkins/docker/ubuntu16.04/Dockerfile
index 1a9434c..da8836e 100644
--- a/tool/jenkins/docker/ubuntu16.04/Dockerfile
+++ b/tool/jenkins/docker/ubuntu16.04/Dockerfile
@@ -12,7 +12,7 @@ RUN apt-get update \
&& apt-get install -y --no-install-recommends git g++ cmake libprotobuf-dev libopenblas-dev protobuf-compiler python-dev python-pip swig wget openssh-server\
&& apt-get clean && apt-get autoremove && apt-get autoclean \
&& rm -rf /var/lib/apt/lists/* \
- && pip --no-cache-dir install -U pip wheel numpy setuptools unittest-xml-reporting protobuf>=2.5.0,<3
+ && pip --no-cache-dir install -U pip wheel numpy setuptools unittest-xml-reporting protobuf
# set environment
ENV CPLUS_INCLUDE_PATH /usr/local/lib/python2.7/dist-packages/numpy/core/include:${CPLUS_INCLUDE_PATH}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/330c8792/tool/jenkins/jenkins_wheel.sh
----------------------------------------------------------------------
diff --git a/tool/jenkins/jenkins_wheel.sh b/tool/jenkins/jenkins_wheel.sh
index f131f68..1c8af27 100644
--- a/tool/jenkins/jenkins_wheel.sh
+++ b/tool/jenkins/jenkins_wheel.sh
@@ -28,16 +28,16 @@ echo kernal version: `uname -a`
echo CUDA version: $CUDA_VERSION
echo CUDNN version: $CUDNN_VERSION
echo OS name: $OS_VERSION
-COMMIT=`git rev-parse HEAD`
+COMMIT=`git rev-parse --short HEAD`
echo COMMIT HASH: $COMMIT
# set parameters
CUDA="OFF"
CUDNN="OFF"
-FOLDER=$COMMIT/$OS_VERSION-cpp/
+FOLDER=$BUILD_NUMBER/$COMMIT/$OS_VERSION-cpp
if [ $1 = "CUDA" ]; then
CUDA="ON"
CUDNN="ON"
- FOLDER=$COMMIT/$OS_VERSION-cuda$CUDA_VERSION-cudnn$CUDNN_VERSION/
+ FOLDER=$BUILD_NUMBER/$COMMIT/$OS_VERSION-cuda$CUDA_VERSION-cudnn$CUDNN_VERSION
fi
echo wheel file folder: build/python/dist/whl/$FOLDER
@@ -58,7 +58,7 @@ python setup.py bdist_wheel
cd dist
mkdir -p $FOLDER
mv *.whl $FOLDER/
-tar czf $FOLDER.tar.gz $FOLDER/*
+tar czf $BUILD_NUMBER.tar.gz $FOLDER/*
# unit test python code
cd ../../../test/python
[3/3] incubator-singa git commit: SINGA-284 - Add python unittest
into Jenkins and link static libs into whl file
Posted by zh...@apache.org.
SINGA-284 - Add python unittest into Jenkins and link static libs into whl file
update python unittest to genreate xml report;
update dockerfile to install unittest-xml-reporting;
update jenkins script to run python unittest;
add USE_MODULES for compiling protobuf and blas
skip cuda test if pysinga is not compiled with cuda
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/e44658aa
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/e44658aa
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/e44658aa
Branch: refs/heads/master
Commit: e44658aa763f9aef9a38c3ea57855e71dd798ef8
Parents: 3f23c0d
Author: wangwei <wa...@comp.nus.edu.sg>
Authored: Thu Dec 15 17:56:23 2016 +0800
Committer: wangwei <wa...@comp.nus.edu.sg>
Committed: Sat Dec 24 09:33:27 2016 +0800
----------------------------------------------------------------------
python/setup.py.in | 9 +++++----
test/python/run.py | 9 +++++++--
test/python/test_optimizer.py | 9 ++++++---
tool/jenkins/README.md | 2 +-
tool/jenkins/docker/ubuntu14.04/Dockerfile | 3 +--
tool/jenkins/docker/ubuntu16.04/Dockerfile | 5 ++---
tool/jenkins/jenkins_wheel.sh | 13 ++++++++-----
7 files changed, 30 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e44658aa/python/setup.py.in
----------------------------------------------------------------------
diff --git a/python/setup.py.in b/python/setup.py.in
index 95550d8..503ac8a 100644
--- a/python/setup.py.in
+++ b/python/setup.py.in
@@ -6,15 +6,15 @@
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
-#
+#
# http://www.apache.org/licenses/LICENSE-2.0
-#
+#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-#
+#
# Always prefer setuptools over distutils
from setuptools import setup
@@ -60,7 +60,8 @@ setup(
install_requires=[
'numpy>=1.11.0',
- 'protobuf>=2.5.0,<3'
+ 'protobuf>=2.5.0,<3',
+ 'unittest-xml-reporting'
],
#List additional groups of dependencies here (e.g. development
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e44658aa/test/python/run.py
----------------------------------------------------------------------
diff --git a/test/python/run.py b/test/python/run.py
index ae33fbd..8f7a45f 100644
--- a/test/python/run.py
+++ b/test/python/run.py
@@ -17,8 +17,13 @@
#
import unittest
+import xmlrunner
loader = unittest.TestLoader()
tests = loader.discover('.')
-testRunner = unittest.runner.TextTestRunner()
-testRunner.run(tests)
+# testRunner = unittest.runner.TextTestRunner()
+with open('unittest.xml', 'wb') as output:
+ testRunner = xmlrunner.XMLTestRunner(output=output,
+ failfast=False,
+ buffer=False)
+ testRunner.run(tests)
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e44658aa/test/python/test_optimizer.py
----------------------------------------------------------------------
diff --git a/test/python/test_optimizer.py b/test/python/test_optimizer.py
index bb3613d..601aada 100644
--- a/test/python/test_optimizer.py
+++ b/test/python/test_optimizer.py
@@ -15,8 +15,6 @@
# specific language governing permissions and limitations
# under the License.
# =============================================================================
-import sys
-import os
import unittest
import numpy as np
@@ -24,8 +22,10 @@ import numpy as np
import singa.tensor as tensor
import singa.optimizer as opt
import singa.device as device
+from singa import singa_wrap
-cuda = device.create_cuda_gpu()
+if singa_wrap.USE_CUDA:
+ cuda = device.create_cuda_gpu()
class TestOptimizer(unittest.TestCase):
@@ -48,6 +48,7 @@ class TestOptimizer(unittest.TestCase):
for i in range(self.W.size()):
self.assertAlmostEqual(w[i], self.np_W[i] - lr * self.np_g[i])
+ @unittest.skipIf(not singa_wrap.USE_CUDA, 'CUDA is not enabled')
def test_sgd_cuda(self):
lr = 0.1
sgd = opt.SGD(lr)
@@ -67,6 +68,7 @@ class TestOptimizer(unittest.TestCase):
for i in range(g.size):
self.assertAlmostEqual(g[i], self.np_g[i] * threshold / nrm)
+ @unittest.skipIf(not singa_wrap.USE_CUDA, 'CUDA is not enabled')
def test_constraint_cuda(self):
threshold = 0.02
self.to_cuda()
@@ -87,6 +89,7 @@ class TestOptimizer(unittest.TestCase):
self.assertAlmostEqual(g[i],
self.np_g[i] + coefficient * self.np_W[i])
+ @unittest.skipIf(not singa_wrap.USE_CUDA, 'CUDA is not enabled')
def test_regularizer_cuda(self):
coefficient = 0.0001
reg = opt.L2Regularizer(coefficient)
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e44658aa/tool/jenkins/README.md
----------------------------------------------------------------------
diff --git a/tool/jenkins/README.md b/tool/jenkins/README.md
index 6bd90df..d20470b 100644
--- a/tool/jenkins/README.md
+++ b/tool/jenkins/README.md
@@ -38,7 +38,7 @@ Create a multi-configuration project and configure project as follows:
### Post-build Actions
* Archive the artifacts - ``build/python/dist/**.whl``
- * Publish JUnit test result report - Test report XMLs - ``**/gtest.xml``
+ * Publish JUnit test result report - Test report XMLs - ``**/gtest.xml, **/unittest.xml``
## Docker Images
We provide in `docker` a number of singa docker images for Jenkins to use as
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e44658aa/tool/jenkins/docker/ubuntu14.04/Dockerfile
----------------------------------------------------------------------
diff --git a/tool/jenkins/docker/ubuntu14.04/Dockerfile b/tool/jenkins/docker/ubuntu14.04/Dockerfile
index b664689..9b974d2 100644
--- a/tool/jenkins/docker/ubuntu14.04/Dockerfile
+++ b/tool/jenkins/docker/ubuntu14.04/Dockerfile
@@ -9,9 +9,8 @@ FROM nvidia/cuda:8.0-cudnn5-devel
# install dependencies
RUN apt-get update && apt-get install -y \
git vim wget build-essential autoconf libtool cmake \
- libprotobuf-dev libopenblas-dev protobuf-compiler \
libpcre3-dev python-dev python-pip && \
- pip install numpy
+ pip install numpy protobuf unittest-xml-reporting
# install swig 3.0
RUN wget http://prdownloads.sourceforge.net/swig/swig-3.0.10.tar.gz && \
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e44658aa/tool/jenkins/docker/ubuntu16.04/Dockerfile
----------------------------------------------------------------------
diff --git a/tool/jenkins/docker/ubuntu16.04/Dockerfile b/tool/jenkins/docker/ubuntu16.04/Dockerfile
index e7835f4..612dd8b 100644
--- a/tool/jenkins/docker/ubuntu16.04/Dockerfile
+++ b/tool/jenkins/docker/ubuntu16.04/Dockerfile
@@ -5,10 +5,9 @@ FROM nvidia/cuda:8.0-cudnn5-devel-ubuntu16.04
# install dependencies
RUN apt-get update && apt-get install -y \
- git vim wget build-essential autoconf libtool cmake \
- libprotobuf-dev libopenblas-dev protobuf-compiler \
+ git wget build-essential autoconf libtool cmake \
libpcre3-dev python-dev python-pip swig && \
- pip install numpy
+ pip install numpy protobuf unittest-xml-reporting
# set environment
ENV CPLUS_INCLUDE_PATH /usr/local/lib/python2.7/dist-packages/numpy/core/include:${CPLUS_INCLUDE_PATH}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/e44658aa/tool/jenkins/jenkins_wheel.sh
----------------------------------------------------------------------
diff --git a/tool/jenkins/jenkins_wheel.sh b/tool/jenkins/jenkins_wheel.sh
index f82d070..c3d075f 100644
--- a/tool/jenkins/jenkins_wheel.sh
+++ b/tool/jenkins/jenkins_wheel.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/usr/bin/env bash
#/**
# *
# * Licensed to the Apache Software Foundation (ASF) under one
@@ -26,7 +26,7 @@ echo workspace: `pwd`
echo OS version: `cat /etc/issue`
echo kernal version: `uname -a`
echo CUDA version: $CUDA_VERSION
-echo CUDNN version: $CUDNN_VERSION
+echo CUDNN version: $CUDNN_VERSION
# set parameters
CUDNN="OFF"
if [ $1 = "CUDNN" ]; then
@@ -37,14 +37,17 @@ rm -rf build
mkdir build
# compile singa c++
cd build
-cmake -DUSE_CUDNN=$CUDNN -DUSE_CUDA=$CUDNN ../
+cmake -DUSE_CUDNN=$CUDNN -DUSE_CUDA=$CUDNN -DUSE_MODULES=ON ../
make
-# unit test
-./bin/test_singa --gtest_output=xml:./../gtest.xml
+# unit test cpp code
+./bin/test_singa --gtest_output=xml:./gtest.xml
# compile pysinga
cd python
python setup.py bdist_wheel
# rename dist
cd dist
mv singa-1.0.1-py2-none-any.whl singa-1.0.0-cp27-none-linux_x86_64.whl
+# unit test python code
+cd ../../../test/python
+PYTHONPATH=../../build/python/ python run.py
echo Job finished...
[2/3] incubator-singa git commit: SINGA-284 - Add python unittest
into Jenkins and link static libs into whl file
Posted by zh...@apache.org.
SINGA-284 - Add python unittest into Jenkins and link static libs into whl file
Update the script to package the whl file into a folder, whose name indicates the version of OS, cuda and cudnn.
Update the jenkins/docker files to avoid installing libprotobuf and openblas
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/761c463d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/761c463d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/761c463d
Branch: refs/heads/master
Commit: 761c463dd8a2aedd59af7f3a936e359080e6828d
Parents: e44658a
Author: wangwei <wa...@comp.nus.edu.sg>
Authored: Fri Dec 23 15:16:19 2016 +0800
Committer: wangwei <wa...@comp.nus.edu.sg>
Committed: Sat Dec 24 09:33:27 2016 +0800
----------------------------------------------------------------------
tool/jenkins/README.md | 8 ++++----
tool/jenkins/docker/ubuntu14.04/Dockerfile | 11 +++++++----
tool/jenkins/docker/ubuntu16.04/Dockerfile | 17 +++++++++++------
tool/jenkins/jenkins_wheel.sh | 21 +++++++++++++++++----
4 files changed, 39 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/761c463d/tool/jenkins/README.md
----------------------------------------------------------------------
diff --git a/tool/jenkins/README.md b/tool/jenkins/README.md
index d20470b..1b5fb7c 100644
--- a/tool/jenkins/README.md
+++ b/tool/jenkins/README.md
@@ -30,19 +30,19 @@ Create a multi-configuration project and configure project as follows:
* Poll SCM - Schedule - ``H/30 * * * *`` (pull every 30 minutes)
### Configuration Matrix
- * User-defined Axis - name ``support`` values ``BLAS CUDNN``
+ * User-defined Axis - name ``lang`` values ``CPP CUDA``
* Slave - name ``env`` Node/label ``tick all nodes``
### Build
- * Execute shell - command - ``bash -ex tool/jenkins/jenkins_wheel.sh $support``
+ * Execute shell - command - ``bash -ex tool/jenkins/jenkins_wheel.sh $lang``
### Post-build Actions
* Archive the artifacts - ``build/python/dist/**.whl``
+ * Send build artifacts over SSH - source files ``build/python/dist/*.tar.gz``, Remove prefix `build/python/dist`, Exec command `tar xf *.tar.gz && rm *.tar.gz`
* Publish JUnit test result report - Test report XMLs - ``**/gtest.xml, **/unittest.xml``
## Docker Images
-We provide in `docker` a number of singa docker images for Jenkins to use as
-slaves.
+We provide in `docker` a number of singa docker images for Jenkins to use as slaves.
## Access Control
Use `Role Strategy Plugin` to give read access for anonymous users.
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/761c463d/tool/jenkins/docker/ubuntu14.04/Dockerfile
----------------------------------------------------------------------
diff --git a/tool/jenkins/docker/ubuntu14.04/Dockerfile b/tool/jenkins/docker/ubuntu14.04/Dockerfile
index 9b974d2..6d96983 100644
--- a/tool/jenkins/docker/ubuntu14.04/Dockerfile
+++ b/tool/jenkins/docker/ubuntu14.04/Dockerfile
@@ -7,10 +7,12 @@
FROM nvidia/cuda:8.0-cudnn5-devel
# install dependencies
-RUN apt-get update && apt-get install -y \
- git vim wget build-essential autoconf libtool cmake \
- libpcre3-dev python-dev python-pip && \
- pip install numpy protobuf unittest-xml-reporting
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends git g++ cmake libprotobuf-dev libopenblas-dev protobuf-compiler python-dev python-pip wget\
+ && apt-get clean && apt-get autoremove && apt-get autoclean \
+ && rm -rf /var/lib/apt/lists/* \
+ && pip --no-cache-dir install -U pip wheel numpy setuptools unittest-xml-reporting protobuf>=2.5.0,<3
+
# install swig 3.0
RUN wget http://prdownloads.sourceforge.net/swig/swig-3.0.10.tar.gz && \
@@ -21,6 +23,7 @@ RUN wget http://prdownloads.sourceforge.net/swig/swig-3.0.10.tar.gz && \
ENV CPLUS_INCLUDE_PATH /usr/local/lib/python2.7/dist-packages/numpy/core/include:${CPLUS_INCLUDE_PATH}
ENV CMAKE_INCLUDE_PATH /usr/local/cuda/include:${CMAKE_INCLUDE_PATH}
ENV CMAKE_LIBRARY_PATH /usr/local/cuda/lib64:${CMAKE_LIBRARY_PATH}
+ENV OS_VERSION ubuntu14.04
# download singa source
RUN git clone https://github.com/apache/incubator-singa.git
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/761c463d/tool/jenkins/docker/ubuntu16.04/Dockerfile
----------------------------------------------------------------------
diff --git a/tool/jenkins/docker/ubuntu16.04/Dockerfile b/tool/jenkins/docker/ubuntu16.04/Dockerfile
index 612dd8b..1a9434c 100644
--- a/tool/jenkins/docker/ubuntu16.04/Dockerfile
+++ b/tool/jenkins/docker/ubuntu16.04/Dockerfile
@@ -4,21 +4,26 @@
FROM nvidia/cuda:8.0-cudnn5-devel-ubuntu16.04
# install dependencies
-RUN apt-get update && apt-get install -y \
- git wget build-essential autoconf libtool cmake \
- libpcre3-dev python-dev python-pip swig && \
- pip install numpy protobuf unittest-xml-reporting
+
+MAINTAINER incubator-singa dev@singa.incubator.apache.org
+
+# install dependencies
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends git g++ cmake libprotobuf-dev libopenblas-dev protobuf-compiler python-dev python-pip swig wget openssh-server\
+ && apt-get clean && apt-get autoremove && apt-get autoclean \
+ && rm -rf /var/lib/apt/lists/* \
+ && pip --no-cache-dir install -U pip wheel numpy setuptools unittest-xml-reporting protobuf>=2.5.0,<3
# set environment
ENV CPLUS_INCLUDE_PATH /usr/local/lib/python2.7/dist-packages/numpy/core/include:${CPLUS_INCLUDE_PATH}
ENV CMAKE_INCLUDE_PATH /usr/local/cuda/include:${CMAKE_INCLUDE_PATH}
ENV CMAKE_LIBRARY_PATH /usr/local/cuda/lib64:${CMAKE_LIBRARY_PATH}
+ENV OS_VERSION ubuntu16.04
# download singa source
RUN git clone https://github.com/apache/incubator-singa.git
-# install ssh service
-RUN apt-get install -y openssh-server
+# config ssh service
RUN mkdir /var/run/sshd
RUN echo 'root:singa' | chpasswd
RUN sed -i 's/PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/761c463d/tool/jenkins/jenkins_wheel.sh
----------------------------------------------------------------------
diff --git a/tool/jenkins/jenkins_wheel.sh b/tool/jenkins/jenkins_wheel.sh
index c3d075f..f131f68 100644
--- a/tool/jenkins/jenkins_wheel.sh
+++ b/tool/jenkins/jenkins_wheel.sh
@@ -27,26 +27,39 @@ echo OS version: `cat /etc/issue`
echo kernal version: `uname -a`
echo CUDA version: $CUDA_VERSION
echo CUDNN version: $CUDNN_VERSION
+echo OS name: $OS_VERSION
+COMMIT=`git rev-parse HEAD`
+echo COMMIT HASH: $COMMIT
# set parameters
+CUDA="OFF"
CUDNN="OFF"
-if [ $1 = "CUDNN" ]; then
+FOLDER=$COMMIT/$OS_VERSION-cpp/
+if [ $1 = "CUDA" ]; then
+ CUDA="ON"
CUDNN="ON"
+ FOLDER=$COMMIT/$OS_VERSION-cuda$CUDA_VERSION-cudnn$CUDNN_VERSION/
fi
+echo wheel file folder: build/python/dist/whl/$FOLDER
+
# setup env
rm -rf build
mkdir build
+
# compile singa c++
cd build
-cmake -DUSE_CUDNN=$CUDNN -DUSE_CUDA=$CUDNN -DUSE_MODULES=ON ../
+cmake -DUSE_CUDNN=$CUDNN -DUSE_CUDA=$CUDA -DUSE_MODULES=ON -DUSE_MODULES=ON ../
make
# unit test cpp code
./bin/test_singa --gtest_output=xml:./gtest.xml
# compile pysinga
cd python
python setup.py bdist_wheel
-# rename dist
+# mv whl file to a folder whose name identifies the OS, CUDA, CUDNN etc.
cd dist
-mv singa-1.0.1-py2-none-any.whl singa-1.0.0-cp27-none-linux_x86_64.whl
+mkdir -p $FOLDER
+mv *.whl $FOLDER/
+tar czf $FOLDER.tar.gz $FOLDER/*
+
# unit test python code
cd ../../../test/python
PYTHONPATH=../../build/python/ python run.py