You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@singa.apache.org by wa...@apache.org on 2016/08/15 16:15:22 UTC
[09/22] incubator-singa git commit: SINGA-237 New documentation files
for SINGA v1.0 - add python installation instruction, remove flask dependency
SINGA-237 New documentation files for SINGA v1.0
- add python installation instruction, remove flask dependency
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/410f238a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/410f238a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/410f238a
Branch: refs/heads/dev
Commit: 410f238af4388c174e9a2725baf40153cacb0915
Parents: d3a57cf
Author: aaronwwf <dc...@gmail.com>
Authored: Sun Aug 14 23:51:03 2016 +0800
Committer: aaronwwf <dc...@gmail.com>
Committed: Mon Aug 15 16:21:23 2016 +0800
----------------------------------------------------------------------
doc/docs/installation.md | 73 +++++++++++-
src/python/setup.py.in | 5 +-
src/python/singa/__init__.py | 240 --------------------------------------
src/python/singa/command.py | 240 ++++++++++++++++++++++++++++++++++++++
4 files changed, 313 insertions(+), 245 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/410f238a/doc/docs/installation.md
----------------------------------------------------------------------
diff --git a/doc/docs/installation.md b/doc/docs/installation.md
index 8ab617f..6bfdee3 100755
--- a/doc/docs/installation.md
+++ b/doc/docs/installation.md
@@ -3,7 +3,7 @@
## Dependencies
### Required
-* Google Protobuf (>=2.5)
+* Google Protobuf (>=2.5,<3)
* BLAS (tested with OpenBLAS >=0.2.10)
* CUDA (tested with 6.5, 7.0 and 7.5)
* CUDNN (v4 and v5)
@@ -52,7 +52,7 @@ Note that if you are using CUDNN, you need to let cmake know the paths to CUDNN,
$ export CMAKE_INCLUDE_PATH=<path to cudnn>/include:$CMAKE_INCLUDE_PATH
$ export CMAKE_LIBRARY_PATH=<path to cudnn>/lib64:$CMAKE_LIBRARY_PATH
-You can use `ccmake ..` to configure the compilation options including using
+You can use `cmake ..` to configure the compilation options including using
LMDB, GLOG, etc.
After compiling SINGA, you can run the unit tests by
@@ -65,5 +65,74 @@ tests, then you have successfully installed SINGA. Please proceed to try the exa
### MacOS
+Currently only Linux OS is officially support.
### Windows
+
+Currently only Linux OS is officially support.
+
+
+# Install SINGA Python Module
+
+SINGA provide a python binding for python programers. Users can either install from source or
+from pre-built wheel file.
+
+## Install from source
+
+### Required
+* python(==2.7)
+* pip(>=1.5)
+* SWIG(>=3.0)
+* numpy(>=1.11.0)
+* Google protobuf(>=2.5,<3)
+
+
+### Configuration
+To build SINGA python package, users should turn on python building switch in cmake config file: "CMakeList.txt"
+
+ OPTION(USE_PYTHON "Generate py wrappers" ON)
+
+### Instructions
+Follow the instructions in the above sections to build SINGA from source,
+
+After that, execute the following commands:
+
+ # under the build directory
+ $ cd python
+ $ sudo pip install .
+
+Then singa package should be installed in the corresponding python library.
+
+## Pip Install from wheel
+
+Install pip if it is not already installed:
+
+ $ sudo apt-get install python-pip python-dev
+
+Then, select the correct binary to install:
+
+ # Ubuntu/Linux 64-bit, CPU only, Python 2.7, Protobuf 2.5
+ $ export SINGA_WHEEL_URL=http://comp.nus.edu.sg/~dbsystem/singa/assets/file/pb2.5/singa-1.0.0-cp27-none-linux_x86_64.whl
+
+ # Ubuntu/Linux 64-bit, CPU only, Python 2.7, Protobuf 2.6
+ $ export SINGA_WHEEL_URL=http://comp.nus.edu.sg/~dbsystem/singa/assets/file/pb2.6/singa-1.0.0-cp27-none-linux_x86_64.whl
+
+ # Ubuntu/Linux 64-bit, GPU enabled, Python 2.7, Protobuf 2.5, CUDA toolkit 7.5 and CuDNN v5
+ $ export SINGA_WHEEL_URL=http://comp.nus.edu.sg/~dbsystem/singa/assets/file/pb2.5-cuda7.5-cudnn5/singa-1.0.0-cp27-none-linux_x86_64.whl
+
+ # Ubuntu/Linux 64-bit, GPU enabled, Python 2.7, Protobuf 2.6, CUDA toolkit 7.5 and CuDNN v5
+ $ export SINGA_WHEEL_URL=http://comp.nus.edu.sg/~dbsystem/singa/assets/file/pb2.6-cuda7.5-cudnn5/singa-1.0.0-cp27-none-linux_x86_64.whl
+
+Install SINGA:
+
+ $ sudo pip install --upgrade $SINGA_WHEEL_URL
+
+### build wheel file from source
+
+Users can build wheel file from source. After build SINGA, execute the following commands:
+
+ # under the build directory
+ $ cd python
+ $ python setup.py bdist_wheel
+
+Then users may get built wheel file under "dist" directory
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/410f238a/src/python/setup.py.in
----------------------------------------------------------------------
diff --git a/src/python/setup.py.in b/src/python/setup.py.in
index d1ac3c9..f2cd9f3 100644
--- a/src/python/setup.py.in
+++ b/src/python/setup.py.in
@@ -42,8 +42,7 @@ setup(
install_requires=[
'numpy>=1.11.0',
- 'protobuf>=2.5.0,<3',
- 'flask>=0.10.1'
+ 'protobuf>=2.5.0,<3'
],
#List additional groups of dependencies here (e.g. development
@@ -75,7 +74,7 @@ setup(
entry_points={
'console_scripts': [
- 'singa=singa:main',
+ 'singa=singa.command:main',
],
},
)
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/410f238a/src/python/singa/__init__.py
----------------------------------------------------------------------
diff --git a/src/python/singa/__init__.py b/src/python/singa/__init__.py
index f14c8c5..e69de29 100644
--- a/src/python/singa/__init__.py
+++ b/src/python/singa/__init__.py
@@ -1,240 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-# =============================================================================
-
-'''
-This script is the main entrance for user to run singa inside a model workspace
-
-To use this script, user sudo install these dependencies: flask pillow and protobuf
-'''
-
-import sys, glob, os, random, shutil, time
-from flask import Flask, request, redirect, url_for
-import numpy as np
-import ConfigParser
-import urllib, traceback
-
-
-from argparse import ArgumentParser
-from argparse import RawDescriptionHelpFormatter
-sys.path.append(os.getcwd())
-
-__all__ = []
-__version__ = 0.1
-__date__ = '2016-07-20'
-__updated__ = '2016-07-20'
-__shortdesc__ = '''
-welcome to singa
-'''
-
-app = Flask(__name__)
-config = ConfigParser.RawConfigParser()
-service = {}
-data_path = "data_"
-parameter_path = "parameter_"
-
-debug = False
-
-class CLIError(Exception):
- '''Generic exception to raise and log different fatal errors.'''
- def __init__(self, msg):
- super(CLIError).__init__(type(self))
- self.msg = "E: %s" % msg
- def __str__(self):
- return self.msg
- def __unicode__(self):
- return self.msg
-
-def main(argv=None): # IGNORE:C0111
- '''Command line options.'''
-
- from . import device
-
- if argv is None:
- argv = sys.argv
- else:
- sys.argv.extend(argv)
-
- program_name = os.path.basename(sys.argv[0])
- program_version = "v%s" % __version__
- program_build_date = str(__updated__)
- program_version_message = '%%(prog)s %s (%s)' % (program_version, program_build_date)
- program_shortdesc = __shortdesc__
- program_license = '''%s
-
- Created by dbsystem group on %s.
- Copyright 2016 NUS School of Computing. All rights reserved.
-
- Licensed under the Apache License 2.0
- http://www.apache.org/licenses/LICENSE-2.0
-
- Distributed on an "AS IS" basis without warranties
- or conditions of any kind, either express or implied.
-
-USAGE
-''' % (program_shortdesc, str(__date__))
-
- global debug
-
- try:
- # Setup argument parser
- parser = ArgumentParser(description=program_license, formatter_class=RawDescriptionHelpFormatter)
- parser.add_argument("-p", "--port", dest="port", default=5000, help="the port to listen to, default is 5000")
- parser.add_argument("-param", "--parameter", dest="parameter", help="the parameter file path to be loaded")
- parser.add_argument("-D", "--debug", dest="debug", action="store_true", help="whether need to debug")
- parser.add_argument("-R", "--reload", dest="reload_data", action="store_true", help="whether need to reload data")
- parser.add_argument("-C", "--cpu", dest="use_cpu", action="store_true", help="Using cpu or not, default is using gpu")
- parser.add_argument("-m", "--mode", dest="mode", choices=['train','test','serve'], default='serve', help="On Which mode (train,test,serve) to run singa")
- parser.add_argument('-V', '--version', action='version', version=program_version_message)
-
- # Process arguments
- args = parser.parse_args()
-
- port = args.port
- parameter_file = args.parameter
- mode = args.mode
- need_reload = args.reload_data
- use_cpu = args.use_cpu
- debug = args.debug
-
- #prepare data files
- config.read('file.cfg')
- file_prepare(need_reload)
-
-
- import network as net
- model = net.create()
-
- #load parameter
- parameter_file=get_parameter(parameter_file)
-
- if parameter_file:
- print "load parameter file: %s" % parameter_file
- model.load(parameter_file)
-
- if use_cpu:
- raise CLIError("Currently cpu is not support!")
- else:
- print "runing with gpu"
- d = device.create_cuda_gpu()
-
- model.to_device(d)
-
- if mode == "serve":
- print "runing singa in serve mode, listen to port: %s " % port
- global service
- from serve import Service
- service =Service(model,d)
-
- app.debug = debug
- app.run(host='0.0.0.0', port= port)
- elif mode == "train":
- print "runing singa in train mode"
- global trainer
- from train import Trainer
- trainer= Trainer(model,d)
- if not parameter_file:
- trainer.initialize()
- trainer.train()
- else:
- raise CLIError("Currently only serve mode is surpported!")
- return 0
- except KeyboardInterrupt:
- ### handle keyboard interrupt ###
- return 0
- except Exception, e:
- if debug:
- traceback.print_exc()
- raise(e)
- indent = len(program_name) * " "
- sys.stderr.write(program_name + ": " + str(e) + "\n")
- sys.stderr.write(indent + " for help use --help \n\n")
- return 2
-
-def file_prepare(reload_data=False):
- '''
- download all files and generate data.py
- '''
- if not reload_data and os.path.exists("data_.py"):
- return
-
- print "download file"
- #clean data
- shutil.rmtree("data_.py",ignore_errors=True)
- shutil.rmtree("data_",ignore_errors=True)
-
- data_py=open("data_.py",'w')
- data_py.write("#%s" % "This file is Generated by SINGA, please don't edit\n\n")
- if config.has_section("data"):
- file_list = config.items("data")
- #download files
- for f in file_list:
- name,path=download_file(f[0],f[1],data_path)
- data_py.write("%s=\"%s\"\n" % (name,path))
-
- data_py.flush()
- data_py.close()
-
- if config.has_section("parameter"):
- parameter_list = config.items("parameter")
- for p in parameter_list:
- download_file(p[0],p[1],parameter_path)
-
-def download_file(name,path,dest):
- '''
- download one file to dest
- '''
- if not os.path.exists(dest):
- os.makedirs(dest)
- if (path.startswith('http')):
- file_name = path.split('/')[-1]
- target = os.path.join(dest,file_name)
- urllib.urlretrieve(path,target)
- return name,target
-
-
-def get_parameter(file_name=None):
- '''
- get the paticular file name or get the last parameter file
- '''
- if not os.path.exists(parameter_path):
- os.makedirs(parameter_path)
- return
-
- if file_name:
- return os.path.join(parameter_path,file_name)
-
- parameter_list = [ os.path.join(parameter_path,f) for f in os.listdir(parameter_path)]
- if len(parameter_list)==0:
- return
- parameter_list.sort()
-
- return parameter_list[-1]
-
-@app.route("/")
-def index():
- return "Hello SINGA User!"
-
-@app.route('/predict', methods=['POST'])
-def predict():
- if request.method == 'POST':
- try:
- response=service.serve(request)
- except Exception as e:
- return e
- return response
- return "error, should be post request"
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/410f238a/src/python/singa/command.py
----------------------------------------------------------------------
diff --git a/src/python/singa/command.py b/src/python/singa/command.py
new file mode 100644
index 0000000..f14c8c5
--- /dev/null
+++ b/src/python/singa/command.py
@@ -0,0 +1,240 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# =============================================================================
+
+'''
+This script is the main entrance for user to run singa inside a model workspace
+
+To use this script, user sudo install these dependencies: flask pillow and protobuf
+'''
+
+import sys, glob, os, random, shutil, time
+from flask import Flask, request, redirect, url_for
+import numpy as np
+import ConfigParser
+import urllib, traceback
+
+
+from argparse import ArgumentParser
+from argparse import RawDescriptionHelpFormatter
+sys.path.append(os.getcwd())
+
+__all__ = []
+__version__ = 0.1
+__date__ = '2016-07-20'
+__updated__ = '2016-07-20'
+__shortdesc__ = '''
+welcome to singa
+'''
+
+app = Flask(__name__)
+config = ConfigParser.RawConfigParser()
+service = {}
+data_path = "data_"
+parameter_path = "parameter_"
+
+debug = False
+
+class CLIError(Exception):
+ '''Generic exception to raise and log different fatal errors.'''
+ def __init__(self, msg):
+ super(CLIError).__init__(type(self))
+ self.msg = "E: %s" % msg
+ def __str__(self):
+ return self.msg
+ def __unicode__(self):
+ return self.msg
+
+def main(argv=None): # IGNORE:C0111
+ '''Command line options.'''
+
+ from . import device
+
+ if argv is None:
+ argv = sys.argv
+ else:
+ sys.argv.extend(argv)
+
+ program_name = os.path.basename(sys.argv[0])
+ program_version = "v%s" % __version__
+ program_build_date = str(__updated__)
+ program_version_message = '%%(prog)s %s (%s)' % (program_version, program_build_date)
+ program_shortdesc = __shortdesc__
+ program_license = '''%s
+
+ Created by dbsystem group on %s.
+ Copyright 2016 NUS School of Computing. All rights reserved.
+
+ Licensed under the Apache License 2.0
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Distributed on an "AS IS" basis without warranties
+ or conditions of any kind, either express or implied.
+
+USAGE
+''' % (program_shortdesc, str(__date__))
+
+ global debug
+
+ try:
+ # Setup argument parser
+ parser = ArgumentParser(description=program_license, formatter_class=RawDescriptionHelpFormatter)
+ parser.add_argument("-p", "--port", dest="port", default=5000, help="the port to listen to, default is 5000")
+ parser.add_argument("-param", "--parameter", dest="parameter", help="the parameter file path to be loaded")
+ parser.add_argument("-D", "--debug", dest="debug", action="store_true", help="whether need to debug")
+ parser.add_argument("-R", "--reload", dest="reload_data", action="store_true", help="whether need to reload data")
+ parser.add_argument("-C", "--cpu", dest="use_cpu", action="store_true", help="Using cpu or not, default is using gpu")
+ parser.add_argument("-m", "--mode", dest="mode", choices=['train','test','serve'], default='serve', help="On Which mode (train,test,serve) to run singa")
+ parser.add_argument('-V', '--version', action='version', version=program_version_message)
+
+ # Process arguments
+ args = parser.parse_args()
+
+ port = args.port
+ parameter_file = args.parameter
+ mode = args.mode
+ need_reload = args.reload_data
+ use_cpu = args.use_cpu
+ debug = args.debug
+
+ #prepare data files
+ config.read('file.cfg')
+ file_prepare(need_reload)
+
+
+ import network as net
+ model = net.create()
+
+ #load parameter
+ parameter_file=get_parameter(parameter_file)
+
+ if parameter_file:
+ print "load parameter file: %s" % parameter_file
+ model.load(parameter_file)
+
+ if use_cpu:
+ raise CLIError("Currently cpu is not support!")
+ else:
+ print "runing with gpu"
+ d = device.create_cuda_gpu()
+
+ model.to_device(d)
+
+ if mode == "serve":
+ print "runing singa in serve mode, listen to port: %s " % port
+ global service
+ from serve import Service
+ service =Service(model,d)
+
+ app.debug = debug
+ app.run(host='0.0.0.0', port= port)
+ elif mode == "train":
+ print "runing singa in train mode"
+ global trainer
+ from train import Trainer
+ trainer= Trainer(model,d)
+ if not parameter_file:
+ trainer.initialize()
+ trainer.train()
+ else:
+ raise CLIError("Currently only serve mode is surpported!")
+ return 0
+ except KeyboardInterrupt:
+ ### handle keyboard interrupt ###
+ return 0
+ except Exception, e:
+ if debug:
+ traceback.print_exc()
+ raise(e)
+ indent = len(program_name) * " "
+ sys.stderr.write(program_name + ": " + str(e) + "\n")
+ sys.stderr.write(indent + " for help use --help \n\n")
+ return 2
+
+def file_prepare(reload_data=False):
+ '''
+ download all files and generate data.py
+ '''
+ if not reload_data and os.path.exists("data_.py"):
+ return
+
+ print "download file"
+ #clean data
+ shutil.rmtree("data_.py",ignore_errors=True)
+ shutil.rmtree("data_",ignore_errors=True)
+
+ data_py=open("data_.py",'w')
+ data_py.write("#%s" % "This file is Generated by SINGA, please don't edit\n\n")
+ if config.has_section("data"):
+ file_list = config.items("data")
+ #download files
+ for f in file_list:
+ name,path=download_file(f[0],f[1],data_path)
+ data_py.write("%s=\"%s\"\n" % (name,path))
+
+ data_py.flush()
+ data_py.close()
+
+ if config.has_section("parameter"):
+ parameter_list = config.items("parameter")
+ for p in parameter_list:
+ download_file(p[0],p[1],parameter_path)
+
+def download_file(name,path,dest):
+ '''
+ download one file to dest
+ '''
+ if not os.path.exists(dest):
+ os.makedirs(dest)
+ if (path.startswith('http')):
+ file_name = path.split('/')[-1]
+ target = os.path.join(dest,file_name)
+ urllib.urlretrieve(path,target)
+ return name,target
+
+
+def get_parameter(file_name=None):
+ '''
+ get the paticular file name or get the last parameter file
+ '''
+ if not os.path.exists(parameter_path):
+ os.makedirs(parameter_path)
+ return
+
+ if file_name:
+ return os.path.join(parameter_path,file_name)
+
+ parameter_list = [ os.path.join(parameter_path,f) for f in os.listdir(parameter_path)]
+ if len(parameter_list)==0:
+ return
+ parameter_list.sort()
+
+ return parameter_list[-1]
+
+@app.route("/")
+def index():
+ return "Hello SINGA User!"
+
+@app.route('/predict', methods=['POST'])
+def predict():
+ if request.method == 'POST':
+ try:
+ response=service.serve(request)
+ except Exception as e:
+ return e
+ return response
+ return "error, should be post request"