You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@singa.apache.org by wa...@apache.org on 2015/12/10 15:05:09 UTC
[08/12] incubator-singa git commit: SINGA-108 Add Python wrapper to
singa
SINGA-108 Add Python wrapper to singa
Modify paths in generatepy.sh.
This script is only for developers that want to add more python APIs.
Delete files in src dir. Modify singa-run.sh
Delete chonho's code.
Add singa.py into tool/python/.
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/3e58760b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/3e58760b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/3e58760b
Branch: refs/heads/master
Commit: 3e58760b42a134d9dad3d0f931811fa384cc0567
Parents: ef5b655
Author: xiezl <xi...@comp.nus.edu.sg>
Authored: Mon Dec 7 11:40:37 2015 +0800
Committer: xiezl <xi...@comp.nus.edu.sg>
Committed: Tue Dec 8 14:55:57 2015 +0800
----------------------------------------------------------------------
Makefile.am | 4 +-
bin/singa-run.sh | 10 +-
include/singa/driver.h | 12 +-
src/driver.cc | 2 -
src/driver.i | 18 --
src/generatepy.sh | 7 -
src/singa.py | 15 -
tool/python/README.md | 191 -----------
tool/python/__init__.py | 0
tool/python/datasets/__init__.py | 0
tool/python/datasets/cifar10.py | 31 --
tool/python/datasets/mnist.py | 29 --
tool/python/datasets/rnnlm.py | 23 --
tool/python/ex_cifar10_cnn.py | 36 ---
tool/python/ex_mnist_mlp.py | 29 --
tool/python/ex_rnnlm.py | 23 --
tool/python/model.py | 580 ----------------------------------
tool/python/singa.py | 20 ++
tool/python/singa/generatepy.sh | 11 +-
tool/python/utils/__init__.py | 0
tool/python/utils/message.py | 50 ---
tool/python/utils/utility.py | 45 ---
22 files changed, 42 insertions(+), 1094 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3e58760b/Makefile.am
----------------------------------------------------------------------
diff --git a/Makefile.am b/Makefile.am
index a09a12f..25f481c 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -228,7 +228,7 @@ clean-local:
rm -rf $(PROTO_SRCS) $(PROTO_HDRS)
rm -rf $(PROTO_PYS)
rm -rf rat_check
- rm -rf tool/pb2
+ rm -rf tool/python/pb2
rat:
@if test ! -z '$(shell command -v java 2>/dev/null)'; then \
@@ -246,9 +246,11 @@ rat:
$(NVCC) $(MSHADOW_FLAGS) $(CUDA_CFLAGS) $(CUDA_LDFLAGS) $(CUDA_LIBS) -I$(top_srcdir)/include -std=c++11 -G -c -o $@ $<
install-pyLTLIBRARIES: $(py_LTLIBRARIES)
+ touch tool/python/singa/__init__.py
cp -f .libs/_driver.so tool/python/singa/
uninstall-pyLTLIBRARIES:
+ rm -f tool/python/singa/__init__.py
rm -f tool/python/singa/_driver.so
test: singatest
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3e58760b/bin/singa-run.sh
----------------------------------------------------------------------
diff --git a/bin/singa-run.sh b/bin/singa-run.sh
index 1dce339..548f8f1 100755
--- a/bin/singa-run.sh
+++ b/bin/singa-run.sh
@@ -33,8 +33,7 @@ usage="Usage: singa-run.sh -conf <job config file> [ other arguments ]\n
# parse arguments
# make sure we have '-conf' and remove '-exec'
-#exe=./singa
-exe=./src/singa.py
+exe=./singa
while [ $# != 0 ]; do
if [ $1 == "-exec" ]; then
shift
@@ -82,12 +81,9 @@ host_file=$log_dir/job.hosts
./singatool genhost $job_conf 1>$host_file || exit 1
# set command to run singa
-#singa_run="$exe $args -conf $job_conf \
-# -singa_conf $SINGA_HOME/conf/singa.conf \
-# -singa_job $job_id"
-singa_run="python $exe $args -conf $job_conf \
+singa_run="$exe $args -conf $job_conf \
-singa_conf $SINGA_HOME/conf/singa.conf \
- -singa_job $job_id"
+ -singa_job $job_id"
singa_sshrun="cd $SINGA_HOME; $singa_run"
# ssh and start singa processes
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3e58760b/include/singa/driver.h
----------------------------------------------------------------------
diff --git a/include/singa/driver.h b/include/singa/driver.h
index c7d2606..1c73db1 100644
--- a/include/singa/driver.h
+++ b/include/singa/driver.h
@@ -56,9 +56,19 @@ class Driver {
* files.
* @param[in] job_conf job configuration.
*/
- void Train(bool resume, const std::string str);
void Train(bool resume, const JobProto& job_conf);
/**
+ * Used for python binding. Users can also directly call it as a C++ API.
+ *
+ * It completes the functions as defined above but accept serialized string
+ * parameters.
+ *
+ * @param[in] resume if true resume the training from the latest checkpoint
+ * files.
+ * @param[in] str serialized string recorded job configuration.
+ */
+ void Train(bool resume, const std::string str);
+ /**
* Create workers and servers to conduct the training.
*
* @param[in] job_conf job configuration with all necessary fields set (e.g.,
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3e58760b/src/driver.cc
----------------------------------------------------------------------
diff --git a/src/driver.cc b/src/driver.cc
index 1181b99..816e0df 100644
--- a/src/driver.cc
+++ b/src/driver.cc
@@ -153,8 +153,6 @@ void Driver::Init(int argc, char **argv) {
void Driver::Train(bool resume, const std::string str){
JobProto job_conf;
job_conf.ParseFromString(str);
- printf("STR IS !@#!#!@# %s\n",str.c_str());
- printf("!!!!!!!!!!!!!!!!%s\n",job_conf.cluster().workspace().c_str());
Train(resume,job_conf);
}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3e58760b/src/driver.i
----------------------------------------------------------------------
diff --git a/src/driver.i b/src/driver.i
deleted file mode 100644
index d1154a8..0000000
--- a/src/driver.i
+++ /dev/null
@@ -1,18 +0,0 @@
-%module driver
-%include "std_vector.i"
-%include "std_string.i"
-%include "argcargv.i"
-%apply (int ARGC, char **ARGV) { (int argc, char **argv) }
-%{
-#include "../include/singa/driver.h"
-%}
-
-namespace singa{
-using std::vector;
-class Driver{
-public:
-void Train(bool resume, const std::string job_conf);
-void Init(int argc, char **argv);
-};
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3e58760b/src/generatepy.sh
----------------------------------------------------------------------
diff --git a/src/generatepy.sh b/src/generatepy.sh
deleted file mode 100644
index e1bbeee..0000000
--- a/src/generatepy.sh
+++ /dev/null
@@ -1,7 +0,0 @@
-
-swig -c++ -python driver.i
-g++ -fPIC driver.cc driver_wrap.cxx -shared -o _driver.so
--L/home/zhongle/incubator-singa/.libs/ -lsinga -DMSHADOW_USE_CUDA=0
--DMSHADOW_USE_CBLAS=1 -DMSHADOW_USE_MKL=0 -DUSE_GPU -std=c++11
--I/usr/cuda-7.5/include -I/home/zhongle/local/include -I../include
--I/usr/include/python2.7/
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3e58760b/src/singa.py
----------------------------------------------------------------------
diff --git a/src/singa.py b/src/singa.py
deleted file mode 100644
index 2201f67..0000000
--- a/src/singa.py
+++ /dev/null
@@ -1,15 +0,0 @@
-import os
-import sys
-import string
-import driver
-from optparse import OptionParser
-
-if __name__ == '__main__':
- d = driver.Driver();
- d.Init(sys.argv);
- i = sys.argv.index("-conf")
- s = open(sys.argv[i+1], 'r').read()
- s = str(s)
- print s
- d.Train(False, s)
-
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3e58760b/tool/python/README.md
----------------------------------------------------------------------
diff --git a/tool/python/README.md b/tool/python/README.md
deleted file mode 100644
index f1b3f4d..0000000
--- a/tool/python/README.md
+++ /dev/null
@@ -1,191 +0,0 @@
- SINGAROOT/tool
- |-- pb2 (has job_pb2.py)
- |-- python
- |-- model.py
- |-- ex_cifar10_cnn.py
- |-- ex_mnist_mlp.py
- |-- datasets
- |-- cifar10.py
- |-- mnist.py
- |-- utils
- |-- utility.py
- |-- message.py
-
-### Layer class (inherited)
-
-* Data
-* Dense
-* Activation
-* Convolution2D
-* MaxPooling2D
-* AvgPooling2D
-* LRN2D
-* Dropout
-
-### Other classes
-
-* Store
-* Parameter
-* SGD
-* Cluster
-
-### Model class
-
-* Model class has `jobconf` (JobProto) and `layers` (layer list)
-
-Methods in Model class
-
-* add
- * add Layer into Model
-
-* compile
- * set Updater (i.e., optimizer) and Cluster (i.e., topology) components
-
-* fit
- * set Training data and parameter values for the training
- * (optional) set Validatiaon data and parameter values
- * set Train_one_batch component
-
-* evaluate
- * set Testing data and parameter values for the testing
- * run singa (train/test/validation) via a command script
- * recieve train/test/validation results, e.g., accuracy
- * [IN PROGRESS] run singa via a wrapper for Driver class
-
-* [IN PROGRESS] run singa with checkpoint
-* [IN PROGRESS] run singa for particular tasks, e.g., classification/prediction
-
-
-## MLP Example
-
-An example (to generate job.conf for mnist)
-
-```
-X_train, X_test, workspace = mnist.load_data()
-
-m = Sequential('mlp') # inherited from Model
-
-par = Parameter(init='uniform', low=-0.05, high=0.05)
-m.add(Dense(2500, w_param=par, b_param=par))
-m.add(Activation('tanh'))
-m.add(Dense(2000, w_param=par, b_param=par, activation='tanh'))
-m.add(Dense(1500, w_param=par, b_param=par, activation='tanh'))
-m.add(Dense(1000, w_param=par, b_param=par, activation='tanh'))
-m.add(Dense(500, w_param=par, b_param=par, activation='tanh'))
-m.add(Dense(10, w_param=par, b_param=par, activation='softmax'))
-
-sgd = SGD(lr=0.001, lr_type='step')
-topo = Cluster(workspace)
-m.compile(loss='categorical_crossentropy', optimizer=sgd, cluster=topo)
-m.fit(X_train, train_steps=1000, disp_freq=10)
-result = m.evaluate(X_test, batch_size=100, test_steps=10, test_freq=60)
-```
-
-
-## CNN Example
-
-An example (to generate job.conf for cifar10)
-
-```
-X_train, X_test, workspace = cifar10.load_data()
-
-m = Sequential('cnn')
-
-parw = Parameter(init='gauss', std=0.0001)
-parb = Parameter(init='const', value=0)
-m.add(Convolution(32, 5, 1, 2, w_param=parw, b_param=parb, b_lr=2))
-m.add(MaxPooling2D(pool_size(3,3), stride=2))
-m.add(Activation('relu'))
-m.add(LRN2D(3, alpha=0.00005, beta=0.75))
-
-m.add(Convolution(32, 5, 1, 2, w_param=parw, b_param=parb))
-m.add(Activation('relu'))
-m.add(AvgPooling2D(pool_size(3,3), stride=2))
-m.add(LRN2D(3, alpha=0.00005, beta=0.75))
-
-m.add(Convolution(64, 5, 1, 2, w_param=parw, b_param=parb.setval(lr_scale=1)))
-m.add(Activation('relu'))
-m.add(AvgPooling2D(pool_size(3,3), stride=2))
-
-parw.setval(wd_scale=250)
-parb.setval(lr_scale=2, wd_scale=0)
-m.add(Dense(10, w_param=parw, b_param=parb, activation='softmax'))
-
-sgd = SGD(decay=0.004, lr_type='fixed', step=(0,60000,65000), step_lr=(0.001,0.0001,0.00001))
-topo = Cluster(workspace)
-m.compile(updater=sgd, cluster=topo)
-m.fit(X_train, 1000, disp_freq=30)
-result = m.evaluate(X_test, 1000, test_steps=30, test_freq=300)
-```
-
-### TIPS
-
-Hidden layers for MLP can be written as
-```
-par = Param(init='uniform', low=-0.05, high=0.05)
-for n in [2500, 2000, 1500, 1000, 500]:
- m.add(Dense(n, w_param=par, b_param=par, activation='tanh'))
-m.add(Dense(10, w_param=par, b_param=par, activation='softmax'))
-```
-
-Alternative ways to write the hidden layers
-```
-m.add(Dense(2500, w_param=par, b_param=par))
-m.add(Activation('tanh'))
-```
-```
-m.add(Dense(2500, init='uniform', activation='softmax'))
-```
-```
-m.add(Dense(2500, w_param=Param(init='uniform'), b_param=Param(init='gaussian')))
-```
-
-Alternative ways to add Data layer
-```
-X_train, X_test = mnist.load_data() // parameter values are set in load_data()
-m.fit(X_train, ...) // Data layer for training is added
-m.evaluate(X_test, ...) // Data layer for testing is added
-```
-```
-X_train, X_test = mnist.load_data() // parameter values are set in load_data()
-m.add(X_train) // explicitly add Data layer
-m.add(X_test) // explicitly add Data layer
-```
-```
-store = Store(path='train.bin', batch_size=64, ...) // parameter values are set explicitly
-m.add(Data(load='recordinput', phase='train', conf=store)) // Data layer is added
-store = Store(path='test.bin', batch_size=100, ...) // parameter values are set explicitly
-m.add(Data(load='recordinput', phase='test', conf=store)) // Data layer is added
-```
-
-### Parameter class
-
-Users need to set parameter configuration and initial values. For example,
-
-* Parameter configuration
- * lr = (float) // learning rate
- * wd = (float) // weight decay
-
-* Parameter initialization
- * init = (string) // one of the types, 'uniform', 'constant', 'gaussian'
- * for uniform [default]
- * high = (float)
- * low = (float)
- * for constant
- * value = (float)
- * for gaussian
- * mean = (float)
- * std = (float)
-
-Several ways to set Parameter values
-```
-parw = Parameter(lr=2, wd=10, init='constant', value=0)
-m.add(Dense(10, w_param=parw, ...)
-```
-```
-parw = Parameter(init='constant', value=0)
-m.add(Dense(10, w_param=parw, w_lr=2, w_wd=10, ...)
-```
-
-
-
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3e58760b/tool/python/__init__.py
----------------------------------------------------------------------
diff --git a/tool/python/__init__.py b/tool/python/__init__.py
deleted file mode 100644
index e69de29..0000000
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3e58760b/tool/python/datasets/__init__.py
----------------------------------------------------------------------
diff --git a/tool/python/datasets/__init__.py b/tool/python/datasets/__init__.py
deleted file mode 100644
index e69de29..0000000
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3e58760b/tool/python/datasets/cifar10.py
----------------------------------------------------------------------
diff --git a/tool/python/datasets/cifar10.py b/tool/python/datasets/cifar10.py
deleted file mode 100644
index c959186..0000000
--- a/tool/python/datasets/cifar10.py
+++ /dev/null
@@ -1,31 +0,0 @@
-#!/usr/bin/env python
-from model import *
-
-def load_data(
- workspace = 'examples/cifar10',
- path_mean = 'examples/cifar10/image_mean.bin',
- backend = 'kvfile',
- batchsize = 64,
- random = 5000,
- shape = (3, 32, 32),
- std = 127.5,
- mean = 127.5
- ):
-
- path_train = workspace + '/train_data.bin'
- path_test = workspace + '/test_data.bin'
-
- store = Store(path=path_train, mean_file=path_mean, backend=backend,
- random_skip=random, batchsize=batchsize,
- shape=shape)
-
- data_train = Data(load='recordinput', phase='train', conf=store)
-
- store = Store(path=path_test, mean_file=path_mean, backend=backend,
- batchsize=batchsize,
- shape=shape)
-
- data_test = Data(load='recordinput', phase='test', conf=store)
-
- return data_train, data_test, workspace
-
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3e58760b/tool/python/datasets/mnist.py
----------------------------------------------------------------------
diff --git a/tool/python/datasets/mnist.py b/tool/python/datasets/mnist.py
deleted file mode 100644
index ee04ae9..0000000
--- a/tool/python/datasets/mnist.py
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/usr/bin/env python
-from model import *
-
-def load_data(
- workspace = 'examples/mnist',
- backend = 'kvfile',
- random = 5000,
- batchsize = 64,
- shape = 784,
- std = 127.5,
- mean = 127.5
- ):
-
- path_train = workspace + '/train_data.bin'
- path_test = workspace + '/test_data.bin'
-
- store = Store(path=path_train, backend=backend,
- random_skip=random,
- batchsize=batchsize, shape=shape,
- std_value=std, mean_value=mean)
- data_train = Data(load='recordinput', phase='train', conf=store)
-
- store = Store(path=path_test, backend=backend,
- batchsize=batchsize, shape=shape,
- std_value=std, mean_value=mean)
- data_test = Data(load='recordinput', phase='test', conf=store)
-
- return data_train, data_test, workspace
-
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3e58760b/tool/python/datasets/rnnlm.py
----------------------------------------------------------------------
diff --git a/tool/python/datasets/rnnlm.py b/tool/python/datasets/rnnlm.py
deleted file mode 100644
index 7a2a3d5..0000000
--- a/tool/python/datasets/rnnlm.py
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/usr/bin/env python
-from model import *
-
-def load_data(
- workspace = 'examples/rnnlm',
- backend = 'kvfile',
- max_window = 10
- ):
-
- path_train = workspace + '/train_data.bin'
- path_valid = workspace + '/valid_data.bin'
- path_test = workspace + '/test_data.bin'
-
-
- data_train = Data(load='kData', phase='train', path=path_train, backend=backend, max_window=max_window)
-
- data_valid = Data(load='kData', phase='val', path=path_valid, max_window=max_window)
-
- #store = Store(path=path_test, backend=backen)
- #data_test = Data(load='recordinput', phase='test', conf=store)
-
- return data_train, data_valid, workspace
-
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3e58760b/tool/python/ex_cifar10_cnn.py
----------------------------------------------------------------------
diff --git a/tool/python/ex_cifar10_cnn.py b/tool/python/ex_cifar10_cnn.py
deleted file mode 100644
index fa96b48..0000000
--- a/tool/python/ex_cifar10_cnn.py
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/usr/bin/env python
-from model import *
-from datasets import cifar10
-
-X_train, X_test, workspace = cifar10.load_data()
-
-m = Sequential('cnn', label=False)
-
-parw = Parameter(init='gaussian', std=0.0001)
-parb = Parameter(init='constant', value=0)
-m.add(Convolution2D(32, 5, 1, 2, w_param=parw, b_param=parb, b_lr=2))
-m.add(MaxPooling2D(pool_size=(3,3), stride=2))
-m.add(Activation('relu'))
-m.add(LRN2D(3, alpha=0.00005, beta=0.75))
-
-parw.update(std=0.01)
-m.add(Convolution2D(32, 5, 1, 2, w_param=parw, b_param=parb))
-m.add(Activation('relu'))
-m.add(AvgPooling2D(pool_size=(3,3), stride=2))
-m.add(LRN2D(3, alpha=0.00005, beta=0.75))
-
-m.add(Convolution2D(64, 5, 1, 2, w_param=parw, b_param=parb, b_lr=1))
-m.add(Activation('relu'))
-m.add(AvgPooling2D(pool_size=(3,3), stride=2))
-
-m.add(Dense(10, w_param=parw, w_wd=250, b_param=parb, b_lr=2, b_wd=0, activation='softmax'))
-
-sgd = SGD(decay=0.004, lr_type='fixed', step=(0,60000,65000), step_lr=(0.001,0.0001,0.00001))
-topo = Cluster(workspace)
-m.compile(optimizer=sgd, cluster=topo)
-m.fit(X_train, train_steps=1000, disp_freq=30)
-result = m.evaluate(X_test, test_steps=10, test_freq=300)
-
-#print
-#m.display()
-
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3e58760b/tool/python/ex_mnist_mlp.py
----------------------------------------------------------------------
diff --git a/tool/python/ex_mnist_mlp.py b/tool/python/ex_mnist_mlp.py
deleted file mode 100644
index 2d135d8..0000000
--- a/tool/python/ex_mnist_mlp.py
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/usr/bin/env python
-from model import *
-from datasets import mnist
-
-X_train, X_test, workspace = mnist.load_data()
-
-m = Sequential('mlp')
-
-par = Parameter(init='uniform', range=0.05)
-m.add(Dense(2500, w_param=par, b_param=par, activation='tanh'))
-m.add(Dense(2000, w_param=par, b_param=par, activation='tanh'))
-m.add(Dense(1500, w_param=par, b_param=par, activation='tanh'))
-m.add(Dense(1000, w_param=par, b_param=par, activation='tanh'))
-m.add(Dense(500, w_param=par, b_param=par, activation='tanh'))
-m.add(Dense(10, w_param=par, b_param=par, activation='softmax'))
-
-sgd = SGD(lr=0.001, lr_type='step')
-topo = Cluster(workspace)
-m.compile(loss='categorical_crossentropy', optimizer=sgd, cluster=topo)
-m.fit(X_train, train_steps=100, disp_freq=10)
-result = m.evaluate(X_test, batch_size=100, test_steps=10)
-
-#TODO---- classify/predict for new data
-#label = m.predict(data_new, ...)
-#-------
-
-#print
-#m.display()
-
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3e58760b/tool/python/ex_rnnlm.py
----------------------------------------------------------------------
diff --git a/tool/python/ex_rnnlm.py b/tool/python/ex_rnnlm.py
deleted file mode 100644
index 69cc7b0..0000000
--- a/tool/python/ex_rnnlm.py
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/usr/bin/env python
-from model import *
-from datasets import rnnlm
-
-vocab_size = 3720
-
-X_train, X_valid, workspace = rnnlm.load_data()
-
-m = Sequential('rnnlm', label=False)
-
-parw = Parameter(init='uniform', range=0.3)
-m.add(Embedding(in_dim=vocab_size, out_dim=15, w_param=parw))
-m.add(RNNLM(1, w_param=parw))
-
-sgd = SGD(lr_type='fixed', step=(0,48810,56945,65080,73215), step_lr=(0.1,0.05,0.025,0.0125,0.00625))
-topo = Cluster(workspace)
-m.compile(loss='user_loss_rnnlm', in_dim=vocab_size, nclass=100, optimizer=sgd, cluster=topo)
-
-m.fit(X_train, train_steps=81350, disp_freq=8135)
-result = m.evaluate(X_valid, validate_steps=683, validate_freq=8135, execpath='examples/rnnlm/rnnlm.bin')
-
-#print
-#m.display()
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3e58760b/tool/python/model.py
----------------------------------------------------------------------
diff --git a/tool/python/model.py b/tool/python/model.py
deleted file mode 100644
index a558a18..0000000
--- a/tool/python/model.py
+++ /dev/null
@@ -1,580 +0,0 @@
-#!/usr/bin/env python
-import os, sys, re, subprocess
-from utils.utility import *
-from utils.message import *
-from google.protobuf import text_format
-#sys.path.append(os.path.join(os.path.dirname(__file__), '../pb2'))
-#from job_pb2 import *
-#from rnnlm_pb2 import *
-
-class Model(object):
-
- def __init__(self, name='my model', label=False):
- self.jobconf = Message('Job', name=name).proto
- self.layers = []
- self.label = label
-
- def add(self, layer):
- self.layers.append(layer)
-
- def compile(self, optimizer=None, cluster=None, loss='categorical_crossentropy', topk=1, **kwargs):
- '''
- required
- optimizer = (Updater) // updater settings, e.g., SGD
- cluster = (Cluster) // cluster settings
- optional
- loss = (string) // name of loss function type
- topk = (int) // the number of results considered to compute accuracy
- '''
- assert optimizer != None, 'optimizer (Updater) should be set'
- assert cluster != None, 'cluster (Cluster) should be set'
- setval(self.jobconf, updater=optimizer.proto)
- setval(self.jobconf, cluster=cluster.proto)
-
- # take care of loss function layer
- assert loss != None, 'loss should be set'
- if hasattr(self.layers[len(self.layers)-1], 'mask'):
- ly = self.layers[len(self.layers)-1].mask
- else:
- ly = self.layers[len(self.layers)-1].layer
-
- if ly.type == enumLayerType('softmax'):
- # revise the last layer
- if loss == 'categorical_crossentropy':
- setval(ly, type=enumLayerType('softmaxloss'))
- setval(ly.softmaxloss_conf, topk=topk)
- elif loss == 'mean_squared_error':
- setval(ly, type=enumLayerType('euclideanloss'))
- else:
- # add new layer
- if loss == 'categorical_crossentropy':
- self.add(Activation('softmaxloss', topk=topk))
- elif loss == 'mean_squared_error':
- self.add(Activation('euclideanloss'))
- elif loss == 'user_loss_rnnlm': # user-defined loss layer for rnnlm
- self.add(UserLossRNNLM(nclass=kwargs['nclass'], vocab_size=kwargs['in_dim']))
-
- def build(self):
- net = NetProto()
- slyname = self.layers[0].layer.name
- for i in range(len(self.layers)):
- ly = net.layer.add()
- ly.CopyFrom(self.layers[i].layer)
- lastly = ly
- if self.layers[i].is_datalayer == True:
- continue
- getattr(ly, 'srclayers').append(slyname)
- slyname = ly.name
- if hasattr(self.layers[i], 'mask'):
- mly = net.layer.add()
- mly.CopyFrom(self.layers[i].mask)
- getattr(mly, 'srclayers').append(slyname)
- slyname = mly.name
- lastly = mly
-
- if self.label == True:
- label_layer = Layer(name='label', type=kLabel)
- ly = net.layer.add()
- ly.CopyFrom(label_layer.layer)
- getattr(ly, 'srclayers').append(self.layers[0].layer.name)
- getattr(lastly, 'srclayers').append(label_layer.layer.name)
- else:
- getattr(lastly, 'srclayers').append(self.layers[0].layer.name)
-
- setval(self.jobconf, neuralnet=net)
-
- def fit(self, **kwargs):
- pass
-
- def evaluate(self, **kwargs):
- pass
-
- def display(self):
- print text_format.MessageToString(self.jobconf)
-
-
-class Sequential(Model):
- def __init__(self, name='my model', label=False):
- super(Sequential, self).__init__(name=name, label=label)
-
- def exist_datalayer(self, phase):
- if self.layers[0].layer.include != enumPhase(phase): # train
- if self.layers[1].layer.include != enumPhase(phase): # test
- if self.layers[2].layer.include != enumPhase(phase): # valid
- return False
- return True
-
- def fit(self, data=None, train_steps=0, **fields):
- '''
- required
- data = (Data) // Data class object for training data
- train_steps = (int) // the number of training, i.e., epoch
- optional
- **fields (KEY=VALUE)
- batch_size = (int) // batch size for training data
- disp_freq = (int) // frequency to display training info
- disp_after = (int) // display after this number
- validate_data = (Data) // validation data, specified in load_data()
- validate_freq = (int) // frequency of validation
- validate_steps = (int) // total number of steps for validation
- validate_after = (int) // start validation after this number
- checkpoint_path = (string) // path to checkpoint file
- checkpoint_freq = (int) // frequency for checkpoint
- checkpoint_after = (int) // start checkpointing after this number
- '''
- assert data != None, 'Training data shold be set'
- assert train_steps != 0, 'Training steps shold be set'
-
- if 'batch_size' in fields: # if new value is set, replace the batch_size
- setval(data.layer.store_conf, batchsize=fields['batch_size'])
-
- if self.exist_datalayer('train') == False:
- self.layers.insert(0, data)
-
- if 'validate_data' in fields:
- self.layers.insert(1, fields['validate_data'])
-
- setval(self.jobconf, train_steps=train_steps)
- setval(self.jobconf, **fields)
-
- # set Train_one_batch component, using backprogapation
- setval(self.jobconf, train_one_batch=Algorithm(type=enumAlgType('bp')).proto)
-
- def evaluate(self, data=None, execpath='', **fields):
- '''
- required
- data = (Data) // Data class object for testing data
- optional
- test_steps = (int) // the number of testing
- **fields (KEY=VALUE)
- batch_size = (int) // batch size for testing data
- test_freq = (int) // frequency of testing
- test_steps = (int) // total number of steps for testing
- test_after = (int) // start testing after this number of steps
- '''
- assert data != None, 'Testing data shold be set'
-
- if 'batch_size' in fields: # if new value is set, replace the batch_size
- setval(data.layer.store_conf, batchsize=fields['batch_size'])
-
- if self.exist_datalayer('test') == False:
- self.layers.insert(1, data)
-
- setval(self.jobconf, **fields)
-
- self.build() # construct Nneuralnet Component
-
- with open('job.conf', 'w') as f:
- f.write(text_format.MessageToString(self.jobconf))
-
- #self.display()
- self.result = SingaRun(execpath=execpath)
- return self.result
-
-class Store(object):
- def __init__(self, **kwargs):
- '''
- **kwargs
- path = (string) // path to dataset
- backend = (string) //
- batch_size = (int) // batch size of dataset
- shape = (int) //
-
- '''
- self.proto = Message('Store', **kwargs).proto
-
-class Algorithm(object):
- def __init__(self, type=enumAlgType('bp'), **kwargs):
- alg = Message('Alg', alg=type, **kwargs).proto
- if type == enumAlgType('cd'):
- setval(alg.cd_conf, **kwargs)
- self.proto = alg
-
-class SGD(object):
- def __init__(self, lr=0.01, lr_type=None,
- decay=0, momentum=0,
- step=(0), step_lr=(0.01), **fields):
- '''
- required
- lr = (float) // base learning rate
- lr_type = (string) // type of learning rate
- optional
- **fields (KEY=VALUE)
- decay = (float) // weight decay
- momentum = (float) // momentum
-
- '''
- assert lr
- assert lr_type != None, 'type of learning rate should be specified'
-
- upd = Message('Updater', type=kSGD, **fields).proto
- setval(upd.learning_rate, base_lr=lr)
- if decay > 0:
- setval(upd, weight_decay=decay)
- if momentum > 0:
- setval(upd, momentum=momentum)
-
- if lr_type == 'step':
- cp = Message('Step', change_freq=60, gamma=0.997)
- setval(upd.learning_rate, type=kStep, step_conf=cp.proto)
- elif lr_type == 'fixed':
- cp = Message('FixedStep', step=step, step_lr=step_lr)
- setval(upd.learning_rate, type=kFixedStep, fixedstep_conf=cp.proto)
- elif lr_type == 'linear':
- cp = Message('Linear', change_freq=10, final_lr=0.1)
- setval(upd.learning_rate, type=kLinear, linear_conf=cp.proto)
- self.proto = upd
-
-
-class Cluster(object):
- def __init__(self, workspace=None,
- nworker_groups=1, nserver_groups=1,
- nworkers_per_group=1, nservers_per_group=1,
- nworkers_per_procs=1, nservers_per_procs=1,
- **fields):
- '''
- required
- workspace = (string) // workspace path
- optional
- nworker_groups = (int)
- nserver_groups = (int)
- nworkers_per_group = (int)
- nservers_per_group = (int)
- nworkers_per_procs = (int)
- nservers_per_procs = (int)
- **fields
- server_worker_separate = (bool)
- '''
- assert workspace != None, 'need to set workspace'
- self.proto = Message('Cluster', workspace=workspace).proto
- # optional
- self.proto.nworker_groups = nworker_groups
- self.proto.nserver_groups = nserver_groups
- self.proto.nworkers_per_group = nworkers_per_group
- self.proto.nservers_per_group = nservers_per_group
- self.proto.nworkers_per_procs = nworkers_per_procs
- self.proto.nservers_per_procs = nservers_per_procs
- # other fields
- setval(self.proto, **fields)
-
-#TODO make this internally used
-class Parameter(object):
-
- def __init__(self, **kwargs):
- '''
- optional
- **kwargs
- name = (string) // parameter name
- lr = (float) // learning rate
- wd = (float) // weight decay
- init = (string) // initialized type {'constant','uniform','gaussian'}
- value = (int) // value for 'constant'
- range = (float) // [low, high] for 'uniform', low=-range, high=range
- low = (float) // low value for 'uniform'
- high = (float) // high value for 'uniform'
- mean = (float) // mean for 'gaussian'
- std = (float) // std for 'gaussian'
- '''
- fields = {'lr_scale' : kwargs['lr'] if 'lr' in kwargs else 1,
- 'wd_scale' : kwargs['wd'] if 'wd' in kwargs else 1
- }
- self.param = Message('Param', **fields).proto
-
- if not 'name' in kwargs:
- setval(self.param, name=generateName('param', 1))
- else:
- setval(self.param, name=kwargs['name'])
-
- if 'range' in kwargs:
- kwargs['low'] = -float(kwargs['range'])
- kwargs['high'] = float(kwargs['range'])
-
- if 'init' in kwargs:
- pg = Message('ParamGen', type=enumInitMethod(kwargs['init']), **kwargs)
- del kwargs['init']
- else: # default: uniform
- pg = Message('ParamGen', type=enumInitMethod(kwargs['uniform']))
- setval(self.param, init=pg.proto)
-
-
- #if 'param_init' in kwargs:
- # setval(self.param, init=kwargs['param_init'].proto)
- # del kwargs['param_init']
-
- def update(self, **fields):
- setval(self.param, **fields)
- setval(self.param.init, **fields)
-
-
-class Layer(object):
- def __init__(self, **kwargs):
- self.layer = Message('Layer', **kwargs).proto
- # required
- if not 'name' in kwargs:
- setval(self.layer, name=generateName('layer', 1))
-
- # srclayers are set in Model.build()
- self.is_datalayer = False
-
- def setParamField(self, param, pname, **kwargs):
- # param: ParamProto
- if pname == 'w':
- field = {'lr_scale' : kwargs['w_lr'] if 'w_lr' in kwargs else param.lr_scale,
- 'wd_scale' : kwargs['w_wd'] if 'w_wd' in kwargs else param.wd_scale
- }
- elif pname == 'b':
- field = {'lr_scale' : kwargs['b_lr'] if 'b_lr' in kwargs else param.lr_scale,
- 'wd_scale' : kwargs['b_wd'] if 'b_wd' in kwargs else param.wd_scale
- }
- setval(param, name=generateName(pname), **field)
-
-class Data(Layer):
- def __init__(self, load, phase='train', conf=None, **kwargs):
- assert load != None, 'data type should be specified'
- if load == 'kData':
- super(Data, self).__init__(name=generateName('data'), user_type=load)
- else:
- self.layer_type = enumLayerType(load)
- super(Data, self).__init__(name=generateName('data'), type=self.layer_type)
-
- # include/exclude
- setval(self.layer, include=enumPhase(phase))
- #setval(self.layer, exclude=kTest if phase=='train' else kTrain)
-
- if conf == None:
- if load == 'kData':
- setval(self.layer.Extensions[data_conf], **kwargs)
- else:
- setval(self.layer.store_conf, **kwargs)
- else:
- setval(self.layer, store_conf=conf.proto)
- self.is_datalayer = True
-
-class Convolution2D(Layer):
- def __init__(self, nb_filter=0, kernel=0, stride=1, pad=0,
- init='uniform', w_param=None, b_param=None,
- activation=None, **kwargs):
- '''
- required
- nb_filter = (int) // the number of filters
- kernel = (int) // the size of filter
- optional
- stride = (int) // the size of stride
- pad = (int) // the size of padding
- xxxxx
-
- '''
- assert nb_filter > 0 and kernel > 0, 'should be set as positive int'
- super(Convolution2D, self).__init__(name=generateName('conv',1), type=kCConvolution)
- fields = {'num_filters' : nb_filter,
- 'kernel' : kernel,
- 'stride' : stride,
- 'pad' : pad}
- setval(self.layer.convolution_conf, **fields)
-
- # parameter w
- if w_param == None:
- w_param = Parameter(name=generateName('w'), init=init) # default: uniform
- else:
- field = {'lr_scale' : kwargs['w_lr'] if 'w_lr' in kwargs else w_param.param.lr_scale,
- 'wd_scale' : kwargs['w_wd'] if 'w_wd' in kwargs else w_param.param.wd_scale
- }
- setval(w_param.param, name=generateName('w'), **field)
- setval(w_param.param.init, **kwargs)
- setval(self.layer, param=w_param.param)
-
- # parameter b
- if b_param == None:
- b_param = Parameter(name=generateName('b'), init=init) # default: uniform
- else:
- field = {'lr_scale' : kwargs['b_lr'] if 'b_lr' in kwargs else b_param.param.lr_scale,
- 'wd_scale' : kwargs['b_wd'] if 'b_wd' in kwargs else b_param.param.wd_scale
- }
- setval(b_param.param, name=generateName('b'), **field)
- setval(self.layer, param=b_param.param)
-
- # following layers: e.g., activation, dropout, etc.
- if activation:
- self.mask = Activation(activation=activation).layer
-
-class MaxPooling2D(Layer):
- def __init__(self, pool_size=None, stride=1, ignore_border=True, **kwargs):
- '''
- required
- pool_size = (int|tuple) // the size for pooling
- optional
- stride = (int) // the size of striding
- ignore_border = (bool) // flag for padding
- **kwargs // fields for Layer class
- '''
- assert pool_size != None, 'pool_size is required'
- if type(pool_size) == int:
- pool_size = (pool_size, pool_size)
- assert type(pool_size) == tuple and \
- pool_size[0] == pool_size[1], 'pool size should be square in Singa'
- super(MaxPooling2D, self).__init__(name=generateName('pool'), type=kCPooling, **kwargs)
- fields = {'pool' : PoolingProto().MAX,
- 'kernel' : pool_size[0],
- 'stride' : stride,
- 'pad' : 0 if ignore_border else 1}
- setval(self.layer.pooling_conf, **fields)
-
-class AvgPooling2D(Layer):
- def __init__(self, pool_size=None, stride=1, ignore_border=True, **kwargs):
- '''
- required
- pool_size = (int|tuple) // size for pooling
- optional
- stride = (int) // size of striding
- ignore_border = (bool) // flag for padding
- **kwargs // fields for Layer class
- '''
- assert pool_size != None, 'pool_size is required'
- if type(pool_size) == int:
- pool_size = (pool_size, pool_size)
- assert type(pool_size) == tuple and \
- pool_size[0] == pool_size[1], 'pool size should be square in Singa'
- super(AvgPooling2D, self).__init__(name=generateName('pool'), type=kCPooling, **kwargs)
- self.layer.pooling_conf.pool = PoolingProto().AVG
- fields = {'pool' : PoolingProto().AVG,
- 'kernel' : pool_size[0],
- 'stride' : stride,
- 'pad' : 0 if ignore_border else 1}
- setval(self.layer.pooling_conf, **fields)
-
-class LRN2D(Layer):
- def __init__(self, size=0, alpha=1e-4, k=1, beta=0.75, **kwargs):
- super(LRN2D, self).__init__(name=generateName('norm'), type=kLRN)
- # required
- assert size != 0, 'local size should be set'
- self.layer.lrn_conf.local_size = size
- setval(self.layer.lrn_conf, alpha=alpha, knorm=k, beta=beta, **kwargs)
-
-class Dense(Layer):
- def __init__(self, output_dim=0, activation=None,
- init='uniform', w_param=None, b_param=None, input_dim=None,
- **kwargs):
- '''
- required
- output_dim = (int)
- optional
- activation = (string)
- **kwargs
- w_lr
- w_wd
-
- '''
- assert output_dim > 0, 'output_dim should be set'
- super(Dense, self).__init__(type=kInnerProduct, **kwargs)
- self.layer.innerproduct_conf.num_output = output_dim # required
-
- # parameter w
- if w_param == None:
- w_param = Parameter(name=generateName('w'), init=init) # default: uniform
- else:
- self.setParamField(w_param.param, 'w', **kwargs)
- setval(self.layer, param=w_param.param)
-
- # parameter b
- if b_param == None:
- b_param = Parameter(name=generateName('b'), init=init) # default: uniform
- else:
- self.setParamField(b_param.param, 'b', **kwargs)
- setval(self.layer, param=b_param.param)
-
- # following layers: e.g., activation, dropout, etc.
- if activation:
- self.mask = Activation(activation=activation).layer
-
-class Activation(Layer):
- def __init__(self, activation='stanh', topk=1):
- self.name = activation
- #TODO better way?
- if activation == 'tanh': activation = 'stanh'
- self.layer_type = enumLayerType(activation)
- super(Activation, self).__init__(name=generateName(self.name), type=self.layer_type)
- if activation == 'softmaxloss':
- self.layer.softmaxloss_conf.topk = topk
-
-class Dropout(Layer):
- def __init__(self, ratio=0.5):
- self.name = 'dropout'
- self.layer_type = kDropout
- super(Dropout, self).__init__(name=generateName(self.name), type=self.layer_type)
- self.layer.dropout_conf.dropout_ratio = ratio
-
-
-class RGB(Layer):
- def __init__(self, meanfile=None, **kwargs):
- assert meanfile != None, 'meanfile should be specified'
- self.name = 'rgb'
- self.layer_type = kRGBImage
- super(RGB, self).__init__(name=generateName(self.name), type=self.layer_type)
- self.layer.rgbimage_conf.meanfile = meanfile
-
-class Embedding(Layer):
- def __init__(self, in_dim, out_dim, w_param=None, **kwargs):
- super(Embedding, self).__init__(name=generateName('embedding',1), user_type='kEmbedding')
- fields = { 'vocab_size': in_dim,
- 'word_dim': out_dim }
- setval(self.layer.Extensions[embedding_conf], **fields)
- if w_param == None:
- w_param = Parameter(name=generateName('w'), init=init) # default: uniform
- else:
- self.setParamField(w_param.param, 'w', **kwargs)
- setval(self.layer, param=w_param.param)
-
-class RNNLM(Layer):
- def __init__(self, dim, w_param=None, **kwargs):
- super(RNNLM, self).__init__(name=generateName('hidden',1), user_type='kHidden')
- if w_param == None:
- w_param = Parameter(name=generateName('w'), init=init) # default: uniform
- else:
- self.setParamField(w_param.param, 'w', **kwargs)
- setval(self.layer, param=w_param.param)
-
-class UserLossRNNLM(Layer):
- def __init__(self, **kwargs):
- super(UserLossRNNLM, self).__init__(name=generateName('loss',1), user_type='kLoss')
- self.layer.Extensions[loss_conf].nclass = kwargs['nclass']
- self.layer.Extensions[loss_conf].vocab_size = kwargs['vocab_size']
- setval(self.layer, param=Parameter(name=generateName('w'), init='uniform', range=0.3).param)
- setval(self.layer, param=Parameter(name=generateName('w',1), init='uniform', range=0.3).param)
-
-
-#TODO run singa training/testing via a wrapper for Driver
-def SingaRun(execpath=''):
- SINGAROOT = '../../'
- conf = 'job.conf'
- if execpath=='':
- cmd = '../../bin/singa-run.sh ' \
- + '-conf %s ' % conf
- else:
- cmd = '../../bin/singa-run.sh ' \
- + '-conf %s ' % conf \
- + '-exec %s ' % execpath
-
- procs = subprocess.Popen(cmd.strip().split(' '), stdout = subprocess.PIPE, stderr = subprocess.STDOUT)
-
- resultDic = {}
- output = iter(procs.stdout.readline, '')
- for line in output:
- print line[:-1]
- line = re.findall(r'[\w|*.*]+', line)
- if 'accuracy' in line:
- step = line[line.index('step')+1]
- acc = line[line.index('accuracy')+1]
- loss = line[line.index('loss')+1]
- #print 'Step: ', line[idx_step+1], 'Acc: ', acc, 'Loss: ', loss
- resultDic.setdefault(step,{})['acc'] = acc
- resultDic.setdefault(step,{})['loss'] = loss
- elif 'Train' in line:
- step = line[line.index('step')+1]
- loss = line[line.index('loss')+1]
- ppl = line[line.index('ppl')+1]
- resultDic.setdefault(step,{})['loss'] = loss
- resultDic.setdefault(step,{})['ppl'] = ppl
-
- #TODO better format to store??
- return resultDic
-
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3e58760b/tool/python/singa.py
----------------------------------------------------------------------
diff --git a/tool/python/singa.py b/tool/python/singa.py
new file mode 100755
index 0000000..4c41729
--- /dev/null
+++ b/tool/python/singa.py
@@ -0,0 +1,20 @@
+#!/usr/bin/env python
+
+
+import os
+import sys
+import string
+import pb2.job_pb2 as job_pb2
+import singa.driver as driver
+from google.protobuf.text_format import Merge
+
+if __name__ == '__main__':
+ i = sys.argv.index("-conf")
+ s = open(sys.argv[i+1], 'r').read()
+ s = str(s)
+ j = job_pb2.JobProto()
+ Merge(s,j)
+ b = j.SerializeToString()
+ d = driver.Driver()
+ d.Init(sys.argv)
+ d.Train(False,b)
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3e58760b/tool/python/singa/generatepy.sh
----------------------------------------------------------------------
diff --git a/tool/python/singa/generatepy.sh b/tool/python/singa/generatepy.sh
index bb4a3af..2a9b2e4 100755
--- a/tool/python/singa/generatepy.sh
+++ b/tool/python/singa/generatepy.sh
@@ -1,8 +1,7 @@
-
+#!/usr/bin/env bash
swig -c++ -python driver.i
-g++ -fPIC /home/zhongle/incubator-singa/src/driver.cc driver_wrap.cxx -shared -o _driver.so \
- -L/home/zhongle/incubator-singa/.libs/ -lsinga -DMSHADOW_USE_CUDA=0 \
- -DMSHADOW_USE_CBLAS=1 -DMSHADOW_USE_MKL=0 -DUSE_GPU -std=c++11 \
- -I/usr/cuda-7.5/include -I/home/zhongle/local/include \
- -I/home/zhongle/incubator-singa/include \
+g++ -fPIC /../../../src/driver.cc driver_wrap.cxx -shared -o _driver.so \
+ -L../../../.libs/ -lsinga -DMSHADOW_USE_CUDA=0 \
+ -DMSHADOW_USE_CBLAS=1 -DMSHADOW_USE_MKL=0 -std=c++11 \
+ -I/../../../include \
-I/usr/include/python2.7/
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3e58760b/tool/python/utils/__init__.py
----------------------------------------------------------------------
diff --git a/tool/python/utils/__init__.py b/tool/python/utils/__init__.py
deleted file mode 100644
index e69de29..0000000
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3e58760b/tool/python/utils/message.py
----------------------------------------------------------------------
diff --git a/tool/python/utils/message.py b/tool/python/utils/message.py
deleted file mode 100644
index 10a1a18..0000000
--- a/tool/python/utils/message.py
+++ /dev/null
@@ -1,50 +0,0 @@
-#!/usr/bin/env python
-import sys, os
-from utility import *
-sys.path.append(os.path.join(os.path.dirname(__file__),'../../pb2'))
-
-module_list=[]
-
-# import all modules in dir singa_root/too/pb2, except common, singa and __init__
-for f in os.listdir(os.path.join(os.path.dirname(__file__),'../../pb2')):
- if (f.endswith(".pyc")):
- continue
- if(f == "__init__.py" or f == "common_pb2.py" or f == "singa_pb2.py" ):
- continue
- module_name = f.split('.')[0]
- module=__import__(module_name)
- module_list.append(module)
- for func_name in dir(module):
- if not func_name.startswith("__"):
- globals()[func_name] = getattr(module,func_name)
-
-class Message(object):
- def __init__(self,protoname,**kwargs):
- for module in module_list:
- if hasattr(module,protoname+"Proto"):
- class_ = getattr(module,protoname+"Proto")
- self.proto = class_()
- return setval(self.proto,**kwargs)
- raise Exception('invalid protoname')
-
-enumDict_=dict()
-
-#get all enum type list in modules
-for module in module_list:
- for enumtype in module.DESCRIPTOR.enum_types_by_name:
- tempDict=enumDict_[enumtype]=dict()
- for name in getattr(module,enumtype).DESCRIPTOR.values_by_name:
- tempDict[name[1:].lower()]=getattr(module,name)
-
-def make_function(enumtype):
- def _function(key):
- return enumDict_[enumtype][key]
- return _function
-
-current_module = sys.modules[__name__]
-
-#def all the enumtypes
-for module in module_list:
- for enumtype in module.DESCRIPTOR.enum_types_by_name:
- setattr(current_module,"enum"+enumtype,make_function(enumtype))
-
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3e58760b/tool/python/utils/utility.py
----------------------------------------------------------------------
diff --git a/tool/python/utils/utility.py b/tool/python/utils/utility.py
deleted file mode 100644
index 42840c9..0000000
--- a/tool/python/utils/utility.py
+++ /dev/null
@@ -1,45 +0,0 @@
-#!/usr/bin/env python
-
-layerid = 0
-paramid = 0
-
-def generateName(label, op=0):
- global layerid, paramid
- num = layerid
- if label == 'layer':
- if op ==1: layerid += 1
- num = layerid
- elif label == 'param':
- if op ==1: paramid += 1
- num = paramid
- else:
- if op ==1: layerid += 1
- num = layerid
-
- return '{0}{1}'.format(label, num)
-
-
-def setval(proto, **kwargs):
- for k,v in kwargs.items():
- #print 'kv: ', k, ', ', v
- if hasattr(proto, k):
- flabel = proto.DESCRIPTOR.fields_by_name[k].label
- ftype = proto.DESCRIPTOR.fields_by_name[k].type
-
- fattr = getattr(proto, k)
- if flabel == 3: # repeated field
- if ftype == 11: # message type
- fattr = fattr.add()
- fattr.MergeFrom(v)
- else:
- if type(v) == list or type(v) == tuple:
- for i in range(len(v)):
- fattr.append(v[i])
- else:
- fattr.append(v)
- else:
- if ftype == 11: # message type
- fattr = getattr(proto,k)
- fattr.MergeFrom(v)
- else:
- setattr(proto, k, v)