You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@singa.apache.org by wa...@apache.org on 2016/01/01 13:29:10 UTC
[01/10] incubator-singa git commit: SINGA-81 Add Python Helper,
which enables users to construct a model (JobProto) and run Singa in
Python
Repository: incubator-singa
Updated Branches:
refs/heads/master 74a28dcb4 -> 9ff176c30
SINGA-81 Add Python Helper, which enables users to construct a model (JobProto) and run Singa in Python
- Updated model.py
. add "device" field in evaluate() to run singa on gpu for only test
. modify "fixed" to "fixedstep" for kFixedStep of leaning rate type
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/3c127300
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/3c127300
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/3c127300
Branch: refs/heads/master
Commit: 3c12730017ba21b03d2845fe2f9a7c72916d3d19
Parents: 8b69cad
Author: chonho <le...@comp.nus.edu.sg>
Authored: Tue Dec 29 10:46:37 2015 +0800
Committer: chonho <le...@comp.nus.edu.sg>
Committed: Fri Jan 1 15:59:14 2016 +0800
----------------------------------------------------------------------
tool/python/singa/model.py | 26 ++++++++++++++++++++------
1 file changed, 20 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3c127300/tool/python/singa/model.py
----------------------------------------------------------------------
diff --git a/tool/python/singa/model.py b/tool/python/singa/model.py
index 51c4126..d68d143 100644
--- a/tool/python/singa/model.py
+++ b/tool/python/singa/model.py
@@ -184,13 +184,15 @@ class Model(object):
def evaluate(self, data=None, alg='bp',
- checkpoint_path=None, execpath='', **fields):
+ checkpoint_path=None, execpath='', device=None, **fields):
'''
required
data = (Data) // Data class object for testing data
optional
- checkpoint_path = (list) // checkpoint path is necessary only for testing
- execpaths = (string) // path to user's own executable
+ alg = (string) // algorithm type, (backpropagation at default)
+ checkpoint_path = (list) // checkpoint path is necessary only for testing
+ execpaths = (string) // path to user's own executable
+ device = (int/list) // a list of gpu ids
**fields (KEY=VALUE)
batch_size = (int) // batch size for testing data
test_freq = (int) // frequency of testing
@@ -222,6 +224,11 @@ class Model(object):
# set Train_one_batch component, using backprogapation at default
setval(self.jobconf, train_one_batch=Algorithm(type=enumAlgType(alg)).proto)
+ # use of cudnn
+ if device != None:
+ setval(self.jobconf, gpu=device)
+ self.cudnn = True
+
self.build() # construct Nneuralnet Component
#--- generate job.conf file for debug purpose
@@ -342,7 +349,13 @@ class Updater(object):
def __init__(self, upd_type, lr, lr_type,
decay, momentum,
step, step_lr, **fields):
-
+ '''
+ required
+ upd_type = (enum) // enum type of updater
+ lr = (float) // base learning rate
+ optional
+ lr_type = (string) // type of the learning rate (Fixed at default)
+ '''
upd = Message('Updater', type=upd_type, **fields).proto
setval(upd.learning_rate, base_lr=lr)
if decay > 0:
@@ -355,14 +368,16 @@ class Updater(object):
elif lr_type == 'step':
cp = Message('Step', change_freq=60, gamma=0.997)
setval(upd.learning_rate, type=kStep, step_conf=cp.proto)
- elif lr_type == 'fixed':
+ elif lr_type == 'fixedstep':
cp = Message('FixedStep', step=step, step_lr=step_lr)
setval(upd.learning_rate, type=kFixedStep, fixedstep_conf=cp.proto)
elif lr_type == 'linear':
cp = Message('Linear', change_freq=10, final_lr=0.1)
setval(upd.learning_rate, type=kLinear, linear_conf=cp.proto)
+
self.proto = upd
+
class SGD(Updater):
def __init__(self, lr=0.01, lr_type=None,
decay=0, momentum=0,
@@ -377,7 +392,6 @@ class SGD(Updater):
step = (int/list) // steps
step_lr = (float/list) // learning rate after the steps
**fields (KEY=VALUE)
-
'''
assert lr
super(SGD, self).__init__(upd_type=kSGD,
[10/10] incubator-singa git commit: SINGA-81 Add Python Helper
Posted by wa...@apache.org.
SINGA-81 Add Python Helper
- Revise src/driver.cc to skip the case of no conf file
- Change import path for datasets in example python codes
- Format python codes and Add license message
- Update README.md
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/9ff176c3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/9ff176c3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/9ff176c3
Branch: refs/heads/master
Commit: 9ff176c304efddb9169d20f7c213be9c222edf17
Parents: 2a6645c
Author: chonho <le...@comp.nus.edu.sg>
Authored: Thu Dec 31 15:10:26 2015 +0800
Committer: Wei Wang <wa...@comp.nus.edu.sg>
Committed: Fri Jan 1 20:13:25 2016 +0800
----------------------------------------------------------------------
src/driver.cc | 4 +-
tool/python/README.md | 138 +--
tool/python/examples/cifar10_cnn.py | 2 +-
tool/python/examples/cifar10_cnn_cudnn.py | 2 +-
tool/python/examples/cifar10_cnn_parameter.py | 2 +-
tool/python/examples/cluster.conf | 6 -
tool/python/examples/mnist_ae.py | 3 +-
tool/python/examples/mnist_mlp.py | 2 +-
tool/python/examples/mnist_mlp_cudnn.py | 34 -
tool/python/examples/mnist_mlp_test.py | 2 +-
tool/python/examples/mnist_rbm1.py | 2 +-
tool/python/examples/mnist_rbm1_parameter.py | 26 -
tool/python/examples/mnist_rbm2.py | 2 +-
tool/python/examples/mnist_rbm2_parameter.py | 28 -
tool/python/examples/mnist_rbm3.py | 2 +-
tool/python/examples/mnist_rbm3_parameter.py | 28 -
tool/python/examples/mnist_rbm4.py | 2 +-
tool/python/examples/mnist_rbm4_parameter.py | 27 -
tool/python/singa.py | 32 +-
tool/python/singa/initializations.py | 97 +-
tool/python/singa/layer.py | 672 +++++++-----
tool/python/singa/model.py | 1072 +++++++++++---------
tool/python/singa/parameter.py | 225 ++--
tool/python/singa/utils/message.py | 112 +-
tool/python/singa/utils/utility.py | 124 ++-
25 files changed, 1393 insertions(+), 1253 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9ff176c3/src/driver.cc
----------------------------------------------------------------------
diff --git a/src/driver.cc b/src/driver.cc
index 67b5c28..c17ea82 100644
--- a/src/driver.cc
+++ b/src/driver.cc
@@ -60,8 +60,8 @@ void Driver::Init(int argc, char **argv) {
SetupLog(singa_conf_.log_dir(), "driver");
// job conf passed by users as "-conf <path>"
arg_pos = ArgPos(argc, argv, "-conf");
- CHECK_NE(arg_pos, -1);
- ReadProtoFromTextFile(argv[arg_pos+1], &job_conf_);
+ if (arg_pos != -1)
+ ReadProtoFromTextFile(argv[arg_pos+1], &job_conf_);
// register layers
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9ff176c3/tool/python/README.md
----------------------------------------------------------------------
diff --git a/tool/python/README.md b/tool/python/README.md
index 1133fa4..02e7fd1 100644
--- a/tool/python/README.md
+++ b/tool/python/README.md
@@ -10,29 +10,29 @@
|-- utils
|-- utility.py
|-- message.py
+ |-- examples
+ |-- cifar10_cnn.py, mnist_mlp.py, , mnist_rbm1.py, mnist_ae.py, etc.
|-- datasets
|-- cifar10.py
|-- mnist.py
- |-- rbm.py
- |-- ae.py
- |-- examples
- |-- cifar10_cnn.py, mnist_mlp.py, , mnist_rbm1.py, rnnlm_usermodel.py, etc.
### How to Run
```
-bin/singa-run.sh -exe user_main.py -conf cluster.conf
+bin/singa-run.sh -exec user_main.py
```
-The python code, e.g., `user_main.py`, would create the JobProto object and pass it to Driver::Train.
-Currently, ./bin/singa-run.sh needs to get the cluster topology, hence we still need to pass a `cluster.conf` to it.
-The cluster.conf has the configuration for a JobProto with all other fields empty except the cluster field.
+The python code, e.g., user_main.py, would create the JobProto object and pass it to Driver::Train.
-Note that `workspace` field in ClusterProto can be set in either (i) cluster.conf or (ii) python code.
-
-#### Examples
+For example,
```
cd SINGA_ROOT
-bin/singa-run.sh -exe tool/python/examples/cifar10_cnn.py -conf tool/python/examples/cluster.conf
+bin/singa-run.sh -exec tool/python/examples/cifar10_cnn.py
+```
+
+Note that, in order to use the Python Helper feature, users need to add the following option
+```
+./configure --enable-python --with-python=PYTHON_DIR
```
+where PYTHON_DIR has Python.h
### Layer class (inherited)
@@ -47,15 +47,6 @@ bin/singa-run.sh -exe tool/python/examples/cifar10_cnn.py -conf tool/python/exam
* RBM
* Autoencoder
-#### for user defined layers (IN PROGRESS)
-
-The following classes are designed to construct user-defined layers for RNNLM example.
-
-* Embedding
-* RNNLM
-* UserLossRNNLM
-
-
### Model class
* Model class has `jobconf` (JobProto) and `layers` (layer list)
@@ -92,60 +83,57 @@ fit() and evaluate() return train/test results, a dictionary containing
* 'ppl' for ppl
* 'se' for squred error
-<<<<<<< HEAD
-=======
+#### To run Singa on GPU
+
+Users need to set a list of gpu ids to `device` field in fit() or evaluate().
+
+For example,
+```
+gpu_id = [0]
+m.fit(X_train, nb_epoch=100, with_test=True, device=gpu_id)
+```
+
+
### Parameter class
Users need to set parameter and initial values. For example,
-* Parameter
- * lr = (float) // learning rate
- * wd = (float) // weight decay
-
-* Parameter initialization
- * init = (string) // one of the types, 'uniform', 'constant', 'gaussian'
- * for uniform [default]
- * high = (float)
- * low = (float)
- * for constant
- * value = (float)
- * for gaussian
- * mean = (float)
- * std = (float)
+* Parameter (fields in Param proto)
+ * lr = (float) // learning rate multiplier, used to scale the learning rate when updating parameters.
+ * wd = (float) // weight decay multiplier, used to scale the weight decay when updating parameters.
+
+* Parameter initialization (fields in ParamGen proto)
+ * init = (string) // one of the types, 'uniform', 'constant', 'gaussian'
+ * high = (float) // for 'uniform'
+ * low = (float) // for 'uniform'
+ * value = (float) // for 'constant'
+ * mean = (float) // for 'gaussian'
+ * std = (float) // for 'gaussian'
* Weight (`w_param`) is 'gaussian' with mean=0, std=0.01 at default
* Bias (`b_param`) is 'constant' with value=0 at default
* How to update the parameter fields
- * for updating Weight, put `w_` in front of field name
- * for updating Bias, put `b_` in front of field name
+ * for updating Weight, put `w_` in front of field name
+ * for updating Bias, put `b_` in front of field name
Several ways to set Parameter values
```
-m.add(Dense(10, w_mean=1, w_std=0.1, w_lr=2, w_wd=10, ...)
+parw = Parameter(lr=2, wd=10, init='gaussian', std=0.1)
+parb = Parameter(lr=1, wd=0, init='constant', value=0)
+m.add(Convolution2D(10, w_param=parw, b_param=parb, ...)
```
```
-parw = Parameter(lr=2, wd=10, init='constant', value=0)
-m.add(Dense(10, w_param=parw, ...)
+m.add(Dense(10, w_mean=1, w_std=0.1, w_lr=2, w_wd=10, ...)
```
```
-parw = Parameter(init='constant', value=0)
-m.add(Dense(10, w_param=parw, w_lr=2, w_wd=10, ...)
+parw = Parameter(init='constant', mean=0)
+m.add(Dense(10, w_param=parw, w_lr=1, w_wd=1, b_value=1, ...)
```
-#### To run Singa on GPU
-
-Users need to set a list of gpu ids to `device` field in fit() or evaluate().
-For example,
-```
-gpu_id = [0]
-m.fit(X_train, nb_epoch=100, with_test=True, device=gpu_id)
-```
-
->>>>>>> cb1ffb4... SINGA-81 Add Python Helper
#### Other classes
* Store
@@ -311,48 +299,6 @@ store = Store(path='test.bin', batch_size=100, ...) // parameter values a
m.add(Data(load='recordinput', phase='test', conf=store)) // Data layer is added
```
-<<<<<<< HEAD
-### Parameter class
-
-Users need to set parameter and initial values. For example,
-
-* Parameter
- * lr = (float) // learning rate
- * wd = (float) // weight decay
-
-* Parameter initialization
- * init = (string) // one of the types, 'uniform', 'constant', 'gaussian'
- * for uniform [default]
- * high = (float)
- * low = (float)
- * for constant
- * value = (float)
- * for gaussian
- * mean = (float)
- * std = (float)
-
-* Weight (w_param) is gaussian with mean=0, std=0.01 at default
-
-* Bias (b_param) is constant with value=0 at default
-
-* How to update the parameter fields
- * for updating Weight, put 'w_' in front of field name
- * for updating Bias, put 'b_' in front of field name
-
-Several ways to set Parameter values
-```
-m.add(Dense(10, w_mean=1, w_std=0.1, w_lr=2, w_wd=10, ...)
-```
-```
-parw = Parameter(lr=2, wd=10, init='constant', value=0)
-m.add(Dense(10, w_param=parw, ...)
-```
-```
-parw = Parameter(init='constant', value=0)
-m.add(Dense(10, w_param=parw, w_lr=2, w_wd=10, ...)
-```
-=======
->>>>>>> cb1ffb4... SINGA-81 Add Python Helper
### Cases to run singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9ff176c3/tool/python/examples/cifar10_cnn.py
----------------------------------------------------------------------
diff --git a/tool/python/examples/cifar10_cnn.py b/tool/python/examples/cifar10_cnn.py
index 859a9a4..9e8e588 100755
--- a/tool/python/examples/cifar10_cnn.py
+++ b/tool/python/examples/cifar10_cnn.py
@@ -2,7 +2,7 @@
import sys, os
sys.path.append(os.path.join(os.path.dirname(__file__),'..'))
from singa.model import *
-from singa.datasets import cifar10
+from examples.datasets import cifar10
X_train, X_test, workspace = cifar10.load_data()
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9ff176c3/tool/python/examples/cifar10_cnn_cudnn.py
----------------------------------------------------------------------
diff --git a/tool/python/examples/cifar10_cnn_cudnn.py b/tool/python/examples/cifar10_cnn_cudnn.py
index d4f4b7c..4269f37 100755
--- a/tool/python/examples/cifar10_cnn_cudnn.py
+++ b/tool/python/examples/cifar10_cnn_cudnn.py
@@ -2,7 +2,7 @@
import sys, os
sys.path.append(os.path.join(os.path.dirname(__file__),'..'))
from singa.model import *
-from singa.datasets import cifar10
+from examples.datasets import cifar10
X_train, X_test, workspace = cifar10.load_data()
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9ff176c3/tool/python/examples/cifar10_cnn_parameter.py
----------------------------------------------------------------------
diff --git a/tool/python/examples/cifar10_cnn_parameter.py b/tool/python/examples/cifar10_cnn_parameter.py
index 4144fa5..39bdb43 100755
--- a/tool/python/examples/cifar10_cnn_parameter.py
+++ b/tool/python/examples/cifar10_cnn_parameter.py
@@ -2,7 +2,7 @@
import sys, os
sys.path.append(os.path.join(os.path.dirname(__file__),'..'))
from singa.model import *
-from singa.datasets import cifar10
+from examples.datasets import cifar10
X_train, X_test, workspace = cifar10.load_data()
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9ff176c3/tool/python/examples/cluster.conf
----------------------------------------------------------------------
diff --git a/tool/python/examples/cluster.conf b/tool/python/examples/cluster.conf
deleted file mode 100644
index 16623d8..0000000
--- a/tool/python/examples/cluster.conf
+++ /dev/null
@@ -1,6 +0,0 @@
-cluster {
- nworker_groups: 1
- nserver_groups: 1
- nworkers_per_group: 1
- nworkers_per_procs: 1
-}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9ff176c3/tool/python/examples/mnist_ae.py
----------------------------------------------------------------------
diff --git a/tool/python/examples/mnist_ae.py b/tool/python/examples/mnist_ae.py
index 0b7e590..4528d9c 100755
--- a/tool/python/examples/mnist_ae.py
+++ b/tool/python/examples/mnist_ae.py
@@ -2,7 +2,7 @@
import sys, os
sys.path.append(os.path.join(os.path.dirname(__file__),'..'))
from singa.model import *
-from singa.datasets import mnist
+from examples.datasets import mnist
# Sample parameter values for Autoencoder example
rbmid = 4
@@ -18,7 +18,6 @@ m = Sequential('autoencoder', sys.argv)
hid_dim = [1000, 500, 250, 30]
m.add(Autoencoder(hid_dim, out_dim=784, activation='sigmoid', param_share=True))
-
agd = AdaGrad(lr=0.01)
topo = Cluster(workspace)
m.compile(loss='mean_squared_error', optimizer=agd, cluster=topo)
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9ff176c3/tool/python/examples/mnist_mlp.py
----------------------------------------------------------------------
diff --git a/tool/python/examples/mnist_mlp.py b/tool/python/examples/mnist_mlp.py
index da5ccce..1bd5bc5 100755
--- a/tool/python/examples/mnist_mlp.py
+++ b/tool/python/examples/mnist_mlp.py
@@ -2,7 +2,7 @@
import sys, os
sys.path.append(os.path.join(os.path.dirname(__file__),'..'))
from singa.model import *
-from singa.datasets import mnist
+from examples.datasets import mnist
# Sample parameter values for Mnist MLP example
pvalues = {'batchsize' : 64, 'shape' : 784, 'random_skip' : 5000,
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9ff176c3/tool/python/examples/mnist_mlp_cudnn.py
----------------------------------------------------------------------
diff --git a/tool/python/examples/mnist_mlp_cudnn.py b/tool/python/examples/mnist_mlp_cudnn.py
deleted file mode 100755
index d418950..0000000
--- a/tool/python/examples/mnist_mlp_cudnn.py
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/usr/bin/env python
-import sys, os
-sys.path.append(os.path.join(os.path.dirname(__file__),'..'))
-from singa.model import *
-from singa.datasets import mnist
-
-# Sample parameter values for Mnist MLP example
-pvalues = {'batchsize' : 64, 'shape' : 784, 'random_skip' : 5000,
- 'std_value' : 127.5, 'mean_value' : 127.5}
-X_train, X_test, workspace = mnist.load_data(**pvalues)
-
-m = Sequential('mlp', argv=sys.argv)
-
-''' Weight and Bias are initialized by
- uniform distribution with scale=0.05 at default
-'''
-m.add(Dense(2500, init='uniform', activation='tanh'))
-m.add(Dense(2000, init='uniform', activation='tanh'))
-m.add(Dense(1500, init='uniform', activation='tanh'))
-m.add(Dense(1000, init='uniform', activation='tanh'))
-m.add(Dense(500, init='uniform', activation='tanh'))
-m.add(Dense(10, init='uniform', activation='softmax'))
-
-sgd = SGD(lr=0.001, lr_type='step')
-topo = Cluster(workspace)
-m.compile(loss='categorical_crossentropy', optimizer=sgd, cluster=topo)
-
-gpu_id = [0]
-m.fit(X_train, nb_epoch=100, with_test=True, device=gpu_id)
-result = m.evaluate(X_test, batch_size=100, test_steps=10)
-
-#e.g., display result
-#for k, v in sorted(result.items(), key=lambda x: x[0]):
-# print k, v
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9ff176c3/tool/python/examples/mnist_mlp_test.py
----------------------------------------------------------------------
diff --git a/tool/python/examples/mnist_mlp_test.py b/tool/python/examples/mnist_mlp_test.py
index 67cf3b3..de5b9c0 100755
--- a/tool/python/examples/mnist_mlp_test.py
+++ b/tool/python/examples/mnist_mlp_test.py
@@ -2,7 +2,7 @@
import sys, os
sys.path.append(os.path.join(os.path.dirname(__file__),'..'))
from singa.model import *
-from singa.datasets import mnist
+from examples.datasets import mnist
# Sample parameter values for Mnist MLP example
pvalues = {'batchsize' : 64, 'shape' : 784,
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9ff176c3/tool/python/examples/mnist_rbm1.py
----------------------------------------------------------------------
diff --git a/tool/python/examples/mnist_rbm1.py b/tool/python/examples/mnist_rbm1.py
index 765be8b..6cf249c 100755
--- a/tool/python/examples/mnist_rbm1.py
+++ b/tool/python/examples/mnist_rbm1.py
@@ -2,7 +2,7 @@
import sys, os
sys.path.append(os.path.join(os.path.dirname(__file__),'..'))
from singa.model import *
-from singa.datasets import mnist
+from examples.datasets import mnist
rbmid = 1
pvalues = {'batchsize' : 100, 'shape' : 784, 'std_value' : 255}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9ff176c3/tool/python/examples/mnist_rbm1_parameter.py
----------------------------------------------------------------------
diff --git a/tool/python/examples/mnist_rbm1_parameter.py b/tool/python/examples/mnist_rbm1_parameter.py
deleted file mode 100755
index 54fe421..0000000
--- a/tool/python/examples/mnist_rbm1_parameter.py
+++ /dev/null
@@ -1,26 +0,0 @@
-#!/usr/bin/env python
-import sys, os
-sys.path.append(os.path.join(os.path.dirname(__file__),'..'))
-from singa.model import *
-from singa.datasets import mnist
-
-rbmid = 1
-pvalues = {'batchsize' : 100, 'shape' : 784, 'std_value' : 255}
-X_train, X_test, workspace = mnist.load_data(
- workspace = 'examples/rbm/rbm1',
- nb_rbm = rbmid,
- checkpoint_steps = 6000,
- **pvalues)
-
-m = Energy('rbm'+str(rbmid), sys.argv)
-
-parw = Parameter(init='gaussian', mean=0, std=0.1)
-parb = Parameter(wd=0, init='constant', value=0)
-m.add(RBM(1000, w_param=parw, b_param=parb))
-
-sgd = SGD(lr=0.1, decay=0.0002, momentum=0.8)
-topo = Cluster(workspace)
-m.compile(optimizer=sgd, cluster=topo)
-m.fit(X_train, alg='cd', nb_epoch=6000)
-#result = m.evaluate(X_test, test_steps=100, test_freq=500)
-
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9ff176c3/tool/python/examples/mnist_rbm2.py
----------------------------------------------------------------------
diff --git a/tool/python/examples/mnist_rbm2.py b/tool/python/examples/mnist_rbm2.py
index f4d187f..b5db36e 100755
--- a/tool/python/examples/mnist_rbm2.py
+++ b/tool/python/examples/mnist_rbm2.py
@@ -2,7 +2,7 @@
import sys, os
sys.path.append(os.path.join(os.path.dirname(__file__),'..'))
from singa.model import *
-from singa.datasets import mnist
+from examples.datasets import mnist
rbmid = 2
pvalues = {'batchsize' : 100, 'shape' : 784, 'std_value' : 255}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9ff176c3/tool/python/examples/mnist_rbm2_parameter.py
----------------------------------------------------------------------
diff --git a/tool/python/examples/mnist_rbm2_parameter.py b/tool/python/examples/mnist_rbm2_parameter.py
deleted file mode 100755
index 9837836..0000000
--- a/tool/python/examples/mnist_rbm2_parameter.py
+++ /dev/null
@@ -1,28 +0,0 @@
-#!/usr/bin/env python
-import sys, os
-sys.path.append(os.path.join(os.path.dirname(__file__),'..'))
-from singa.model import *
-from singa.datasets import mnist
-
-rbmid = 2
-pvalues = {'batchsize' : 100, 'shape' : 784, 'std_value' : 255}
-X_train, X_test, workspace = mnist.load_data(
- workspace = 'examples/rbm/rbm2',
- nb_rbm = rbmid,
- checkpoint_steps = 6000,
- **pvalues)
-
-m = Energy('rbm'+str(rbmid), sys.argv)
-
-out_dim = [1000, 500]
-parw = Parameter(init='gaussian', mean=0, std=0.1)
-parb = Parameter(wd=0, init='constant', value=0)
-m.add(RBM(out_dim, w_param=parw, b_param=parb))
-
-
-sgd = SGD(lr=0.1, decay=0.0002, momentum=0.8)
-topo = Cluster(workspace)
-m.compile(optimizer=sgd, cluster=topo)
-m.fit(X_train, alg='cd', nb_epoch=6000)
-#result = m.evaluate(X_test, test_steps=100, test_freq=500)
-
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9ff176c3/tool/python/examples/mnist_rbm3.py
----------------------------------------------------------------------
diff --git a/tool/python/examples/mnist_rbm3.py b/tool/python/examples/mnist_rbm3.py
index 48bbe38..be1b789 100755
--- a/tool/python/examples/mnist_rbm3.py
+++ b/tool/python/examples/mnist_rbm3.py
@@ -2,7 +2,7 @@
import sys, os
sys.path.append(os.path.join(os.path.dirname(__file__),'..'))
from singa.model import *
-from singa.datasets import mnist
+from examples.datasets import mnist
rbmid = 3
pvalues = {'batchsize' : 100, 'shape' : 784, 'std_value' : 255}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9ff176c3/tool/python/examples/mnist_rbm3_parameter.py
----------------------------------------------------------------------
diff --git a/tool/python/examples/mnist_rbm3_parameter.py b/tool/python/examples/mnist_rbm3_parameter.py
deleted file mode 100755
index 6c9a378..0000000
--- a/tool/python/examples/mnist_rbm3_parameter.py
+++ /dev/null
@@ -1,28 +0,0 @@
-#!/usr/bin/env python
-import sys, os
-sys.path.append(os.path.join(os.path.dirname(__file__),'..'))
-from singa.model import *
-from singa.datasets import mnist
-
-rbmid = 3
-pvalues = {'batchsize' : 100, 'shape' : 784, 'std_value' : 255}
-X_train, X_test, workspace = mnist.load_data(
- workspace = 'examples/rbm/rbm3',
- nb_rbm = rbmid,
- checkpoint_steps = 6000,
- **pvalues)
-
-m = Energy('rbm'+str(rbmid), sys.argv)
-
-out_dim = [1000, 500, 250]
-parw = Parameter(init='gaussian', mean=0, std=0.1)
-parb = Parameter(wd=0, init='constant', value=0)
-m.add(RBM(out_dim, w_param=parw, b_param=parb))
-
-
-sgd = SGD(lr=0.1, decay=0.0002, momentum=0.8)
-topo = Cluster(workspace)
-m.compile(optimizer=sgd, cluster=topo)
-m.fit(X_train, alg='cd', nb_epoch=6000)
-#result = m.evaluate(X_test, test_steps=100, test_freq=500)
-
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9ff176c3/tool/python/examples/mnist_rbm4.py
----------------------------------------------------------------------
diff --git a/tool/python/examples/mnist_rbm4.py b/tool/python/examples/mnist_rbm4.py
index f12b739..a4ee995 100755
--- a/tool/python/examples/mnist_rbm4.py
+++ b/tool/python/examples/mnist_rbm4.py
@@ -2,7 +2,7 @@
import sys, os
sys.path.append(os.path.join(os.path.dirname(__file__),'..'))
from singa.model import *
-from singa.datasets import mnist
+from examples.datasets import mnist
rbmid = 4
pvalues = {'batchsize' : 100, 'shape' : 784, 'std_value' : 255}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9ff176c3/tool/python/examples/mnist_rbm4_parameter.py
----------------------------------------------------------------------
diff --git a/tool/python/examples/mnist_rbm4_parameter.py b/tool/python/examples/mnist_rbm4_parameter.py
deleted file mode 100755
index 2a7be1d..0000000
--- a/tool/python/examples/mnist_rbm4_parameter.py
+++ /dev/null
@@ -1,27 +0,0 @@
-#!/usr/bin/env python
-import sys, os
-sys.path.append(os.path.join(os.path.dirname(__file__),'..'))
-from singa.model import *
-from singa.datasets import mnist
-
-rbmid = 4
-pvalues = {'batchsize' : 100, 'shape' : 784, 'std_value' : 255}
-X_train, X_test, workspace = mnist.load_data(
- workspace = 'examples/rbm/rbm'+rbmid,
- nb_rbm = rbmid,
- checkpoint_steps = 6000,
- **pvalues)
-
-m = Energy('rbm'+str(rbmid), sys.argv)
-
-out_dim = [1000, 500, 250, 30]
-parw = Parameter(init='gaussian', mean=0, std=0.1)
-parb = Parameter(wd=0, init='constant', value=0)
-m.add(RBM(out_dim, w_param=parw, b_param=parb))
-
-sgd = SGD(lr=0.1, decay=0.0002, momentum=0.8)
-topo = Cluster(workspace)
-m.compile(optimizer=sgd, cluster=topo)
-m.fit(X_train, alg='cd', nb_epoch=6000)
-#result = m.evaluate(X_test, test_steps=100, test_freq=500)
-
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9ff176c3/tool/python/singa.py
----------------------------------------------------------------------
diff --git a/tool/python/singa.py b/tool/python/singa.py
index 986a6b8..e44e94d 100755
--- a/tool/python/singa.py
+++ b/tool/python/singa.py
@@ -21,8 +21,6 @@
#*
#*************************************************************/
-
-
import os
import sys
import string
@@ -31,18 +29,18 @@ import singa.driver as driver
from google.protobuf.text_format import Merge
if __name__ == '__main__':
- """Invoke the training program using this python script.
- ./bin/singa-run.sh -exec tool/python/singa.py -conf examples/cifar10/job.conf
- """"
-
- i = sys.argv.index("-conf")
- s = open(sys.argv[i+1], 'r').read()
- s = str(s)
- j = job_pb2.JobProto()
- Merge(s,j)
- b = j.SerializeToString()
- d = driver.Driver()
- d.InitLog(sys.argv[0])
- d.Init(sys.argv)
- d.Train(False,b)
- #d.Test(b)
+ """Invoke the training program using this python script.
+ ./bin/singa-run.sh -exec tool/python/singa.py -conf examples/cifar10/job.conf
+ """
+
+ i = sys.argv.index('-conf')
+ s = open(sys.argv[i+1], 'r').read()
+ s = str(s)
+ j = job_pb2.JobProto()
+ Merge(s, j)
+ b = j.SerializeToString()
+ d = driver.Driver()
+ d.InitLog(sys.argv[0])
+ d.Init(sys.argv)
+ d.Train(False, b)
+ #d.Test(b)
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9ff176c3/tool/python/singa/initializations.py
----------------------------------------------------------------------
diff --git a/tool/python/singa/initializations.py b/tool/python/singa/initializations.py
index f3037cd..f016f1f 100644
--- a/tool/python/singa/initializations.py
+++ b/tool/python/singa/initializations.py
@@ -1,34 +1,67 @@
#!/usr/bin/env python
-def get(identifier, **kwargs):
-
- field = {}
-
- if identifier == 'none':
- return
-
- if identifier == 'uniform':
- scale = kwargs['scale'] if 'scale' in kwargs else 0.05
- names = ['low', 'high']
- values = [-scale, scale]
-
- elif identifier == 'constant':
- names = ['value']
- values = [0]
-
- elif identifier == 'gaussian':
- names = ['mean', 'std']
- values = [0, 0.01]
-
- elif identifier == 'conv2d':
- names = ['stride', 'pad']
- values = [1, 0]
-
- elif identifier == 'lrn2d':
- names = ['alpha', 'beta', 'knorm']
- values = [1, 0.75, 1]
-
- for i in range(len(names)):
- field[names[i]] = kwargs[names[i]] if names[i] in kwargs else values[i]
-
- return field
+#/************************************************************
+#*
+#* Licensed to the Apache Software Foundation (ASF) under one
+#* or more contributor license agreements. See the NOTICE file
+#* distributed with this work for additional information
+#* regarding copyright ownership. The ASF licenses this file
+#* to you under the Apache License, Version 2.0 (the
+#* "License"); you may not use this file except in compliance
+#* with the License. You may obtain a copy of the License at
+#*
+#* http://www.apache.org/licenses/LICENSE-2.0
+#*
+#* Unless required by applicable law or agreed to in writing,
+#* software distributed under the License is distributed on an
+#* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#* KIND, either express or implied. See the License for the
+#* specific language governing permissions and limitations
+#* under the License.
+#*
+#*************************************************************/
+
+'''
+This module pre-defines initial value for fields
+'''
+
+def get_init_values(identifier, **kwargs):
+ '''
+ This method returns field, a set of key-value pairs, that
+ key is specified by identifier and values are initialized.
+ '''
+
+ field = {}
+
+ if identifier == 'none':
+ return
+
+ if identifier == 'uniform':
+ scale = kwargs['scale'] if 'scale' in kwargs else 0.05
+ names = ['low', 'high']
+ values = [-scale, scale]
+
+ elif identifier == 'constant':
+ names = ['value']
+ values = [0]
+
+ elif identifier == 'gaussian':
+ names = ['mean', 'std']
+ values = [0, 0.01]
+
+ elif identifier == 'conv2d':
+ names = ['stride', 'pad']
+ values = [1, 0]
+
+ elif identifier == 'lrn2d':
+ names = ['alpha', 'beta', 'knorm']
+ values = [1, 0.75, 1]
+
+ elif identifier == 'dropout':
+ names = ['ratio']
+ values = [0.5]
+
+ for i in range(len(names)):
+ field[names[i]] = kwargs[names[i]] if names[i] in kwargs else values[i]
+
+ return field
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9ff176c3/tool/python/singa/layer.py
----------------------------------------------------------------------
diff --git a/tool/python/singa/layer.py b/tool/python/singa/layer.py
index 1744d50..b391d26 100644
--- a/tool/python/singa/layer.py
+++ b/tool/python/singa/layer.py
@@ -1,300 +1,440 @@
#!/usr/bin/env python
-from parameter import *
-from utils.utility import *
-from utils.message import *
+
+#/************************************************************
+#*
+#* Licensed to the Apache Software Foundation (ASF) under one
+#* or more contributor license agreements. See the NOTICE file
+#* distributed with this work for additional information
+#* regarding copyright ownership. The ASF licenses this file
+#* to you under the Apache License, Version 2.0 (the
+#* "License"); you may not use this file except in compliance
+#* with the License. You may obtain a copy of the License at
+#*
+#* http://www.apache.org/licenses/LICENSE-2.0
+#*
+#* Unless required by applicable law or agreed to in writing,
+#* software distributed under the License is distributed on an
+#* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#* KIND, either express or implied. See the License for the
+#* specific language governing permissions and limitations
+#* under the License.
+#*
+#*************************************************************/
+
+'''
+This script includes Layer class and its subclasses that
+users can configure different types of layers for their model.
+'''
+
+from singa.parameter import Parameter, set_param_field
+from singa.initializations import get_init_values
+from singa.utils.utility import setval, generate_name
+from singa.utils.message import *
from google.protobuf import text_format
class Layer(object):
- def __init__(self, **kwargs):
- self.layer = Message('Layer', **kwargs).proto
- # required
- if not 'name' in kwargs:
- setval(self.layer, name=generateName('layer', 1))
- # srclayers are set in Model.build()
- self.is_datalayer = False
+ def __init__(self, **kwargs):
+ '''
+ **kwargs (KEY=VALUE)
+ partition_dim = (int) // partition dimension for net
+ '''
+
+ self.layer = Message('Layer', **kwargs).proto
+ # required
+ if not 'name' in kwargs:
+ setval(self.layer, name=generate_name('layer', 1))
+
+ # srclayers are set in Model.build()
+ self.is_datalayer = False
class Data(Layer):
- def __init__(self, load, phase='train', checkpoint=None,
- conf=None, **kwargs):
- assert load != None, 'data type should be specified'
- if load == 'kData':
- super(Data, self).__init__(name=generateName('data'), user_type=load)
- else:
- self.layer_type = enumLayerType(load)
- super(Data, self).__init__(name=generateName('data'), type=self.layer_type)
-
- # include/exclude
- setval(self.layer, include=enumPhase(phase))
- #setval(self.layer, exclude=kTest if phase=='train' else kTrain)
-
- if conf == None:
- if load == 'kData':
- setval(self.layer.Extensions[data_conf], **kwargs)
- else:
- setval(self.layer.store_conf, **kwargs)
- else:
- setval(self.layer, store_conf=conf.proto)
- self.is_datalayer = True
-
- self.checkpoint = checkpoint # checkpoint for training data
+
+ def __init__(self, load, phase='train', checkpoint=None,
+ conf=None, **kwargs):
+ '''
+ required
+ load = (string) // type of data
+ optional
+ phase = (string) // phase of data layer
+ checkpoint = (string) // checkpoint path
+ conf = (Store) // Store object
+ **kwargs (KEY=VALUE)
+ partition_dim = (int) // partition dimension for net
+ '''
+
+ assert load != None, 'data type should be specified'
+ if load == 'kData':
+ super(Data, self).__init__(name=generate_name('data'),
+ user_type=load)
+ else:
+ self.layer_type = enumLayerType(load)
+ super(Data, self).__init__(name=generate_name('data'),
+ type=self.layer_type)
+ self.is_datalayer = True
+
+ # include/exclude
+ setval(self.layer, include=enumPhase(phase))
+ #setval(self.layer, exclude=kTest if phase=='train' else kTrain)
+
+ if conf == None:
+ if load == 'kData':
+ setval(self.layer.Extensions[data_conf], **kwargs)
+ else:
+ setval(self.layer.store_conf, **kwargs)
+ else:
+ setval(self.layer, store_conf=conf.proto)
+
+ self.checkpoint = checkpoint # checkpoint for training data
class Convolution2D(Layer):
- def __init__(self, nb_filter=0, kernel=0, stride=1, pad=0,
- init=None, w_param=None, b_param=None,
- activation=None, **kwargs):
- '''
- required
- nb_filter = (int) // the number of filters
- kernel = (int) // the size of filter
- optional
- stride = (int) // the size of stride
- pad = (int) // the size of padding
- '''
- assert nb_filter > 0 and kernel > 0, 'should be set as positive int'
- super(Convolution2D, self).__init__(name=generateName('conv',1), type=kCConvolution)
- fields = {'num_filters' : nb_filter,
- 'kernel' : kernel,
- 'stride' : stride,
- 'pad' : pad}
- setval(self.layer.convolution_conf, **fields)
-
- # parameter w
- if w_param == None:
- self.init = 'gaussian' if init==None else init
- w_param = Parameter(init=self.init)
- setParamField(w_param.param, 'w', True, **kwargs)
- setval(self.layer, param=w_param.param)
-
- # parameter b
- if b_param == None:
- self.init = 'constant' if init==None else init
- b_param = Parameter(init=self.init) # default: constant
- setParamField(b_param.param, 'b', True, **kwargs)
- setval(self.layer, param=b_param.param)
-
- # following layers: e.g., activation, dropout, etc.
- if activation:
- self.mask = Activation(activation=activation).layer
+
+ def __init__(self, nb_filter=0, kernel=0, stride=1, pad=0,
+ init=None, w_param=None, b_param=None,
+ activation=None, **kwargs):
+ '''
+ required
+ nb_filter = (int) // the number of filters
+ kernel = (int) // the size of filter
+ optional
+ stride = (int) // the size of stride
+ pad = (int) // the size of padding
+ init = (string) // 'unirom', 'gaussian', 'constant'
+ w_param = (Parameter) // Parameter object for weight
+ b_param = (Parameter) // Parameter object for bias
+ **kwargs (KEY=VALUE)
+ w_lr = (float) // learning rate multiplier for weight, used to
+ // scale the learning rate when updating parameters.
+ w_wd = (float) // weight decay multiplier for weight, used to
+ // scale the weight decay when updating parameters.
+ b_lr = (float) // learning rate multiplier for bias
+ b_wd = (float) // weight decay multiplier for bias
+ '''
+
+ assert nb_filter > 0 and kernel > 0, 'should be set as positive int'
+ super(Convolution2D, self).__init__(name=generate_name('conv', 1),
+ type=kCConvolution)
+ fields = {'num_filters' : nb_filter,
+ 'kernel' : kernel,
+ 'stride' : stride,
+ 'pad' : pad}
+ setval(self.layer.convolution_conf, **fields)
+
+ # parameter w
+ if w_param == None:
+ self.init = 'gaussian' if init == None else init
+ w_param = Parameter(init=self.init)
+ set_param_field(w_param.param, 'w', True, **kwargs)
+ setval(self.layer, param=w_param.param)
+
+ # parameter b
+ if b_param == None:
+ self.init = 'constant' if init == None else init
+ b_param = Parameter(init=self.init) # default: constant
+ set_param_field(b_param.param, 'b', True, **kwargs)
+ setval(self.layer, param=b_param.param)
+
+ # following layers: e.g., activation, dropout, etc.
+ if activation:
+ self.mask = Activation(activation=activation).layer
class MaxPooling2D(Layer):
- def __init__(self, pool_size=None, stride=1, ignore_border=True, **kwargs):
- '''
- required
- pool_size = (int|tuple) // the size for pooling
- optional
- stride = (int) // the size of striding
- ignore_border = (bool) // flag for padding
- **kwargs // fields for Layer class
- '''
- assert pool_size != None, 'pool_size is required'
- if type(pool_size) == int:
- pool_size = (pool_size, pool_size)
- assert type(pool_size) == tuple and \
- pool_size[0] == pool_size[1], 'pool size should be square in Singa'
- super(MaxPooling2D, self).__init__(name=generateName('pool'), type=kCPooling, **kwargs)
- fields = {'pool' : PoolingProto().MAX,
- 'kernel' : pool_size[0],
- 'stride' : stride,
- 'pad' : 0 if ignore_border else 1}
- setval(self.layer.pooling_conf, **fields)
+
+ def __init__(self, pool_size=None,
+ stride=1, ignore_border=True, **kwargs):
+ '''
+ Max Pooling layer
+
+ required
+ pool_size = (int|tuple) // the size for pooling
+ optional
+ stride = (int) // the size of striding
+ ignore_border = (bool) // flag for padding
+ **kwargs // fields for Layer class
+ '''
+
+ assert pool_size != None, 'pool_size is required'
+ if type(pool_size) == int:
+ pool_size = (pool_size, pool_size)
+ assert type(pool_size) == tuple and pool_size[0] == pool_size[1], \
+ 'pool size should be square in Singa'
+ super(MaxPooling2D, self).__init__(name=generate_name('pool'),
+ type=kCPooling, **kwargs)
+ fields = {'pool' : PoolingProto().MAX,
+ 'kernel' : pool_size[0],
+ 'stride' : stride,
+ 'pad' : 0 if ignore_border else 1}
+ setval(self.layer.pooling_conf, **fields)
class AvgPooling2D(Layer):
- def __init__(self, pool_size=None, stride=1, ignore_border=True, **kwargs):
- '''
- required
- pool_size = (int|tuple) // size for pooling
- optional
- stride = (int) // size of striding
- ignore_border = (bool) // flag for padding
- **kwargs // fields for Layer class
- '''
- assert pool_size != None, 'pool_size is required'
- if type(pool_size) == int:
- pool_size = (pool_size, pool_size)
- assert type(pool_size) == tuple and \
- pool_size[0] == pool_size[1], 'pool size should be square in Singa'
- super(AvgPooling2D, self).__init__(name=generateName('pool'), type=kCPooling, **kwargs)
- self.layer.pooling_conf.pool = PoolingProto().AVG
- fields = {'pool' : PoolingProto().AVG,
- 'kernel' : pool_size[0],
- 'stride' : stride,
- 'pad' : 0 if ignore_border else 1}
- setval(self.layer.pooling_conf, **fields)
+
+ def __init__(self, pool_size=None,
+ stride=1, ignore_border=True, **kwargs):
+ '''
+ required
+ pool_size = (int|tuple) // size for pooling
+ optional
+ stride = (int) // size of striding
+ ignore_border = (bool) // flag for padding
+ **kwargs // fields for Layer class
+ '''
+
+ assert pool_size != None, 'pool_size is required'
+ if type(pool_size) == int:
+ pool_size = (pool_size, pool_size)
+ assert type(pool_size) == tuple and pool_size[0] == pool_size[1], \
+ 'pool size should be square in Singa'
+ super(AvgPooling2D, self).__init__(name=generate_name('pool'),
+ type=kCPooling, **kwargs)
+ self.layer.pooling_conf.pool = PoolingProto().AVG
+ fields = {'pool' : PoolingProto().AVG,
+ 'kernel' : pool_size[0],
+ 'stride' : stride,
+ 'pad' : 0 if ignore_border else 1}
+ setval(self.layer.pooling_conf, **fields)
class LRN2D(Layer):
- def __init__(self, size=0, **kwargs):
- super(LRN2D, self).__init__(name=generateName('norm'), type=kLRN)
- # required
- assert size != 0, 'local size should be set'
- self.layer.lrn_conf.local_size = size
- init_value = initializations.get('lrn2d', **kwargs)
- setval(self.layer.lrn_conf, **init_value)
+
+ def __init__(self, size=0, **kwargs):
+ '''
+ required
+ size = (int) // local size
+ '''
+
+ super(LRN2D, self).__init__(name=generate_name('norm'), type=kLRN)
+ # required
+ assert size != 0, 'local size should be set'
+ self.layer.lrn_conf.local_size = size
+ init_values = get_init_values('lrn2d', **kwargs)
+ setval(self.layer.lrn_conf, **init_values)
class Activation(Layer):
- def __init__(self, activation='stanh', topk=1):
- self.name = activation
- if activation == 'tanh': activation = 'stanh' # <-- better way to set?
- self.layer_type = enumLayerType(activation)
- super(Activation, self).__init__(name=generateName(self.name), type=self.layer_type)
- if activation == 'softmaxloss':
- self.layer.softmaxloss_conf.topk = topk
-
-class Dropout(Layer):
- def __init__(self, ratio=0.5):
- self.name = 'dropout'
- self.layer_type = kDropout
- super(Dropout, self).__init__(name=generateName(self.name), type=self.layer_type)
- self.layer.dropout_conf.dropout_ratio = ratio
+
+ def __init__(self, activation='stanh', topk=1):
+ '''
+ required
+ activation = (string)
+ optional
+ topk = (int) // the number of results
+ '''
+
+ self.name = activation
+ if activation == 'tanh': activation = 'stanh' # <-- better way to set?
+ self.layer_type = enumLayerType(activation)
+ super(Activation, self).__init__(name=generate_name(self.name),
+ type=self.layer_type)
+ if activation == 'softmaxloss':
+ self.layer.softmaxloss_conf.topk = topk
+
+class Dropout(Layer):
+
+ def __init__(self, ratio=0.5):
+ '''
+ required
+ ratio = (float) // ratio of drop out nodes
+ '''
+
+ self.name = 'dropout'
+ self.layer_type = enumLayerType(self.name)
+ super(Dropout, self).__init__(name=generate_name(self.name),
+ type=self.layer_type)
+ self.layer.dropout_conf.dropout_ratio = ratio
class RGB(Layer):
- def __init__(self, meanfile=None, **kwargs):
- assert meanfile != None, 'meanfile should be specified'
- self.name = 'rgb'
- self.layer_type = kRGBImage
- super(RGB, self).__init__(name=generateName(self.name), type=self.layer_type)
- self.layer.rgbimage_conf.meanfile = meanfile
+
+ def __init__(self, meanfile=None, **kwargs):
+ '''
+ required
+ meanfile = (string) // path to meanfile (depreciated)
+ '''
+
+ assert meanfile != None, 'meanfile should be specified'
+ self.name = 'rgb'
+ self.layer_type = kRGBImage
+ super(RGB, self).__init__(name=generate_name(self.name),
+ type=self.layer_type)
+ self.layer.rgbimage_conf.meanfile = meanfile
class Dense(Layer):
- def __init__(self, output_dim=0, activation=None,
- init=None, w_param=None, b_param=None, input_dim=None,
- **kwargs):
- '''
- required
- output_dim = (int)
- optional
- activation = (string)
- init = (string) // 'unirom', 'gaussian', 'constant'
- **kwargs
- w_lr = (float) // learning rate for w
- w_wd = (float) // weight decay for w
- b_lr = (float) // learning rate for b
- b_wd = (float) // weight decay for b
- '''
- # required
- assert output_dim > 0, 'output_dim should be set'
- super(Dense, self).__init__(type=kInnerProduct, **kwargs)
- self.layer.innerproduct_conf.num_output = output_dim
- if 'transpose' in kwargs:
- self.layer.innerproduct_conf.transpose = kwargs['transpose']
-
- # parameter w (default: gaussian)
- if w_param == None:
- self.init = 'gaussian' if init==None else init
- w_param = Parameter(init=self.init)
- setParamField(w_param.param, 'w', False, **kwargs)
- setval(self.layer, param=w_param.param)
-
- # parameter b (default: constant)
- if b_param == None:
- self.init = 'constant' if init==None else init
- b_param = Parameter(init=self.init)
- setParamField(b_param.param, 'b', False, **kwargs)
- setval(self.layer, param=b_param.param)
-
- # following layers: e.g., activation, dropout, etc.
- if activation:
- self.mask = Activation(activation=activation).layer
-
-
-''' Class to deal with multiple layers
+
+ def __init__(self, output_dim=0, activation=None,
+ init=None, w_param=None, b_param=None, input_dim=None,
+ **kwargs):
+ '''
+ required
+ output_dim = (int)
+ optional
+ activation = (string)
+ init = (string) // 'unirom', 'gaussian', 'constant'
+ w_param = (Parameter) // Parameter object for weight
+ b_param = (Parameter) // Parameter object for bias
+ **kwargs
+ w_lr = (float) // learning rate multiplier for weight, used to
+ // scale the learning rate when updating parameters.
+ w_wd = (float) // weight decay multiplier for weight, used to
+ // scale the weight decay when updating parameters.
+ b_lr = (float) // learning rate multiplier for bias
+ b_wd = (float) // weight decay multiplier for bias
+ '''
+ # required
+ assert output_dim > 0, 'output_dim should be set'
+ super(Dense, self).__init__(type=kInnerProduct, **kwargs)
+ self.layer.innerproduct_conf.num_output = output_dim
+ if 'transpose' in kwargs:
+ self.layer.innerproduct_conf.transpose = kwargs['transpose']
+
+ # parameter w (default: gaussian)
+ if w_param == None:
+ self.init = 'gaussian' if init == None else init
+ w_param = Parameter(init=self.init)
+ set_param_field(w_param.param, 'w', False, **kwargs)
+ setval(self.layer, param=w_param.param)
+
+ # parameter b (default: constant)
+ if b_param == None:
+ self.init = 'constant' if init == None else init
+ b_param = Parameter(init=self.init)
+ set_param_field(b_param.param, 'b', False, **kwargs)
+ setval(self.layer, param=b_param.param)
+
+ # following layers: e.g., activation, dropout, etc.
+ if activation:
+ self.mask = Activation(activation=activation).layer
+
+
+''' Classes to deal with multiple layers
'''
class Autoencoder(object):
- def __init__(self, hid_dim=None, out_dim=0, activation=None,
- param_share=True, **kwargs):
- # required
- assert out_dim > 0, 'out_dim should be set'
- self.out_dim = out_dim
- assert hid_dim != None, 'hid_dim should be set'
- self.hid_dim = [hid_dim] if type(hid_dim)==int else hid_dim
-
- self.layer_type = 'AutoEncoder'
- self.activation = activation
- self.param_share = param_share
+
+ def __init__(self, hid_dim=None, out_dim=0,
+ activation=None, param_share=True):
+ '''
+ Generate a set of layers (like MLP) for encoder and decoder
+ The layers are expanded and added in Sequential.add()
+
+ required
+ hid_dim = (int/list) // the number of nodes in hidden layers
+ out_dim = (int) // the number of nodes in the top layer
+ optional
+ activation = (string)
+ param_share = (bool) // to share params in encoder and decoder
+ '''
+
+ # required
+ assert out_dim > 0, 'out_dim should be set'
+ self.out_dim = out_dim
+ assert hid_dim != None, 'hid_dim should be set'
+ self.hid_dim = [hid_dim] if type(hid_dim) == int else hid_dim
+
+ self.layer_type = 'AutoEncoder'
+ self.activation = activation
+ self.param_share = param_share
class RBM(Layer):
- def __init__(self, out_dim=None, w_param=None, b_param=None, sampling=None, **kwargs):
- '''
- Generate layers (like MLP) according to the number of elements in out_dim, and
- on top of it, two layers RBMVis and RBMHid with bidirectional connection
-
- required
- out_dim = (int) or (int list) // the number of hidden nodes
- optional
- sampling = (string)
- '''
- assert out_dim > 0, 'out_dim should be set'
- self.out_dim = [out_dim] if type(out_dim)==int else out_dim
-
- self.name = kwargs['name'] if 'name' in kwargs else 'RBMVis'
- self.layer_type = kwargs['type'] if 'type' in kwargs else kRBMVis
- super(RBM, self).__init__(name=generateName(self.name, withnumber=False), type=self.layer_type)
- setval(self.layer.rbm_conf, hdim=self.out_dim[-1])
- if self.layer_type == kRBMHid and sampling != None:
- if sampling == 'gaussian':
- setval(self.layer.rbm_conf, gaussian=True)
-
- # parameter w
- if w_param == None:
- w_param = Parameter(init='gaussian', **kwargs)
- setParamField(w_param.param, 'w', withnumber=False, level=len(self.out_dim), **kwargs)
- else:
- if self.layer_type == kRBMHid:
- del kwargs['name']
- else:
- setParamField(w_param.param, 'w', withnumber=False, level=len(self.out_dim), **kwargs)
- setval(self.layer, param=w_param.param)
-
- # parameter b
- if b_param == None:
- b_param = Parameter(init='constant', **kwargs)
- setParamField(b_param.param, 'b', withnumber=False, level=len(self.out_dim), **kwargs)
- else:
- if self.layer_type == kRBMHid:
- pass
- else:
- setParamField(b_param.param, 'b', withnumber=False, level=len(self.out_dim), **kwargs)
- setval(self.layer, param=b_param.param)
-
- if self.layer_type == kRBMVis:
- wname = w_param.param.name
- parw = Parameter(name=wname+"_", init='none', share_from=wname)
- bname = b_param.param.name
- parb = Parameter(name=bname+"2", wd=0, init='constant')
- self.bidirect = RBM(self.out_dim, name='RBMHid', type=kRBMHid,
- w_param=parw, b_param=parb, sampling=sampling).layer
-
-
+
+ def __init__(self, out_dim=None, w_param=None, b_param=None,
+ sampling=None, **kwargs):
+ '''
+ Generate a set of layers (like MLP) according to the number of elements
+ in out_dim, and on top of it, two layers RBMVis and RBMHid with
+ bidirectional connection
+ The layers are expanded and added in Energy.add()
+
+ required
+ out_dim = (int) or (int list) // the number of hidden nodes
+ optional
+ w_param = (Parameter) // Parameter object for weight
+ b_param = (Parameter) // Parameter object for bias
+ sampling = (string)
+ '''
+
+ assert out_dim > 0, 'out_dim should be set'
+ self.out_dim = [out_dim] if type(out_dim) == int else out_dim
+
+ self.name = kwargs['name'] if 'name' in kwargs else 'RBMVis'
+ self.layer_type = kwargs['type'] if 'type' in kwargs else kRBMVis
+ super(RBM, self).__init__(name=generate_name(self.name,
+ withnumber=False), type=self.layer_type)
+ setval(self.layer.rbm_conf, hdim=self.out_dim[-1])
+ if self.layer_type == kRBMHid and sampling != None:
+ if sampling == 'gaussian':
+ setval(self.layer.rbm_conf, gaussian=True)
+
+ # parameter w
+ if w_param == None:
+ w_param = Parameter(init='gaussian', **kwargs)
+ set_param_field(w_param.param, 'w', withnumber=False,
+ level=len(self.out_dim), **kwargs)
+ else:
+ if self.layer_type == kRBMHid:
+ del kwargs['name']
+ else:
+ set_param_field(w_param.param, 'w', withnumber=False,
+ level=len(self.out_dim), **kwargs)
+ setval(self.layer, param=w_param.param)
+
+ # parameter b
+ if b_param == None:
+ b_param = Parameter(init='constant', **kwargs)
+ set_param_field(b_param.param, 'b', withnumber=False,
+ level=len(self.out_dim), **kwargs)
+ else:
+ if self.layer_type == kRBMHid:
+ pass
+ else:
+ set_param_field(b_param.param, 'b', withnumber=False,
+ level=len(self.out_dim), **kwargs)
+ setval(self.layer, param=b_param.param)
+
+ if self.layer_type == kRBMVis:
+ wname = w_param.param.name
+ parw = Parameter(name=wname+"_", init='none', share_from=wname)
+ bname = b_param.param.name
+ parb = Parameter(name=bname+"2", wd=0, init='constant')
+ self.bidirect = RBM(self.out_dim, name='RBMHid', type=kRBMHid,
+ w_param=parw, b_param=parb, sampling=sampling).layer
+
class Embedding(Layer):
- def __init__(self, in_dim, out_dim, w_param=None, **kwargs):
- super(Embedding, self).__init__(name=generateName('embedding',1), user_type='kEmbedding')
- fields = { 'vocab_size': in_dim,
- 'word_dim': out_dim }
- setval(self.layer.Extensions[embedding_conf], **fields)
- if w_param == None:
- w_param = Parameter(name=generateName('w'), init=init) # default: uniform
- else:
- setParamField(w_param.param, 'w', True, **kwargs)
- setval(self.layer, param=w_param.param)
-
+
+ def __init__(self, in_dim, out_dim, w_param=None, **kwargs):
+
+ super(Embedding, self).__init__(name=generate_name('embedding', 1),
+ user_type='kEmbedding')
+ fields = {'vocab_size': in_dim,
+ 'word_dim': out_dim}
+ setval(self.layer.Extensions[embedding_conf], **fields)
+ if w_param == None:
+ # default: uniform
+ w_param = Parameter(name=generate_name('w'), init=init)
+ else:
+ set_param_field(w_param.param, 'w', True, **kwargs)
+ setval(self.layer, param=w_param.param)
+
class RNNLM(Layer):
- def __init__(self, dim, w_param=None, **kwargs):
- super(RNNLM, self).__init__(name=generateName('hidden',1), user_type='kHidden')
- if w_param == None:
- w_param = Parameter(name=generateName('w'), init=init) # default: uniform
- else:
- setParamField(w_param.param, 'w', True, **kwargs)
- setval(self.layer, param=w_param.param)
+
+ def __init__(self, dim, w_param=None, **kwargs):
+
+ super(RNNLM, self).__init__(name=generate_name('hidden', 1),
+ user_type='kHidden')
+ if w_param == None:
+ # default: uniform
+ w_param = Parameter(name=generate_name('w'), init=init)
+ else:
+ set_param_field(w_param.param, 'w', True, **kwargs)
+ setval(self.layer, param=w_param.param)
class UserLossRNNLM(Layer):
- def __init__(self, **kwargs):
- super(UserLossRNNLM, self).__init__(name=generateName('loss',1), user_type='kLoss')
- self.layer.Extensions[loss_conf].nclass = kwargs['nclass']
- self.layer.Extensions[loss_conf].vocab_size = kwargs['vocab_size']
- setval(self.layer, param=Parameter(name=generateName('w'), init='uniform', scale=0.3).param)
- setval(self.layer, param=Parameter(name=generateName('w',1), init='uniform', scale=0.3).param)
+ def __init__(self, **kwargs):
+ super(UserLossRNNLM, self).__init__(name=generate_name('loss', 1),
+ user_type='kLoss')
+ self.layer.Extensions[loss_conf].nclass = kwargs['nclass']
+ self.layer.Extensions[loss_conf].vocab_size = kwargs['vocab_size']
+ setval(self.layer, param=Parameter(name=generate_name('w'),
+ init='uniform', scale=0.3).param)
+ setval(self.layer, param=Parameter(name=generate_name('w', 1),
+ init='uniform', scale=0.3).param)
[03/10] incubator-singa git commit: SINGA-81 Add Python Helper,
which enables users to construct a model (JobProto) and run Singa in
Python
Posted by wa...@apache.org.
SINGA-81 Add Python Helper, which enables users to construct a model (JobProto) and run Singa in Python
Add implementation of Test(string str) in driver.cc
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/94435ebd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/94435ebd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/94435ebd
Branch: refs/heads/master
Commit: 94435ebde81ccf7deb19b42eebf5bfa5fbc4e7cb
Parents: ceb0665
Author: xiezl <xi...@comp.nus.edu.sg>
Authored: Mon Dec 28 16:56:28 2015 +0800
Committer: chonho <le...@comp.nus.edu.sg>
Committed: Fri Jan 1 15:59:14 2016 +0800
----------------------------------------------------------------------
src/driver.cc | 6 ++++++
1 file changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/94435ebd/src/driver.cc
----------------------------------------------------------------------
diff --git a/src/driver.cc b/src/driver.cc
index 1bb9335..67b5c28 100644
--- a/src/driver.cc
+++ b/src/driver.cc
@@ -179,6 +179,12 @@ void Driver::Train(bool resume, const JobProto& job_conf) {
Train(job);
}
+void Driver::Test(const std::string str) {
+ JobProto job_conf;
+ job_conf.ParseFromString(str);
+ Test(job_conf);
+}
+
void Driver::Test(const JobProto& job_conf) {
Cluster::Setup(job_id_, singa_conf_, job_conf.cluster());
Cluster::Get()->Register(getpid(), "localhost");
[08/10] incubator-singa git commit: SINGA-81 Add Python Helper
Posted by wa...@apache.org.
SINGA-81 Add Python Helper
Add comments for some functions in model.py.
Remove the rnnlm related code, which could be added later when it can be run sucessfully using python.
Move datasets/ into examples as they are used mainly by the examples.
Update .gitinore to ignore the pb2 foder in tool/python/.
TODO add comments for other methods in files under singa/ folder.
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/8914750e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/8914750e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/8914750e
Branch: refs/heads/master
Commit: 8914750e8c6d6fd0d9d0d8aed53fd775a1367b88
Parents: 3c12730
Author: Wei Wang <wa...@comp.nus.edu.sg>
Authored: Tue Dec 29 11:36:28 2015 +0800
Committer: chonho <le...@comp.nus.edu.sg>
Committed: Fri Jan 1 15:59:15 2016 +0800
----------------------------------------------------------------------
tool/python/examples/cifar10_cnn.py | 4 +-
tool/python/examples/cifar10_cnn_cudnn.py | 4 +-
.../python/examples/cifar10_cnn_cudnn_hybrid.py | 34 -----
tool/python/examples/cifar10_cnn_parameter.py | 4 +-
tool/python/examples/datasets/__init__.py | 0
tool/python/examples/datasets/cifar10.py | 34 +++++
tool/python/examples/datasets/mnist.py | 32 +++++
tool/python/examples/rnnlm_usermodel.py | 22 ----
tool/python/singa.py | 26 ++--
tool/python/singa/datasets/__init__.py | 0
tool/python/singa/datasets/cifar10.py | 34 -----
tool/python/singa/datasets/mnist.py | 32 -----
tool/python/singa/datasets/rnnlm.py | 20 ---
tool/python/singa/model.py | 132 +++++++++++--------
14 files changed, 164 insertions(+), 214 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8914750e/tool/python/examples/cifar10_cnn.py
----------------------------------------------------------------------
diff --git a/tool/python/examples/cifar10_cnn.py b/tool/python/examples/cifar10_cnn.py
index 9ef552b..859a9a4 100755
--- a/tool/python/examples/cifar10_cnn.py
+++ b/tool/python/examples/cifar10_cnn.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
import sys, os
-sys.path.append(os.path.join(os.path.dirname(__file__),'..'))
+sys.path.append(os.path.join(os.path.dirname(__file__),'..'))
from singa.model import *
from singa.datasets import cifar10
@@ -24,7 +24,7 @@ m.add(AvgPooling2D(pool_size=(3,3), stride=2))
m.add(Dense(10, w_wd=250, b_lr=2, b_wd=0, activation='softmax'))
-sgd = SGD(decay=0.004, lr_type='fixed', step=(0,60000,65000), step_lr=(0.001,0.0001,0.00001))
+sgd = SGD(decay=0.004, lr_type='manual', step=(0,60000,65000), step_lr=(0.001,0.0001,0.00001))
topo = Cluster(workspace)
m.compile(loss='categorical_crossentropy', optimizer=sgd, cluster=topo)
m.fit(X_train, nb_epoch=1000, with_test=True)
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8914750e/tool/python/examples/cifar10_cnn_cudnn.py
----------------------------------------------------------------------
diff --git a/tool/python/examples/cifar10_cnn_cudnn.py b/tool/python/examples/cifar10_cnn_cudnn.py
index e3c5c49..d4f4b7c 100755
--- a/tool/python/examples/cifar10_cnn_cudnn.py
+++ b/tool/python/examples/cifar10_cnn_cudnn.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
import sys, os
-sys.path.append(os.path.join(os.path.dirname(__file__),'..'))
+sys.path.append(os.path.join(os.path.dirname(__file__),'..'))
from singa.model import *
from singa.datasets import cifar10
@@ -24,7 +24,7 @@ m.add(AvgPooling2D(pool_size=(3,3), stride=2))
m.add(Dense(10, w_wd=250, b_lr=2, b_wd=0, activation='softmax'))
-sgd = SGD(decay=0.004, lr_type='fixed', step=(0,60000,65000), step_lr=(0.001,0.0001,0.00001))
+sgd = SGD(decay=0.004, lr_type='manual', step=(0,60000,65000), step_lr=(0.001,0.0001,0.00001))
topo = Cluster(workspace)
m.compile(loss='categorical_crossentropy', optimizer=sgd, cluster=topo)
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8914750e/tool/python/examples/cifar10_cnn_cudnn_hybrid.py
----------------------------------------------------------------------
diff --git a/tool/python/examples/cifar10_cnn_cudnn_hybrid.py b/tool/python/examples/cifar10_cnn_cudnn_hybrid.py
deleted file mode 100755
index f5e4c27..0000000
--- a/tool/python/examples/cifar10_cnn_cudnn_hybrid.py
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/usr/bin/env python
-import sys, os
-sys.path.append(os.path.join(os.path.dirname(__file__),'..'))
-from singa.model import *
-from singa.datasets import cifar10
-
-X_train, X_test, workspace = cifar10.load_data()
-
-m = Sequential('cifar10-cnn', sys.argv)
-
-m.add(Convolution2D(32, 5, 1, 2, w_std=0.0001, b_lr=2))
-m.add(MaxPooling2D(pool_size=(3,3), stride=2))
-m.add(Activation('relu'))
-m.add(LRN2D(3, alpha=0.00005, beta=0.75))
-
-m.add(Convolution2D(32, 5, 1, 2, b_lr=2))
-m.add(Activation('relu'))
-m.add(AvgPooling2D(pool_size=(3,3), stride=2))
-m.add(LRN2D(3, alpha=0.00005, beta=0.75))
-
-m.add(Convolution2D(64, 5, 1, 2))
-m.add(Activation('relu'))
-m.add(AvgPooling2D(pool_size=(3,3), stride=2))
-
-m.add(Dense(10, w_wd=250, b_lr=2, b_wd=0, activation='softmax'))
-
-sgd = SGD(decay=0.004, lr_type='fixed', step=(0,60000,65000), step_lr=(0.001,0.0001,0.00001))
-topo = Cluster(workspace, nworkers_per_group=2, nworkers_per_procs=2)
-m.compile(loss='categorical_crossentropy', optimizer=sgd, cluster=topo)
-
-gpu_id = [0,1]
-m.fit(X_train, nb_epoch=10000, with_test=True, device=gpu_id)
-result = m.evaluate(X_test, test_steps=0, test_freq=200)
-
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8914750e/tool/python/examples/cifar10_cnn_parameter.py
----------------------------------------------------------------------
diff --git a/tool/python/examples/cifar10_cnn_parameter.py b/tool/python/examples/cifar10_cnn_parameter.py
index dd03f5c..4144fa5 100755
--- a/tool/python/examples/cifar10_cnn_parameter.py
+++ b/tool/python/examples/cifar10_cnn_parameter.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
import sys, os
-sys.path.append(os.path.join(os.path.dirname(__file__),'..'))
+sys.path.append(os.path.join(os.path.dirname(__file__),'..'))
from singa.model import *
from singa.datasets import cifar10
@@ -27,7 +27,7 @@ m.add(AvgPooling2D(pool_size=(3,3), stride=2))
m.add(Dense(10, w_param=parw, w_wd=250, b_param=parb, b_lr=2, b_wd=0, activation='softmax'))
-sgd = SGD(decay=0.004, lr_type='fixed', step=(0,60000,65000), step_lr=(0.001,0.0001,0.00001))
+sgd = SGD(decay=0.004, lr_type='manual', step=(0,60000,65000), step_lr=(0.001,0.0001,0.00001))
topo = Cluster(workspace)
m.compile(loss='categorical_crossentropy', optimizer=sgd, cluster=topo)
m.fit(X_train, nb_epoch=100, with_test=True)
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8914750e/tool/python/examples/datasets/__init__.py
----------------------------------------------------------------------
diff --git a/tool/python/examples/datasets/__init__.py b/tool/python/examples/datasets/__init__.py
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8914750e/tool/python/examples/datasets/cifar10.py
----------------------------------------------------------------------
diff --git a/tool/python/examples/datasets/cifar10.py b/tool/python/examples/datasets/cifar10.py
new file mode 100644
index 0000000..65bcd60
--- /dev/null
+++ b/tool/python/examples/datasets/cifar10.py
@@ -0,0 +1,34 @@
+#!/usr/bin/env python
+from singa.model import *
+
+def load_data(
+ workspace = None,
+ backend = 'kvfile',
+ batchsize = 64,
+ random = 5000,
+ shape = (3, 32, 32),
+ std = 127.5,
+ mean = 127.5
+ ):
+
+ # using cifar10 dataset
+ data_dir = 'examples/cifar10'
+ path_train = data_dir + '/train_data.bin'
+ path_test = data_dir + '/test_data.bin'
+ path_mean = data_dir + '/image_mean.bin'
+ if workspace == None: workspace = data_dir
+
+ store = Store(path=path_train, mean_file=path_mean, backend=backend,
+ random_skip=random, batchsize=batchsize,
+ shape=shape)
+
+ data_train = Data(load='recordinput', phase='train', conf=store)
+
+ store = Store(path=path_test, mean_file=path_mean, backend=backend,
+ batchsize=batchsize,
+ shape=shape)
+
+ data_test = Data(load='recordinput', phase='test', conf=store)
+
+ return data_train, data_test, workspace
+
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8914750e/tool/python/examples/datasets/mnist.py
----------------------------------------------------------------------
diff --git a/tool/python/examples/datasets/mnist.py b/tool/python/examples/datasets/mnist.py
new file mode 100644
index 0000000..c8695ec
--- /dev/null
+++ b/tool/python/examples/datasets/mnist.py
@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+from singa.model import *
+
+def load_data(
+ workspace = None,
+ backend = 'kvfile',
+ nb_rbm = 0, # the number of layers for RBM and Autoencoder
+ checkpoint_steps = 0,
+ **pvalues
+ ):
+
+ # using mnist dataset
+ data_dir = 'examples/mnist'
+ path_train = data_dir + '/train_data.bin'
+ path_test = data_dir + '/test_data.bin'
+ if workspace == None: workspace = data_dir
+
+ # checkpoint path to load
+ checkpoint_list = None
+ if checkpoint_steps > 0:
+ workerid = 0
+ checkpoint_list = []
+ for i in range(nb_rbm-1, 0, -1):
+ checkpoint_list.append('examples/rbm/rbm{0}/checkpoint/step{1}-worker{2}'.format(str(i),checkpoint_steps,workerid))
+
+ store = Store(path=path_train, backend=backend, **pvalues)
+ data_train = Data(load='recordinput', phase='train', conf=store, checkpoint=checkpoint_list)
+
+ store = Store(path=path_test, backend=backend, **pvalues)
+ data_test = Data(load='recordinput', phase='test', conf=store)
+
+ return data_train, data_test, workspace
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8914750e/tool/python/examples/rnnlm_usermodel.py
----------------------------------------------------------------------
diff --git a/tool/python/examples/rnnlm_usermodel.py b/tool/python/examples/rnnlm_usermodel.py
deleted file mode 100755
index 1b49321..0000000
--- a/tool/python/examples/rnnlm_usermodel.py
+++ /dev/null
@@ -1,22 +0,0 @@
-#!/usr/bin/env python
-import sys, os
-sys.path.append(os.path.join(os.path.dirname(__file__),'..'))
-from singa.model import *
-from singa.datasets import rnnlm
-
-vocab_size = 3720
-
-X_train, X_valid, workspace = rnnlm.load_data()
-
-m = Sequential('rnnlm', sys.argv)
-
-parw = Parameter(init='uniform', range=0.3)
-m.add(Embedding(in_dim=vocab_size, out_dim=15, w_param=parw))
-m.add(RNNLM(1, w_param=parw))
-
-sgd = SGD(lr_type='fixed', step=(0,48810,56945,65080,73215), step_lr=(0.1,0.05,0.025,0.0125,0.00625))
-topo = Cluster(workspace)
-m.compile(loss='user_loss_rnnlm', in_dim=vocab_size, nclass=100, optimizer=sgd, cluster=topo)
-
-m.fit(X_train, validate=X_valid, validate_steps=683, nb_epoch=81350, execpath='examples/rnnlm/rnnlm.bin')
-#result = m.evaluate(X_valid, validate_steps=683, validate_freq=8135, execpath='examples/rnnlm/rnnlm.bin')
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8914750e/tool/python/singa.py
----------------------------------------------------------------------
diff --git a/tool/python/singa.py b/tool/python/singa.py
index 6d7fbdf..986a6b8 100755
--- a/tool/python/singa.py
+++ b/tool/python/singa.py
@@ -31,14 +31,18 @@ import singa.driver as driver
from google.protobuf.text_format import Merge
if __name__ == '__main__':
- i = sys.argv.index("-conf")
- s = open(sys.argv[i+1], 'r').read()
- s = str(s)
- j = job_pb2.JobProto()
- Merge(s,j)
- b = j.SerializeToString()
- d = driver.Driver()
- d.InitLog(sys.argv[0])
- d.Init(sys.argv)
-# d.Train(False,b)
- d.Test(b)
+ """Invoke the training program using this python script.
+ ./bin/singa-run.sh -exec tool/python/singa.py -conf examples/cifar10/job.conf
+ """"
+
+ i = sys.argv.index("-conf")
+ s = open(sys.argv[i+1], 'r').read()
+ s = str(s)
+ j = job_pb2.JobProto()
+ Merge(s,j)
+ b = j.SerializeToString()
+ d = driver.Driver()
+ d.InitLog(sys.argv[0])
+ d.Init(sys.argv)
+ d.Train(False,b)
+ #d.Test(b)
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8914750e/tool/python/singa/datasets/__init__.py
----------------------------------------------------------------------
diff --git a/tool/python/singa/datasets/__init__.py b/tool/python/singa/datasets/__init__.py
deleted file mode 100644
index e69de29..0000000
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8914750e/tool/python/singa/datasets/cifar10.py
----------------------------------------------------------------------
diff --git a/tool/python/singa/datasets/cifar10.py b/tool/python/singa/datasets/cifar10.py
deleted file mode 100644
index 65bcd60..0000000
--- a/tool/python/singa/datasets/cifar10.py
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/usr/bin/env python
-from singa.model import *
-
-def load_data(
- workspace = None,
- backend = 'kvfile',
- batchsize = 64,
- random = 5000,
- shape = (3, 32, 32),
- std = 127.5,
- mean = 127.5
- ):
-
- # using cifar10 dataset
- data_dir = 'examples/cifar10'
- path_train = data_dir + '/train_data.bin'
- path_test = data_dir + '/test_data.bin'
- path_mean = data_dir + '/image_mean.bin'
- if workspace == None: workspace = data_dir
-
- store = Store(path=path_train, mean_file=path_mean, backend=backend,
- random_skip=random, batchsize=batchsize,
- shape=shape)
-
- data_train = Data(load='recordinput', phase='train', conf=store)
-
- store = Store(path=path_test, mean_file=path_mean, backend=backend,
- batchsize=batchsize,
- shape=shape)
-
- data_test = Data(load='recordinput', phase='test', conf=store)
-
- return data_train, data_test, workspace
-
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8914750e/tool/python/singa/datasets/mnist.py
----------------------------------------------------------------------
diff --git a/tool/python/singa/datasets/mnist.py b/tool/python/singa/datasets/mnist.py
deleted file mode 100644
index c8695ec..0000000
--- a/tool/python/singa/datasets/mnist.py
+++ /dev/null
@@ -1,32 +0,0 @@
-#!/usr/bin/env python
-from singa.model import *
-
-def load_data(
- workspace = None,
- backend = 'kvfile',
- nb_rbm = 0, # the number of layers for RBM and Autoencoder
- checkpoint_steps = 0,
- **pvalues
- ):
-
- # using mnist dataset
- data_dir = 'examples/mnist'
- path_train = data_dir + '/train_data.bin'
- path_test = data_dir + '/test_data.bin'
- if workspace == None: workspace = data_dir
-
- # checkpoint path to load
- checkpoint_list = None
- if checkpoint_steps > 0:
- workerid = 0
- checkpoint_list = []
- for i in range(nb_rbm-1, 0, -1):
- checkpoint_list.append('examples/rbm/rbm{0}/checkpoint/step{1}-worker{2}'.format(str(i),checkpoint_steps,workerid))
-
- store = Store(path=path_train, backend=backend, **pvalues)
- data_train = Data(load='recordinput', phase='train', conf=store, checkpoint=checkpoint_list)
-
- store = Store(path=path_test, backend=backend, **pvalues)
- data_test = Data(load='recordinput', phase='test', conf=store)
-
- return data_train, data_test, workspace
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8914750e/tool/python/singa/datasets/rnnlm.py
----------------------------------------------------------------------
diff --git a/tool/python/singa/datasets/rnnlm.py b/tool/python/singa/datasets/rnnlm.py
deleted file mode 100644
index ef8142a..0000000
--- a/tool/python/singa/datasets/rnnlm.py
+++ /dev/null
@@ -1,20 +0,0 @@
-#!/usr/bin/env python
-from singa.model import *
-
-def load_data(
- workspace = 'examples/rnnlm',
- backend = 'kvfile',
- max_window = 10
- ):
-
- path_train = workspace + '/train_data.bin'
- path_valid = workspace + '/valid_data.bin'
- path_test = workspace + '/test_data.bin'
-
-
- data_train = Data(load='kData', phase='train', path=path_train, backend=backend, max_window=max_window)
-
- data_valid = Data(load='kData', phase='val', path=path_valid, max_window=max_window)
-
- return data_train, data_valid, workspace
-
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8914750e/tool/python/singa/model.py
----------------------------------------------------------------------
diff --git a/tool/python/singa/model.py b/tool/python/singa/model.py
index d68d143..29db70e 100644
--- a/tool/python/singa/model.py
+++ b/tool/python/singa/model.py
@@ -1,8 +1,8 @@
#!/usr/bin/env python
import sys, re, subprocess
from layer import *
-from utils.utility import *
-from utils.message import *
+from utils.utility import *
+from utils.message import *
from google.protobuf import text_format
class Model(object):
@@ -14,14 +14,14 @@ class Model(object):
argv // pass sys.argv to source
label = (bool) // exist label layer (depreciated)
'''
- self.jobconf = Message('Job', name=name).proto
+ self.jobconf = Message('Job', name=name).proto
self.layers = []
self.label = label
self.argv = argv
self.result = None
self.last_checkpoint_path = None
self.cudnn = False
-
+
def exist_datalayer(self, phase):
for ly in self.layers:
if enumPhase(phase) in ly.layer.include:
@@ -38,7 +38,7 @@ class Model(object):
topk = (int) // the number of results considered to compute accuracy
'''
assert optimizer != None, 'optimizer (Updater component) should be set'
- assert cluster != None, 'cluster (Cluster component) should be set'
+ assert cluster != None, 'cluster (Cluster component) should be set'
setval(self.jobconf, updater=optimizer.proto)
setval(self.jobconf, cluster=cluster.proto)
@@ -56,7 +56,7 @@ class Model(object):
# revise the last layer
if loss == 'categorical_crossentropy':
setval(ly, type=enumLayerType('softmaxloss'))
- setval(ly.softmaxloss_conf, topk=topk)
+ setval(ly.softmaxloss_conf, topk=topk)
elif loss == 'mean_squared_error':
setval(ly, type=enumLayerType('euclideanloss'))
else:
@@ -72,7 +72,7 @@ class Model(object):
'''
construct neuralnet proto
'''
- net = NetProto()
+ net = NetProto()
slyname = self.layers[0].layer.name
for i in range(len(self.layers)):
ly = net.layer.add()
@@ -95,7 +95,7 @@ class Model(object):
# deal with label layer (depreciated)
if self.label == True:
- label_layer = Layer(name='label', type=kLabel)
+ label_layer = Layer(name='label', type=kLabel)
ly = net.layer.add()
ly.CopyFrom(label_layer.layer)
getattr(ly, 'srclayers').append(self.layers[0].layer.name)
@@ -108,7 +108,7 @@ class Model(object):
# use of cudnn
if self.cudnn == True:
- self.setCudnnLayerType(net)
+ self.setCudnnLayerType(net)
setval(self.jobconf, neuralnet=net)
@@ -127,7 +127,7 @@ class Model(object):
batch_size = (int) // batch size for training data
train_steps = (int) // the number of steps for training, i.e., epoch
disp_freq = (int) // frequency to display training info
- disp_after = (int) // display after this number
+ disp_after = (int) // display after this number
validate_data = (Data) // validation data, specified in load_data()
validate_freq = (int) // frequency of validation
validate_steps = (int) // total number of steps for validation
@@ -143,7 +143,7 @@ class Model(object):
setval(data.layer.store_conf, batchsize=fields['batch_size'])
# insert layer for training
- if self.exist_datalayer('train') == False:
+ if self.exist_datalayer('train') == False:
self.layers.insert(0, data)
setval(self.jobconf, train_steps=nb_epoch)
setval(self.jobconf, disp_freq=nb_epoch/10)
@@ -163,8 +163,8 @@ class Model(object):
# save model parameter (i.e., checkpoint_path)
setval(self.jobconf, checkpoint_freq=nb_epoch)
self.last_checkpoint_path = '{0}/step{1}-worker0'.format(
- self.jobconf.cluster.workspace, nb_epoch)
-
+ self.jobconf.cluster.workspace, nb_epoch)
+
# set Train_one_batch component, using backprogapation at default
setval(self.jobconf, train_one_batch=Algorithm(type=enumAlgType(alg)).proto)
@@ -174,7 +174,7 @@ class Model(object):
self.cudnn = True
# start to run singa for training
- if with_test == False:
+ if with_test == False:
self.build() # construct Nneuralnet Component
#self.display()
return SingaRun(jobproto=self.jobconf, argv=self.argv, execpath=execpath)
@@ -191,13 +191,13 @@ class Model(object):
optional
alg = (string) // algorithm type, (backpropagation at default)
checkpoint_path = (list) // checkpoint path is necessary only for testing
- execpaths = (string) // path to user's own executable
+ execpaths = (string) // path to user's own executable
device = (int/list) // a list of gpu ids
**fields (KEY=VALUE)
batch_size = (int) // batch size for testing data
test_freq = (int) // frequency of testing
- test_steps = (int) // total number of steps for testing
- test_after = (int) // start testing after this number of steps
+ test_steps = (int) // total number of steps for testing
+ test_after = (int) // start testing after this number of steps
'''
assert data != None, 'Testing data should be set'
is_testonly = False
@@ -206,11 +206,11 @@ class Model(object):
setval(data.layer.store_conf, batchsize=fields['batch_size'])
# insert layer for testing
- if self.exist_datalayer('test') == False:
+ if self.exist_datalayer('test') == False:
self.layers.insert(0, data)
# loading checkpoint if singa runs only for testing
- if self.exist_datalayer('train') == False:
+ if self.exist_datalayer('train') == False:
is_testonly = True
if checkpoint_path == None:
print 'checkpoint_path has not been specified'
@@ -220,7 +220,7 @@ class Model(object):
steps = fields['test_steps'] if 'test_steps' in fields else 10
setval(self.jobconf, test_steps=steps)
setval(self.jobconf, **fields)
-
+
# set Train_one_batch component, using backprogapation at default
setval(self.jobconf, train_one_batch=Algorithm(type=enumAlgType(alg)).proto)
@@ -231,16 +231,16 @@ class Model(object):
self.build() # construct Nneuralnet Component
- #--- generate job.conf file for debug purpose
+ #--- generate job.conf file for debug purpose
#filename = 'job.conf'
#with open(filename, 'w') as f:
# f.write(text_format.MessageToString(self.jobconf.cluster))
#self.display()
- #--- run singa ---
+ #--- run singa ---
return SingaRun(jobproto=self.jobconf, argv=self.argv, execpath=execpath, testmode=is_testonly)
#return SingaRun_script(filename=filename, execpath=execpath)
-
+
def display(self):
''' print out job proto
@@ -260,13 +260,13 @@ class Model(object):
elif ly_type == kSoftmaxLoss: cudnn_ly_type = kCudnnSoftmaxLoss
elif ly_type == kSTanh:
cudnn_ly_type = kCudnnActivation
- net.layer[i].activation_conf.type = STANH
+ net.layer[i].activation_conf.type = STANH
elif ly_type == kSigmoid:
cudnn_ly_type = kCudnnActivation
- net.layer[i].activation_conf.type = SIGMOID
+ net.layer[i].activation_conf.type = SIGMOID
elif ly_type == kReLU:
cudnn_ly_type = kCudnnActivation
- net.layer[i].activation_conf.type = RELU
+ net.layer[i].activation_conf.type = RELU
net.layer[i].type = cudnn_ly_type
@@ -277,7 +277,7 @@ class Energy(Model):
def add(self, layer):
if hasattr(layer, 'layer_type'):
if layer.layer_type == kRBMVis:
- dim = 0
+ dim = 0
for i in range(1, len(layer.out_dim)):
parw = Parameter(name='w', init='none', level=i)
parb = Parameter(name='b', init='none', level=i)
@@ -293,7 +293,7 @@ class Sequential(Model):
def add(self, layer):
if hasattr(layer, 'layer_type'):
if layer.layer_type == 'AutoEncoder':
- dim = 0
+ dim = 0
if layer.param_share == True:
# Encoding
for i in range(1, len(layer.hid_dim)+1):
@@ -331,9 +331,9 @@ class Store(object):
'''
**kwargs
path = (string) // path to dataset
- backend = (string) //
+ backend = (string) //
batch_size = (int) // batch size of dataset
- shape = (int) //
+ shape = (int) //
'''
self.proto = Message('Store', **kwargs).proto
@@ -357,23 +357,23 @@ class Updater(object):
lr_type = (string) // type of the learning rate (Fixed at default)
'''
upd = Message('Updater', type=upd_type, **fields).proto
- setval(upd.learning_rate, base_lr=lr)
+ setval(upd.learning_rate, base_lr=lr)
if decay > 0:
- setval(upd, weight_decay=decay)
+ setval(upd, weight_decay=decay)
if momentum > 0:
- setval(upd, momentum=momentum)
+ setval(upd, momentum=momentum)
- if lr_type == None:
- setval(upd.learning_rate, type=kFixed)
+ if lr_type == None or lr_type == "fixed":
+ setval(upd.learning_rate, type=kFixed)
elif lr_type == 'step':
cp = Message('Step', change_freq=60, gamma=0.997)
- setval(upd.learning_rate, type=kStep, step_conf=cp.proto)
- elif lr_type == 'fixedstep':
+ setval(upd.learning_rate, type=kStep, step_conf=cp.proto)
+ elif lr_type == 'manual':
cp = Message('FixedStep', step=step, step_lr=step_lr)
- setval(upd.learning_rate, type=kFixedStep, fixedstep_conf=cp.proto)
+ setval(upd.learning_rate, type=kFixedStep, fixedstep_conf=cp.proto)
elif lr_type == 'linear':
cp = Message('Linear', change_freq=10, final_lr=0.1)
- setval(upd.learning_rate, type=kLinear, linear_conf=cp.proto)
+ setval(upd.learning_rate, type=kLinear, linear_conf=cp.proto)
self.proto = upd
@@ -422,6 +422,15 @@ class AdaGrad(Updater):
class Cluster(object):
+ """ Specify the cluster topology, e.g., number of workers/servers.
+
+ Currently we need to create this object in the .py file and also provide a
+ cluster configuration file to the command line. TODO(wangwei) update SINGA
+ code to eliminate the requirement of the cluster configuration file for
+ training on a single node or the cluster object in the pyfile for training
+ in a cluster.
+ """
+
def __init__(self, workspace=None,
nworker_groups=1, nserver_groups=1,
nworkers_per_group=1, nservers_per_group=1,
@@ -443,65 +452,78 @@ class Cluster(object):
assert workspace != None, 'need to set workspace'
self.proto = Message('Cluster', workspace=workspace).proto
# optional
- self.proto.nworker_groups = nworker_groups
- self.proto.nserver_groups = nserver_groups
- self.proto.nworkers_per_group = nworkers_per_group
- self.proto.nservers_per_group = nservers_per_group
- self.proto.nworkers_per_procs = nworkers_per_procs
- self.proto.nservers_per_procs = nservers_per_procs
+ self.proto.nworker_groups = nworker_groups
+ self.proto.nserver_groups = nserver_groups
+ self.proto.nworkers_per_group = nworkers_per_group
+ self.proto.nservers_per_group = nservers_per_group
+ self.proto.nworkers_per_procs = nworkers_per_procs
+ self.proto.nservers_per_procs = nservers_per_procs
# other fields
setval(self.proto, **fields)
def StoreResults(lines):
+ """ Parsing metrics from each line in the log file.
- resultDic = {}
+ TODO(wangwei) format the log string to make them uniform for easy parsing
+ Another approach is creating a protobuf message for metrics, which can be
+ used for dumping metrics to string and loading perf string back to messages.
+ """
+
+ resultDic = {}
for line in lines:
line = re.findall(r'[\w|*.*]+', line)
if 'Train' in line:
step = line[line.index('step')+1]
if 'accuracy' in line:
- resultDic.setdefault(step,{})['acc'] = line[line.index('accuracy')+1]
+ resultDic.setdefault(step,{})['acc'] = line[line.index('accuracy')+1]
if 'loss' in line:
- resultDic.setdefault(step,{})['loss'] = line[line.index('loss')+1]
+ resultDic.setdefault(step,{})['loss'] = line[line.index('loss')+1]
if 'ppl' in line:
- resultDic.setdefault(step,{})['ppl'] = line[line.index('ppl')+1]
+ resultDic.setdefault(step,{})['ppl'] = line[line.index('ppl')+1]
if 'Squared' in line:
- resultDic.setdefault(step,{})['se'] = line[line.index('Squared')+2]
+ resultDic.setdefault(step,{})['se'] = line[line.index('Squared')+2]
return resultDic
def SingaRun(jobproto='', argv=[], execpath='', testmode=False):
import singa.driver as driver
d = driver.Driver()
- d.InitLog(argv[0])
+ d.InitLog(argv[0])
d.Init(argv)
if testmode == True:
d.Test(jobproto.SerializeToString())
else:
d.Train(False, jobproto.SerializeToString())
+ # Get the performance from the latest log file.
+ # TODO(wangwei) the log file would be overwritten by other running instance of
+ # the same program, e.g., lt-singa
logfile = '/tmp/singa-log/{0}.ERROR'.format(argv[0].split('/')[-1])
fin = open(logfile, 'r')
result = StoreResults(fin.readlines())
-
+
return result
def SingaRun_script(filename='', execpath=''):
+ """
+ Deprecated.
+ Generate the job conf file and run the shell command.
+ """
SINGAROOT = '../../../'
conf = 'examples/' + filename
if execpath=='':
cmd = SINGAROOT+'bin/singa-run.sh ' \
- + '-conf %s ' % conf
+ + '-conf %s ' % conf
else:
cmd = SINGAROOT+'bin/singa-run.sh ' \
+ '-conf %s ' % conf \
- + '-exec %s ' % execpath
+ + '-exec %s ' % execpath
procs = subprocess.Popen(cmd.strip().split(' '), stdout = subprocess.PIPE, stderr = subprocess.STDOUT)
- resultDic = {}
+ resultDic = {}
outputlines = iter(procs.stdout.readline, '')
resultDic = StoreResults(outputlines)
[07/10] incubator-singa git commit: SINGA-81 Add Python Helper
Posted by wa...@apache.org.
SINGA-81 Add Python Helper
- Update README.md
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/d47ed4b3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/d47ed4b3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/d47ed4b3
Branch: refs/heads/master
Commit: d47ed4b3aac0552458f41727d72c846dd971be2d
Parents: 8914750
Author: chonho <le...@comp.nus.edu.sg>
Authored: Thu Dec 31 11:07:27 2015 +0800
Committer: chonho <le...@comp.nus.edu.sg>
Committed: Fri Jan 1 15:59:15 2016 +0800
----------------------------------------------------------------------
tool/python/README.md | 71 +++++++++++++++++++++++++++++++++++++++++-----
1 file changed, 64 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/d47ed4b3/tool/python/README.md
----------------------------------------------------------------------
diff --git a/tool/python/README.md b/tool/python/README.md
index 4308ed1..1133fa4 100644
--- a/tool/python/README.md
+++ b/tool/python/README.md
@@ -22,11 +22,11 @@
```
bin/singa-run.sh -exe user_main.py -conf cluster.conf
```
-The python code, i.e., `user_main.py`, would create the JobProto object and pass it to Driver::Train.
+The python code, e.g., `user_main.py`, would create the JobProto object and pass it to Driver::Train.
Currently, ./bin/singa-run.sh needs to get the cluster topology, hence we still need to pass a `cluster.conf` to it.
The cluster.conf has the configuration for a JobProto with all other fields empty except the cluster field.
-Note that 'workspace' field in ClusterProto can be set in either (i) cluster.conf or (ii) python code.
+Note that `workspace` field in ClusterProto can be set in either (i) cluster.conf or (ii) python code.
#### Examples
```
@@ -83,7 +83,7 @@ Methods in Model class
#### Results
-fit() and evaluate() return `result`, a dictionary containing
+fit() and evaluate() return train/test results, a dictionary containing
* [key]: step number
* [value]: a list of dictionay
@@ -92,13 +92,67 @@ fit() and evaluate() return `result`, a dictionary containing
* 'ppl' for ppl
* 'se' for squred error
+<<<<<<< HEAD
+=======
+### Parameter class
+
+Users need to set parameter and initial values. For example,
+
+* Parameter
+ * lr = (float) // learning rate
+ * wd = (float) // weight decay
+
+* Parameter initialization
+ * init = (string) // one of the types, 'uniform', 'constant', 'gaussian'
+ * for uniform [default]
+ * high = (float)
+ * low = (float)
+ * for constant
+ * value = (float)
+ * for gaussian
+ * mean = (float)
+ * std = (float)
+
+* Weight (`w_param`) is 'gaussian' with mean=0, std=0.01 at default
+
+* Bias (`b_param`) is 'constant' with value=0 at default
+
+* How to update the parameter fields
+ * for updating Weight, put `w_` in front of field name
+ * for updating Bias, put `b_` in front of field name
+
+Several ways to set Parameter values
+```
+m.add(Dense(10, w_mean=1, w_std=0.1, w_lr=2, w_wd=10, ...)
+```
+```
+parw = Parameter(lr=2, wd=10, init='constant', value=0)
+m.add(Dense(10, w_param=parw, ...)
+```
+```
+parw = Parameter(init='constant', value=0)
+m.add(Dense(10, w_param=parw, w_lr=2, w_wd=10, ...)
+```
+
+
+#### To run Singa on GPU
+
+Users need to set a list of gpu ids to `device` field in fit() or evaluate().
+
+For example,
+```
+gpu_id = [0]
+m.fit(X_train, nb_epoch=100, with_test=True, device=gpu_id)
+```
+
+>>>>>>> cb1ffb4... SINGA-81 Add Python Helper
#### Other classes
* Store
* Algorithm
* Updater
-* SGD (inherited)
-* AdaGrad (inherited)
+* SGD
+* AdaGrad
* Cluster
@@ -150,7 +204,7 @@ m.add(AvgPooling2D(pool_size=(3,3), stride=2))
m.add(Dense(10, w_wd=250, b_lr=2, b_wd=0, activation='softmax'))
-sgd = SGD(decay=0.004, lr_type='fixed', step=(0,60000,65000), step_lr=(0.001,0.0001,0.00001))
+sgd = SGD(decay=0.004, lr_type='manual', step=(0,60000,65000), step_lr=(0.001,0.0001,0.00001))
topo = Cluster(workspace)
m.compile(updater=sgd, cluster=topo)
m.fit(X_train, nb_epoch=1000, with_test=True)
@@ -257,6 +311,7 @@ store = Store(path='test.bin', batch_size=100, ...) // parameter values a
m.add(Data(load='recordinput', phase='test', conf=store)) // Data layer is added
```
+<<<<<<< HEAD
### Parameter class
Users need to set parameter and initial values. For example,
@@ -296,8 +351,10 @@ m.add(Dense(10, w_param=parw, ...)
parw = Parameter(init='constant', value=0)
m.add(Dense(10, w_param=parw, w_lr=2, w_wd=10, ...)
```
+=======
+>>>>>>> cb1ffb4... SINGA-81 Add Python Helper
-### Example to run singa
+### Cases to run singa
(1) Run singa for training
```
[06/10] incubator-singa git commit: SINGA-81 Add Python Helper
Posted by wa...@apache.org.
SINGA-81 Add Python Helper
update singa-run.sh to support running in a single process without job.conf
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/2a6645cf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/2a6645cf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/2a6645cf
Branch: refs/heads/master
Commit: 2a6645cf2b7b625795b58b619e6cc544c08b63ff
Parents: d47ed4b
Author: WANG Sheng <wa...@gmail.com>
Authored: Thu Dec 31 11:02:22 2015 +0800
Committer: chonho <le...@comp.nus.edu.sg>
Committed: Fri Jan 1 15:59:15 2016 +0800
----------------------------------------------------------------------
bin/singa-run.sh | 44 +++++++++++++++++++++++---------------------
src/utils/cluster_rt.cc | 26 ++++++++++++++------------
src/utils/tool.cc | 2 +-
3 files changed, 38 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/2a6645cf/bin/singa-run.sh
----------------------------------------------------------------------
diff --git a/bin/singa-run.sh b/bin/singa-run.sh
index 548f8f1..836342e 100755
--- a/bin/singa-run.sh
+++ b/bin/singa-run.sh
@@ -23,16 +23,14 @@
# run a singa job
#
-usage="Usage: singa-run.sh -conf <job config file> [ other arguments ]\n
- -resume : if want to recover a job\n
- -exec <path to mysinga> : if want to use own singa driver\n
- ### NOTICE ###\n
- if you are using model.conf + cluster.conf,\n
- please see how to combine them to a job.conf:\n
- http://singa.incubator.apache.org/quick-start.html"
+usage="Usage: singa-run.sh [ arguments ]\n
+ -exec <binary or python script> : if want to use own singa driver\n
+ -conf <job config file> : need cluster conf if train in a cluster
+ -resume : if want to recover a job"
# parse arguments
-# make sure we have '-conf' and remove '-exec'
+# extract and remove '-exec' and '-conf'
+# other arguments remain untouched
exe=./singa
while [ $# != 0 ]; do
if [ $1 == "-exec" ]; then
@@ -46,23 +44,23 @@ while [ $# != 0 ]; do
fi
shift
done
-if [ -z $conf ]; then
- echo -e $usage
- exit 1
-fi
# get environment variables
. `dirname "${BASH_SOURCE-$0}"`/singa-env.sh
# change conf to an absolute path
-conf_dir=`dirname "$conf"`
-conf_dir=`cd "$conf_dir">/dev/null; pwd`
-conf_base=`basename "$conf"`
-job_conf=$conf_dir/$conf_base
-if [ ! -f $job_conf ]; then
- echo $job_conf not exists
- exit 1
+if [ ! -z $conf ]; then
+ conf_dir=`dirname "$conf"`
+ conf_dir=`cd "$conf_dir">/dev/null; pwd`
+ conf_base=`basename "$conf"`
+ job_conf=$conf_dir/$conf_base
+ if [ ! -f $job_conf ]; then
+ echo $job_conf not exists
+ exit 1
+ fi
fi
+
+# go to singa home to execute binary
cd $SINGA_HOME
# generate unique job id
@@ -81,9 +79,13 @@ host_file=$log_dir/job.hosts
./singatool genhost $job_conf 1>$host_file || exit 1
# set command to run singa
-singa_run="$exe $args -conf $job_conf \
+singa_run="$exe $args \
-singa_conf $SINGA_HOME/conf/singa.conf \
- -singa_job $job_id"
+ -singa_job $job_id"
+# add -conf if exists
+if [ ! -z $job_conf ]; then
+ singa_run="$singa_run -conf $job_conf"
+fi
singa_sshrun="cd $SINGA_HOME; $singa_run"
# ssh and start singa processes
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/2a6645cf/src/utils/cluster_rt.cc
----------------------------------------------------------------------
diff --git a/src/utils/cluster_rt.cc b/src/utils/cluster_rt.cc
index 5ac3812..cdf8aab 100644
--- a/src/utils/cluster_rt.cc
+++ b/src/utils/cluster_rt.cc
@@ -367,19 +367,21 @@ bool JobManager::GenerateJobID(int* id) {
bool JobManager::GenerateHostList(const char* host_file, const char* job_file,
vector<string>* list) {
+ int nprocs = 1;
// compute required #process from job conf
- ClusterProto cluster;
- google::protobuf::TextFormat::ParseFromString(ExtractClusterConf(job_file),
- &cluster);
- int nworker_procs = cluster.nworker_groups() * cluster.nworkers_per_group()
- / cluster.nworkers_per_procs();
- int nserver_procs = cluster.nserver_groups() * cluster.nservers_per_group()
- / cluster.nservers_per_procs();
- int nprocs = 0;
- if (cluster.server_worker_separate())
- nprocs = nworker_procs + nserver_procs;
- else
- nprocs = std::max(nworker_procs, nserver_procs);
+ if (job_file != nullptr) {
+ ClusterProto cluster;
+ google::protobuf::TextFormat::ParseFromString(ExtractClusterConf(job_file),
+ &cluster);
+ int nworker_procs = cluster.nworker_groups() * cluster.nworkers_per_group()
+ / cluster.nworkers_per_procs();
+ int nserver_procs = cluster.nserver_groups() * cluster.nservers_per_group()
+ / cluster.nservers_per_procs();
+ if (cluster.server_worker_separate())
+ nprocs = nworker_procs + nserver_procs;
+ else
+ nprocs = std::max(nworker_procs, nserver_procs);
+ }
// get available host list from global conf
std::ifstream hostfile(host_file);
if (!hostfile.is_open()) {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/2a6645cf/src/utils/tool.cc
----------------------------------------------------------------------
diff --git a/src/utils/tool.cc b/src/utils/tool.cc
index 18df826..4b50214 100644
--- a/src/utils/tool.cc
+++ b/src/utils/tool.cc
@@ -147,7 +147,7 @@ int main(int argc, char **argv) {
else if (!strcmp(argv[1], "create"))
stat = create();
else if (!strcmp(argv[1], "genhost"))
- stat = (argc > 2) ? genhost(argv[2]) : ARG_ERR;
+ stat = (argc > 2) ? genhost(argv[2]) : genhost(nullptr);
else if (!strcmp(argv[1], "list"))
stat = list(false);
else if (!strcmp(argv[1], "listall"))
[09/10] incubator-singa git commit: SINGA-81 Add Python Helper
Posted by wa...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9ff176c3/tool/python/singa/model.py
----------------------------------------------------------------------
diff --git a/tool/python/singa/model.py b/tool/python/singa/model.py
index 29db70e..6ad9422 100644
--- a/tool/python/singa/model.py
+++ b/tool/python/singa/model.py
@@ -1,532 +1,612 @@
#!/usr/bin/env python
+
+#/************************************************************
+#*
+#* Licensed to the Apache Software Foundation (ASF) under one
+#* or more contributor license agreements. See the NOTICE file
+#* distributed with this work for additional information
+#* regarding copyright ownership. The ASF licenses this file
+#* to you under the Apache License, Version 2.0 (the
+#* "License"); you may not use this file except in compliance
+#* with the License. You may obtain a copy of the License at
+#*
+#* http://www.apache.org/licenses/LICENSE-2.0
+#*
+#* Unless required by applicable law or agreed to in writing,
+#* software distributed under the License is distributed on an
+#* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#* KIND, either express or implied. See the License for the
+#* specific language governing permissions and limitations
+#* under the License.
+#*
+#*************************************************************/
+
+'''
+This script includes Model class and its subclasses that
+users can configure model parameter.
+'''
+
import sys, re, subprocess
-from layer import *
-from utils.utility import *
-from utils.message import *
+from singa.layer import *
+from singa.utils.utility import *
+from singa.utils.message import *
from google.protobuf import text_format
class Model(object):
-
- def __init__(self, name='my model', argv=[], label=False):
- '''
- optional
- name = (string) // name of model/job
- argv // pass sys.argv to source
- label = (bool) // exist label layer (depreciated)
- '''
- self.jobconf = Message('Job', name=name).proto
- self.layers = []
- self.label = label
- self.argv = argv
- self.result = None
- self.last_checkpoint_path = None
- self.cudnn = False
-
- def exist_datalayer(self, phase):
- for ly in self.layers:
- if enumPhase(phase) in ly.layer.include:
- return True
- return False
-
- def compile(self, optimizer=None, cluster=None, loss=None, topk=1, **kwargs):
- '''
- required
- optimizer = (Updater) // updater settings, e.g., SGD
- cluster = (Cluster) // cluster settings
- optional
- loss = (string) // name of loss function type
- topk = (int) // the number of results considered to compute accuracy
- '''
- assert optimizer != None, 'optimizer (Updater component) should be set'
- assert cluster != None, 'cluster (Cluster component) should be set'
- setval(self.jobconf, updater=optimizer.proto)
- setval(self.jobconf, cluster=cluster.proto)
-
- # take care of loss function layer
- if loss == None:
- print 'loss layer is not set'
- else:
- if hasattr(self.layers[-1], 'mask'):
- ly = self.layers[-1].mask
- else:
- ly = self.layers[-1].layer
-
- # take care of the last layer
- if ly.type == enumLayerType('softmax'):
- # revise the last layer
- if loss == 'categorical_crossentropy':
- setval(ly, type=enumLayerType('softmaxloss'))
- setval(ly.softmaxloss_conf, topk=topk)
- elif loss == 'mean_squared_error':
- setval(ly, type=enumLayerType('euclideanloss'))
- else:
- # add new layer
- if loss == 'categorical_crossentropy':
- self.add(Activation('softmaxloss', topk=topk))
- elif loss == 'mean_squared_error':
- self.add(Activation('euclideanloss'))
- elif loss == 'user_loss_rnnlm': # user-defined loss layer for rnnlm
- self.add(UserLossRNNLM(nclass=kwargs['nclass'], vocab_size=kwargs['in_dim']))
-
- def build(self):
+ ''' Configure model parameter
+ - add(): add layer
+ - compile(): specify Updater and Cluster protos
+ - build(): construct a model (i.e., NetProto)
+ - fit(): run singa for training
+ - evaluate(): run singa for testing
'''
- construct neuralnet proto
- '''
- net = NetProto()
- slyname = self.layers[0].layer.name
- for i in range(len(self.layers)):
- ly = net.layer.add()
- ly.CopyFrom(self.layers[i].layer)
- lastly = ly
- if self.layers[i].is_datalayer == True:
- continue
- getattr(ly, 'srclayers').append(slyname)
- slyname = ly.name
- if hasattr(self.layers[i], 'mask'):
- mly = net.layer.add()
- mly.CopyFrom(self.layers[i].mask)
- getattr(mly, 'srclayers').append(slyname)
- slyname = mly.name
- lastly = mly
- if hasattr(self.layers[i], 'bidirect'):
- bly = net.layer.add()
- bly.CopyFrom(self.layers[i].bidirect)
- getattr(bly, 'srclayers').append(slyname)
-
- # deal with label layer (depreciated)
- if self.label == True:
- label_layer = Layer(name='label', type=kLabel)
- ly = net.layer.add()
- ly.CopyFrom(label_layer.layer)
- getattr(ly, 'srclayers').append(self.layers[0].layer.name)
- getattr(lastly, 'srclayers').append(label_layer.layer.name)
- else:
- if lastly.name == 'RBMVis':
- getattr(lastly, 'srclayers').append(bly.name)
- else:
- getattr(lastly, 'srclayers').append(self.layers[0].layer.name)
-
- # use of cudnn
- if self.cudnn == True:
- self.setCudnnLayerType(net)
-
- setval(self.jobconf, neuralnet=net)
-
- def fit(self, data=None, alg='bp', nb_epoch=0,
- with_test=False, execpath='', device=None, **fields):
- '''
- required
- data = (Data) // Data class object for training data
- alg = (string) // algorithm, e.g., 'bp', 'cd'
- nb_epoch = (int) // the number of training steps
- optional
- with_test = (bool) // flag if singa runs for test data
- execpath = (string) // path to user own singa (executable file)
- device = (int/list) // a list of gpu ids
- **fields (KEY=VALUE)
- batch_size = (int) // batch size for training data
- train_steps = (int) // the number of steps for training, i.e., epoch
- disp_freq = (int) // frequency to display training info
- disp_after = (int) // display after this number
- validate_data = (Data) // validation data, specified in load_data()
- validate_freq = (int) // frequency of validation
- validate_steps = (int) // total number of steps for validation
- validate_after = (int) // start validation after this number
- checkpoint_path = (string) // path to checkpoint file
- checkpoint_freq = (int) // frequency for checkpoint
- checkpoint_after = (int) // start checkpointing after this number
- '''
- assert data != None, 'Training data shold be set'
- assert nb_epoch > 0, 'Training steps shold be set'
-
- if 'batch_size' in fields: # if new value is set, replace the batch_size
- setval(data.layer.store_conf, batchsize=fields['batch_size'])
-
- # insert layer for training
- if self.exist_datalayer('train') == False:
- self.layers.insert(0, data)
- setval(self.jobconf, train_steps=nb_epoch)
- setval(self.jobconf, disp_freq=nb_epoch/10)
- if 'disp_freq' in fields:
- setval(self.jobconf, disp_freq=fields['disp_freq'])
-
- if 'validate_data' in fields:
- self.layers.insert(1, fields['validate_data'])
- setval(self.jobconf, validate_freq=nb_epoch/10)
-
- setval(self.jobconf, **fields)
-
- # loading checkpoint if it is set
- if data.checkpoint != None:
- setval(self.jobconf, checkpoint_path=data.checkpoint)
-
- # save model parameter (i.e., checkpoint_path)
- setval(self.jobconf, checkpoint_freq=nb_epoch)
- self.last_checkpoint_path = '{0}/step{1}-worker0'.format(
- self.jobconf.cluster.workspace, nb_epoch)
-
- # set Train_one_batch component, using backprogapation at default
- setval(self.jobconf, train_one_batch=Algorithm(type=enumAlgType(alg)).proto)
-
- # use of cudnn
- if device != None:
- setval(self.jobconf, gpu=device)
- self.cudnn = True
-
- # start to run singa for training
- if with_test == False:
- self.build() # construct Nneuralnet Component
- #self.display()
- return SingaRun(jobproto=self.jobconf, argv=self.argv, execpath=execpath)
- else:
- # run singa in evaluate() with test data
- pass
+ def __init__(self, name='my model', argv=None, label=False):
+ '''
+ optional
+ name = (string) // name of model/job
+ argv // pass sys.argv to source
+ label = (bool) // exist label layer (depreciated)
+ '''
+ self.jobconf = Message('Job', name=name).proto
+ self.layers = []
+ self.label = label
+ self.argv = argv
+ self.result = None
+ self.last_checkpoint_path = None
+ self.cudnn = False
+
+ def add(self, layer):
+ '''
+ add layer
+ '''
+ pass
+
+ def exist_datalayer(self, phase):
+ '''
+ check if data layer exists
+ '''
+ for ly in self.layers:
+ if enumPhase(phase) in ly.layer.include:
+ return True
+ return False
+
+ def compile(self, optimizer=None, cluster=None,
+ loss=None, topk=1, **kwargs):
+ '''
+ required
+ optimizer = (Updater) // updater settings, e.g., SGD
+ cluster = (Cluster) // cluster settings
+ optional
+ loss = (string) // name of loss function type
+ topk = (int) // nb of results considered to compute accuracy
+ '''
+ assert optimizer != None, 'optimizer (Updater component) should be set'
+ assert cluster != None, 'cluster (Cluster component) should be set'
+ setval(self.jobconf, updater=optimizer.proto)
+ setval(self.jobconf, cluster=cluster.proto)
+
+ # take care of loss function layer
+ if loss == None:
+ print 'loss layer is not set'
+ else:
+ if hasattr(self.layers[-1], 'mask'):
+ ly = self.layers[-1].mask
+ else:
+ ly = self.layers[-1].layer
+
+ # take care of the last layer
+ if ly.type == enumLayerType('softmax'):
+ # revise the last layer
+ if loss == 'categorical_crossentropy':
+ setval(ly, type=enumLayerType('softmaxloss'))
+ setval(ly.softmaxloss_conf, topk=topk)
+ elif loss == 'mean_squared_error':
+ setval(ly, type=enumLayerType('euclideanloss'))
+ else:
+ # add new layer
+ if loss == 'categorical_crossentropy':
+ self.add(Activation('softmaxloss', topk=topk))
+ elif loss == 'mean_squared_error':
+ self.add(Activation('euclideanloss'))
+ elif loss == 'user_loss_rnnlm': # user-defined loss layer
+ self.add(UserLossRNNLM(nclass=kwargs['nclass'],
+ vocab_size=kwargs['in_dim']))
+
+ def build(self):
+ '''
+ construct neuralnet proto
+ '''
+ net = NetProto()
+ slyname = self.layers[0].layer.name
+ for i in range(len(self.layers)):
+ ly = net.layer.add()
+ ly.CopyFrom(self.layers[i].layer)
+ lastly = ly
+ if self.layers[i].is_datalayer == True:
+ continue
+ getattr(ly, 'srclayers').append(slyname)
+ slyname = ly.name
+ if hasattr(self.layers[i], 'mask'):
+ mly = net.layer.add()
+ mly.CopyFrom(self.layers[i].mask)
+ getattr(mly, 'srclayers').append(slyname)
+ slyname = mly.name
+ lastly = mly
+ if hasattr(self.layers[i], 'bidirect'):
+ bly = net.layer.add()
+ bly.CopyFrom(self.layers[i].bidirect)
+ getattr(bly, 'srclayers').append(slyname)
+
+ # deal with label layer (depreciated)
+ if self.label == True:
+ label_layer = Layer(name='label', type=kLabel)
+ ly = net.layer.add()
+ ly.CopyFrom(label_layer.layer)
+ getattr(ly, 'srclayers').append(self.layers[0].layer.name)
+ getattr(lastly, 'srclayers').append(label_layer.layer.name)
+ else:
+ if lastly.name == 'RBMVis':
+ getattr(lastly, 'srclayers').append(bly.name)
+ else:
+ getattr(lastly, 'srclayers').append(self.layers[0].layer.name)
+
+ # use of cudnn
+ if self.cudnn == True:
+ self.set_cudnn_layer_type(net)
+
+ setval(self.jobconf, neuralnet=net)
+
+ def fit(self, data=None, alg='bp', nb_epoch=0,
+ with_test=False, execpath='', device=None, **fields):
+ '''
+ required
+ data = (Data) // Data class object for training data
+ alg = (string) // algorithm, e.g., 'bp', 'cd'
+ nb_epoch = (int) // the number of training steps
+ optional
+ with_test = (bool) // flag if singa runs for test data
+ execpath = (string) // path to user own singa (executable file)
+ device = (int/list) // a list of gpu ids
+ **fields (KEY=VALUE)
+ batch_size = (int) // batch size for training data
+ train_steps = (int) // nb of steps for training, i.e., epoch
+ disp_freq = (int) // frequency to display training info
+ disp_after = (int) // display after this number
+ validate_data = (Data) // valid data, specified in load_data()
+ validate_freq = (int) // frequency of validation
+ validate_steps = (int) // total number of steps for validation
+ validate_after = (int) // start validation after this number
+ checkpoint_path = (string) // path to checkpoint file
+ checkpoint_freq = (int) // frequency for checkpoint
+ checkpoint_after = (int) // start checkpointing after this number
+ '''
+ assert data != None, 'Training data shold be set'
+ assert nb_epoch > 0, 'Training steps shold be set'
+
+ if 'batch_size' in fields: # if new value is set, replace it
+ setval(data.layer.store_conf, batchsize=fields['batch_size'])
+
+ # insert layer for training
+ if self.exist_datalayer('train') == False:
+ self.layers.insert(0, data)
+ setval(self.jobconf, train_steps=nb_epoch)
+ setval(self.jobconf, disp_freq=nb_epoch/10)
+ if 'disp_freq' in fields:
+ setval(self.jobconf, disp_freq=fields['disp_freq'])
+
+ if 'validate_data' in fields:
+ self.layers.insert(1, fields['validate_data'])
+ setval(self.jobconf, validate_freq=nb_epoch/10)
+
+ setval(self.jobconf, **fields)
+
+ # loading checkpoint if it is set
+ if data.checkpoint != None:
+ setval(self.jobconf, checkpoint_path=data.checkpoint)
+
+ # save model parameter (i.e., checkpoint_path)
+ setval(self.jobconf, checkpoint_freq=nb_epoch)
+ self.last_checkpoint_path = '{0}/step{1}-worker0'.format(
+ self.jobconf.cluster.workspace, nb_epoch)
+
+ # set Train_one_batch component, using backprogapation at default
+ setval(self.jobconf,
+ train_one_batch=Algorithm(type=enumAlgType(alg)).proto)
+
+ # use of cudnn
+ if device != None:
+ setval(self.jobconf, gpu=device)
+ self.cudnn = True
+
+ # start to run singa for training
+ if with_test == False:
+ self.build() # construct Nneuralnet Component
+ #self.display()
+ return SingaRun(jobproto=self.jobconf,
+ argv=self.argv, execpath=execpath)
+ else:
+ # run singa in evaluate() with test data
+ pass
+
+ def evaluate(self, data=None, alg='bp',
+ checkpoint_path=None, execpath='', device=None, **fields):
+ '''
+ required
+ data = (Data) // Data class object for testing data
+ optional
+ alg = (string) // algorithm type, (bp at default)
+ checkpoint_path = (list) // checkpoint path
+ execpaths = (string) // path to user's own executable
+ device = (int/list) // a list of gpu ids
+ **fields (KEY=VALUE)
+ batch_size = (int) // batch size for testing data
+ test_freq = (int) // frequency of testing
+ test_steps = (int) // total number of steps for testing
+ test_after = (int) // start testing after this number of steps
+ '''
+ assert data != None, 'Testing data should be set'
+ is_testonly = False
+
+ if 'batch_size' in fields: # if new value is set, replace it
+ setval(data.layer.store_conf, batchsize=fields['batch_size'])
+
+ # insert layer for testing
+ if self.exist_datalayer('test') == False:
+ self.layers.insert(0, data)
+
+ # loading checkpoint if singa runs only for testing
+ if self.exist_datalayer('train') == False:
+ is_testonly = True
+ if checkpoint_path == None:
+ print 'checkpoint_path has not been specified'
+ else:
+ setval(self.jobconf, checkpoint_path=checkpoint_path)
+
+ steps = fields['test_steps'] if 'test_steps' in fields else 10
+ setval(self.jobconf, test_steps=steps)
+ setval(self.jobconf, **fields)
+
+ # set Train_one_batch component, using backprogapation at default
+ setval(self.jobconf,
+ train_one_batch=Algorithm(type=enumAlgType(alg)).proto)
+
+ # use of cudnn
+ if device != None:
+ setval(self.jobconf, gpu=device)
+ self.cudnn = True
+
+ self.build() # construct Nneuralnet Component
+
+ #--- generate job.conf file for debug purpose
+ #filename = 'job.conf'
+ #with open(filename, 'w') as f:
+ # f.write(text_format.MessageToString(self.jobconf.cluster))
+ #self.display()
+
+ #--- run singa ---
+ return SingaRun(jobproto=self.jobconf,
+ argv=self.argv, execpath=execpath, testmode=is_testonly)
+ #return SingaRun_script(filename=filename, execpath=execpath)
+
+
+ def display(self):
+ ''' print out job proto
+ '''
+ print text_format.MessageToString(self.jobconf)
+
+ def set_cudnn_layer_type(self, net):
+ ''' convert LayerType to CdunnLayerType
+ '''
+ for i in range(len(net.layer)):
+ ly_type = net.layer[i].type
+ cudnn_ly_type = ly_type
+ if ly_type == kCConvolution: cudnn_ly_type = kCudnnConv
+ elif ly_type == kCPooling: cudnn_ly_type = kCudnnPool
+ elif ly_type == kLRN: cudnn_ly_type = kCudnnLRN
+ elif ly_type == kSoftmax: cudnn_ly_type = kCudnnSoftmax
+ elif ly_type == kSoftmaxLoss: cudnn_ly_type = kCudnnSoftmaxLoss
+ elif ly_type == kSTanh:
+ cudnn_ly_type = kCudnnActivation
+ net.layer[i].activation_conf.type = STANH
+ elif ly_type == kSigmoid:
+ cudnn_ly_type = kCudnnActivation
+ net.layer[i].activation_conf.type = SIGMOID
+ elif ly_type == kReLU:
+ cudnn_ly_type = kCudnnActivation
+ net.layer[i].activation_conf.type = RELU
+ net.layer[i].type = cudnn_ly_type
- def evaluate(self, data=None, alg='bp',
- checkpoint_path=None, execpath='', device=None, **fields):
- '''
- required
- data = (Data) // Data class object for testing data
- optional
- alg = (string) // algorithm type, (backpropagation at default)
- checkpoint_path = (list) // checkpoint path is necessary only for testing
- execpaths = (string) // path to user's own executable
- device = (int/list) // a list of gpu ids
- **fields (KEY=VALUE)
- batch_size = (int) // batch size for testing data
- test_freq = (int) // frequency of testing
- test_steps = (int) // total number of steps for testing
- test_after = (int) // start testing after this number of steps
+class Energy(Model):
+ ''' energy model
'''
- assert data != None, 'Testing data should be set'
- is_testonly = False
-
- if 'batch_size' in fields: # if new value is set, replace the batch_size
- setval(data.layer.store_conf, batchsize=fields['batch_size'])
-
- # insert layer for testing
- if self.exist_datalayer('test') == False:
- self.layers.insert(0, data)
-
- # loading checkpoint if singa runs only for testing
- if self.exist_datalayer('train') == False:
- is_testonly = True
- if checkpoint_path == None:
- print 'checkpoint_path has not been specified'
- else:
- setval(self.jobconf, checkpoint_path=checkpoint_path)
- steps = fields['test_steps'] if 'test_steps' in fields else 10
- setval(self.jobconf, test_steps=steps)
- setval(self.jobconf, **fields)
+ def __init__(self, name='my model', argv=[], label=False):
+ super(Energy, self).__init__(name=name, argv=argv, label=label)
+
+ def add(self, layer):
+ if hasattr(layer, 'layer_type'):
+ if layer.layer_type == kRBMVis:
+ dim = 0
+ for i in range(1, len(layer.out_dim)):
+ parw = Parameter(name='w', init='none', level=i)
+ parb = Parameter(name='b', init='none', level=i)
+ dim = layer.out_dim[i-1]
+ self.layers.append(Dense(dim, w_param=parw, b_param=parb,
+ activation='sigmoid'))
+ self.layers.append(layer)
- # set Train_one_batch component, using backprogapation at default
- setval(self.jobconf, train_one_batch=Algorithm(type=enumAlgType(alg)).proto)
-
- # use of cudnn
- if device != None:
- setval(self.jobconf, gpu=device)
- self.cudnn = True
-
- self.build() # construct Nneuralnet Component
-
- #--- generate job.conf file for debug purpose
- #filename = 'job.conf'
- #with open(filename, 'w') as f:
- # f.write(text_format.MessageToString(self.jobconf.cluster))
- #self.display()
-
- #--- run singa ---
- return SingaRun(jobproto=self.jobconf, argv=self.argv, execpath=execpath, testmode=is_testonly)
- #return SingaRun_script(filename=filename, execpath=execpath)
-
-
- def display(self):
- ''' print out job proto
- '''
- print text_format.MessageToString(self.jobconf)
-
- def setCudnnLayerType(self, net):
- ''' convert LayerType to CdunnLayerType
+class Sequential(Model):
+ ''' sequential model
'''
- for i in range(len(net.layer)):
- ly_type = net.layer[i].type
- cudnn_ly_type = ly_type
- if ly_type == kCConvolution: cudnn_ly_type = kCudnnConv
- elif ly_type == kCPooling: cudnn_ly_type = kCudnnPool
- elif ly_type == kLRN: cudnn_ly_type = kCudnnLRN
- elif ly_type == kSoftmax: cudnn_ly_type = kCudnnSoftmax
- elif ly_type == kSoftmaxLoss: cudnn_ly_type = kCudnnSoftmaxLoss
- elif ly_type == kSTanh:
- cudnn_ly_type = kCudnnActivation
- net.layer[i].activation_conf.type = STANH
- elif ly_type == kSigmoid:
- cudnn_ly_type = kCudnnActivation
- net.layer[i].activation_conf.type = SIGMOID
- elif ly_type == kReLU:
- cudnn_ly_type = kCudnnActivation
- net.layer[i].activation_conf.type = RELU
- net.layer[i].type = cudnn_ly_type
-
-
-class Energy(Model):
- def __init__(self, name='my model', argv=[], label=False):
- super(Energy, self).__init__(name=name, argv=argv, label=label)
-
- def add(self, layer):
- if hasattr(layer, 'layer_type'):
- if layer.layer_type == kRBMVis:
- dim = 0
- for i in range(1, len(layer.out_dim)):
- parw = Parameter(name='w', init='none', level=i)
- parb = Parameter(name='b', init='none', level=i)
- dim = layer.out_dim[i-1]
- self.layers.append(Dense(dim, w_param=parw, b_param=parb, activation='sigmoid'))
- self.layers.append(layer)
-
-class Sequential(Model):
- def __init__(self, name='my model', argv=[], label=False):
- super(Sequential, self).__init__(name=name, argv=argv, label=label)
-
- def add(self, layer):
- if hasattr(layer, 'layer_type'):
- if layer.layer_type == 'AutoEncoder':
- dim = 0
- if layer.param_share == True:
- # Encoding
- for i in range(1, len(layer.hid_dim)+1):
- parw = Parameter(name='w', init='none', level=i)
- parb = Parameter(name='b', init='none', level=i)
- dim = layer.hid_dim[i-1]
- if i == len(layer.hid_dim): activation = None
- else: activation = layer.activation
- self.layers.append(Dense(dim, w_param=parw, b_param=parb, activation=activation))
- # Decoding
- for i in range(len(layer.hid_dim), 0, -1):
- parw = Parameter(name=generateName('w',2), init='none')
- parb = Parameter(name=generateName('b',2), init='none')
- setval(parw.param, share_from='w'+str(i))
- setval(parb.param, name='b'+str(i))
- if i == 1: dim = layer.out_dim
- else: dim = layer.hid_dim[i-2]
- self.layers.append(Dense(dim, w_param=parw, b_param=parb, activation=layer.activation, transpose=True))
+ def __init__(self, name='my model', argv=[], label=False):
+ super(Sequential, self).__init__(name=name, argv=argv, label=label)
+
+ def add(self, layer):
+ if hasattr(layer, 'layer_type'):
+ if layer.layer_type == 'AutoEncoder':
+ dim = 0
+ if layer.param_share == True:
+ # Encoding
+ for i in range(1, len(layer.hid_dim)+1):
+ parw = Parameter(name='w',
+ init='none', level=i)
+ parb = Parameter(name='b',
+ init='none', level=i)
+ dim = layer.hid_dim[i-1]
+ if i == len(layer.hid_dim): activation = None
+ else: activation = layer.activation
+ self.layers.append(Dense(dim,
+ w_param=parw, b_param=parb,
+ activation=activation))
+ # Decoding
+ for i in range(len(layer.hid_dim), 0, -1):
+ parw = Parameter(name=generate_name('w', 2),
+ init='none')
+ parb = Parameter(name=generate_name('b', 2),
+ init='none')
+ setval(parw.param, share_from='w'+str(i))
+ setval(parb.param, name='b'+str(i))
+ if i == 1: dim = layer.out_dim
+ else: dim = layer.hid_dim[i-2]
+ self.layers.append(Dense(dim,
+ w_param=parw, b_param=parb,
+ activation=layer.activation,
+ transpose=True))
+ else:
+ # MLP
+ for i in range(1, len(layer.hid_dim)+2):
+ parw = Parameter(name='w',
+ init='none', level=i)
+ parb = Parameter(name='b',
+ init='none', level=i)
+ if i == len(layer.hid_dim)+1: dim = layer.out_dim
+ else: dim = layer.hid_dim[i-1]
+ self.layers.append(Dense(dim,
+ w_param=parw, b_param=parb,
+ activation=layer.activation))
+ else:
+ self.layers.append(layer)
else:
- # MLP
- for i in range(1, len(layer.hid_dim)+2):
- parw = Parameter(name='w', init='none', level=i)
- parb = Parameter(name='b', init='none', level=i)
- if i == len(layer.hid_dim)+1: dim = layer.out_dim
- else: dim = layer.hid_dim[i-1]
- self.layers.append(Dense(dim, w_param=parw, b_param=parb, activation=layer.activation))
- else:
- self.layers.append(layer)
- else:
- self.layers.append(layer)
+ self.layers.append(layer)
class Store(object):
- def __init__(self, **kwargs):
- '''
- **kwargs
- path = (string) // path to dataset
- backend = (string) //
- batch_size = (int) // batch size of dataset
- shape = (int) //
- '''
- self.proto = Message('Store', **kwargs).proto
+ def __init__(self, **kwargs):
+ '''
+ **kwargs
+ path = (string) // path to dataset
+ backend = (string) //
+ batch_size = (int) // batch size of dataset
+ shape = (int) //
+ '''
+ self.proto = Message('Store', **kwargs).proto
class Algorithm(object):
- def __init__(self, type=enumAlgType('bp'), **kwargs):
- alg = Message('Alg', alg=type, **kwargs).proto
- if type == enumAlgType('cd'):
- setval(alg.cd_conf, **kwargs)
- self.proto = alg
+
+ def __init__(self, type=enumAlgType('bp'), **kwargs):
+ '''
+ type = (string) // type of algorithm, bp at default
+ '''
+ alg = Message('Alg', alg=type, **kwargs).proto
+ if type == enumAlgType('cd'):
+ setval(alg.cd_conf, **kwargs)
+ self.proto = alg
class Updater(object):
- def __init__(self, upd_type, lr, lr_type,
- decay, momentum,
- step, step_lr, **fields):
- '''
- required
- upd_type = (enum) // enum type of updater
- lr = (float) // base learning rate
- optional
- lr_type = (string) // type of the learning rate (Fixed at default)
- '''
- upd = Message('Updater', type=upd_type, **fields).proto
- setval(upd.learning_rate, base_lr=lr)
- if decay > 0:
- setval(upd, weight_decay=decay)
- if momentum > 0:
- setval(upd, momentum=momentum)
-
- if lr_type == None or lr_type == "fixed":
- setval(upd.learning_rate, type=kFixed)
- elif lr_type == 'step':
- cp = Message('Step', change_freq=60, gamma=0.997)
- setval(upd.learning_rate, type=kStep, step_conf=cp.proto)
- elif lr_type == 'manual':
- cp = Message('FixedStep', step=step, step_lr=step_lr)
- setval(upd.learning_rate, type=kFixedStep, fixedstep_conf=cp.proto)
- elif lr_type == 'linear':
- cp = Message('Linear', change_freq=10, final_lr=0.1)
- setval(upd.learning_rate, type=kLinear, linear_conf=cp.proto)
-
- self.proto = upd
+
+ def __init__(self, upd_type, lr, lr_type,
+ decay, momentum,
+ step, step_lr, **fields):
+ '''
+ required
+ upd_type = (enum) // enum type of updater
+ lr = (float) // base learning rate
+ optional
+ lr_type = (string) // type of the learning rate (Fixed at default)
+ '''
+ upd = Message('Updater', type=upd_type, **fields).proto
+ setval(upd.learning_rate, base_lr=lr)
+ if decay > 0:
+ setval(upd, weight_decay=decay)
+ if momentum > 0:
+ setval(upd, momentum=momentum)
+
+ if lr_type == None or lr_type == "fixed":
+ setval(upd.learning_rate, type=kFixed)
+ elif lr_type == 'step':
+ cp = Message('Step', change_freq=60, gamma=0.997)
+ setval(upd.learning_rate, type=kStep, step_conf=cp.proto)
+ elif lr_type == 'manual':
+ cp = Message('FixedStep', step=step, step_lr=step_lr)
+ setval(upd.learning_rate, type=kFixedStep, fixedstep_conf=cp.proto)
+ elif lr_type == 'linear':
+ cp = Message('Linear', change_freq=10, final_lr=0.1)
+ setval(upd.learning_rate, type=kLinear, linear_conf=cp.proto)
+
+ self.proto = upd
class SGD(Updater):
- def __init__(self, lr=0.01, lr_type=None,
- decay=0, momentum=0,
- step=(0), step_lr=(0.01), **fields):
- '''
- required
- lr = (float) // base learning rate
- optional
- lr_type = (string) // type of learning rate, 'Fixed' at default
- decay = (float) // weight decay
- momentum = (float) // momentum
- step = (int/list) // steps
- step_lr = (float/list) // learning rate after the steps
- **fields (KEY=VALUE)
- '''
- assert lr
- super(SGD, self).__init__(upd_type=kSGD,
- lr=lr, lr_type=lr_type,
- decay=decay, momentum=momentum,
- step=step, step_lr=step_lr, **fields)
+
+ def __init__(self, lr=0.01, lr_type=None,
+ decay=0, momentum=0,
+ step=(0), step_lr=(0.01), **fields):
+ '''
+ required
+ lr = (float) // base learning rate
+ optional
+ lr_type = (string) // type of learning rate, 'Fixed' at default
+ decay = (float) // weight decay
+ momentum = (float) // momentum
+ step = (int/list) // steps
+ step_lr = (float/list) // learning rate after the steps
+ **fields (KEY=VALUE)
+ '''
+ assert lr
+ super(SGD, self).__init__(upd_type=kSGD,
+ lr=lr, lr_type=lr_type,
+ decay=decay, momentum=momentum,
+ step=step, step_lr=step_lr, **fields)
class AdaGrad(Updater):
- def __init__(self, lr=0.01, lr_type=None,
- decay=0, momentum=0,
- step=(0), step_lr=(0.01), **fields):
- '''
- required
- lr = (float) // base learning rate
- optional
- lr_type = (string) // type of learning rate, 'Fixed' at default
- decay = (float) // weight decay
- momentum = (float) // momentum
- step = (int/list) // steps
- step_lr = (float/list) // learning rate after the steps
- **fields (KEY=VALUE)
- '''
- assert lr
- super(AdaGrad, self).__init__(upd_type=kAdaGrad,
- lr=lr, lr_type=lr_type,
- decay=decay, momentum=momentum,
- step=step, step_lr=step_lr, **fields)
+ def __init__(self, lr=0.01, lr_type=None,
+ decay=0, momentum=0,
+ step=(0), step_lr=(0.01), **fields):
+ '''
+ required
+ lr = (float) // base learning rate
+ optional
+ lr_type = (string) // type of learning rate, 'Fixed' at default
+ decay = (float) // weight decay
+ momentum = (float) // momentum
+ step = (int/list) // steps
+ step_lr = (float/list) // learning rate after the steps
+ **fields (KEY=VALUE)
+ '''
+ assert lr
+ super(AdaGrad, self).__init__(upd_type=kAdaGrad,
+ lr=lr, lr_type=lr_type,
+ decay=decay, momentum=momentum,
+ step=step, step_lr=step_lr, **fields)
class Cluster(object):
- """ Specify the cluster topology, e.g., number of workers/servers.
-
- Currently we need to create this object in the .py file and also provide a
- cluster configuration file to the command line. TODO(wangwei) update SINGA
- code to eliminate the requirement of the cluster configuration file for
- training on a single node or the cluster object in the pyfile for training
- in a cluster.
- """
-
- def __init__(self, workspace=None,
- nworker_groups=1, nserver_groups=1,
- nworkers_per_group=1, nservers_per_group=1,
- nworkers_per_procs=1, nservers_per_procs=1,
- **fields):
- '''
- required
- workspace = (string) // workspace path
- optional
- nworker_groups = (int)
- nserver_groups = (int)
- nworkers_per_group = (int)
- nservers_per_group = (int)
- nworkers_per_procs = (int)
- nservers_per_procs = (int)
- **fields
- server_worker_separate = (bool)
- '''
- assert workspace != None, 'need to set workspace'
- self.proto = Message('Cluster', workspace=workspace).proto
- # optional
- self.proto.nworker_groups = nworker_groups
- self.proto.nserver_groups = nserver_groups
- self.proto.nworkers_per_group = nworkers_per_group
- self.proto.nservers_per_group = nservers_per_group
- self.proto.nworkers_per_procs = nworkers_per_procs
- self.proto.nservers_per_procs = nservers_per_procs
- # other fields
- setval(self.proto, **fields)
-
+ """ Specify the cluster topology, e.g., number of workers/servers.
+
+ Currently we need to create this object in the .py file and also provide a
+ cluster configuration file to the command line. TODO(wangwei) update SINGA
+ code to eliminate the requirement of the cluster configuration file for
+ training on a single node or the cluster object in the pyfile for training
+ in a cluster.
+ """
+
+ def __init__(self, workspace=None,
+ nworker_groups=1, nserver_groups=1,
+ nworkers_per_group=1, nservers_per_group=1,
+ nworkers_per_procs=1, nservers_per_procs=1,
+ **fields):
+ '''
+ required
+ workspace = (string) // workspace path
+ optional
+ nworker_groups = (int)
+ nserver_groups = (int)
+ nworkers_per_group = (int)
+ nservers_per_group = (int)
+ nworkers_per_procs = (int)
+ nservers_per_procs = (int)
+ **fields
+ server_worker_separate = (bool)
+ '''
+ assert workspace != None, 'need to set workspace'
+ self.proto = Message('Cluster', workspace=workspace).proto
+ # optional
+ self.proto.nworker_groups = nworker_groups
+ self.proto.nserver_groups = nserver_groups
+ self.proto.nworkers_per_group = nworkers_per_group
+ self.proto.nservers_per_group = nservers_per_group
+ self.proto.nworkers_per_procs = nworkers_per_procs
+ self.proto.nservers_per_procs = nservers_per_procs
+ # other fields
+ setval(self.proto, **fields)
def StoreResults(lines):
- """ Parsing metrics from each line in the log file.
-
- TODO(wangwei) format the log string to make them uniform for easy parsing
- Another approach is creating a protobuf message for metrics, which can be
- used for dumping metrics to string and loading perf string back to messages.
- """
-
- resultDic = {}
- for line in lines:
- line = re.findall(r'[\w|*.*]+', line)
- if 'Train' in line:
- step = line[line.index('step')+1]
- if 'accuracy' in line:
- resultDic.setdefault(step,{})['acc'] = line[line.index('accuracy')+1]
- if 'loss' in line:
- resultDic.setdefault(step,{})['loss'] = line[line.index('loss')+1]
- if 'ppl' in line:
- resultDic.setdefault(step,{})['ppl'] = line[line.index('ppl')+1]
- if 'Squared' in line:
- resultDic.setdefault(step,{})['se'] = line[line.index('Squared')+2]
- return resultDic
-
-def SingaRun(jobproto='', argv=[], execpath='', testmode=False):
-
- import singa.driver as driver
- d = driver.Driver()
- d.InitLog(argv[0])
- d.Init(argv)
- if testmode == True:
- d.Test(jobproto.SerializeToString())
- else:
- d.Train(False, jobproto.SerializeToString())
-
- # Get the performance from the latest log file.
- # TODO(wangwei) the log file would be overwritten by other running instance of
- # the same program, e.g., lt-singa
- logfile = '/tmp/singa-log/{0}.ERROR'.format(argv[0].split('/')[-1])
- fin = open(logfile, 'r')
- result = StoreResults(fin.readlines())
-
- return result
+ """ Parsing metrics from each line in the log file.
+
+ TODO(wangwei) format the log string to make them uniform for easy parsing
+ Another approach is creating a protobuf message for metrics, which can be
+ used for dumping metrics to string and loading perf string back to messages.
+ """
+
+ resultDic = {}
+ for line in lines:
+ line = re.findall(r'[\w|*.*]+', line)
+ if 'Train' in line:
+ step = line[line.index('step')+1]
+ if 'accuracy' in line:
+ resultDic.setdefault(step, {})['acc'] \
+ = line[line.index('accuracy')+1]
+ if 'loss' in line:
+ resultDic.setdefault(step, {})['loss'] \
+ = line[line.index('loss')+1]
+ if 'ppl' in line:
+ resultDic.setdefault(step, {})['ppl'] \
+ = line[line.index('ppl')+1]
+ if 'Squared' in line:
+ resultDic.setdefault(step, {})['se'] \
+ = line[line.index('Squared')+2]
+ return resultDic
+
+def SingaRun(jobproto='', argv=None, execpath='', testmode=False):
+ """
+ Run Singa and receive the training/test results.
+ """
+
+ import singa.driver as driver
+ d = driver.Driver()
+ d.InitLog(argv[0])
+ d.Init(argv)
+ if testmode == True:
+ d.Test(jobproto.SerializeToString())
+ else:
+ d.Train(False, jobproto.SerializeToString())
+
+ # Get the performance from the latest log file.
+ # TODO(wangwei) the log file would be overwritten by other running instance
+ # of the same program, e.g., lt-singa
+ logfile = '/tmp/singa-log/{0}.ERROR'.format(argv[0].split('/')[-1])
+ fin = open(logfile, 'r')
+ result = StoreResults(fin.readlines())
+
+ return result
def SingaRun_script(filename='', execpath=''):
- """
- Deprecated.
- Generate the job conf file and run the shell command.
- """
- SINGAROOT = '../../../'
- conf = 'examples/' + filename
- if execpath=='':
- cmd = SINGAROOT+'bin/singa-run.sh ' \
- + '-conf %s ' % conf
- else:
- cmd = SINGAROOT+'bin/singa-run.sh ' \
- + '-conf %s ' % conf \
- + '-exec %s ' % execpath
-
- procs = subprocess.Popen(cmd.strip().split(' '), stdout = subprocess.PIPE, stderr = subprocess.STDOUT)
-
- resultDic = {}
- outputlines = iter(procs.stdout.readline, '')
- resultDic = StoreResults(outputlines)
-
- #TODO better format to store the result??
- return resultDic
+ """
+ Deprecated.
+ Generate the job conf file and run the shell command.
+ """
+ SINGAROOT = '../../../'
+ conf = 'examples/' + filename
+ if execpath == '':
+ cmd = SINGAROOT+'bin/singa-run.sh ' \
+ + '-conf %s ' % conf
+ else:
+ cmd = SINGAROOT+'bin/singa-run.sh ' \
+ + '-conf %s ' % conf \
+ + '-exec %s ' % execpath
+
+ procs = subprocess.Popen(cmd.strip().split(' '),
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT)
+
+ resultDic = {}
+ outputlines = iter(procs.stdout.readline, '')
+ resultDic = StoreResults(outputlines)
+
+ #TODO better format to store the result??
+ return resultDic
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9ff176c3/tool/python/singa/parameter.py
----------------------------------------------------------------------
diff --git a/tool/python/singa/parameter.py b/tool/python/singa/parameter.py
index 3b33243..14ad852 100644
--- a/tool/python/singa/parameter.py
+++ b/tool/python/singa/parameter.py
@@ -1,105 +1,140 @@
#!/usr/bin/env python
-import initializations
-from utils.utility import *
-from utils.message import *
+
+#/************************************************************
+#*
+#* Licensed to the Apache Software Foundation (ASF) under one
+#* or more contributor license agreements. See the NOTICE file
+#* distributed with this work for additional information
+#* regarding copyright ownership. The ASF licenses this file
+#* to you under the Apache License, Version 2.0 (the
+#* "License"); you may not use this file except in compliance
+#* with the License. You may obtain a copy of the License at
+#*
+#* http://www.apache.org/licenses/LICENSE-2.0
+#*
+#* Unless required by applicable law or agreed to in writing,
+#* software distributed under the License is distributed on an
+#* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#* KIND, either express or implied. See the License for the
+#* specific language governing permissions and limitations
+#* under the License.
+#*
+#*************************************************************/
+
+'''
+This script includes Parameter class and a method, named set_param_field
+that users can configure Param and ParamGen protos.
+'''
+
+from singa.initializations import get_init_values
+from singa.utils.utility import setval, generate_name
+from singa.utils.message import *
from google.protobuf import text_format
+
class Parameter(object):
- def __init__(self, **kwargs):
- '''
- optional
- **kwargs
- name = (string) // parameter name
- lr = (float) // learning rate
- wd = (float) // weight decay
- init = (string) // initialization type {'constant','uniform','gaussian'}
- value = (int) // value for 'constant'
- scale = (float) // [low, high] for 'uniform', low=-scale, high=scale
- low = (float) // low value for 'uniform'
- high = (float) // high value for 'uniform'
- mean = (float) // mean for 'gaussian'
- std = (float) // std for 'gaussian'
- '''
- fields = {'lr_scale' : kwargs['lr'] if 'lr' in kwargs else 1,
- 'wd_scale' : kwargs['wd'] if 'wd' in kwargs else 1
- }
- self.param = Message('Param', **fields).proto
+ def __init__(self, **kwargs):
+ '''
+ optional
+ **kwargs
+ name = (string) // parameter name
+ lr = (float) // learning rate multiplier
+ wd = (float) // weight decay multiplier
+ init = (string) // init type {'constant','uniform','gaussian'}
+ value = (int) // value for 'constant'
+ scale = (float) // [low=-scale, high=scale] for 'uniform'
+ low = (float) // low value for 'uniform'
+ high = (float) // high value for 'uniform'
+ mean = (float) // mean for 'gaussian'
+ std = (float) // std for 'gaussian'
+ '''
+ fields = {'lr_scale' : kwargs['lr'] if 'lr' in kwargs else 1,
+ 'wd_scale' : kwargs['wd'] if 'wd' in kwargs else 1
+ }
+ self.param = Message('Param', **fields).proto
- if not 'name' in kwargs:
- setval(self.param, name=generateName('param', 1))
- else:
- pname = kwargs['name']
- # parameter name for RBM
- if 'level' in kwargs:
- pname += str(kwargs['level'])
- if pname[0] == 'b':
- pname += '2'
- setval(self.param, name=pname)
-
- if 'share_from' in kwargs:
- setval(self.param, share_from=kwargs['share_from'])
-
- if 'init' in kwargs:
- init_values = initializations.get(kwargs['init'], **kwargs)
-
- if not kwargs['init'] == 'none':
- pg = Message('ParamGen', type=enumInitMethod(kwargs['init']), **init_values)
- del kwargs['init']
- setval(self.param, init=pg.proto)
- else: # default: uniform
- pg = Message('ParamGen', type=enumInitMethod('uniform'))
- setval(self.param, init=pg.proto)
-
- def update(self, **fields):
- setval(self.param, **fields)
- setval(self.param.init, **fields)
-
-
-def setParamField(param, pname, changename=False, withnumber=True, **kwargs):
- ''' param = (ParamProto)
+ if not 'name' in kwargs:
+ setval(self.param, name=generate_name('param', 1))
+ else:
+ pname = kwargs['name']
+ # parameter name for RBM
+ if 'level' in kwargs:
+ pname += str(kwargs['level'])
+ if pname[0] == 'b':
+ pname += '2'
+ setval(self.param, name=pname)
+
+ if 'share_from' in kwargs:
+ setval(self.param, share_from=kwargs['share_from'])
+
+ if 'init' in kwargs:
+ init_values = get_init_values(kwargs['init'], **kwargs)
+ if not kwargs['init'] == 'none':
+ pgen = Message('ParamGen', type=enumInitMethod(kwargs['init']),
+ **init_values)
+ del kwargs['init']
+ setval(self.param, init=pgen.proto)
+ else: # default: uniform
+ pgen = Message('ParamGen', type=enumInitMethod('uniform'))
+ setval(self.param, init=pgen.proto)
+
+ def update(self, **fields):
+ setval(self.param, **fields)
+ setval(self.param.init, **fields)
+
+
+def set_param_field(param, pname, changename=False, withnumber=True, **kwargs):
+ '''
+ param = (ParamProto)
pname = (string) // 'w' for wiehgt, or 'b' for bias
changename = (bool) // update parameter name if True
withnumber = (bool) // add layer number if True
**kwargs
- '''
- assert pname == 'w' or pname == 'b', 'pname should be w or b'
-
- lr = param.lr_scale
- wd = param.wd_scale
- initkv = {}
-
- if pname == 'w':
- if 'w_lr' in kwargs:
- lr = kwargs['w_lr']
- del kwargs['w_lr']
- if 'w_wd' in kwargs:
- wd = kwargs['w_wd']
- del kwargs['w_wd']
- for k, v in kwargs.items():
- if k.startswith('w_'):
- initkv[k[2:]] = v
-
- elif pname == 'b':
- if 'b_lr' in kwargs:
- lr = kwargs['b_lr']
- del kwargs['b_lr']
- if 'b_wd' in kwargs:
- wd = kwargs['b_wd']
- del kwargs['b_wd']
- for k, v in kwargs.items():
- if k.startswith('b_'):
- initkv[k[2:]] = v
-
- field = {'lr_scale' : lr, 'wd_scale' : wd}
-
- # Set/update parameter fields
- if param.name.startswith('param') or changename==True:
- if 'level' in kwargs: # parameter name for RBM
- pname += str(kwargs['level'])
- setval(param, name=generateName(pname, withnumber=withnumber), **field)
- else:
- setval(param, **field)
-
- # Set/update parameter init fields
- setval(param.init, **initkv)
+ w_lr = (float) // learning rate multiplier for weight, used to
+ // scale the learning rate when updating parameters.
+ w_wd = (float) // weight decay multiplier for weight, used to
+ // scale the weight decay when updating parameters.
+ b_lr = (float) // learning rate multiplier for bias
+ b_wd = (float) // weight decay multiplier for bias
+ '''
+ assert pname == 'w' or pname == 'b', 'pname should be w or b'
+
+ lr_ = param.lr_scale
+ wd_ = param.wd_scale
+ initkv = {}
+
+ if pname == 'w':
+ if 'w_lr' in kwargs:
+ lr_ = kwargs['w_lr']
+ del kwargs['w_lr']
+ if 'w_wd' in kwargs:
+ wd_ = kwargs['w_wd']
+ del kwargs['w_wd']
+ for key, val in kwargs.items():
+ if key.startswith('w_'):
+ initkv[key[2:]] = val
+
+ elif pname == 'b':
+ if 'b_lr' in kwargs:
+ lr_ = kwargs['b_lr']
+ del kwargs['b_lr']
+ if 'b_wd' in kwargs:
+ wd_ = kwargs['b_wd']
+ del kwargs['b_wd']
+ for key, val in kwargs.items():
+ if key.startswith('b_'):
+ initkv[key[2:]] = val
+
+ field = {'lr_scale' : lr_, 'wd_scale' : wd_}
+
+ # Set/update parameter fields
+ if param.name.startswith('param') or changename == True:
+ if 'level' in kwargs: # parameter name for RBM
+ pname += str(kwargs['level'])
+ setval(param, name=generate_name(pname, withnumber=withnumber), **field)
+ else:
+ setval(param, **field)
+
+ # Set/update parameter init fields
+ setval(param.init, **initkv)
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9ff176c3/tool/python/singa/utils/message.py
----------------------------------------------------------------------
diff --git a/tool/python/singa/utils/message.py b/tool/python/singa/utils/message.py
index 251a377..bfa9ef2 100644
--- a/tool/python/singa/utils/message.py
+++ b/tool/python/singa/utils/message.py
@@ -1,56 +1,80 @@
-#!/usr/bin/env python
-import sys, os
-from utility import *
-sys.path.append(os.path.join(os.path.dirname(__file__),'../../pb2'))
+#!/usr/bin/env python
+
+#/************************************************************
+#*
+#* Licensed to the Apache Software Foundation (ASF) under one
+#* or more contributor license agreements. See the NOTICE file
+#* distributed with this work for additional information
+#* regarding copyright ownership. The ASF licenses this file
+#* to you under the Apache License, Version 2.0 (the
+#* "License"); you may not use this file except in compliance
+#* with the License. You may obtain a copy of the License at
+#*
+#* http://www.apache.org/licenses/LICENSE-2.0
+#*
+#* Unless required by applicable law or agreed to in writing,
+#* software distributed under the License is distributed on an
+#* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#* KIND, either express or implied. See the License for the
+#* specific language governing permissions and limitations
+#* under the License.
+#*
+#*************************************************************/
+
+import sys, os
+from utility import *
+sys.path.append(os.path.join(os.path.dirname(__file__), '../../pb2'))
'''
- - This script reads proto files in ../../pb2, generated by protocol buffer compiler
- - Message class creates an object for proto and sets the fields specified by kwargs
- - enumXXX function returns enum values of name XXX
+This script reads proto files in ../../pb2, generated by proto buffer compiler.
+ - Message class creates an object for proto and sets initial vlaues for
+ the fields, specified by kwargs
+ - make_function method generates a method named enumInitMethod that returns
+ enum values of given enum type, defined in the proto files
'''
-module_list=[]
-
-# import all modules in dir singa_root/too/pb2, except common, singa and __init__
-for f in os.listdir(os.path.join(os.path.dirname(__file__),'../../pb2')):
- if (f.endswith(".pyc")):
- continue
- if(f == "__init__.py" or f == "common_pb2.py" or f == "singa_pb2.py" ):
- continue
- module_name = f.split('.')[0]
- module=__import__(module_name)
- module_list.append(module)
- for func_name in dir(module):
- if not func_name.startswith("__"):
- globals()[func_name] = getattr(module,func_name)
+MODULE_LIST = []
+
+# import all modules in dir singa_root/tool/python/pb2
+# except common, singa, and __init__
+for f in os.listdir(os.path.join(os.path.dirname(__file__), '../../pb2')):
+ if (f.endswith(".pyc")):
+ continue
+ if(f == "__init__.py" or f == "common_pb2.py" or f == "singa_pb2.py"):
+ continue
+ module_name = f.split('.')[0]
+ module_obj = __import__(module_name)
+ MODULE_LIST.append(module_obj)
+ for func_name in dir(module_obj):
+ if not func_name.startswith("__"):
+ globals()[func_name] = getattr(module_obj, func_name)
class Message(object):
- def __init__(self,protoname,**kwargs):
- for module in module_list:
- if hasattr(module,protoname+"Proto"):
- class_ = getattr(module,protoname+"Proto")
- self.proto = class_()
- return setval(self.proto,**kwargs)
- raise Exception('invalid protoname')
-
-enumDict_=dict()
-
-#get all enum type list in modules
-for module in module_list:
- for enumtype in module.DESCRIPTOR.enum_types_by_name:
- tempDict=enumDict_[enumtype]=dict()
- for name in getattr(module,enumtype).DESCRIPTOR.values_by_name:
- tempDict[name[1:].lower()]=getattr(module,name)
+ def __init__(self, protoname, **kwargs):
+ for module_obj in MODULE_LIST:
+ if hasattr(module_obj, protoname+"Proto"):
+ class_ = getattr(module_obj, protoname+"Proto")
+ self.proto = class_()
+ return setval(self.proto, **kwargs)
+ raise Exception('invalid protoname')
+
+enumDict_ = dict()
+
+#get all enum type list in the modules
+for module_obj in MODULE_LIST:
+ for enumtype in module_obj.DESCRIPTOR.enum_types_by_name:
+ tempDict = enumDict_[enumtype] = dict()
+ for name in getattr(module_obj, enumtype).DESCRIPTOR.values_by_name:
+ tempDict[name[1:].lower()] = getattr(module_obj, name)
def make_function(enumtype):
- def _function(key):
- return enumDict_[enumtype][key]
- return _function
+ def _function(key):
+ return enumDict_[enumtype][key]
+ return _function
current_module = sys.modules[__name__]
#def all the enumtypes
-for module in module_list:
- for enumtype in module.DESCRIPTOR.enum_types_by_name:
- setattr(current_module,"enum"+enumtype,make_function(enumtype))
-
+for module_obj in MODULE_LIST:
+ for enumtype in module_obj.DESCRIPTOR.enum_types_by_name:
+ setattr(current_module, "enum"+enumtype, make_function(enumtype))
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/9ff176c3/tool/python/singa/utils/utility.py
----------------------------------------------------------------------
diff --git a/tool/python/singa/utils/utility.py b/tool/python/singa/utils/utility.py
index 93d2f7f..8c6c455 100644
--- a/tool/python/singa/utils/utility.py
+++ b/tool/python/singa/utils/utility.py
@@ -1,50 +1,84 @@
#!/usr/bin/env python
-layerid = 0
-paramid = 0
-
-def generateName(label, op=0, withnumber=True):
- global layerid, paramid
- num = layerid
- if label == 'layer':
- if op ==1: layerid += 1
- num = layerid
- elif label == 'param':
- if op ==1: paramid += 1
- num = paramid
- else:
- if op ==1: layerid += 1
- num = layerid
- if op ==2:
- num = layerid+1
-
- if withnumber == False:
- return '{0}'.format(label)
-
- return '{0}{1}'.format(label, num)
+#/************************************************************
+#*
+#* Licensed to the Apache Software Foundation (ASF) under one
+#* or more contributor license agreements. See the NOTICE file
+#* distributed with this work for additional information
+#* regarding copyright ownership. The ASF licenses this file
+#* to you under the Apache License, Version 2.0 (the
+#* "License"); you may not use this file except in compliance
+#* with the License. You may obtain a copy of the License at
+#*
+#* http://www.apache.org/licenses/LICENSE-2.0
+#*
+#* Unless required by applicable law or agreed to in writing,
+#* software distributed under the License is distributed on an
+#* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#* KIND, either express or implied. See the License for the
+#* specific language governing permissions and limitations
+#* under the License.
+#*
+#*************************************************************/
+'''
+This script includes methods to
+(1) generate name of layer, parameter, etc.
+(2) set field values for proto.
+'''
+
+LAYERID = 0
+PARAMID = 0
+
+def generate_name(label, option=0, withnumber=True):
+ ''' This method returns name of layer or parameter with unique id.
+ option: 1 to increase id number
+ withnumber: True to concatenate number to name
+ '''
+
+ global LAYERID, PARAMID
+ num = LAYERID
+ if label == 'layer':
+ if option == 1: LAYERID += 1
+ num = LAYERID
+ elif label == 'param':
+ if option == 1: PARAMID += 1
+ num = PARAMID
+ else:
+ if option == 1: LAYERID += 1
+ num = LAYERID
+ if option == 2:
+ num = LAYERID+1
+
+ if withnumber == False:
+ return '{0}'.format(label)
+
+ return '{0}{1}'.format(label, num)
def setval(proto, **kwargs):
- for k,v in kwargs.items():
- #print 'kv: ', k, ', ', v
- if hasattr(proto, k):
- flabel = proto.DESCRIPTOR.fields_by_name[k].label
- ftype = proto.DESCRIPTOR.fields_by_name[k].type
-
- fattr = getattr(proto, k)
- if flabel == 3: # repeated field
- if ftype == 11: # message type
- fattr = fattr.add()
- fattr.MergeFrom(v)
- else:
- if type(v) == list or type(v) == tuple:
- for i in range(len(v)):
- fattr.append(v[i])
- else:
- fattr.append(v)
- else:
- if ftype == 11: # message type
- fattr = getattr(proto,k)
- fattr.MergeFrom(v)
- else:
- setattr(proto, k, v)
+ ''' This method sets field values for give proto.
+ '''
+
+ for key, val in kwargs.items():
+ #print 'kv: ', k, ', ', v
+ if hasattr(proto, key):
+ flabel = proto.DESCRIPTOR.fields_by_name[key].label
+ ftype = proto.DESCRIPTOR.fields_by_name[key].type
+
+ fattr = getattr(proto, key)
+ if flabel == 3: # repeated field
+ if ftype == 11: # message type
+ fattr = fattr.add()
+ fattr.MergeFrom(val)
+ else:
+ if type(val) == list or type(val) == tuple:
+ for i in range(len(val)):
+ fattr.append(val[i])
+ else:
+ fattr.append(val)
+ else:
+ if ftype == 11: # message type
+ fattr = getattr(proto, key)
+ fattr.MergeFrom(val)
+ else:
+ setattr(proto, key, val)
[02/10] incubator-singa git commit: SINGA-81 Add Python Helper,
which enables users to construct a model (JobProto) and run Singa in
Python
Posted by wa...@apache.org.
SINGA-81 Add Python Helper, which enables users to construct a model (JobProto) and run Singa in Python
- Update examples for cudnn
. cifar10_cnn_cudnn.py
. mnist_mlp_cudnn.py
- Now, mnist_ae.py works properly
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/8b69cadb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/8b69cadb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/8b69cadb
Branch: refs/heads/master
Commit: 8b69cadb7ca015eaee1a666a541d2e6185255554
Parents: 94435eb
Author: chonho <le...@comp.nus.edu.sg>
Authored: Mon Dec 28 13:31:30 2015 +0800
Committer: chonho <le...@comp.nus.edu.sg>
Committed: Fri Jan 1 15:59:14 2016 +0800
----------------------------------------------------------------------
tool/python/examples/cifar10_cnn_cudnn.py | 4 +--
.../python/examples/cifar10_cnn_cudnn_hybrid.py | 34 ++++++++++++++++++++
tool/python/singa/model.py | 26 ++++++++-------
3 files changed, 51 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8b69cadb/tool/python/examples/cifar10_cnn_cudnn.py
----------------------------------------------------------------------
diff --git a/tool/python/examples/cifar10_cnn_cudnn.py b/tool/python/examples/cifar10_cnn_cudnn.py
index e08610a..e3c5c49 100755
--- a/tool/python/examples/cifar10_cnn_cudnn.py
+++ b/tool/python/examples/cifar10_cnn_cudnn.py
@@ -29,6 +29,6 @@ topo = Cluster(workspace)
m.compile(loss='categorical_crossentropy', optimizer=sgd, cluster=topo)
gpu_id = [0]
-m.fit(X_train, nb_epoch=1000, with_test=True, device=gpu_id)
-result = m.evaluate(X_test, test_steps=100, test_freq=300)
+m.fit(X_train, nb_epoch=7000, with_test=True, device=gpu_id)
+result = m.evaluate(X_test, test_steps=100, test_freq=1000)
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8b69cadb/tool/python/examples/cifar10_cnn_cudnn_hybrid.py
----------------------------------------------------------------------
diff --git a/tool/python/examples/cifar10_cnn_cudnn_hybrid.py b/tool/python/examples/cifar10_cnn_cudnn_hybrid.py
new file mode 100755
index 0000000..f5e4c27
--- /dev/null
+++ b/tool/python/examples/cifar10_cnn_cudnn_hybrid.py
@@ -0,0 +1,34 @@
+#!/usr/bin/env python
+import sys, os
+sys.path.append(os.path.join(os.path.dirname(__file__),'..'))
+from singa.model import *
+from singa.datasets import cifar10
+
+X_train, X_test, workspace = cifar10.load_data()
+
+m = Sequential('cifar10-cnn', sys.argv)
+
+m.add(Convolution2D(32, 5, 1, 2, w_std=0.0001, b_lr=2))
+m.add(MaxPooling2D(pool_size=(3,3), stride=2))
+m.add(Activation('relu'))
+m.add(LRN2D(3, alpha=0.00005, beta=0.75))
+
+m.add(Convolution2D(32, 5, 1, 2, b_lr=2))
+m.add(Activation('relu'))
+m.add(AvgPooling2D(pool_size=(3,3), stride=2))
+m.add(LRN2D(3, alpha=0.00005, beta=0.75))
+
+m.add(Convolution2D(64, 5, 1, 2))
+m.add(Activation('relu'))
+m.add(AvgPooling2D(pool_size=(3,3), stride=2))
+
+m.add(Dense(10, w_wd=250, b_lr=2, b_wd=0, activation='softmax'))
+
+sgd = SGD(decay=0.004, lr_type='fixed', step=(0,60000,65000), step_lr=(0.001,0.0001,0.00001))
+topo = Cluster(workspace, nworkers_per_group=2, nworkers_per_procs=2)
+m.compile(loss='categorical_crossentropy', optimizer=sgd, cluster=topo)
+
+gpu_id = [0,1]
+m.fit(X_train, nb_epoch=10000, with_test=True, device=gpu_id)
+result = m.evaluate(X_test, test_steps=0, test_freq=200)
+
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8b69cadb/tool/python/singa/model.py
----------------------------------------------------------------------
diff --git a/tool/python/singa/model.py b/tool/python/singa/model.py
index ade0b85..51c4126 100644
--- a/tool/python/singa/model.py
+++ b/tool/python/singa/model.py
@@ -11,6 +11,7 @@ class Model(object):
'''
optional
name = (string) // name of model/job
+ argv // pass sys.argv to source
label = (bool) // exist label layer (depreciated)
'''
self.jobconf = Message('Job', name=name).proto
@@ -227,7 +228,7 @@ class Model(object):
#filename = 'job.conf'
#with open(filename, 'w') as f:
# f.write(text_format.MessageToString(self.jobconf.cluster))
- self.display()
+ #self.display()
#--- run singa ---
return SingaRun(jobproto=self.jobconf, argv=self.argv, execpath=execpath, testmode=is_testonly)
@@ -285,8 +286,8 @@ class Sequential(Model):
def add(self, layer):
if hasattr(layer, 'layer_type'):
if layer.layer_type == 'AutoEncoder':
+ dim = 0
if layer.param_share == True:
- dim = 0
# Encoding
for i in range(1, len(layer.hid_dim)+1):
parw = Parameter(name='w', init='none', level=i)
@@ -368,11 +369,13 @@ class SGD(Updater):
step=(0), step_lr=(0.01), **fields):
'''
required
- lr = (float) // base learning rate
+ lr = (float) // base learning rate
optional
- lr_type = (string) // type of learning rate, 'Fixed' at default
- decay = (float) // weight decay
- momentum = (float) // momentum
+ lr_type = (string) // type of learning rate, 'Fixed' at default
+ decay = (float) // weight decay
+ momentum = (float) // momentum
+ step = (int/list) // steps
+ step_lr = (float/list) // learning rate after the steps
**fields (KEY=VALUE)
'''
@@ -388,13 +391,14 @@ class AdaGrad(Updater):
step=(0), step_lr=(0.01), **fields):
'''
required
- lr = (float) // base learning rate
+ lr = (float) // base learning rate
optional
- lr_type = (string) // type of learning rate, 'Fixed' at default
- decay = (float) // weight decay
- momentum = (float) // momentum
+ lr_type = (string) // type of learning rate, 'Fixed' at default
+ decay = (float) // weight decay
+ momentum = (float) // momentum
+ step = (int/list) // steps
+ step_lr = (float/list) // learning rate after the steps
**fields (KEY=VALUE)
-
'''
assert lr
super(AdaGrad, self).__init__(upd_type=kAdaGrad,
[05/10] incubator-singa git commit: SINGA-81 Add Python Helper,
which enables users to construct a model (JobProto) and run Singa in
Python
Posted by wa...@apache.org.
SINGA-81 Add Python Helper, which enables users to construct a model (JobProto) and run Singa in Python
- Add wrapper API "Driver::Train(const std::string Job_conf)" for python.
- Add wrapper API "Driver::Test(const std::string Job_conf)" for python.
- Python codes (1) construct a model (JobProto), and (2) run singa
- Users are supposed to generate 'usermodel.py'
. examples are provided, e.g., cifar10_cnn.py, mnist_mlp.py, mnist_rbm.py
. 'cluster.conf' is required to maintain cluster information
- Users are supposed to run it as follows. e.g.,
{code}
cd SINGA_ROOT
bin/singa-run.sh -conf tool/python/examples/cluster.conf -exe tool/python/examples/mnist_mlp.py
{code}
- Note: in job.proto, 'required' rule of the following fields should be changed to 'optional'
. JobProto: name, neuralnet, train_one_batch, updater, train_steps
. ClusterProto: workspace
. workspace field can be set in either (i) cluster.conf or (ii) python code
- __init__.py is required in the following directories
. singa
. singa/utils
. singa/datasets
. examples
- Add StoreResult() that takes care of training results
. in SingaRun() called by fit() or evaluate()
. read logfile
. store accuracy, loss, ppl, se, etc. in dictionary format
- Parameter initialization
. Parameter class is internally used
. Weight follows gaussian distribution at default
. Bias follows constant at default
. As an option, users can explicitly specify parameter (e.g., *_parameter.py)
- Removed dataset/ae.py and dataset/rbm.py
. RBM and Autoencoder examples use Mnist dataset
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/7d43e273
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/7d43e273
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/7d43e273
Branch: refs/heads/master
Commit: 7d43e27330581c3eecbd44a04f0c8691c3502ec6
Parents: 74a28dc
Author: chonho <le...@comp.nus.edu.sg>
Authored: Wed Dec 23 12:09:23 2015 +0800
Committer: chonho <le...@comp.nus.edu.sg>
Committed: Fri Jan 1 15:59:14 2016 +0800
----------------------------------------------------------------------
include/singa/driver.h | 9 +
src/proto/job.proto | 3 +-
tool/python/README.md | 322 +++++++++++++++
tool/python/examples/__init__.py | 0
tool/python/examples/cifar10_cnn.py | 32 ++
tool/python/examples/cifar10_cnn_parameter.py | 35 ++
tool/python/examples/cluster.conf | 6 +
tool/python/examples/mnist_ae.py | 27 ++
tool/python/examples/mnist_mlp.py | 33 ++
tool/python/examples/mnist_mlp_parameter.py | 28 ++
tool/python/examples/mnist_mlp_test.py | 30 ++
tool/python/examples/mnist_rbm1.py | 24 ++
tool/python/examples/mnist_rbm1_parameter.py | 26 ++
tool/python/examples/mnist_rbm2.py | 25 ++
tool/python/examples/mnist_rbm2_parameter.py | 28 ++
tool/python/examples/mnist_rbm3.py | 25 ++
tool/python/examples/mnist_rbm3_parameter.py | 28 ++
tool/python/examples/mnist_rbm4.py | 25 ++
tool/python/examples/mnist_rbm4_parameter.py | 27 ++
tool/python/examples/rnnlm_usermodel.py | 22 +
tool/python/singa.py | 3 +-
tool/python/singa/__init__.py | 0
tool/python/singa/datasets/__init__.py | 0
tool/python/singa/datasets/cifar10.py | 34 ++
tool/python/singa/datasets/mnist.py | 32 ++
tool/python/singa/datasets/rnnlm.py | 20 +
tool/python/singa/driver.i | 1 +
tool/python/singa/driver.py | 1 +
tool/python/singa/driver_wrap.cxx | 33 ++
tool/python/singa/initializations.py | 34 ++
tool/python/singa/layer.py | 300 ++++++++++++++
tool/python/singa/model.py | 455 +++++++++++++++++++++
tool/python/singa/parameter.py | 105 +++++
tool/python/singa/utils/__init__.py | 0
tool/python/singa/utils/message.py | 56 +++
tool/python/singa/utils/utility.py | 50 +++
36 files changed, 1877 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7d43e273/include/singa/driver.h
----------------------------------------------------------------------
diff --git a/include/singa/driver.h b/include/singa/driver.h
index f28f8c6..fb5a33a 100644
--- a/include/singa/driver.h
+++ b/include/singa/driver.h
@@ -93,6 +93,15 @@ class Driver {
*/
void Test(const JobProto& job_conf);
/**
+ * Used for python binding. Users can also directly call it as a C++ API.
+ *
+ * It completes the functions as defined above but accept serialized string
+ * parameters.
+ *
+ * @param[in] str serialized string recorded job configuration.
+ */
+ void Test(const std::string str);
+ /**
* Setting the checkpoint field of the job configuration to resume training.
*
* The checkpoint folder will be searched to get the files for the latest
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7d43e273/src/proto/job.proto
----------------------------------------------------------------------
diff --git a/src/proto/job.proto b/src/proto/job.proto
index 98c03f1..22d4bc5 100644
--- a/src/proto/job.proto
+++ b/src/proto/job.proto
@@ -150,7 +150,8 @@ message ClusterProto {
optional int32 nworkers_per_procs = 5 [default = 1];
optional int32 nservers_per_procs = 6 [default = 1];
// local workspace for checkpoint files and vis files
- required string workspace = 10;
+ //required string workspace = 10;
+ optional string workspace = 10;
// servers and workers in different processes?
optional bool server_worker_separate = 20 [default = false];
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7d43e273/tool/python/README.md
----------------------------------------------------------------------
diff --git a/tool/python/README.md b/tool/python/README.md
new file mode 100644
index 0000000..4308ed1
--- /dev/null
+++ b/tool/python/README.md
@@ -0,0 +1,322 @@
+## SINGA-81 Add Python Helper, which enables users to construct a model (JobProto) and run Singa in Python
+
+ SINGAROOT/tool/python
+ |-- pb2 (has job_pb2.py)
+ |-- singa
+ |-- model.py
+ |-- layer.py
+ |-- parameter.py
+ |-- initialization.py
+ |-- utils
+ |-- utility.py
+ |-- message.py
+ |-- datasets
+ |-- cifar10.py
+ |-- mnist.py
+ |-- rbm.py
+ |-- ae.py
+ |-- examples
+ |-- cifar10_cnn.py, mnist_mlp.py, , mnist_rbm1.py, rnnlm_usermodel.py, etc.
+
+### How to Run
+```
+bin/singa-run.sh -exe user_main.py -conf cluster.conf
+```
+The python code, i.e., `user_main.py`, would create the JobProto object and pass it to Driver::Train.
+Currently, ./bin/singa-run.sh needs to get the cluster topology, hence we still need to pass a `cluster.conf` to it.
+The cluster.conf has the configuration for a JobProto with all other fields empty except the cluster field.
+
+Note that 'workspace' field in ClusterProto can be set in either (i) cluster.conf or (ii) python code.
+
+#### Examples
+```
+cd SINGA_ROOT
+bin/singa-run.sh -exe tool/python/examples/cifar10_cnn.py -conf tool/python/examples/cluster.conf
+```
+
+### Layer class (inherited)
+
+* Data
+* Dense
+* Activation
+* Convolution2D
+* MaxPooling2D
+* AvgPooling2D
+* LRN2D
+* Dropout
+* RBM
+* Autoencoder
+
+#### for user defined layers (IN PROGRESS)
+
+The following classes are designed to construct user-defined layers for RNNLM example.
+
+* Embedding
+* RNNLM
+* UserLossRNNLM
+
+
+### Model class
+
+* Model class has `jobconf` (JobProto) and `layers` (layer list)
+
+Methods in Model class
+
+* add
+ * add Layer into Model
+ * 2 subclasses: Sequential model and Energy model
+
+* compile
+ * set Updater (i.e., optimizer) and Cluster (i.e., topology) components
+
+* fit
+ * set Training data and parameter values for the training
+ * (optional) set Validatiaon data and parameter values
+ * set Train_one_batch component
+ * specify `with_test` field if a user wants to run singa with test data simultaneously.
+ * [TODO] recieve train/validation results, e.g., accuracy, loss, ppl, etc.
+
+* evaluate
+ * set Testing data and parameter values for the testing
+ * specify `checkpoint_path` field if a user want to run singa only for testing.
+ * [TODO] recieve test results, e.g., accuracy, loss, ppl, etc.
+
+#### Results
+
+fit() and evaluate() return `result`, a dictionary containing
+
+* [key]: step number
+* [value]: a list of dictionay
+ * 'acc' for accuracy
+ * 'loss' for loss
+ * 'ppl' for ppl
+ * 'se' for squred error
+
+#### Other classes
+
+* Store
+* Algorithm
+* Updater
+* SGD (inherited)
+* AdaGrad (inherited)
+* Cluster
+
+
+## MLP Example
+
+An example (to generate job.conf for mnist)
+
+```
+X_train, X_test, workspace = mnist.load_data()
+
+m = Sequential('mlp', sys.argv)
+
+m.add(Dense(2500, init='uniform', activation='tanh'))
+m.add(Dense(2000, init='uniform', activation='tanh'))
+m.add(Dense(1500, init='uniform', activation='tanh'))
+m.add(Dense(1000, init='uniform', activation='tanh'))
+m.add(Dense(500, init='uniform', activation='tanh'))
+m.add(Dense(10, init='uniform', activation='softmax'))
+
+sgd = SGD(lr=0.001, lr_type='step')
+topo = Cluster(workspace)
+m.compile(loss='categorical_crossentropy', optimizer=sgd, cluster=topo)
+m.fit(X_train, nb_epoch=1000, with_test=True)
+result = m.evaluate(X_test, batch_size=100, test_steps=10, test_freq=60)
+```
+
+## CNN Example
+
+An example (to generate job.conf for cifar10)
+
+```
+X_train, X_test, workspace = cifar10.load_data()
+
+m = Sequential('cnn', sys.argv)
+
+m.add(Convolution2D(32, 5, 1, 2, w_std=0.0001, b_lr=2))
+m.add(MaxPooling2D(pool_size=(3,3), stride=2))
+m.add(Activation('relu'))
+m.add(LRN2D(3, alpha=0.00005, beta=0.75))
+
+m.add(Convolution2D(32, 5, 1, 2, b_lr=2))
+m.add(Activation('relu'))
+m.add(AvgPooling2D(pool_size=(3,3), stride=2))
+m.add(LRN2D(3, alpha=0.00005, beta=0.75))
+
+m.add(Convolution2D(64, 5, 1, 2))
+m.add(Activation('relu'))
+m.add(AvgPooling2D(pool_size=(3,3), stride=2))
+
+m.add(Dense(10, w_wd=250, b_lr=2, b_wd=0, activation='softmax'))
+
+sgd = SGD(decay=0.004, lr_type='fixed', step=(0,60000,65000), step_lr=(0.001,0.0001,0.00001))
+topo = Cluster(workspace)
+m.compile(updater=sgd, cluster=topo)
+m.fit(X_train, nb_epoch=1000, with_test=True)
+result = m.evaluate(X_test, 1000, test_steps=30, test_freq=300)
+```
+
+
+## RBM Example
+```
+rbmid = 3
+X_train, X_test, workspace = mnist.load_data(nb_rbm=rbmid)
+m = Energy('rbm'+str(rbmid), sys.argv)
+
+out_dim = [1000, 500, 250]
+m.add(RBM(out_dim, w_std=0.1, b_wd=0))
+
+sgd = SGD(lr=0.1, decay=0.0002, momentum=0.8)
+topo = Cluster(workspace)
+m.compile(optimizer=sgd, cluster=topo)
+m.fit(X_train, alg='cd', nb_epoch=6000)
+```
+
+## AutoEncoder Example
+```
+rbmid = 4
+X_train, X_test, workspace = mnist.load_data(nb_rbm=rbmid+1)
+m = Sequential('autoencoder', sys.argv)
+
+hid_dim = [1000, 500, 250, 30]
+m.add(Autoencoder(hid_dim, out_dim=784, activation='sigmoid', param_share=True))
+
+agd = AdaGrad(lr=0.01)
+topo = Cluster(workspace)
+m.compile(loss='mean_squared_error', optimizer=agd, cluster=topo)
+m.fit(X_train, alg='bp', nb_epoch=12200)
+```
+
+### TIPS
+
+Hidden layers for MLP can be written as
+```
+for n in [2500, 2000, 1500, 1000, 500]:
+ m.add(Dense(n, init='uniform', activation='tanh'))
+m.add(Dense(10, init='uniform', activation='softmax'))
+```
+
+Activation layer can be specified separately
+```
+m.add(Dense(2500, init='uniform'))
+m.add(Activation('tanh'))
+```
+
+Users can explicity specify weight and bias, and their values
+
+for example of MLP
+```
+par = Parameter(init='uniform', scale=0.05)
+m.add(Dense(2500, w_param=par, b_param=par, activation='tanh'))
+m.add(Dense(2000, w_param=par, b_param=par, activation='tanh'))
+m.add(Dense(1500, w_param=par, b_param=par, activation='tanh'))
+m.add(Dense(1000, w_param=par, b_param=par, activation='tanh'))
+m.add(Dense(500, w_param=par, b_param=par, activation='tanh'))
+m.add(Dense(10, w_param=par, b_param=par, activation='softmax'))
+```
+
+for example of Cifar10
+```
+parw = Parameter(init='gauss', std=0.0001)
+parb = Parameter(init='const', value=0)
+m.add(Convolution(32, 5, 1, 2, w_param=parw, b_param=parb, b_lr=2))
+m.add(MaxPooling2D(pool_size(3,3), stride=2))
+m.add(Activation('relu'))
+m.add(LRN2D(3, alpha=0.00005, beta=0.75))
+
+parw.update(std=0.01)
+m.add(Convolution(32, 5, 1, 2, w_param=parw, b_param=parb))
+m.add(Activation('relu'))
+m.add(AvgPooling2D(pool_size(3,3), stride=2))
+m.add(LRN2D(3, alpha=0.00005, beta=0.75))
+
+m.add(Convolution(64, 5, 1, 2, w_param=parw, b_param=parb, b_lr=1))
+m.add(Activation('relu'))
+m.add(AvgPooling2D(pool_size(3,3), stride=2))
+
+m.add(Dense(10, w_param=parw, w_wd=250, b_param=parb, b_lr=2, b_wd=0, activation='softmax'))
+```
+
+
+Alternative ways to add Data layer
+```
+X_train, X_test = mnist.load_data() // parameter values are set in load_data()
+m.fit(X_train, ...) // Data layer for training is added
+m.evaluate(X_test, ...) // Data layer for testing is added
+```
+```
+X_train, X_test = mnist.load_data() // parameter values are set in load_data()
+m.add(X_train) // explicitly add Data layer
+m.add(X_test) // explicitly add Data layer
+```
+```
+store = Store(path='train.bin', batch_size=64, ...) // parameter values are set explicitly
+m.add(Data(load='recordinput', phase='train', conf=store)) // Data layer is added
+store = Store(path='test.bin', batch_size=100, ...) // parameter values are set explicitly
+m.add(Data(load='recordinput', phase='test', conf=store)) // Data layer is added
+```
+
+### Parameter class
+
+Users need to set parameter and initial values. For example,
+
+* Parameter
+ * lr = (float) // learning rate
+ * wd = (float) // weight decay
+
+* Parameter initialization
+ * init = (string) // one of the types, 'uniform', 'constant', 'gaussian'
+ * for uniform [default]
+ * high = (float)
+ * low = (float)
+ * for constant
+ * value = (float)
+ * for gaussian
+ * mean = (float)
+ * std = (float)
+
+* Weight (w_param) is gaussian with mean=0, std=0.01 at default
+
+* Bias (b_param) is constant with value=0 at default
+
+* How to update the parameter fields
+ * for updating Weight, put 'w_' in front of field name
+ * for updating Bias, put 'b_' in front of field name
+
+Several ways to set Parameter values
+```
+m.add(Dense(10, w_mean=1, w_std=0.1, w_lr=2, w_wd=10, ...)
+```
+```
+parw = Parameter(lr=2, wd=10, init='constant', value=0)
+m.add(Dense(10, w_param=parw, ...)
+```
+```
+parw = Parameter(init='constant', value=0)
+m.add(Dense(10, w_param=parw, w_lr=2, w_wd=10, ...)
+```
+
+### Example to run singa
+
+(1) Run singa for training
+```
+m.fit(X_train, nb_epoch=1000)
+```
+
+(2) Run singa for training and validation
+```
+m.fit(X_train, validate_data=X_valid, nb_epoch=1000)
+```
+
+(3) Run singa for test while training
+```
+m.fit(X_train, nb_epoch=1000, with_test=True)
+result = m.evaluate(X_test, batch_size=100, test_steps=100)
+```
+
+(4) Run singa for test only
+Assume a checkpoint exists after training
+```
+result = m.evaluate(X_test, batch_size=100, checkpoint_path=workspace+'/checkpoint/step100-worker0')
+```
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7d43e273/tool/python/examples/__init__.py
----------------------------------------------------------------------
diff --git a/tool/python/examples/__init__.py b/tool/python/examples/__init__.py
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7d43e273/tool/python/examples/cifar10_cnn.py
----------------------------------------------------------------------
diff --git a/tool/python/examples/cifar10_cnn.py b/tool/python/examples/cifar10_cnn.py
new file mode 100755
index 0000000..9ef552b
--- /dev/null
+++ b/tool/python/examples/cifar10_cnn.py
@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+import sys, os
+sys.path.append(os.path.join(os.path.dirname(__file__),'..'))
+from singa.model import *
+from singa.datasets import cifar10
+
+X_train, X_test, workspace = cifar10.load_data()
+
+m = Sequential('cifar10-cnn', sys.argv)
+
+m.add(Convolution2D(32, 5, 1, 2, w_std=0.0001, b_lr=2))
+m.add(MaxPooling2D(pool_size=(3,3), stride=2))
+m.add(Activation('relu'))
+m.add(LRN2D(3, alpha=0.00005, beta=0.75))
+
+m.add(Convolution2D(32, 5, 1, 2, b_lr=2))
+m.add(Activation('relu'))
+m.add(AvgPooling2D(pool_size=(3,3), stride=2))
+m.add(LRN2D(3, alpha=0.00005, beta=0.75))
+
+m.add(Convolution2D(64, 5, 1, 2))
+m.add(Activation('relu'))
+m.add(AvgPooling2D(pool_size=(3,3), stride=2))
+
+m.add(Dense(10, w_wd=250, b_lr=2, b_wd=0, activation='softmax'))
+
+sgd = SGD(decay=0.004, lr_type='fixed', step=(0,60000,65000), step_lr=(0.001,0.0001,0.00001))
+topo = Cluster(workspace)
+m.compile(loss='categorical_crossentropy', optimizer=sgd, cluster=topo)
+m.fit(X_train, nb_epoch=1000, with_test=True)
+result = m.evaluate(X_test, test_steps=100, test_freq=300)
+
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7d43e273/tool/python/examples/cifar10_cnn_parameter.py
----------------------------------------------------------------------
diff --git a/tool/python/examples/cifar10_cnn_parameter.py b/tool/python/examples/cifar10_cnn_parameter.py
new file mode 100755
index 0000000..dd03f5c
--- /dev/null
+++ b/tool/python/examples/cifar10_cnn_parameter.py
@@ -0,0 +1,35 @@
+#!/usr/bin/env python
+import sys, os
+sys.path.append(os.path.join(os.path.dirname(__file__),'..'))
+from singa.model import *
+from singa.datasets import cifar10
+
+X_train, X_test, workspace = cifar10.load_data()
+
+m = Sequential('cifar10-cnn', sys.argv)
+
+parw = Parameter(init='gaussian', std=0.0001)
+parb = Parameter(init='constant')
+m.add(Convolution2D(32, 5, 1, 2, w_param=parw, b_param=parb, b_lr=2))
+m.add(MaxPooling2D(pool_size=(3,3), stride=2))
+m.add(Activation('relu'))
+m.add(LRN2D(3, alpha=0.00005, beta=0.75))
+
+parw.update(std=0.01)
+m.add(Convolution2D(32, 5, 1, 2, w_param=parw, b_param=parb))
+m.add(Activation('relu'))
+m.add(AvgPooling2D(pool_size=(3,3), stride=2))
+m.add(LRN2D(3, alpha=0.00005, beta=0.75))
+
+m.add(Convolution2D(64, 5, 1, 2, w_param=parw, b_param=parb, b_lr=1))
+m.add(Activation('relu'))
+m.add(AvgPooling2D(pool_size=(3,3), stride=2))
+
+m.add(Dense(10, w_param=parw, w_wd=250, b_param=parb, b_lr=2, b_wd=0, activation='softmax'))
+
+sgd = SGD(decay=0.004, lr_type='fixed', step=(0,60000,65000), step_lr=(0.001,0.0001,0.00001))
+topo = Cluster(workspace)
+m.compile(loss='categorical_crossentropy', optimizer=sgd, cluster=topo)
+m.fit(X_train, nb_epoch=100, with_test=True)
+result = m.evaluate(X_test, test_steps=10, test_freq=300)
+
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7d43e273/tool/python/examples/cluster.conf
----------------------------------------------------------------------
diff --git a/tool/python/examples/cluster.conf b/tool/python/examples/cluster.conf
new file mode 100644
index 0000000..16623d8
--- /dev/null
+++ b/tool/python/examples/cluster.conf
@@ -0,0 +1,6 @@
+cluster {
+ nworker_groups: 1
+ nserver_groups: 1
+ nworkers_per_group: 1
+ nworkers_per_procs: 1
+}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7d43e273/tool/python/examples/mnist_ae.py
----------------------------------------------------------------------
diff --git a/tool/python/examples/mnist_ae.py b/tool/python/examples/mnist_ae.py
new file mode 100755
index 0000000..0b7e590
--- /dev/null
+++ b/tool/python/examples/mnist_ae.py
@@ -0,0 +1,27 @@
+#!/usr/bin/env python
+import sys, os
+sys.path.append(os.path.join(os.path.dirname(__file__),'..'))
+from singa.model import *
+from singa.datasets import mnist
+
+# Sample parameter values for Autoencoder example
+rbmid = 4
+pvalues = {'batchsize' : 100, 'shape' : 784, 'std_value' : 255}
+X_train, X_test, workspace = mnist.load_data(
+ workspace = 'examples/rbm/autoencoder',
+ nb_rbm = rbmid+1,
+ checkpoint_steps = 6000,
+ **pvalues)
+
+m = Sequential('autoencoder', sys.argv)
+
+hid_dim = [1000, 500, 250, 30]
+m.add(Autoencoder(hid_dim, out_dim=784, activation='sigmoid', param_share=True))
+
+
+agd = AdaGrad(lr=0.01)
+topo = Cluster(workspace)
+m.compile(loss='mean_squared_error', optimizer=agd, cluster=topo)
+m.fit(X_train, alg='bp', nb_epoch=12200, with_test=True)
+result = m.evaluate(X_test, test_steps=100, test_freq=1000)
+
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7d43e273/tool/python/examples/mnist_mlp.py
----------------------------------------------------------------------
diff --git a/tool/python/examples/mnist_mlp.py b/tool/python/examples/mnist_mlp.py
new file mode 100755
index 0000000..da5ccce
--- /dev/null
+++ b/tool/python/examples/mnist_mlp.py
@@ -0,0 +1,33 @@
+#!/usr/bin/env python
+import sys, os
+sys.path.append(os.path.join(os.path.dirname(__file__),'..'))
+from singa.model import *
+from singa.datasets import mnist
+
+# Sample parameter values for Mnist MLP example
+pvalues = {'batchsize' : 64, 'shape' : 784, 'random_skip' : 5000,
+ 'std_value' : 127.5, 'mean_value' : 127.5}
+X_train, X_test, workspace = mnist.load_data(**pvalues)
+
+m = Sequential('mlp', argv=sys.argv)
+
+''' Weight and Bias are initialized by
+ uniform distribution with scale=0.05 at default
+'''
+m.add(Dense(2500, init='uniform', activation='tanh'))
+m.add(Dense(2000, init='uniform', activation='tanh'))
+m.add(Dense(1500, init='uniform', activation='tanh'))
+m.add(Dense(1000, init='uniform', activation='tanh'))
+m.add(Dense(500, init='uniform', activation='tanh'))
+m.add(Dense(10, init='uniform', activation='softmax'))
+
+sgd = SGD(lr=0.001, lr_type='step')
+topo = Cluster(workspace)
+m.compile(loss='categorical_crossentropy', optimizer=sgd, cluster=topo)
+
+m.fit(X_train, nb_epoch=100, with_test=True)
+result = m.evaluate(X_test, batch_size=100, test_steps=10)
+
+#e.g., display result
+#for k, v in sorted(result.items(), key=lambda x: x[0]):
+# print k, v
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7d43e273/tool/python/examples/mnist_mlp_parameter.py
----------------------------------------------------------------------
diff --git a/tool/python/examples/mnist_mlp_parameter.py b/tool/python/examples/mnist_mlp_parameter.py
new file mode 100755
index 0000000..24fc960
--- /dev/null
+++ b/tool/python/examples/mnist_mlp_parameter.py
@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+import sys, os
+sys.path.append(os.path.join(os.path.dirname(__file__),'..'))
+from singa.model import *
+from singa.datasets import mnist
+
+# Sample parameter values for Mnist MLP example
+pvalues = {'batchsize' : 64, 'shape' : 784,
+ 'random_skip' : 5000,
+ 'std_value' : 127.5, 'mean_value' : 127.5}
+X_train, X_test, workspace = mnist.load_data(**pvalues)
+
+m = Sequential('mlp', argv=sys.argv)
+
+par = Parameter(init='uniform', scale=0.05)
+m.add(Dense(2500, w_param=par, b_param=par, activation='tanh'))
+m.add(Dense(2000, w_param=par, b_param=par, activation='tanh'))
+m.add(Dense(1500, w_param=par, b_param=par, activation='tanh'))
+m.add(Dense(1000, w_param=par, b_param=par, activation='tanh'))
+m.add(Dense(500, w_param=par, b_param=par, activation='tanh'))
+m.add(Dense(10, w_param=par, b_param=par, activation='softmax'))
+
+sgd = SGD(lr=0.001, lr_type='step')
+topo = Cluster(workspace)
+m.compile(loss='categorical_crossentropy', optimizer=sgd, cluster=topo)
+
+m.fit(X_train, nb_epoch=100, with_test=True)
+result = m.evaluate(X_test, batch_size=100, test_steps=10)
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7d43e273/tool/python/examples/mnist_mlp_test.py
----------------------------------------------------------------------
diff --git a/tool/python/examples/mnist_mlp_test.py b/tool/python/examples/mnist_mlp_test.py
new file mode 100755
index 0000000..67cf3b3
--- /dev/null
+++ b/tool/python/examples/mnist_mlp_test.py
@@ -0,0 +1,30 @@
+#!/usr/bin/env python
+import sys, os
+sys.path.append(os.path.join(os.path.dirname(__file__),'..'))
+from singa.model import *
+from singa.datasets import mnist
+
+# Sample parameter values for Mnist MLP example
+pvalues = {'batchsize' : 64, 'shape' : 784,
+ 'std_value' : 127.5, 'mean_value' : 127.5}
+X_train, X_test, workspace = mnist.load_data(**pvalues)
+
+m = Sequential('mlp', argv=sys.argv)
+
+m.add(Dense(2500, init='uniform', activation='tanh'))
+m.add(Dense(2000, init='uniform', activation='tanh'))
+m.add(Dense(1500, init='uniform', activation='tanh'))
+m.add(Dense(1000, init='uniform', activation='tanh'))
+m.add(Dense(500, init='uniform', activation='tanh'))
+m.add(Dense(10, init='uniform', activation='softmax'))
+
+sgd = SGD(lr=0.001, lr_type='step')
+topo = Cluster(workspace)
+m.compile(loss='categorical_crossentropy', optimizer=sgd, cluster=topo)
+
+''' For doing test only, normally users sets checkpoint path
+ e.g., assume that checkpoint exists by
+ m.fit(X_train, nb_epoch=100, checkpoint_freq=100)
+'''
+path = workspace+'/checkpoint/step100-worker0'
+result = m.evaluate(X_test, batch_size=100, test_steps=100, checkpoint_path=path)
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7d43e273/tool/python/examples/mnist_rbm1.py
----------------------------------------------------------------------
diff --git a/tool/python/examples/mnist_rbm1.py b/tool/python/examples/mnist_rbm1.py
new file mode 100755
index 0000000..765be8b
--- /dev/null
+++ b/tool/python/examples/mnist_rbm1.py
@@ -0,0 +1,24 @@
+#!/usr/bin/env python
+import sys, os
+sys.path.append(os.path.join(os.path.dirname(__file__),'..'))
+from singa.model import *
+from singa.datasets import mnist
+
+rbmid = 1
+pvalues = {'batchsize' : 100, 'shape' : 784, 'std_value' : 255}
+X_train, X_test, workspace = mnist.load_data(
+ workspace = 'examples/rbm/rbm1',
+ nb_rbm = rbmid,
+ checkpoint_steps = 6000,
+ **pvalues)
+
+m = Energy('rbm'+str(rbmid), sys.argv)
+
+m.add(RBM(1000, w_std=0.1, b_wd=0))
+
+sgd = SGD(lr=0.1, decay=0.0002, momentum=0.8)
+topo = Cluster(workspace)
+m.compile(optimizer=sgd, cluster=topo)
+m.fit(X_train, alg='cd', nb_epoch=6000)
+#result = m.evaluate(X_test, test_steps=100, test_freq=500)
+
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7d43e273/tool/python/examples/mnist_rbm1_parameter.py
----------------------------------------------------------------------
diff --git a/tool/python/examples/mnist_rbm1_parameter.py b/tool/python/examples/mnist_rbm1_parameter.py
new file mode 100755
index 0000000..54fe421
--- /dev/null
+++ b/tool/python/examples/mnist_rbm1_parameter.py
@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+import sys, os
+sys.path.append(os.path.join(os.path.dirname(__file__),'..'))
+from singa.model import *
+from singa.datasets import mnist
+
+rbmid = 1
+pvalues = {'batchsize' : 100, 'shape' : 784, 'std_value' : 255}
+X_train, X_test, workspace = mnist.load_data(
+ workspace = 'examples/rbm/rbm1',
+ nb_rbm = rbmid,
+ checkpoint_steps = 6000,
+ **pvalues)
+
+m = Energy('rbm'+str(rbmid), sys.argv)
+
+parw = Parameter(init='gaussian', mean=0, std=0.1)
+parb = Parameter(wd=0, init='constant', value=0)
+m.add(RBM(1000, w_param=parw, b_param=parb))
+
+sgd = SGD(lr=0.1, decay=0.0002, momentum=0.8)
+topo = Cluster(workspace)
+m.compile(optimizer=sgd, cluster=topo)
+m.fit(X_train, alg='cd', nb_epoch=6000)
+#result = m.evaluate(X_test, test_steps=100, test_freq=500)
+
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7d43e273/tool/python/examples/mnist_rbm2.py
----------------------------------------------------------------------
diff --git a/tool/python/examples/mnist_rbm2.py b/tool/python/examples/mnist_rbm2.py
new file mode 100755
index 0000000..f4d187f
--- /dev/null
+++ b/tool/python/examples/mnist_rbm2.py
@@ -0,0 +1,25 @@
+#!/usr/bin/env python
+import sys, os
+sys.path.append(os.path.join(os.path.dirname(__file__),'..'))
+from singa.model import *
+from singa.datasets import mnist
+
+rbmid = 2
+pvalues = {'batchsize' : 100, 'shape' : 784, 'std_value' : 255}
+X_train, X_test, workspace = mnist.load_data(
+ workspace = 'examples/rbm/rbm2',
+ nb_rbm = rbmid,
+ checkpoint_steps = 6000,
+ **pvalues)
+
+m = Energy('rbm'+str(rbmid), sys.argv)
+
+out_dim = [1000, 500]
+m.add(RBM(out_dim, w_std=0.1, b_wd=0))
+
+sgd = SGD(lr=0.1, decay=0.0002, momentum=0.8)
+topo = Cluster(workspace)
+m.compile(optimizer=sgd, cluster=topo)
+m.fit(X_train, alg='cd', nb_epoch=6000)
+#result = m.evaluate(X_test, test_steps=100, test_freq=500)
+
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7d43e273/tool/python/examples/mnist_rbm2_parameter.py
----------------------------------------------------------------------
diff --git a/tool/python/examples/mnist_rbm2_parameter.py b/tool/python/examples/mnist_rbm2_parameter.py
new file mode 100755
index 0000000..9837836
--- /dev/null
+++ b/tool/python/examples/mnist_rbm2_parameter.py
@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+import sys, os
+sys.path.append(os.path.join(os.path.dirname(__file__),'..'))
+from singa.model import *
+from singa.datasets import mnist
+
+rbmid = 2
+pvalues = {'batchsize' : 100, 'shape' : 784, 'std_value' : 255}
+X_train, X_test, workspace = mnist.load_data(
+ workspace = 'examples/rbm/rbm2',
+ nb_rbm = rbmid,
+ checkpoint_steps = 6000,
+ **pvalues)
+
+m = Energy('rbm'+str(rbmid), sys.argv)
+
+out_dim = [1000, 500]
+parw = Parameter(init='gaussian', mean=0, std=0.1)
+parb = Parameter(wd=0, init='constant', value=0)
+m.add(RBM(out_dim, w_param=parw, b_param=parb))
+
+
+sgd = SGD(lr=0.1, decay=0.0002, momentum=0.8)
+topo = Cluster(workspace)
+m.compile(optimizer=sgd, cluster=topo)
+m.fit(X_train, alg='cd', nb_epoch=6000)
+#result = m.evaluate(X_test, test_steps=100, test_freq=500)
+
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7d43e273/tool/python/examples/mnist_rbm3.py
----------------------------------------------------------------------
diff --git a/tool/python/examples/mnist_rbm3.py b/tool/python/examples/mnist_rbm3.py
new file mode 100755
index 0000000..48bbe38
--- /dev/null
+++ b/tool/python/examples/mnist_rbm3.py
@@ -0,0 +1,25 @@
+#!/usr/bin/env python
+import sys, os
+sys.path.append(os.path.join(os.path.dirname(__file__),'..'))
+from singa.model import *
+from singa.datasets import mnist
+
+rbmid = 3
+pvalues = {'batchsize' : 100, 'shape' : 784, 'std_value' : 255}
+X_train, X_test, workspace = mnist.load_data(
+ workspace = 'examples/rbm/rbm3',
+ nb_rbm = rbmid,
+ checkpoint_steps = 6000,
+ **pvalues)
+
+m = Energy('rbm'+str(rbmid), sys.argv)
+
+out_dim = [1000, 500, 250]
+m.add(RBM(out_dim, w_std=0.1, b_wd=0))
+
+sgd = SGD(lr=0.1, decay=0.0002, momentum=0.8)
+topo = Cluster(workspace)
+m.compile(optimizer=sgd, cluster=topo)
+m.fit(X_train, alg='cd', nb_epoch=6000)
+#result = m.evaluate(X_test, test_steps=100, test_freq=500)
+
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7d43e273/tool/python/examples/mnist_rbm3_parameter.py
----------------------------------------------------------------------
diff --git a/tool/python/examples/mnist_rbm3_parameter.py b/tool/python/examples/mnist_rbm3_parameter.py
new file mode 100755
index 0000000..6c9a378
--- /dev/null
+++ b/tool/python/examples/mnist_rbm3_parameter.py
@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+import sys, os
+sys.path.append(os.path.join(os.path.dirname(__file__),'..'))
+from singa.model import *
+from singa.datasets import mnist
+
+rbmid = 3
+pvalues = {'batchsize' : 100, 'shape' : 784, 'std_value' : 255}
+X_train, X_test, workspace = mnist.load_data(
+ workspace = 'examples/rbm/rbm3',
+ nb_rbm = rbmid,
+ checkpoint_steps = 6000,
+ **pvalues)
+
+m = Energy('rbm'+str(rbmid), sys.argv)
+
+out_dim = [1000, 500, 250]
+parw = Parameter(init='gaussian', mean=0, std=0.1)
+parb = Parameter(wd=0, init='constant', value=0)
+m.add(RBM(out_dim, w_param=parw, b_param=parb))
+
+
+sgd = SGD(lr=0.1, decay=0.0002, momentum=0.8)
+topo = Cluster(workspace)
+m.compile(optimizer=sgd, cluster=topo)
+m.fit(X_train, alg='cd', nb_epoch=6000)
+#result = m.evaluate(X_test, test_steps=100, test_freq=500)
+
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7d43e273/tool/python/examples/mnist_rbm4.py
----------------------------------------------------------------------
diff --git a/tool/python/examples/mnist_rbm4.py b/tool/python/examples/mnist_rbm4.py
new file mode 100755
index 0000000..f12b739
--- /dev/null
+++ b/tool/python/examples/mnist_rbm4.py
@@ -0,0 +1,25 @@
+#!/usr/bin/env python
+import sys, os
+sys.path.append(os.path.join(os.path.dirname(__file__),'..'))
+from singa.model import *
+from singa.datasets import mnist
+
+rbmid = 4
+pvalues = {'batchsize' : 100, 'shape' : 784, 'std_value' : 255}
+X_train, X_test, workspace = mnist.load_data(
+ workspace = 'examples/rbm/rbm'+str(rbmid),
+ nb_rbm = rbmid,
+ checkpoint_steps = 6000,
+ **pvalues)
+
+m = Energy('rbm'+str(rbmid), sys.argv)
+
+out_dim = [1000, 500, 250, 30]
+m.add(RBM(out_dim, sampling='gaussian', w_std=0.1, b_wd=0))
+
+sgd = SGD(lr=0.001, decay=0.0002, momentum=0.8)
+topo = Cluster(workspace)
+m.compile(optimizer=sgd, cluster=topo)
+m.fit(X_train, alg='cd', nb_epoch=6000)
+#result = m.evaluate(X_test, test_steps=100, test_freq=500)
+
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7d43e273/tool/python/examples/mnist_rbm4_parameter.py
----------------------------------------------------------------------
diff --git a/tool/python/examples/mnist_rbm4_parameter.py b/tool/python/examples/mnist_rbm4_parameter.py
new file mode 100755
index 0000000..2a7be1d
--- /dev/null
+++ b/tool/python/examples/mnist_rbm4_parameter.py
@@ -0,0 +1,27 @@
+#!/usr/bin/env python
+import sys, os
+sys.path.append(os.path.join(os.path.dirname(__file__),'..'))
+from singa.model import *
+from singa.datasets import mnist
+
+rbmid = 4
+pvalues = {'batchsize' : 100, 'shape' : 784, 'std_value' : 255}
+X_train, X_test, workspace = mnist.load_data(
+ workspace = 'examples/rbm/rbm'+rbmid,
+ nb_rbm = rbmid,
+ checkpoint_steps = 6000,
+ **pvalues)
+
+m = Energy('rbm'+str(rbmid), sys.argv)
+
+out_dim = [1000, 500, 250, 30]
+parw = Parameter(init='gaussian', mean=0, std=0.1)
+parb = Parameter(wd=0, init='constant', value=0)
+m.add(RBM(out_dim, w_param=parw, b_param=parb))
+
+sgd = SGD(lr=0.1, decay=0.0002, momentum=0.8)
+topo = Cluster(workspace)
+m.compile(optimizer=sgd, cluster=topo)
+m.fit(X_train, alg='cd', nb_epoch=6000)
+#result = m.evaluate(X_test, test_steps=100, test_freq=500)
+
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7d43e273/tool/python/examples/rnnlm_usermodel.py
----------------------------------------------------------------------
diff --git a/tool/python/examples/rnnlm_usermodel.py b/tool/python/examples/rnnlm_usermodel.py
new file mode 100755
index 0000000..1b49321
--- /dev/null
+++ b/tool/python/examples/rnnlm_usermodel.py
@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+import sys, os
+sys.path.append(os.path.join(os.path.dirname(__file__),'..'))
+from singa.model import *
+from singa.datasets import rnnlm
+
+vocab_size = 3720
+
+X_train, X_valid, workspace = rnnlm.load_data()
+
+m = Sequential('rnnlm', sys.argv)
+
+parw = Parameter(init='uniform', range=0.3)
+m.add(Embedding(in_dim=vocab_size, out_dim=15, w_param=parw))
+m.add(RNNLM(1, w_param=parw))
+
+sgd = SGD(lr_type='fixed', step=(0,48810,56945,65080,73215), step_lr=(0.1,0.05,0.025,0.0125,0.00625))
+topo = Cluster(workspace)
+m.compile(loss='user_loss_rnnlm', in_dim=vocab_size, nclass=100, optimizer=sgd, cluster=topo)
+
+m.fit(X_train, validate=X_valid, validate_steps=683, nb_epoch=81350, execpath='examples/rnnlm/rnnlm.bin')
+#result = m.evaluate(X_valid, validate_steps=683, validate_freq=8135, execpath='examples/rnnlm/rnnlm.bin')
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7d43e273/tool/python/singa.py
----------------------------------------------------------------------
diff --git a/tool/python/singa.py b/tool/python/singa.py
index c179192..6d7fbdf 100755
--- a/tool/python/singa.py
+++ b/tool/python/singa.py
@@ -40,4 +40,5 @@ if __name__ == '__main__':
d = driver.Driver()
d.InitLog(sys.argv[0])
d.Init(sys.argv)
- d.Train(False,b)
+# d.Train(False,b)
+ d.Test(b)
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7d43e273/tool/python/singa/__init__.py
----------------------------------------------------------------------
diff --git a/tool/python/singa/__init__.py b/tool/python/singa/__init__.py
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7d43e273/tool/python/singa/datasets/__init__.py
----------------------------------------------------------------------
diff --git a/tool/python/singa/datasets/__init__.py b/tool/python/singa/datasets/__init__.py
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7d43e273/tool/python/singa/datasets/cifar10.py
----------------------------------------------------------------------
diff --git a/tool/python/singa/datasets/cifar10.py b/tool/python/singa/datasets/cifar10.py
new file mode 100644
index 0000000..65bcd60
--- /dev/null
+++ b/tool/python/singa/datasets/cifar10.py
@@ -0,0 +1,34 @@
+#!/usr/bin/env python
+from singa.model import *
+
+def load_data(
+ workspace = None,
+ backend = 'kvfile',
+ batchsize = 64,
+ random = 5000,
+ shape = (3, 32, 32),
+ std = 127.5,
+ mean = 127.5
+ ):
+
+ # using cifar10 dataset
+ data_dir = 'examples/cifar10'
+ path_train = data_dir + '/train_data.bin'
+ path_test = data_dir + '/test_data.bin'
+ path_mean = data_dir + '/image_mean.bin'
+ if workspace == None: workspace = data_dir
+
+ store = Store(path=path_train, mean_file=path_mean, backend=backend,
+ random_skip=random, batchsize=batchsize,
+ shape=shape)
+
+ data_train = Data(load='recordinput', phase='train', conf=store)
+
+ store = Store(path=path_test, mean_file=path_mean, backend=backend,
+ batchsize=batchsize,
+ shape=shape)
+
+ data_test = Data(load='recordinput', phase='test', conf=store)
+
+ return data_train, data_test, workspace
+
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7d43e273/tool/python/singa/datasets/mnist.py
----------------------------------------------------------------------
diff --git a/tool/python/singa/datasets/mnist.py b/tool/python/singa/datasets/mnist.py
new file mode 100644
index 0000000..c8695ec
--- /dev/null
+++ b/tool/python/singa/datasets/mnist.py
@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+from singa.model import *
+
+def load_data(
+ workspace = None,
+ backend = 'kvfile',
+ nb_rbm = 0, # the number of layers for RBM and Autoencoder
+ checkpoint_steps = 0,
+ **pvalues
+ ):
+
+ # using mnist dataset
+ data_dir = 'examples/mnist'
+ path_train = data_dir + '/train_data.bin'
+ path_test = data_dir + '/test_data.bin'
+ if workspace == None: workspace = data_dir
+
+ # checkpoint path to load
+ checkpoint_list = None
+ if checkpoint_steps > 0:
+ workerid = 0
+ checkpoint_list = []
+ for i in range(nb_rbm-1, 0, -1):
+ checkpoint_list.append('examples/rbm/rbm{0}/checkpoint/step{1}-worker{2}'.format(str(i),checkpoint_steps,workerid))
+
+ store = Store(path=path_train, backend=backend, **pvalues)
+ data_train = Data(load='recordinput', phase='train', conf=store, checkpoint=checkpoint_list)
+
+ store = Store(path=path_test, backend=backend, **pvalues)
+ data_test = Data(load='recordinput', phase='test', conf=store)
+
+ return data_train, data_test, workspace
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7d43e273/tool/python/singa/datasets/rnnlm.py
----------------------------------------------------------------------
diff --git a/tool/python/singa/datasets/rnnlm.py b/tool/python/singa/datasets/rnnlm.py
new file mode 100644
index 0000000..ef8142a
--- /dev/null
+++ b/tool/python/singa/datasets/rnnlm.py
@@ -0,0 +1,20 @@
+#!/usr/bin/env python
+from singa.model import *
+
+def load_data(
+ workspace = 'examples/rnnlm',
+ backend = 'kvfile',
+ max_window = 10
+ ):
+
+ path_train = workspace + '/train_data.bin'
+ path_valid = workspace + '/valid_data.bin'
+ path_test = workspace + '/test_data.bin'
+
+
+ data_train = Data(load='kData', phase='train', path=path_train, backend=backend, max_window=max_window)
+
+ data_valid = Data(load='kData', phase='val', path=path_valid, max_window=max_window)
+
+ return data_train, data_valid, workspace
+
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7d43e273/tool/python/singa/driver.i
----------------------------------------------------------------------
diff --git a/tool/python/singa/driver.i b/tool/python/singa/driver.i
index 56599b6..f756d57 100644
--- a/tool/python/singa/driver.i
+++ b/tool/python/singa/driver.i
@@ -37,6 +37,7 @@ public:
void Train(bool resume, const std::string job_conf);
void Init(int argc, char **argv);
void InitLog(char* arg);
+void Test(const std::string job_conf);
};
}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7d43e273/tool/python/singa/driver.py
----------------------------------------------------------------------
diff --git a/tool/python/singa/driver.py b/tool/python/singa/driver.py
index c5f3b4f..c1aac7f 100644
--- a/tool/python/singa/driver.py
+++ b/tool/python/singa/driver.py
@@ -107,6 +107,7 @@ class Driver(_object):
def Train(self, *args): return _driver.Driver_Train(self, *args)
def Init(self, *args): return _driver.Driver_Init(self, *args)
def InitLog(self, *args): return _driver.Driver_InitLog(self, *args)
+ def Test(self, *args): return _driver.Driver_Test(self, *args)
def __init__(self):
this = _driver.new_Driver()
try: self.this.append(this)
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7d43e273/tool/python/singa/driver_wrap.cxx
----------------------------------------------------------------------
diff --git a/tool/python/singa/driver_wrap.cxx b/tool/python/singa/driver_wrap.cxx
index 99756f1..e6e6de2 100644
--- a/tool/python/singa/driver_wrap.cxx
+++ b/tool/python/singa/driver_wrap.cxx
@@ -4579,6 +4579,38 @@ fail:
}
+SWIGINTERN PyObject *_wrap_Driver_Test(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+ PyObject *resultobj = 0;
+ singa::Driver *arg1 = (singa::Driver *) 0 ;
+ std::string arg2 ;
+ void *argp1 = 0 ;
+ int res1 = 0 ;
+ PyObject * obj0 = 0 ;
+ PyObject * obj1 = 0 ;
+
+ if (!PyArg_ParseTuple(args,(char *)"OO:Driver_Test",&obj0,&obj1)) SWIG_fail;
+ res1 = SWIG_ConvertPtr(obj0, &argp1,SWIGTYPE_p_singa__Driver, 0 | 0 );
+ if (!SWIG_IsOK(res1)) {
+ SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "Driver_Test" "', argument " "1"" of type '" "singa::Driver *""'");
+ }
+ arg1 = reinterpret_cast< singa::Driver * >(argp1);
+ {
+ std::string *ptr = (std::string *)0;
+ int res = SWIG_AsPtr_std_string(obj1, &ptr);
+ if (!SWIG_IsOK(res) || !ptr) {
+ SWIG_exception_fail(SWIG_ArgError((ptr ? res : SWIG_TypeError)), "in method '" "Driver_Test" "', argument " "2"" of type '" "std::string const""'");
+ }
+ arg2 = *ptr;
+ if (SWIG_IsNewObj(res)) delete ptr;
+ }
+ (arg1)->Test(arg2);
+ resultobj = SWIG_Py_Void();
+ return resultobj;
+fail:
+ return NULL;
+}
+
+
SWIGINTERN PyObject *_wrap_new_Driver(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
PyObject *resultobj = 0;
singa::Driver *result = 0 ;
@@ -4643,6 +4675,7 @@ static PyMethodDef SwigMethods[] = {
{ (char *)"Driver_Train", _wrap_Driver_Train, METH_VARARGS, NULL},
{ (char *)"Driver_Init", _wrap_Driver_Init, METH_VARARGS, NULL},
{ (char *)"Driver_InitLog", _wrap_Driver_InitLog, METH_VARARGS, NULL},
+ { (char *)"Driver_Test", _wrap_Driver_Test, METH_VARARGS, NULL},
{ (char *)"new_Driver", _wrap_new_Driver, METH_VARARGS, NULL},
{ (char *)"delete_Driver", _wrap_delete_Driver, METH_VARARGS, NULL},
{ (char *)"Driver_swigregister", Driver_swigregister, METH_VARARGS, NULL},
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7d43e273/tool/python/singa/initializations.py
----------------------------------------------------------------------
diff --git a/tool/python/singa/initializations.py b/tool/python/singa/initializations.py
new file mode 100644
index 0000000..f3037cd
--- /dev/null
+++ b/tool/python/singa/initializations.py
@@ -0,0 +1,34 @@
+#!/usr/bin/env python
+
+def get(identifier, **kwargs):
+
+ field = {}
+
+ if identifier == 'none':
+ return
+
+ if identifier == 'uniform':
+ scale = kwargs['scale'] if 'scale' in kwargs else 0.05
+ names = ['low', 'high']
+ values = [-scale, scale]
+
+ elif identifier == 'constant':
+ names = ['value']
+ values = [0]
+
+ elif identifier == 'gaussian':
+ names = ['mean', 'std']
+ values = [0, 0.01]
+
+ elif identifier == 'conv2d':
+ names = ['stride', 'pad']
+ values = [1, 0]
+
+ elif identifier == 'lrn2d':
+ names = ['alpha', 'beta', 'knorm']
+ values = [1, 0.75, 1]
+
+ for i in range(len(names)):
+ field[names[i]] = kwargs[names[i]] if names[i] in kwargs else values[i]
+
+ return field
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7d43e273/tool/python/singa/layer.py
----------------------------------------------------------------------
diff --git a/tool/python/singa/layer.py b/tool/python/singa/layer.py
new file mode 100644
index 0000000..1744d50
--- /dev/null
+++ b/tool/python/singa/layer.py
@@ -0,0 +1,300 @@
+#!/usr/bin/env python
+from parameter import *
+from utils.utility import *
+from utils.message import *
+from google.protobuf import text_format
+
+class Layer(object):
+ def __init__(self, **kwargs):
+ self.layer = Message('Layer', **kwargs).proto
+ # required
+ if not 'name' in kwargs:
+ setval(self.layer, name=generateName('layer', 1))
+
+ # srclayers are set in Model.build()
+ self.is_datalayer = False
+
+class Data(Layer):
+ def __init__(self, load, phase='train', checkpoint=None,
+ conf=None, **kwargs):
+ assert load != None, 'data type should be specified'
+ if load == 'kData':
+ super(Data, self).__init__(name=generateName('data'), user_type=load)
+ else:
+ self.layer_type = enumLayerType(load)
+ super(Data, self).__init__(name=generateName('data'), type=self.layer_type)
+
+ # include/exclude
+ setval(self.layer, include=enumPhase(phase))
+ #setval(self.layer, exclude=kTest if phase=='train' else kTrain)
+
+ if conf == None:
+ if load == 'kData':
+ setval(self.layer.Extensions[data_conf], **kwargs)
+ else:
+ setval(self.layer.store_conf, **kwargs)
+ else:
+ setval(self.layer, store_conf=conf.proto)
+ self.is_datalayer = True
+
+ self.checkpoint = checkpoint # checkpoint for training data
+
+
+class Convolution2D(Layer):
+ def __init__(self, nb_filter=0, kernel=0, stride=1, pad=0,
+ init=None, w_param=None, b_param=None,
+ activation=None, **kwargs):
+ '''
+ required
+ nb_filter = (int) // the number of filters
+ kernel = (int) // the size of filter
+ optional
+ stride = (int) // the size of stride
+ pad = (int) // the size of padding
+ '''
+ assert nb_filter > 0 and kernel > 0, 'should be set as positive int'
+ super(Convolution2D, self).__init__(name=generateName('conv',1), type=kCConvolution)
+ fields = {'num_filters' : nb_filter,
+ 'kernel' : kernel,
+ 'stride' : stride,
+ 'pad' : pad}
+ setval(self.layer.convolution_conf, **fields)
+
+ # parameter w
+ if w_param == None:
+ self.init = 'gaussian' if init==None else init
+ w_param = Parameter(init=self.init)
+ setParamField(w_param.param, 'w', True, **kwargs)
+ setval(self.layer, param=w_param.param)
+
+ # parameter b
+ if b_param == None:
+ self.init = 'constant' if init==None else init
+ b_param = Parameter(init=self.init) # default: constant
+ setParamField(b_param.param, 'b', True, **kwargs)
+ setval(self.layer, param=b_param.param)
+
+ # following layers: e.g., activation, dropout, etc.
+ if activation:
+ self.mask = Activation(activation=activation).layer
+
+class MaxPooling2D(Layer):
+ def __init__(self, pool_size=None, stride=1, ignore_border=True, **kwargs):
+ '''
+ required
+ pool_size = (int|tuple) // the size for pooling
+ optional
+ stride = (int) // the size of striding
+ ignore_border = (bool) // flag for padding
+ **kwargs // fields for Layer class
+ '''
+ assert pool_size != None, 'pool_size is required'
+ if type(pool_size) == int:
+ pool_size = (pool_size, pool_size)
+ assert type(pool_size) == tuple and \
+ pool_size[0] == pool_size[1], 'pool size should be square in Singa'
+ super(MaxPooling2D, self).__init__(name=generateName('pool'), type=kCPooling, **kwargs)
+ fields = {'pool' : PoolingProto().MAX,
+ 'kernel' : pool_size[0],
+ 'stride' : stride,
+ 'pad' : 0 if ignore_border else 1}
+ setval(self.layer.pooling_conf, **fields)
+
+class AvgPooling2D(Layer):
+ def __init__(self, pool_size=None, stride=1, ignore_border=True, **kwargs):
+ '''
+ required
+ pool_size = (int|tuple) // size for pooling
+ optional
+ stride = (int) // size of striding
+ ignore_border = (bool) // flag for padding
+ **kwargs // fields for Layer class
+ '''
+ assert pool_size != None, 'pool_size is required'
+ if type(pool_size) == int:
+ pool_size = (pool_size, pool_size)
+ assert type(pool_size) == tuple and \
+ pool_size[0] == pool_size[1], 'pool size should be square in Singa'
+ super(AvgPooling2D, self).__init__(name=generateName('pool'), type=kCPooling, **kwargs)
+ self.layer.pooling_conf.pool = PoolingProto().AVG
+ fields = {'pool' : PoolingProto().AVG,
+ 'kernel' : pool_size[0],
+ 'stride' : stride,
+ 'pad' : 0 if ignore_border else 1}
+ setval(self.layer.pooling_conf, **fields)
+
+class LRN2D(Layer):
+ def __init__(self, size=0, **kwargs):
+ super(LRN2D, self).__init__(name=generateName('norm'), type=kLRN)
+ # required
+ assert size != 0, 'local size should be set'
+ self.layer.lrn_conf.local_size = size
+ init_value = initializations.get('lrn2d', **kwargs)
+ setval(self.layer.lrn_conf, **init_value)
+
+
+class Activation(Layer):
+ def __init__(self, activation='stanh', topk=1):
+ self.name = activation
+ if activation == 'tanh': activation = 'stanh' # <-- better way to set?
+ self.layer_type = enumLayerType(activation)
+ super(Activation, self).__init__(name=generateName(self.name), type=self.layer_type)
+ if activation == 'softmaxloss':
+ self.layer.softmaxloss_conf.topk = topk
+
+class Dropout(Layer):
+ def __init__(self, ratio=0.5):
+ self.name = 'dropout'
+ self.layer_type = kDropout
+ super(Dropout, self).__init__(name=generateName(self.name), type=self.layer_type)
+ self.layer.dropout_conf.dropout_ratio = ratio
+
+
+class RGB(Layer):
+ def __init__(self, meanfile=None, **kwargs):
+ assert meanfile != None, 'meanfile should be specified'
+ self.name = 'rgb'
+ self.layer_type = kRGBImage
+ super(RGB, self).__init__(name=generateName(self.name), type=self.layer_type)
+ self.layer.rgbimage_conf.meanfile = meanfile
+
+class Dense(Layer):
+ def __init__(self, output_dim=0, activation=None,
+ init=None, w_param=None, b_param=None, input_dim=None,
+ **kwargs):
+ '''
+ required
+ output_dim = (int)
+ optional
+ activation = (string)
+ init = (string) // 'unirom', 'gaussian', 'constant'
+ **kwargs
+ w_lr = (float) // learning rate for w
+ w_wd = (float) // weight decay for w
+ b_lr = (float) // learning rate for b
+ b_wd = (float) // weight decay for b
+ '''
+ # required
+ assert output_dim > 0, 'output_dim should be set'
+ super(Dense, self).__init__(type=kInnerProduct, **kwargs)
+ self.layer.innerproduct_conf.num_output = output_dim
+ if 'transpose' in kwargs:
+ self.layer.innerproduct_conf.transpose = kwargs['transpose']
+
+ # parameter w (default: gaussian)
+ if w_param == None:
+ self.init = 'gaussian' if init==None else init
+ w_param = Parameter(init=self.init)
+ setParamField(w_param.param, 'w', False, **kwargs)
+ setval(self.layer, param=w_param.param)
+
+ # parameter b (default: constant)
+ if b_param == None:
+ self.init = 'constant' if init==None else init
+ b_param = Parameter(init=self.init)
+ setParamField(b_param.param, 'b', False, **kwargs)
+ setval(self.layer, param=b_param.param)
+
+ # following layers: e.g., activation, dropout, etc.
+ if activation:
+ self.mask = Activation(activation=activation).layer
+
+
+''' Class to deal with multiple layers
+'''
+class Autoencoder(object):
+ def __init__(self, hid_dim=None, out_dim=0, activation=None,
+ param_share=True, **kwargs):
+ # required
+ assert out_dim > 0, 'out_dim should be set'
+ self.out_dim = out_dim
+ assert hid_dim != None, 'hid_dim should be set'
+ self.hid_dim = [hid_dim] if type(hid_dim)==int else hid_dim
+
+ self.layer_type = 'AutoEncoder'
+ self.activation = activation
+ self.param_share = param_share
+
+class RBM(Layer):
+ def __init__(self, out_dim=None, w_param=None, b_param=None, sampling=None, **kwargs):
+ '''
+ Generate layers (like MLP) according to the number of elements in out_dim, and
+ on top of it, two layers RBMVis and RBMHid with bidirectional connection
+
+ required
+ out_dim = (int) or (int list) // the number of hidden nodes
+ optional
+ sampling = (string)
+ '''
+ assert out_dim > 0, 'out_dim should be set'
+ self.out_dim = [out_dim] if type(out_dim)==int else out_dim
+
+ self.name = kwargs['name'] if 'name' in kwargs else 'RBMVis'
+ self.layer_type = kwargs['type'] if 'type' in kwargs else kRBMVis
+ super(RBM, self).__init__(name=generateName(self.name, withnumber=False), type=self.layer_type)
+ setval(self.layer.rbm_conf, hdim=self.out_dim[-1])
+ if self.layer_type == kRBMHid and sampling != None:
+ if sampling == 'gaussian':
+ setval(self.layer.rbm_conf, gaussian=True)
+
+ # parameter w
+ if w_param == None:
+ w_param = Parameter(init='gaussian', **kwargs)
+ setParamField(w_param.param, 'w', withnumber=False, level=len(self.out_dim), **kwargs)
+ else:
+ if self.layer_type == kRBMHid:
+ del kwargs['name']
+ else:
+ setParamField(w_param.param, 'w', withnumber=False, level=len(self.out_dim), **kwargs)
+ setval(self.layer, param=w_param.param)
+
+ # parameter b
+ if b_param == None:
+ b_param = Parameter(init='constant', **kwargs)
+ setParamField(b_param.param, 'b', withnumber=False, level=len(self.out_dim), **kwargs)
+ else:
+ if self.layer_type == kRBMHid:
+ pass
+ else:
+ setParamField(b_param.param, 'b', withnumber=False, level=len(self.out_dim), **kwargs)
+ setval(self.layer, param=b_param.param)
+
+ if self.layer_type == kRBMVis:
+ wname = w_param.param.name
+ parw = Parameter(name=wname+"_", init='none', share_from=wname)
+ bname = b_param.param.name
+ parb = Parameter(name=bname+"2", wd=0, init='constant')
+ self.bidirect = RBM(self.out_dim, name='RBMHid', type=kRBMHid,
+ w_param=parw, b_param=parb, sampling=sampling).layer
+
+
+class Embedding(Layer):
+ def __init__(self, in_dim, out_dim, w_param=None, **kwargs):
+ super(Embedding, self).__init__(name=generateName('embedding',1), user_type='kEmbedding')
+ fields = { 'vocab_size': in_dim,
+ 'word_dim': out_dim }
+ setval(self.layer.Extensions[embedding_conf], **fields)
+ if w_param == None:
+ w_param = Parameter(name=generateName('w'), init=init) # default: uniform
+ else:
+ setParamField(w_param.param, 'w', True, **kwargs)
+ setval(self.layer, param=w_param.param)
+
+class RNNLM(Layer):
+ def __init__(self, dim, w_param=None, **kwargs):
+ super(RNNLM, self).__init__(name=generateName('hidden',1), user_type='kHidden')
+ if w_param == None:
+ w_param = Parameter(name=generateName('w'), init=init) # default: uniform
+ else:
+ setParamField(w_param.param, 'w', True, **kwargs)
+ setval(self.layer, param=w_param.param)
+
+class UserLossRNNLM(Layer):
+ def __init__(self, **kwargs):
+ super(UserLossRNNLM, self).__init__(name=generateName('loss',1), user_type='kLoss')
+ self.layer.Extensions[loss_conf].nclass = kwargs['nclass']
+ self.layer.Extensions[loss_conf].vocab_size = kwargs['vocab_size']
+ setval(self.layer, param=Parameter(name=generateName('w'), init='uniform', scale=0.3).param)
+ setval(self.layer, param=Parameter(name=generateName('w',1), init='uniform', scale=0.3).param)
+
+
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7d43e273/tool/python/singa/model.py
----------------------------------------------------------------------
diff --git a/tool/python/singa/model.py b/tool/python/singa/model.py
new file mode 100644
index 0000000..1aa62b0
--- /dev/null
+++ b/tool/python/singa/model.py
@@ -0,0 +1,455 @@
+#!/usr/bin/env python
+import sys, re, subprocess
+from layer import *
+from utils.utility import *
+from utils.message import *
+from google.protobuf import text_format
+
+class Model(object):
+
+ def __init__(self, name='my model', argv=[], label=False):
+ '''
+ optional
+ name = (string) // name of model/job
+ label = (bool) // exist label layer (depreciated)
+ '''
+ self.jobconf = Message('Job', name=name).proto
+ self.layers = []
+ self.label = label
+ self.argv = argv
+ self.result = None
+ self.last_checkpoint_path = None
+
+ def exist_datalayer(self, phase):
+ for ly in self.layers:
+ if enumPhase(phase) in ly.layer.include:
+ return True
+ return False
+
+ def compile(self, optimizer=None, cluster=None, loss=None, topk=1, **kwargs):
+ '''
+ required
+ optimizer = (Updater) // updater settings, e.g., SGD
+ cluster = (Cluster) // cluster settings
+ optional
+ loss = (string) // name of loss function type
+ topk = (int) // the number of results considered to compute accuracy
+ '''
+ assert optimizer != None, 'optimizer (Updater component) should be set'
+ assert cluster != None, 'cluster (Cluster component) should be set'
+ setval(self.jobconf, updater=optimizer.proto)
+ setval(self.jobconf, cluster=cluster.proto)
+
+ # take care of loss function layer
+ if loss == None:
+ print 'loss layer is not set'
+ else:
+ if hasattr(self.layers[-1], 'mask'):
+ ly = self.layers[-1].mask
+ else:
+ ly = self.layers[-1].layer
+
+ # take care of the last layer
+ if ly.type == enumLayerType('softmax'):
+ # revise the last layer
+ if loss == 'categorical_crossentropy':
+ setval(ly, type=enumLayerType('softmaxloss'))
+ setval(ly.softmaxloss_conf, topk=topk)
+ elif loss == 'mean_squared_error':
+ setval(ly, type=enumLayerType('euclideanloss'))
+ else:
+ # add new layer
+ if loss == 'categorical_crossentropy':
+ self.add(Activation('softmaxloss', topk=topk))
+ elif loss == 'mean_squared_error':
+ self.add(Activation('euclideanloss'))
+ elif loss == 'user_loss_rnnlm': # user-defined loss layer for rnnlm
+ self.add(UserLossRNNLM(nclass=kwargs['nclass'], vocab_size=kwargs['in_dim']))
+
+ def build(self):
+ '''
+ construct neuralnet proto
+ '''
+ net = NetProto()
+ slyname = self.layers[0].layer.name
+ for i in range(len(self.layers)):
+ ly = net.layer.add()
+ ly.CopyFrom(self.layers[i].layer)
+ lastly = ly
+ if self.layers[i].is_datalayer == True:
+ continue
+ getattr(ly, 'srclayers').append(slyname)
+ slyname = ly.name
+ if hasattr(self.layers[i], 'mask'):
+ mly = net.layer.add()
+ mly.CopyFrom(self.layers[i].mask)
+ getattr(mly, 'srclayers').append(slyname)
+ slyname = mly.name
+ lastly = mly
+ if hasattr(self.layers[i], 'bidirect'):
+ bly = net.layer.add()
+ bly.CopyFrom(self.layers[i].bidirect)
+ getattr(bly, 'srclayers').append(slyname)
+
+ # deal with label layer (depreciated)
+ if self.label == True:
+ label_layer = Layer(name='label', type=kLabel)
+ ly = net.layer.add()
+ ly.CopyFrom(label_layer.layer)
+ getattr(ly, 'srclayers').append(self.layers[0].layer.name)
+ getattr(lastly, 'srclayers').append(label_layer.layer.name)
+ else:
+ if lastly.name == 'RBMVis':
+ getattr(lastly, 'srclayers').append(bly.name)
+ else:
+ getattr(lastly, 'srclayers').append(self.layers[0].layer.name)
+
+ setval(self.jobconf, neuralnet=net)
+
+ def fit(self, data=None, alg='bp', nb_epoch=0,
+ with_test=False, execpath='', **fields):
+ '''
+ required
+ data = (Data) // Data class object for training data
+ alg = (string) // algorithm, e.g., 'bp', 'cd'
+ nb_epoch = (int) // the number of training steps
+ optional
+ with_test = (bool) // flag if singa runs for test data
+ execpath = (string) // path to user own singa (executable file)
+ **fields (KEY=VALUE)
+ batch_size = (int) // batch size for training data
+ train_steps = (int) // the number of steps for training, i.e., epoch
+ disp_freq = (int) // frequency to display training info
+ disp_after = (int) // display after this number
+ validate_data = (Data) // validation data, specified in load_data()
+ validate_freq = (int) // frequency of validation
+ validate_steps = (int) // total number of steps for validation
+ validate_after = (int) // start validation after this number
+ checkpoint_path = (string) // path to checkpoint file
+ checkpoint_freq = (int) // frequency for checkpoint
+ checkpoint_after = (int) // start checkpointing after this number
+ '''
+ assert data != None, 'Training data shold be set'
+ assert nb_epoch > 0, 'Training steps shold be set'
+
+ if 'batch_size' in fields: # if new value is set, replace the batch_size
+ setval(data.layer.store_conf, batchsize=fields['batch_size'])
+
+ # insert layer for training
+ if self.exist_datalayer('train') == False:
+ self.layers.insert(0, data)
+ setval(self.jobconf, train_steps=nb_epoch)
+ setval(self.jobconf, disp_freq=nb_epoch/10)
+ if 'disp_freq' in fields:
+ setval(self.jobconf, disp_freq=fields['disp_freq'])
+
+ if 'validate_data' in fields:
+ self.layers.insert(1, fields['validate_data'])
+ setval(self.jobconf, validate_freq=nb_epoch/10)
+
+ setval(self.jobconf, **fields)
+
+ # loading checkpoint if it is set
+ if data.checkpoint != None:
+ setval(self.jobconf, checkpoint_path=data.checkpoint)
+
+ # save model parameter (i.e., checkpoint_path)
+ setval(self.jobconf, checkpoint_freq=nb_epoch)
+ self.last_checkpoint_path = '{0}/step{1}-worker0'.format(
+ self.jobconf.cluster.workspace, nb_epoch)
+
+ # set Train_one_batch component, using backprogapation at default
+ setval(self.jobconf, train_one_batch=Algorithm(type=enumAlgType(alg)).proto)
+
+ # start to run singa for training
+ if with_test == False:
+ self.build() # construct Nneuralnet Component
+ #self.display()
+ return SingaRun(jobproto=self.jobconf, argv=self.argv, execpath=execpath)
+ else:
+ # run singa in evaluate() with test data
+ pass
+
+
+ def evaluate(self, data=None, alg='bp',
+ checkpoint_path=None, execpath='', **fields):
+ '''
+ required
+ data = (Data) // Data class object for testing data
+ optional
+ checkpoint_path = (list) // checkpoint path is necessary only for testing
+ execpaths = (string) // path to user's own executable
+ **fields (KEY=VALUE)
+ batch_size = (int) // batch size for testing data
+ test_freq = (int) // frequency of testing
+ test_steps = (int) // total number of steps for testing
+ test_after = (int) // start testing after this number of steps
+ '''
+ assert data != None, 'Testing data should be set'
+ is_testonly = False
+
+ if 'batch_size' in fields: # if new value is set, replace the batch_size
+ setval(data.layer.store_conf, batchsize=fields['batch_size'])
+
+ # insert layer for testing
+ if self.exist_datalayer('test') == False:
+ self.layers.insert(0, data)
+
+ # loading checkpoint if singa runs only for testing
+ if self.exist_datalayer('train') == False:
+ is_testonly = True
+ if checkpoint_path == None:
+ print 'checkpoint_path has not been specified'
+ else:
+ setval(self.jobconf, checkpoint_path=checkpoint_path)
+
+ steps = fields['test_steps'] if 'test_steps' in fields else 10
+ setval(self.jobconf, test_steps=steps)
+ setval(self.jobconf, **fields)
+
+ # set Train_one_batch component, using backprogapation at default
+ setval(self.jobconf, train_one_batch=Algorithm(type=enumAlgType(alg)).proto)
+
+ self.build() # construct Nneuralnet Component
+
+ #--- generate job.conf file for debug purpose
+ #filename = 'job.conf'
+ #with open(filename, 'w') as f:
+ # f.write(text_format.MessageToString(self.jobconf.cluster))
+ #self.display()
+
+ #--- run singa ---
+ return SingaRun(jobproto=self.jobconf, argv=self.argv, execpath=execpath, testmode=is_testonly)
+ #return SingaRun_script(filename=filename, execpath=execpath)
+
+
+ def display(self):
+ print text_format.MessageToString(self.jobconf)
+
+
+class Energy(Model):
+ def __init__(self, name='my model', argv=[], label=False):
+ super(Energy, self).__init__(name=name, argv=argv, label=label)
+
+ def add(self, layer):
+ if hasattr(layer, 'layer_type'):
+ if layer.layer_type == kRBMVis:
+ dim = 0
+ for i in range(1, len(layer.out_dim)):
+ parw = Parameter(name='w', init='none', level=i)
+ parb = Parameter(name='b', init='none', level=i)
+ dim = layer.out_dim[i-1]
+ self.layers.append(Dense(dim, w_param=parw, b_param=parb, activation='sigmoid'))
+ self.layers.append(layer)
+
+
+class Sequential(Model):
+ def __init__(self, name='my model', argv=[], label=False):
+ super(Sequential, self).__init__(name=name, argv=argv, label=label)
+
+ def add(self, layer):
+ if hasattr(layer, 'layer_type'):
+ if layer.layer_type == 'AutoEncoder':
+ if layer.param_share == True:
+ dim = 0
+ # Encoding
+ for i in range(1, len(layer.hid_dim)+1):
+ parw = Parameter(name='w', init='none', level=i)
+ parb = Parameter(name='b', init='none', level=i)
+ dim = layer.hid_dim[i-1]
+ if i == len(layer.hid_dim): activation = None
+ else: activation = layer.activation
+ self.layers.append(Dense(dim, w_param=parw, b_param=parb, activation=activation))
+ # Decoding
+ for i in range(len(layer.hid_dim), 0, -1):
+ parw = Parameter(name=generateName('w',2), init='none')
+ parb = Parameter(name=generateName('b',2), init='none')
+ setval(parw.param, share_from='w'+str(i))
+ setval(parb.param, name='b'+str(i))
+ if i == 1: dim = layer.out_dim
+ else: dim = layer.hid_dim[i-2]
+ self.layers.append(Dense(dim, w_param=parw, b_param=parb, activation=layer.activation, transpose=True))
+ else:
+ # MLP
+ for i in range(1, len(layer.hid_dim)+2):
+ parw = Parameter(name='w', init='none', level=i)
+ parb = Parameter(name='b', init='none', level=i)
+ if i == len(layer.hid_dim)+1: dim = layer.out_dim
+ else: dim = layer.hid_dim[i-1]
+ self.layers.append(Dense(dim, w_param=parw, b_param=parb, activation=layer.activation))
+ else:
+ self.layers.append(layer)
+
+
+class Store(object):
+ def __init__(self, **kwargs):
+ '''
+ **kwargs
+ path = (string) // path to dataset
+ backend = (string) //
+ batch_size = (int) // batch size of dataset
+ shape = (int) //
+
+ '''
+ self.proto = Message('Store', **kwargs).proto
+
+class Algorithm(object):
+ def __init__(self, type=enumAlgType('bp'), **kwargs):
+ alg = Message('Alg', alg=type, **kwargs).proto
+ if type == enumAlgType('cd'):
+ setval(alg.cd_conf, **kwargs)
+ self.proto = alg
+
+class Updater(object):
+ def __init__(self, upd_type, lr, lr_type,
+ decay, momentum,
+ step, step_lr, **fields):
+
+ upd = Message('Updater', type=upd_type, **fields).proto
+ setval(upd.learning_rate, base_lr=lr)
+ if decay > 0:
+ setval(upd, weight_decay=decay)
+ if momentum > 0:
+ setval(upd, momentum=momentum)
+
+ if lr_type == None:
+ setval(upd.learning_rate, type=kFixed)
+ elif lr_type == 'step':
+ cp = Message('Step', change_freq=60, gamma=0.997)
+ setval(upd.learning_rate, type=kStep, step_conf=cp.proto)
+ elif lr_type == 'fixed':
+ cp = Message('FixedStep', step=step, step_lr=step_lr)
+ setval(upd.learning_rate, type=kFixedStep, fixedstep_conf=cp.proto)
+ elif lr_type == 'linear':
+ cp = Message('Linear', change_freq=10, final_lr=0.1)
+ setval(upd.learning_rate, type=kLinear, linear_conf=cp.proto)
+ self.proto = upd
+
+class SGD(Updater):
+ def __init__(self, lr=0.01, lr_type=None,
+ decay=0, momentum=0,
+ step=(0), step_lr=(0.01), **fields):
+ '''
+ required
+ lr = (float) // base learning rate
+ optional
+ lr_type = (string) // type of learning rate, 'Fixed' at default
+ decay = (float) // weight decay
+ momentum = (float) // momentum
+ **fields (KEY=VALUE)
+
+ '''
+ assert lr
+ super(SGD, self).__init__(upd_type=kSGD,
+ lr=lr, lr_type=lr_type,
+ decay=decay, momentum=momentum,
+ step=step, step_lr=step_lr, **fields)
+
+class AdaGrad(Updater):
+ def __init__(self, lr=0.01, lr_type=None,
+ decay=0, momentum=0,
+ step=(0), step_lr=(0.01), **fields):
+ '''
+ required
+ lr = (float) // base learning rate
+ optional
+ lr_type = (string) // type of learning rate, 'Fixed' at default
+ decay = (float) // weight decay
+ momentum = (float) // momentum
+ **fields (KEY=VALUE)
+
+ '''
+ assert lr
+ super(AdaGrad, self).__init__(upd_type=kAdaGrad,
+ lr=lr, lr_type=lr_type,
+ decay=decay, momentum=momentum,
+ step=step, step_lr=step_lr, **fields)
+
+
+class Cluster(object):
+ def __init__(self, workspace=None,
+ nworker_groups=1, nserver_groups=1,
+ nworkers_per_group=1, nservers_per_group=1,
+ nworkers_per_procs=1, nservers_per_procs=1,
+ **fields):
+ '''
+ required
+ workspace = (string) // workspace path
+ optional
+ nworker_groups = (int)
+ nserver_groups = (int)
+ nworkers_per_group = (int)
+ nservers_per_group = (int)
+ nworkers_per_procs = (int)
+ nservers_per_procs = (int)
+ **fields
+ server_worker_separate = (bool)
+ '''
+ assert workspace != None, 'need to set workspace'
+ self.proto = Message('Cluster', workspace=workspace).proto
+ # optional
+ self.proto.nworker_groups = nworker_groups
+ self.proto.nserver_groups = nserver_groups
+ self.proto.nworkers_per_group = nworkers_per_group
+ self.proto.nservers_per_group = nservers_per_group
+ self.proto.nworkers_per_procs = nworkers_per_procs
+ self.proto.nservers_per_procs = nservers_per_procs
+ # other fields
+ setval(self.proto, **fields)
+
+
+
+def StoreResults(lines):
+
+ resultDic = {}
+ for line in lines:
+ line = re.findall(r'[\w|*.*]+', line)
+ if 'Train' in line:
+ step = line[line.index('step')+1]
+ if 'accuracy' in line:
+ resultDic.setdefault(step,{})['acc'] = line[line.index('accuracy')+1]
+ if 'loss' in line:
+ resultDic.setdefault(step,{})['loss'] = line[line.index('loss')+1]
+ if 'ppl' in line:
+ resultDic.setdefault(step,{})['ppl'] = line[line.index('ppl')+1]
+ if 'Squared' in line:
+ resultDic.setdefault(step,{})['se'] = line[line.index('Squared')+2]
+ return resultDic
+
+def SingaRun(jobproto='', argv=[], execpath='', testmode=False):
+
+ import singa.driver as driver
+ d = driver.Driver()
+ d.InitLog(argv[0])
+ d.Init(argv)
+ if testmode == True:
+ d.Test(jobproto.SerializeToString())
+ else:
+ d.Train(False, jobproto.SerializeToString())
+
+ logfile = '/tmp/singa-log/{0}.ERROR'.format(argv[0].split('/')[-1])
+ fin = open(logfile, 'r')
+ result = StoreResults(fin.readlines())
+
+ return result
+
+def SingaRun_script(filename='', execpath=''):
+ SINGAROOT = '../../../'
+ conf = 'examples/' + filename
+ if execpath=='':
+ cmd = SINGAROOT+'bin/singa-run.sh ' \
+ + '-conf %s ' % conf
+ else:
+ cmd = SINGAROOT+'bin/singa-run.sh ' \
+ + '-conf %s ' % conf \
+ + '-exec %s ' % execpath
+
+ procs = subprocess.Popen(cmd.strip().split(' '), stdout = subprocess.PIPE, stderr = subprocess.STDOUT)
+
+ resultDic = {}
+ outputlines = iter(procs.stdout.readline, '')
+ resultDic = StoreResults(outputlines)
+
+ #TODO better format to store the result??
+ return resultDic
+
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7d43e273/tool/python/singa/parameter.py
----------------------------------------------------------------------
diff --git a/tool/python/singa/parameter.py b/tool/python/singa/parameter.py
new file mode 100644
index 0000000..3b33243
--- /dev/null
+++ b/tool/python/singa/parameter.py
@@ -0,0 +1,105 @@
+#!/usr/bin/env python
+import initializations
+from utils.utility import *
+from utils.message import *
+from google.protobuf import text_format
+
+class Parameter(object):
+
+ def __init__(self, **kwargs):
+ '''
+ optional
+ **kwargs
+ name = (string) // parameter name
+ lr = (float) // learning rate
+ wd = (float) // weight decay
+ init = (string) // initialization type {'constant','uniform','gaussian'}
+ value = (int) // value for 'constant'
+ scale = (float) // [low, high] for 'uniform', low=-scale, high=scale
+ low = (float) // low value for 'uniform'
+ high = (float) // high value for 'uniform'
+ mean = (float) // mean for 'gaussian'
+ std = (float) // std for 'gaussian'
+ '''
+ fields = {'lr_scale' : kwargs['lr'] if 'lr' in kwargs else 1,
+ 'wd_scale' : kwargs['wd'] if 'wd' in kwargs else 1
+ }
+ self.param = Message('Param', **fields).proto
+
+ if not 'name' in kwargs:
+ setval(self.param, name=generateName('param', 1))
+ else:
+ pname = kwargs['name']
+ # parameter name for RBM
+ if 'level' in kwargs:
+ pname += str(kwargs['level'])
+ if pname[0] == 'b':
+ pname += '2'
+ setval(self.param, name=pname)
+
+ if 'share_from' in kwargs:
+ setval(self.param, share_from=kwargs['share_from'])
+
+ if 'init' in kwargs:
+ init_values = initializations.get(kwargs['init'], **kwargs)
+
+ if not kwargs['init'] == 'none':
+ pg = Message('ParamGen', type=enumInitMethod(kwargs['init']), **init_values)
+ del kwargs['init']
+ setval(self.param, init=pg.proto)
+ else: # default: uniform
+ pg = Message('ParamGen', type=enumInitMethod('uniform'))
+ setval(self.param, init=pg.proto)
+
+ def update(self, **fields):
+ setval(self.param, **fields)
+ setval(self.param.init, **fields)
+
+
+def setParamField(param, pname, changename=False, withnumber=True, **kwargs):
+ ''' param = (ParamProto)
+ pname = (string) // 'w' for wiehgt, or 'b' for bias
+ changename = (bool) // update parameter name if True
+ withnumber = (bool) // add layer number if True
+ **kwargs
+ '''
+ assert pname == 'w' or pname == 'b', 'pname should be w or b'
+
+ lr = param.lr_scale
+ wd = param.wd_scale
+ initkv = {}
+
+ if pname == 'w':
+ if 'w_lr' in kwargs:
+ lr = kwargs['w_lr']
+ del kwargs['w_lr']
+ if 'w_wd' in kwargs:
+ wd = kwargs['w_wd']
+ del kwargs['w_wd']
+ for k, v in kwargs.items():
+ if k.startswith('w_'):
+ initkv[k[2:]] = v
+
+ elif pname == 'b':
+ if 'b_lr' in kwargs:
+ lr = kwargs['b_lr']
+ del kwargs['b_lr']
+ if 'b_wd' in kwargs:
+ wd = kwargs['b_wd']
+ del kwargs['b_wd']
+ for k, v in kwargs.items():
+ if k.startswith('b_'):
+ initkv[k[2:]] = v
+
+ field = {'lr_scale' : lr, 'wd_scale' : wd}
+
+ # Set/update parameter fields
+ if param.name.startswith('param') or changename==True:
+ if 'level' in kwargs: # parameter name for RBM
+ pname += str(kwargs['level'])
+ setval(param, name=generateName(pname, withnumber=withnumber), **field)
+ else:
+ setval(param, **field)
+
+ # Set/update parameter init fields
+ setval(param.init, **initkv)
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7d43e273/tool/python/singa/utils/__init__.py
----------------------------------------------------------------------
diff --git a/tool/python/singa/utils/__init__.py b/tool/python/singa/utils/__init__.py
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7d43e273/tool/python/singa/utils/message.py
----------------------------------------------------------------------
diff --git a/tool/python/singa/utils/message.py b/tool/python/singa/utils/message.py
new file mode 100644
index 0000000..251a377
--- /dev/null
+++ b/tool/python/singa/utils/message.py
@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+import sys, os
+from utility import *
+sys.path.append(os.path.join(os.path.dirname(__file__),'../../pb2'))
+
+'''
+ - This script reads proto files in ../../pb2, generated by protocol buffer compiler
+ - Message class creates an object for proto and sets the fields specified by kwargs
+ - enumXXX function returns enum values of name XXX
+'''
+
+module_list=[]
+
+# import all modules in dir singa_root/too/pb2, except common, singa and __init__
+for f in os.listdir(os.path.join(os.path.dirname(__file__),'../../pb2')):
+ if (f.endswith(".pyc")):
+ continue
+ if(f == "__init__.py" or f == "common_pb2.py" or f == "singa_pb2.py" ):
+ continue
+ module_name = f.split('.')[0]
+ module=__import__(module_name)
+ module_list.append(module)
+ for func_name in dir(module):
+ if not func_name.startswith("__"):
+ globals()[func_name] = getattr(module,func_name)
+
+class Message(object):
+ def __init__(self,protoname,**kwargs):
+ for module in module_list:
+ if hasattr(module,protoname+"Proto"):
+ class_ = getattr(module,protoname+"Proto")
+ self.proto = class_()
+ return setval(self.proto,**kwargs)
+ raise Exception('invalid protoname')
+
+enumDict_=dict()
+
+#get all enum type list in modules
+for module in module_list:
+ for enumtype in module.DESCRIPTOR.enum_types_by_name:
+ tempDict=enumDict_[enumtype]=dict()
+ for name in getattr(module,enumtype).DESCRIPTOR.values_by_name:
+ tempDict[name[1:].lower()]=getattr(module,name)
+
+def make_function(enumtype):
+ def _function(key):
+ return enumDict_[enumtype][key]
+ return _function
+
+current_module = sys.modules[__name__]
+
+#def all the enumtypes
+for module in module_list:
+ for enumtype in module.DESCRIPTOR.enum_types_by_name:
+ setattr(current_module,"enum"+enumtype,make_function(enumtype))
+
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7d43e273/tool/python/singa/utils/utility.py
----------------------------------------------------------------------
diff --git a/tool/python/singa/utils/utility.py b/tool/python/singa/utils/utility.py
new file mode 100644
index 0000000..93d2f7f
--- /dev/null
+++ b/tool/python/singa/utils/utility.py
@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+
+layerid = 0
+paramid = 0
+
+def generateName(label, op=0, withnumber=True):
+ global layerid, paramid
+ num = layerid
+ if label == 'layer':
+ if op ==1: layerid += 1
+ num = layerid
+ elif label == 'param':
+ if op ==1: paramid += 1
+ num = paramid
+ else:
+ if op ==1: layerid += 1
+ num = layerid
+ if op ==2:
+ num = layerid+1
+
+ if withnumber == False:
+ return '{0}'.format(label)
+
+ return '{0}{1}'.format(label, num)
+
+
+def setval(proto, **kwargs):
+ for k,v in kwargs.items():
+ #print 'kv: ', k, ', ', v
+ if hasattr(proto, k):
+ flabel = proto.DESCRIPTOR.fields_by_name[k].label
+ ftype = proto.DESCRIPTOR.fields_by_name[k].type
+
+ fattr = getattr(proto, k)
+ if flabel == 3: # repeated field
+ if ftype == 11: # message type
+ fattr = fattr.add()
+ fattr.MergeFrom(v)
+ else:
+ if type(v) == list or type(v) == tuple:
+ for i in range(len(v)):
+ fattr.append(v[i])
+ else:
+ fattr.append(v)
+ else:
+ if ftype == 11: # message type
+ fattr = getattr(proto,k)
+ fattr.MergeFrom(v)
+ else:
+ setattr(proto, k, v)
[04/10] incubator-singa git commit: SINGA-81 Add Python Helper,
which enables users to construct a model (JobProto) and run Singa in
Python
Posted by wa...@apache.org.
SINGA-81 Add Python Helper, which enables users to construct a model (JobProto) and run Singa in Python
- Update model.py to take care of cudnn
. fit() receives device field, a list of gpu id
. setCudnnLayerType converts LayerType to CdunnLayerType
- Add examples for cudnn
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/ceb06656
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/ceb06656
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/ceb06656
Branch: refs/heads/master
Commit: ceb066560708a6b5e4cbdbbcba095821f18db327
Parents: 7d43e27
Author: chonho <le...@comp.nus.edu.sg>
Authored: Mon Dec 28 13:31:30 2015 +0800
Committer: chonho <le...@comp.nus.edu.sg>
Committed: Fri Jan 1 15:59:14 2016 +0800
----------------------------------------------------------------------
tool/python/examples/cifar10_cnn_cudnn.py | 34 +++++++++++++++
tool/python/examples/mnist_mlp_cudnn.py | 34 +++++++++++++++
tool/python/singa/driver.py | 2 +-
tool/python/singa/driver_wrap.cxx | 57 ++++++++++++++------------
tool/python/singa/model.py | 51 +++++++++++++++++++----
5 files changed, 144 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/ceb06656/tool/python/examples/cifar10_cnn_cudnn.py
----------------------------------------------------------------------
diff --git a/tool/python/examples/cifar10_cnn_cudnn.py b/tool/python/examples/cifar10_cnn_cudnn.py
new file mode 100755
index 0000000..e08610a
--- /dev/null
+++ b/tool/python/examples/cifar10_cnn_cudnn.py
@@ -0,0 +1,34 @@
+#!/usr/bin/env python
+import sys, os
+sys.path.append(os.path.join(os.path.dirname(__file__),'..'))
+from singa.model import *
+from singa.datasets import cifar10
+
+X_train, X_test, workspace = cifar10.load_data()
+
+m = Sequential('cifar10-cnn', sys.argv)
+
+m.add(Convolution2D(32, 5, 1, 2, w_std=0.0001, b_lr=2))
+m.add(MaxPooling2D(pool_size=(3,3), stride=2))
+m.add(Activation('relu'))
+m.add(LRN2D(3, alpha=0.00005, beta=0.75))
+
+m.add(Convolution2D(32, 5, 1, 2, b_lr=2))
+m.add(Activation('relu'))
+m.add(AvgPooling2D(pool_size=(3,3), stride=2))
+m.add(LRN2D(3, alpha=0.00005, beta=0.75))
+
+m.add(Convolution2D(64, 5, 1, 2))
+m.add(Activation('relu'))
+m.add(AvgPooling2D(pool_size=(3,3), stride=2))
+
+m.add(Dense(10, w_wd=250, b_lr=2, b_wd=0, activation='softmax'))
+
+sgd = SGD(decay=0.004, lr_type='fixed', step=(0,60000,65000), step_lr=(0.001,0.0001,0.00001))
+topo = Cluster(workspace)
+m.compile(loss='categorical_crossentropy', optimizer=sgd, cluster=topo)
+
+gpu_id = [0]
+m.fit(X_train, nb_epoch=1000, with_test=True, device=gpu_id)
+result = m.evaluate(X_test, test_steps=100, test_freq=300)
+
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/ceb06656/tool/python/examples/mnist_mlp_cudnn.py
----------------------------------------------------------------------
diff --git a/tool/python/examples/mnist_mlp_cudnn.py b/tool/python/examples/mnist_mlp_cudnn.py
new file mode 100755
index 0000000..d418950
--- /dev/null
+++ b/tool/python/examples/mnist_mlp_cudnn.py
@@ -0,0 +1,34 @@
+#!/usr/bin/env python
+import sys, os
+sys.path.append(os.path.join(os.path.dirname(__file__),'..'))
+from singa.model import *
+from singa.datasets import mnist
+
+# Sample parameter values for Mnist MLP example
+pvalues = {'batchsize' : 64, 'shape' : 784, 'random_skip' : 5000,
+ 'std_value' : 127.5, 'mean_value' : 127.5}
+X_train, X_test, workspace = mnist.load_data(**pvalues)
+
+m = Sequential('mlp', argv=sys.argv)
+
+''' Weight and Bias are initialized by
+ uniform distribution with scale=0.05 at default
+'''
+m.add(Dense(2500, init='uniform', activation='tanh'))
+m.add(Dense(2000, init='uniform', activation='tanh'))
+m.add(Dense(1500, init='uniform', activation='tanh'))
+m.add(Dense(1000, init='uniform', activation='tanh'))
+m.add(Dense(500, init='uniform', activation='tanh'))
+m.add(Dense(10, init='uniform', activation='softmax'))
+
+sgd = SGD(lr=0.001, lr_type='step')
+topo = Cluster(workspace)
+m.compile(loss='categorical_crossentropy', optimizer=sgd, cluster=topo)
+
+gpu_id = [0]
+m.fit(X_train, nb_epoch=100, with_test=True, device=gpu_id)
+result = m.evaluate(X_test, batch_size=100, test_steps=10)
+
+#e.g., display result
+#for k, v in sorted(result.items(), key=lambda x: x[0]):
+# print k, v
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/ceb06656/tool/python/singa/driver.py
----------------------------------------------------------------------
diff --git a/tool/python/singa/driver.py b/tool/python/singa/driver.py
index c1aac7f..d203923 100644
--- a/tool/python/singa/driver.py
+++ b/tool/python/singa/driver.py
@@ -1,5 +1,5 @@
# This file was automatically generated by SWIG (http://www.swig.org).
-# Version 3.0.2
+# Version 2.0.11
#
# Do not make changes to this file unless you know what you are doing--modify
# the SWIG interface file instead.
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/ceb06656/tool/python/singa/driver_wrap.cxx
----------------------------------------------------------------------
diff --git a/tool/python/singa/driver_wrap.cxx b/tool/python/singa/driver_wrap.cxx
index e6e6de2..40ea5ce 100644
--- a/tool/python/singa/driver_wrap.cxx
+++ b/tool/python/singa/driver_wrap.cxx
@@ -1,6 +1,6 @@
/* ----------------------------------------------------------------------------
* This file was automatically generated by SWIG (http://www.swig.org).
- * Version 3.0.2
+ * Version 2.0.11
*
* This file is not intended to be easily readable and contains a number of
* coding conventions designed to improve portability and efficiency. Do not make
@@ -560,14 +560,14 @@ SWIG_MangledTypeQueryModule(swig_module_info *start,
swig_module_info *iter = start;
do {
if (iter->size) {
- size_t l = 0;
- size_t r = iter->size - 1;
+ register size_t l = 0;
+ register size_t r = iter->size - 1;
do {
/* since l+r >= 0, we can (>> 1) instead (/ 2) */
- size_t i = (l + r) >> 1;
+ register size_t i = (l + r) >> 1;
const char *iname = iter->types[i]->name;
if (iname) {
- int compare = strcmp(name, iname);
+ register int compare = strcmp(name, iname);
if (compare == 0) {
return iter->types[i];
} else if (compare < 0) {
@@ -611,7 +611,7 @@ SWIG_TypeQueryModule(swig_module_info *start,
of the str field (the human readable name) */
swig_module_info *iter = start;
do {
- size_t i = 0;
+ register size_t i = 0;
for (; i < iter->size; ++i) {
if (iter->types[i]->str && (SWIG_TypeEquiv(iter->types[i]->str, name)))
return iter->types[i];
@@ -630,10 +630,10 @@ SWIG_TypeQueryModule(swig_module_info *start,
SWIGRUNTIME char *
SWIG_PackData(char *c, void *ptr, size_t sz) {
static const char hex[17] = "0123456789abcdef";
- const unsigned char *u = (unsigned char *) ptr;
- const unsigned char *eu = u + sz;
+ register const unsigned char *u = (unsigned char *) ptr;
+ register const unsigned char *eu = u + sz;
for (; u != eu; ++u) {
- unsigned char uu = *u;
+ register unsigned char uu = *u;
*(c++) = hex[(uu & 0xf0) >> 4];
*(c++) = hex[uu & 0xf];
}
@@ -645,11 +645,11 @@ SWIG_PackData(char *c, void *ptr, size_t sz) {
*/
SWIGRUNTIME const char *
SWIG_UnpackData(const char *c, void *ptr, size_t sz) {
- unsigned char *u = (unsigned char *) ptr;
- const unsigned char *eu = u + sz;
+ register unsigned char *u = (unsigned char *) ptr;
+ register const unsigned char *eu = u + sz;
for (; u != eu; ++u) {
- char d = *(c++);
- unsigned char uu;
+ register char d = *(c++);
+ register unsigned char uu;
if ((d >= '0') && (d <= '9'))
uu = ((d - '0') << 4);
else if ((d >= 'a') && (d <= 'f'))
@@ -1326,7 +1326,7 @@ SWIG_Python_UnpackTuple(PyObject *args, const char *name, Py_ssize_t min, Py_ssi
}
if (!PyTuple_Check(args)) {
if (min <= 1 && max >= 1) {
- int i;
+ register int i;
objs[0] = args;
for (i = 1; i < max; ++i) {
objs[i] = 0;
@@ -1336,7 +1336,7 @@ SWIG_Python_UnpackTuple(PyObject *args, const char *name, Py_ssize_t min, Py_ssi
PyErr_SetString(PyExc_SystemError, "UnpackTuple() argument list is not a tuple");
return 0;
} else {
- Py_ssize_t l = PyTuple_GET_SIZE(args);
+ register Py_ssize_t l = PyTuple_GET_SIZE(args);
if (l < min) {
PyErr_Format(PyExc_TypeError, "%s expected %s%d arguments, got %d",
name, (min == max ? "" : "at least "), (int)min, (int)l);
@@ -1346,7 +1346,7 @@ SWIG_Python_UnpackTuple(PyObject *args, const char *name, Py_ssize_t min, Py_ssi
name, (min == max ? "" : "at most "), (int)max, (int)l);
return 0;
} else {
- int i;
+ register int i;
for (i = 0; i < l; ++i) {
objs[i] = PyTuple_GET_ITEM(args, i);
}
@@ -2461,7 +2461,7 @@ SWIG_Python_NewShadowInstance(SwigPyClientData *data, PyObject *swig_this)
}
} else {
#if PY_VERSION_HEX >= 0x03000000
- inst = ((PyTypeObject*) data->newargs)->tp_new((PyTypeObject*) data->newargs, Py_None, Py_None);
+ inst = PyBaseObject_Type.tp_new((PyTypeObject*) data->newargs, Py_None, Py_None);
if (inst) {
PyObject_SetAttr(inst, SWIG_This(), swig_this);
Py_TYPE(inst)->tp_flags &= ~Py_TPFLAGS_VALID_VERSION_TAG;
@@ -2967,7 +2967,7 @@ static swig_module_info swig_module = {swig_types, 5, 0, 0, 0, 0};
#endif
#define SWIG_name "_driver"
-#define SWIGVERSION 0x030002
+#define SWIGVERSION 0x020011
#define SWIG_VERSION SWIGVERSION
@@ -3060,9 +3060,11 @@ namespace swig {
#include <string>
+#include <stdexcept>
+#include <stddef.h>
-#include <stddef.h>
+ #include <stddef.h>
namespace swig {
@@ -3407,22 +3409,25 @@ SWIG_AsVal_ptrdiff_t (PyObject * obj, ptrdiff_t *val)
}
+#include <stdexcept>
+
+
#include <algorithm>
#include <vector>
+#include <string>
+
+
#include "singa/driver.h"
SWIGINTERN int
SWIG_AsVal_bool (PyObject *obj, bool *val)
{
- int r;
- if (!PyBool_Check(obj))
- return SWIG_ERROR;
- r = PyObject_IsTrue(obj);
+ int r = PyObject_IsTrue(obj);
if (r == -1)
return SWIG_ERROR;
if (val) *val = r ? true : false;
@@ -4735,7 +4740,7 @@ static swig_const_info swig_const_table[] = {
* array with the correct data and linking the correct swig_cast_info
* structures together.
*
- * The generated swig_type_info structures are assigned statically to an initial
+ * The generated swig_type_info structures are assigned staticly to an initial
* array. We just loop through that array, and handle each type individually.
* First we lookup if this type has been already loaded, and if so, use the
* loaded structure instead of the generated one. Then we have to fill in the
@@ -5065,7 +5070,7 @@ extern "C" {
var = var->next;
}
if (res == NULL && !PyErr_Occurred()) {
- PyErr_Format(PyExc_AttributeError, "Unknown C global variable '%s'", n);
+ PyErr_SetString(PyExc_NameError,"Unknown C global variable");
}
return res;
}
@@ -5082,7 +5087,7 @@ extern "C" {
var = var->next;
}
if (res == 1 && !PyErr_Occurred()) {
- PyErr_Format(PyExc_AttributeError, "Unknown C global variable '%s'", n);
+ PyErr_SetString(PyExc_NameError,"Unknown C global variable");
}
return res;
}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/ceb06656/tool/python/singa/model.py
----------------------------------------------------------------------
diff --git a/tool/python/singa/model.py b/tool/python/singa/model.py
index 1aa62b0..ade0b85 100644
--- a/tool/python/singa/model.py
+++ b/tool/python/singa/model.py
@@ -19,6 +19,7 @@ class Model(object):
self.argv = argv
self.result = None
self.last_checkpoint_path = None
+ self.cudnn = False
def exist_datalayer(self, phase):
for ly in self.layers:
@@ -104,18 +105,23 @@ class Model(object):
else:
getattr(lastly, 'srclayers').append(self.layers[0].layer.name)
+ # use of cudnn
+ if self.cudnn == True:
+ self.setCudnnLayerType(net)
+
setval(self.jobconf, neuralnet=net)
def fit(self, data=None, alg='bp', nb_epoch=0,
- with_test=False, execpath='', **fields):
+ with_test=False, execpath='', device=None, **fields):
'''
required
- data = (Data) // Data class object for training data
- alg = (string) // algorithm, e.g., 'bp', 'cd'
- nb_epoch = (int) // the number of training steps
+ data = (Data) // Data class object for training data
+ alg = (string) // algorithm, e.g., 'bp', 'cd'
+ nb_epoch = (int) // the number of training steps
optional
- with_test = (bool) // flag if singa runs for test data
- execpath = (string) // path to user own singa (executable file)
+ with_test = (bool) // flag if singa runs for test data
+ execpath = (string) // path to user own singa (executable file)
+ device = (int/list) // a list of gpu ids
**fields (KEY=VALUE)
batch_size = (int) // batch size for training data
train_steps = (int) // the number of steps for training, i.e., epoch
@@ -161,6 +167,11 @@ class Model(object):
# set Train_one_batch component, using backprogapation at default
setval(self.jobconf, train_one_batch=Algorithm(type=enumAlgType(alg)).proto)
+ # use of cudnn
+ if device != None:
+ setval(self.jobconf, gpu=device)
+ self.cudnn = True
+
# start to run singa for training
if with_test == False:
self.build() # construct Nneuralnet Component
@@ -216,7 +227,7 @@ class Model(object):
#filename = 'job.conf'
#with open(filename, 'w') as f:
# f.write(text_format.MessageToString(self.jobconf.cluster))
- #self.display()
+ self.display()
#--- run singa ---
return SingaRun(jobproto=self.jobconf, argv=self.argv, execpath=execpath, testmode=is_testonly)
@@ -224,8 +235,32 @@ class Model(object):
def display(self):
+ ''' print out job proto
+ '''
print text_format.MessageToString(self.jobconf)
+ def setCudnnLayerType(self, net):
+ ''' convert LayerType to CdunnLayerType
+ '''
+ for i in range(len(net.layer)):
+ ly_type = net.layer[i].type
+ cudnn_ly_type = ly_type
+ if ly_type == kCConvolution: cudnn_ly_type = kCudnnConv
+ elif ly_type == kCPooling: cudnn_ly_type = kCudnnPool
+ elif ly_type == kLRN: cudnn_ly_type = kCudnnLRN
+ elif ly_type == kSoftmax: cudnn_ly_type = kCudnnSoftmax
+ elif ly_type == kSoftmaxLoss: cudnn_ly_type = kCudnnSoftmaxLoss
+ elif ly_type == kSTanh:
+ cudnn_ly_type = kCudnnActivation
+ net.layer[i].activation_conf.type = STANH
+ elif ly_type == kSigmoid:
+ cudnn_ly_type = kCudnnActivation
+ net.layer[i].activation_conf.type = SIGMOID
+ elif ly_type == kReLU:
+ cudnn_ly_type = kCudnnActivation
+ net.layer[i].activation_conf.type = RELU
+ net.layer[i].type = cudnn_ly_type
+
class Energy(Model):
def __init__(self, name='my model', argv=[], label=False):
@@ -277,6 +312,8 @@ class Sequential(Model):
if i == len(layer.hid_dim)+1: dim = layer.out_dim
else: dim = layer.hid_dim[i-1]
self.layers.append(Dense(dim, w_param=parw, b_param=parb, activation=layer.activation))
+ else:
+ self.layers.append(layer)
else:
self.layers.append(layer)