You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@singa.apache.org by wa...@apache.org on 2018/04/12 09:16:07 UTC
[4/4] incubator-singa git commit: SINGA-348 Support autograd MLP
Example
SINGA-348 Support autograd MLP Example
rename some variables; and add more checks for Dummy operations.
move mlp.py into example/autograd; I can run mlp.py without any bugs.
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/060e7dfe
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/060e7dfe
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/060e7dfe
Branch: refs/heads/master
Commit: 060e7dfe1cc847500e4beb115f3b24e923288b3e
Parents: f42d4d0
Author: Wang Wei <dc...@nus.edu.sg>
Authored: Wed Apr 11 21:09:45 2018 +0800
Committer: Wang Wei <dc...@nus.edu.sg>
Committed: Thu Apr 12 16:59:48 2018 +0800
----------------------------------------------------------------------
examples/MLP.py | 83 -------------------------------------------
examples/autograd/mlp.py | 74 ++++++++++++++++++++++++++++++++++++++
python/singa/autograd.py | 33 ++++++++++-------
python/singa/tensor.py | 58 +++++++++++++++++++-----------
4 files changed, 131 insertions(+), 117 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/060e7dfe/examples/MLP.py
----------------------------------------------------------------------
diff --git a/examples/MLP.py b/examples/MLP.py
deleted file mode 100644
index b773efb..0000000
--- a/examples/MLP.py
+++ /dev/null
@@ -1,83 +0,0 @@
-from singa import tensor
-from singa import autograd
-from singa import optimizer
-import numpy as np
-
-
-if __name__ == '__main__':
-
- # prepare training data in numpy array
-
- # generate the boundary
- f = lambda x: (5 * x + 1)
- bd_x = np.linspace(-1., 1, 200)
- bd_y = f(bd_x)
- # generate the training data
- x = np.random.uniform(-1, 1, 400)
- y = f(x) + 2 * np.random.randn(len(x))
- # convert training data to 2d space
- label = np.asarray([5 * a + 1 > b for (a, b) in zip(x, y)])
- data = np.array([[a, b] for (a, b) in zip(x, y)], dtype=np.float32)
-
- def to_categorical(y, num_classes=None):
- '''
- Converts a class vector (integers) to binary class matrix.
-
- Args
- y: class vector to be converted into a matrix
- (integers from 0 to num_classes).
- num_classes: total number of classes.
-
- Return
- A binary matrix representation of the input.
- '''
- y = np.array(y, dtype='int')
- input_shape = y.shape
- if input_shape and input_shape[-1] == 1 and len(input_shape) > 1:
- input_shape = tuple(input_shape[:-1])
- y = y.ravel()
- if not num_classes:
- num_classes = np.max(y) + 1
- n = y.shape[0]
- categorical = np.zeros((n, num_classes))
- categorical[np.arange(n), y] = 1
- output_shape = input_shape + (num_classes,)
- categorical = np.reshape(categorical, output_shape)
- return categorical
-
- label = to_categorical(label, 2).astype(np.float32)
- print('train_data_shape:', data.shape)
- print('train_label_shape:', label.shape)
-
- inputs = tensor.Tensor(data=data, requires_grad=False)
- target = tensor.Tensor(data=label, requires_grad=False)
-
- w0 = tensor.Tensor(shape=(2, 3), requires_grad=True, stores_grad=True)
- w0.gaussian(0.0, 0.1)
- b0 = tensor.Tensor(shape=(1, 3), requires_grad=True, stores_grad=True)
- b0.set_value(0.0)
-
- w1 = tensor.Tensor(shape=(3, 2), requires_grad=True, stores_grad=True)
- w1.gaussian(0.0, 0.1)
- b1 = tensor.Tensor(shape=(1, 2), requires_grad=True, stores_grad=True)
- b1.set_value(0.0)
-
- sgd = optimizer.SGD(0.05)
- # training process
- for i in range(1001):
- x = tensor.matmul(inputs, w0)
- x = tensor.add_bias(x, b0)
- x = tensor.relu(x)
- x = tensor.matmul(x, w1)
- x = tensor.add_bias(x, b1)
- x = tensor.softmax(x)
- loss = tensor.cross_entropy(x, target)
- in_grads = autograd.backward(loss)
-
- for param in in_grads:
- sgd.apply(0, in_grads[param], param, '')
-
- if (i % 100 == 0):
- print('training loss = ', tensor.to_numpy(loss)[0])
-
-
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/060e7dfe/examples/autograd/mlp.py
----------------------------------------------------------------------
diff --git a/examples/autograd/mlp.py b/examples/autograd/mlp.py
new file mode 100644
index 0000000..7352c21
--- /dev/null
+++ b/examples/autograd/mlp.py
@@ -0,0 +1,74 @@
+from singa import tensor
+from singa.tensor import Tensor
+from singa import autograd
+from singa import optimizer
+import numpy as np
+
+
+if __name__ == '__main__':
+
+ # prepare training data in numpy array
+
+ # generate the boundary
+ f = lambda x: (5 * x + 1)
+ bd_x = np.linspace(-1., 1, 200)
+ bd_y = f(bd_x)
+ # generate the training data
+ x = np.random.uniform(-1, 1, 400)
+ y = f(x) + 2 * np.random.randn(len(x))
+ # convert training data to 2d space
+ label = np.asarray([5 * a + 1 > b for (a, b) in zip(x, y)])
+ data = np.array([[a, b] for (a, b) in zip(x, y)], dtype=np.float32)
+
+ def to_categorical(y, num_classes):
+ '''
+ Converts a class vector (integers) to binary class matrix.
+
+ Args
+ y: class vector to be converted into a matrix
+ (integers from 0 to num_classes).
+ num_classes: total number of classes.
+
+ Return
+ A binary matrix representation of the input.
+ '''
+ y = np.array(y, dtype='int')
+ n = y.shape[0]
+ categorical = np.zeros((n, num_classes))
+ categorical[np.arange(n), y] = 1
+ return categorical
+
+ label = to_categorical(label, 2).astype(np.float32)
+ print('train_data_shape:', data.shape)
+ print('train_label_shape:', label.shape)
+
+ inputs = Tensor(data=data)
+ target = Tensor(data=label)
+
+ w0 = Tensor(shape=(2, 3), requires_grad=True, stores_grad=True)
+ w0.gaussian(0.0, 0.1)
+ b0 = Tensor(shape=(1, 3), requires_grad=True, stores_grad=True)
+ b0.set_value(0.0)
+
+ w1 = Tensor(shape=(3, 2), requires_grad=True, stores_grad=True)
+ w1.gaussian(0.0, 0.1)
+ b1 = Tensor(shape=(1, 2), requires_grad=True, stores_grad=True)
+ b1.set_value(0.0)
+
+ sgd = optimizer.SGD(0.05)
+ # training process
+ for i in range(1001):
+ x = tensor.matmul(inputs, w0)
+ x = tensor.add_bias(x, b0)
+ x = tensor.relu(x)
+ x = tensor.matmul(x, w1)
+ x = tensor.add_bias(x, b1)
+ x = tensor.soft_max(x)
+ loss = tensor.cross_entropy(x, target)
+ in_grads = autograd.backward(loss)
+
+ for param in in_grads:
+ sgd.apply(0, in_grads[param], param, '')
+
+ if (i % 100 == 0):
+ print('training loss = ', tensor.to_numpy(loss)[0])
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/060e7dfe/python/singa/autograd.py
----------------------------------------------------------------------
diff --git a/python/singa/autograd.py b/python/singa/autograd.py
index 399fa19..175f8b2 100644
--- a/python/singa/autograd.py
+++ b/python/singa/autograd.py
@@ -1,6 +1,5 @@
from collections import Counter, deque
-from singa import tensor
-
+from .tensor import Tensor, Dummy
def infer_dependency(op):
@@ -23,7 +22,8 @@ def infer_dependency(op):
while len(queue) > 0:
cur_op = queue.pop()
for src_op, _, _, _ in cur_op.src:
- if src_op not in dependency_count:
+ if src_op not in dependency_count and \
+ (not isinstance(src_op, Dummy)):
# dependency[src_op] = [Counter() for _ in src_op.y_id2idx]
dependency_count[src_op] = 0
queue.append(src_op)
@@ -53,7 +53,7 @@ def backward(y, dy=None):
# by default the dy is a tensor with 1.0 for each sample;
if dy is None:
dy = float(1.0)
- elif isinstance(dy, tensor.Tensor):
+ elif isinstance(dy, Tensor):
dy = dy.data
else:
dy = float(dy)
@@ -62,17 +62,22 @@ def backward(y, dy=None):
ready = deque([(y.creator, (dy,))])
not_ready = {} # mapping: op->[dy]
gradients = {} # mapping: x->dx if x.stores_grad
+ if y.stores_grad:
+ gradients[y] = dy
while len(ready) > 0:
op, dys = ready.pop()
- #if not isinstance(op, tensor.Dummy):
+ if not op.requires_grad or isinstance(op, Dummy):
+ continue
+ # if not isinstance(op, tensor.Dummy):
dxs = op._do_backward(*dys)
# TODO src and dx must match
assert len(op.src) == len(dxs), \
'the number of src ops (=%d) and dx (=%d) not match' \
% (len(op.src), len(dxs))
- for (src_op, x_id, param, x_stores_grad), dx in zip(op.src, dxs):
- # x_id is the python id of one input arg of op, denoted as x.
+ for (src_op, x_id, y, y_stores_grad), dx in zip(op.src, dxs):
+ # prefix x is w.r.t op; prefix y is w.r.t src_op.
+ # x_id is the python id of one input arg of src_op, denoted as x.
# y_idx (below) is the index of x among the outputs of src_op.
# not_ready[src_op][y_idx] records the intermediate gradient
# of the y_idx'th output of src_op. 'intermediate gradient'
@@ -81,10 +86,11 @@ def backward(y, dy=None):
# children operations. When src_op is ready, it means that
# the gradient of all its outputs are available, i.e. all children
# operations have been backwarded.
- y_idx = src_op.y_ids[x_id]
+ # y is None if y.stores_grad is false; otherwise it is a Tensor
+ y_idx = src_op.y_id2idx[x_id]
if src_op not in not_ready:
# src_op may have mulitple outputs
- not_ready[src_op] = [None for _ in src_op.y_ids]
+ not_ready[src_op] = [None for _ in src_op.y_id2idx]
not_ready[src_op][y_idx] = dx
else:
dxs = not_ready[src_op]
@@ -94,14 +100,15 @@ def backward(y, dy=None):
# add the gradient from another children operation that
# uses y_idx'th output of src_op as input arg
dxs[y_idx] += dx
- if x_stores_grad:
+ if y_stores_grad:
# store the gradient for final return, e.g. if x is parameter
- gradient = not_ready[src_op][y_idx]
- gradients[param] = tensor.Tensor(device=gradient.device, data=gradient, requires_grad=False)
+ g = not_ready[src_op][y_idx]
+ gradients[y] = Tensor(device=g.device, data=g)
dependency[src_op] -= 1
if src_op.requires_grad is True:
if dependency[src_op] == 0:
- ready.append((src_op, not_ready[src_op]))
+ if not isinstance(src_op, Dummy):
+ ready.append((src_op, not_ready[src_op]))
del not_ready[src_op]
return gradients
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/060e7dfe/python/singa/tensor.py
----------------------------------------------------------------------
diff --git a/python/singa/tensor.py b/python/singa/tensor.py
index f4801a4..70a9302 100644
--- a/python/singa/tensor.py
+++ b/python/singa/tensor.py
@@ -1130,32 +1130,42 @@ class Operation(object):
a Xxxx instance and calls __call__ to do forward. The autograd engine
is able to do backward propagation by calling the backward() of Xxxx
automatically. Notice that the tensors are CTensor. NOT Python Tensor.
- The arguments of forward includes both CTensor instances and other
- types of data; The backward function ONLY supports CTensor args.
+ The arguments of forward() and backward() should only include CTensor args;
'''
def __call__(self, *xs):
return self._do_forward(*xs)
def _do_forward(self, *xs):
+ '''
+ Do not call this function from user code. It is called by __call__().
+
+ Args:
+ xs, Tensor instance(s)
+
+ Returns:
+ Tensor instance(s)
+ '''
# TODO add the pre hook
- # filter out args that are not Tensor instances
- tensor_xs = [x for x in xs if isinstance(x, Tensor)]
+ assert all([isinstance(x, Tensor) for x in xs]), \
+ 'xs should include only Tensor instances'
# need to do backward if any of its input arg needs gradient
- self.requires_grad = any([x.requires_grad for x in tensor_xs])
- # src records info of every input arg that needs gradient
- # the backward() function computes grad only for those arg
+ self.requires_grad = any([x.requires_grad for x in xs])
self.src = []
- for x in tensor_xs:
+ for x in xs:
if x.stores_grad:
+ # store the tensor whose gradient needs be returned in
+ # backward(), e.g. if x is parameter
self.src.append((x.creator, id(x), x, x.stores_grad))
else:
+ # for intermediate tensors, they will be released soon;
+ # no need to store them --> use None
self.src.append((x.creator, id(x), None, x.stores_grad))
- # use the CTensor (data) if the input arg is Tensor
- xs = tuple(x.data if isinstance(x, Tensor) else x for x in xs)
+ # get the CTensor (data) if the input arg is Tensor
+ xs = tuple(x.data for x in xs)
ys = self.forward(*xs)
if not isinstance(ys, tuple):
ys = (ys,)
@@ -1166,7 +1176,7 @@ class Operation(object):
requires_grad=self.requires_grad,
creator=self) for y in ys)
# map from python id to output index
- self.y_ids = {id(y): i for i, y in enumerate(ys)}
+ self.y_id2idx = {id(y): i for i, y in enumerate(ys)}
# TODO add the post hook
return ys
@@ -1180,7 +1190,7 @@ class Operation(object):
'''Forward propagation.
Args:
- xs: input args consisting of CTensors and others.
+ xs: input args consisting of only CTensors.
Returns:
CTensor instance(s)
@@ -1191,7 +1201,7 @@ class Operation(object):
''' Backward propagation.
Args:
- dys: input args consisting of CTensors.
+ dys: input args consisting of only CTensors.
Returns:
CTensor instance(s)
@@ -1209,7 +1219,7 @@ class Dummy(Operation):
def __init__(self, tensor, name=None):
self.name = name
self.src = []
- self.y_ids = {id(tensor): 0}
+ self.y_id2idx = {id(tensor): 0}
self.requires_grad = False
@@ -1268,7 +1278,8 @@ class Matmul(Operation):
Returns:
a tuple for (dx, dw)
'''
- return singa.Mult(dy, self.input[1].T()), singa.Mult(self.input[0].T(), dy)
+ return singa.Mult(dy, self.input[1].T()), \
+ singa.Mult(self.input[0].T(), dy)
def matmul(x, w):
@@ -1279,10 +1290,11 @@ class AddBias(Operation):
'''
Add Bias to each row / column of the Tensor, depending on the parameter axis.
'''
+
def __init__(self, axis=0):
'''
To indicate the calculation axis, 0 for row, 1 for column.
-
+
Args:
axis: 0 or 1, default is 0.
'''
@@ -1327,6 +1339,7 @@ class SoftMax(Operation):
Apply SoftMax for each row of the Tensor or each column of the Tensor
according to the parameter axis.
'''
+
def __init__(self, axis=0):
self.axis = axis
@@ -1372,7 +1385,7 @@ class SoftMax(Operation):
return dx.T()
-def softmax(x, axis=0):
+def soft_max(x, axis=0):
return SoftMax(axis)(x)[0]
@@ -1381,6 +1394,7 @@ class CrossEntropy(Operation):
Calculte CrossEntropy loss for a batch of training data.
'''
+
def forward(self, x, t):
'''
Args:
@@ -1393,10 +1407,12 @@ class CrossEntropy(Operation):
loss = CTensor((1,))
loss_data = -singa.SumAsFloat(singa.__mul__(t, singa.Log(x)))
loss.SetFloatValue(loss_data / x.shape()[0])
+ self.x = x
+ self.t = t
self.input = (x, t)
return loss
- def backward(self, dy):
+ def backward(self, dy=1.0):
'''
Args:
dy (float or CTensor): scalar, accumulate gradient from outside of current network, usually
@@ -1406,8 +1422,8 @@ class CrossEntropy(Operation):
dx (CTensor): data for the dL /dx, L is the loss, x is the output of current network.
note that this is true for dy = 1.0
'''
- dx = singa.__div__(self.input[1], self.input[0])
- dx *= float(-1 / self.input[0].shape()[0])
+ dx = singa.__div__(self.t, self.x)
+ dx *= float(-1 / self.x.shape()[0])
if isinstance(dy, float):
# dtype of dy: float
dx *= dy
@@ -1426,4 +1442,4 @@ def ctensor2numpy(x):
Convert a singa_tensor to numpy_tensor.
'''
np_array = x.GetFloatValue(int(x.Size()))
- return np_array.reshape(x.shape())
\ No newline at end of file
+ return np_array.reshape(x.shape())