You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@singa.apache.org by zh...@apache.org on 2017/05/24 12:12:20 UTC
[1/5] incubator-singa git commit: SINGA-313 Add L2 norm layer
Repository: incubator-singa
Updated Branches:
refs/heads/master 2d1dd4290 -> 3415099a9
SINGA-313 Add L2 norm layer
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/0815391d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/0815391d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/0815391d
Branch: refs/heads/master
Commit: 0815391d0488acbe3f788d71457c28ec4b083dee
Parents: 2d1dd42
Author: wangwei <wa...@comp.nus.edu.sg>
Authored: Mon May 22 11:20:42 2017 +0800
Committer: wangwei <wa...@comp.nus.edu.sg>
Committed: Mon May 22 11:20:42 2017 +0800
----------------------------------------------------------------------
python/singa/layer.py | 33 +++++++++++++++++++++++++++++++++
1 file changed, 33 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/0815391d/python/singa/layer.py
----------------------------------------------------------------------
diff --git a/python/singa/layer.py b/python/singa/layer.py
index caafef0..00b4763 100644
--- a/python/singa/layer.py
+++ b/python/singa/layer.py
@@ -544,6 +544,39 @@ class BatchNormalization(Layer):
self.setup(input_sample_shape)
+class L2Norm(Layer):
+ '''Normalize each sample to have L2 norm = 1'''
+ def __init__(self, name, input_sample_shape, epsilon=1e-8):
+ super(L2Norm, self).__init__(name)
+ self.y = None
+ self.norm = None
+ self.name = name
+ self.epsilon = epsilon
+ self.out_sample_shape = input_sample_shape
+
+ def get_output_sample_shape(self):
+ return self.out_sample_shape
+
+ def forward(self, is_train, x):
+ norm = tensor.sum_columns(tensor.square(x))
+ norm += self.epsilon
+ norm = tensor.sqrt(norm)
+ self.y = x.clone()
+ self.y.div_column(norm)
+
+ if is_train:
+ self.norm = norm
+ return self.y
+
+ def backward(self, is_train, dy):
+ # (dy - y * k) / norm, k = sum(dy * y)
+ k = tensor.sum_columns(tensor.eltwise_mult(dy, self.y))
+ self.y.mult_column(k)
+ dx = dy - self.y
+ dx.div_column(self.norm)
+ return dx, []
+
+
class LRN(Layer):
"""Local response normalization.
[3/5] incubator-singa git commit: SINGA-315 Reduce memory footprint
by Python generator for parameter gradient
Posted by zh...@apache.org.
SINGA-315 Reduce memory footprint by Python generator for parameter gradient
Update the API of net::backward() function.
1. add arguments dy and output. dy for the input gradient tensor(s), e.g. from the loss functions.
output is a list of layer names, whose output gradient tensor(s) would be returned in addition to the param gradient tensor(s).
2. returnes a generator iterator that generates (param_names, param_values, param_grads, out_grads) after processing each layer.
The callee function can update the parameters and release the gradient tensors in layerwise.
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/ea078dca
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/ea078dca
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/ea078dca
Branch: refs/heads/master
Commit: ea078dca9cfafdda9d5a15ae2f2823897d217292
Parents: fa4f631
Author: wangwei <wa...@comp.nus.edu.sg>
Authored: Tue May 23 13:48:39 2017 +0800
Committer: wangwei <wa...@comp.nus.edu.sg>
Committed: Tue May 23 13:48:39 2017 +0800
----------------------------------------------------------------------
python/singa/net.py | 100 ++++++++++++++++++++++++++-----------------
test/python/test_net.py | 24 ++++++++++-
2 files changed, 84 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/ea078dca/python/singa/net.py
----------------------------------------------------------------------
diff --git a/python/singa/net.py b/python/singa/net.py
index 0226864..96a9c79 100644
--- a/python/singa/net.py
+++ b/python/singa/net.py
@@ -169,22 +169,25 @@ class FeedForwardNet(object):
Currently only support nets with a single output layer, and a single
loss objective and metric.
- TODO(wangwei) consider multiple loss objectives and metrics.
+ For multiple outputs (with multiple loss/metric), please manually
+ call forward, compute loss/metric and call backward. backward() is also
+ more memory efficient than this function.
Args:
x: input data, a single input Tensor or a dict: layer name -> Tensor
y: label data, a single input Tensor.
-
Returns:
gradients of parameters and the loss and metric values.
'''
out = self.forward(kTrain, x)
l = self.loss.forward(kTrain, out, y)
+ m = None
if self.metric is not None:
m = self.metric.evaluate(out, y)
- return self.backward(), (l.l1(), m)
- else:
- return self.backward(), (l.l1(), None)
+ grads = [] # store all gradient tensors; memory inefficient
+ for _, _, grad, _ in self.backward():
+ grads.append(grad)
+ return grads[::-1], l.l1(), m
def evaluate(self, x, y):
'''Evaluate the loss and metric of the given data.
@@ -250,22 +253,23 @@ class FeedForwardNet(object):
def forward(self, flag, x, output=[]):
'''Forward the input(s) through every layer.
- If a layer has inputs from other layers and from x, the data from x is
- ordered before the data from other layers, e.g., if layer 1 -> layer 2,
- and x['layer 2'] has data, then the input of layer 2 is
- flatten([x['layer 2'], output of layer 1])
-
Args:
flag: True for training; False for evaluation; could also be
model_pb2.kTrain or model_pb2.kEval, or other values for future
use.
- x: a single SINGA tensor or a dictionary: layer name-> singa tensor
+ x: a single SINGA tensor if there is a single input; otherwise, a
+ dictionary: layer name-> singa tensor, for each layer accepting
+ input data. Do not associate a layer with input tensor if it is
+ connected from another layer. For such case, use a Dummy() layer
+ to accept the input data and connect the dummy layer to this
+ layer.
output(list): a list of layer names whose output would be returned
- in addition to the default output
+ in addition to the default output.
Returns:
- if there is only one output layer, return its output tensor(s);
- else return a dictionary: layer name -> output tensor(s)
+ if there is only one output layer and output arg is empty, return
+ the result from the single output layer; otherwise, return a
+ dictionary: layer name -> output tensor(s)
'''
if self.ordered_layers is None:
self.ordered_layers = self.topo_sort(self.layers, self.src_of_layer)
@@ -321,11 +325,26 @@ class FeedForwardNet(object):
else:
return ret
- def backward(self):
+ def backward(self, dy, output=[]):
'''Run back-propagation after forward-propagation.
+ Args:
+ dy: a single tensor if there is a single loss function; otherwise,
+ a dictionary maps the name of the layer connecting to the loss
+ function -> gradient from the loss function. Do not associate a
+ layer with gradient tensor if it is connecting to another layer.
+ For such case, connect this layer to a Dummy() layer and use the
+ dummy layer to accept the gradient.
+ output(list): a list of layer names whose output gradient would be
+ returned in addition to the param gradient
+
Returns:
- a list of gradient tensor for all parameters
+ a geneartor iterator that generates
+ (param_names, param_values, param_grads, layer_grads) after
+ processing each layer h, where the first three lists are for h
+ and the last item is a dictionary which maps
+ layer name -> its output gradient tensor(s). At the end of this
+ function, the key set includes all layers in the output arg.
'''
if self.dst_of_layer is None:
self.dst_of_layer = {}
@@ -335,29 +354,35 @@ class FeedForwardNet(object):
srcs = self.src_of_layer[cur.name]
for src in srcs:
self.dst_of_layer[src.name].append(cur)
- grad = self.loss.backward()
- if len(grad.shape) > 1:
- grad /= grad.shape[0] # average across the batch
- # print 'grad', grad.l1()
- grads = [grad]
- output_of_layer = {}
- pgrads = []
+ output_of_layer = {} # outputs generated by each layer
+ ret = {} # outputs to return
+ if type(dy) is dict:
+ input_of_layer = dy
+ else:
+ assert isinstance(dy, tensor.Tensor), \
+ 'The inputs of a net should be dict or a single tensor'
+ input_of_layer = {self.ordered_layers[-1].name: dy}
for cur in reversed(self.ordered_layers):
+ inputs = []
+ if cur.name in input_of_layer:
+ if type(input_of_layer[cur.name]) is list:
+ inputs.extend(input_of_layer[cur.name])
+ else:
+ inputs.append(input_of_layer[cur.name])
for dst in self.dst_of_layer[cur.name]:
outputs = output_of_layer[dst.name]
if type(outputs) == list:
assert len(outputs) > 0, \
'the gradient from layer %s is empty' % dst.name
- grads.append(outputs[0])
+ inputs.append(outputs[0])
outputs.pop(0)
else:
- grads.append(outputs)
+ inputs.append(outputs)
output_of_layer[dst.name] = []
# del output_of_layer[dst.name]
- if len(grads) == 1:
- grads = grads[0]
- outs, _pgrads = cur.backward(kTrain, grads)
- pgrads.append(_pgrads)
+ if len(inputs) == 1:
+ inputs = inputs[0]
+ outs, pgrads = cur.backward(kTrain, inputs)
if verbose:
disp_src = '+'.join(
[dst.name for dst in self.dst_of_layer[cur.name]])
@@ -371,12 +396,10 @@ class FeedForwardNet(object):
output_of_layer[cur.name] = outs[::-1]
else:
output_of_layer[cur.name] = outs
- grads = []
-
- ret = []
- for pgrad in reversed(pgrads):
- ret.extend(pgrad)
- return ret
+ if cur.name in output:
+ ret[cur.name] = outs
+ # ret.update(output_of_layer)
+ yield (cur.param_names(), cur.param_values(), pgrads)
def save(self, f, buffer_size=10, use_pickle=False):
'''Save model parameters using io/snapshot.
@@ -391,7 +414,7 @@ class FeedForwardNet(object):
'''
if use_pickle:
params = {}
- # since SINGA>=1.1.1
+ # since SINGA>=1.1.1 (1101)
params['SINGA_VERSION'] = __version__
for (name, val) in zip(self.param_names(), self.param_values()):
val.to_host()
@@ -416,10 +439,10 @@ class FeedForwardNet(object):
version = 0
def get_name(name):
- if version < 1011:
+ if version < 1101:
idx = name.rfind('/')
assert idx > 0, '/ must be in the parameter name'
- name = name[:idx-1] + '_' + name[idx:]
+ name = name[:idx] + '_' + name[idx+1:]
return name
if use_pickle:
@@ -442,7 +465,6 @@ class FeedForwardNet(object):
sp = snapshot.Snapshot(f, False, buffer_size)
params = sp.read()
if 'SINGA_VERSION' in params:
- # for SINGA >= 1.1.1
version = params['SINGA_VERSION']
for name, val in zip(self.param_names(), self.param_values()):
name = get_name(name)
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/ea078dca/test/python/test_net.py
----------------------------------------------------------------------
diff --git a/test/python/test_net.py b/test/python/test_net.py
index 50b976c..aad9b12 100644
--- a/test/python/test_net.py
+++ b/test/python/test_net.py
@@ -74,7 +74,7 @@ class TestFeedForwardNet(unittest.TestCase):
out = tensor.to_numpy(out['split1'])
self.assertAlmostEqual(np.average(out), 2)
- def test_save(self):
+ def test_save_load(self):
ffn = net.FeedForwardNet(loss.SoftmaxCrossEntropy())
ffn.add(layer.Conv2D('conv', 4, 3, input_sample_shape=(3, 12, 12)))
ffn.add(layer.Flatten('flat'))
@@ -88,6 +88,28 @@ class TestFeedForwardNet(unittest.TestCase):
ffn.load('test_snaphost')
ffn.load('test_pickle', use_pickle=True)
+ def test_train_one_batch(self):
+ ffn = net.FeedForwardNet(loss.SoftmaxCrossEntropy())
+ ffn.add(layer.Conv2D('conv', 4, 3, input_sample_shape=(3, 12, 12)))
+ ffn.add(layer.Flatten('flat'))
+ ffn.add(layer.Dense('dense', num_output=4))
+ for pname, pval in zip(ffn.param_names(), ffn.param_values()):
+ pval.set_value(0.1)
+ x = tensor.Tensor((4, 3, 12, 12))
+ x.gaussian(0, 0.01)
+ y = np.asarray([[1, 0, 0],
+ [0, 0, 1],
+ [0, 0, 1],
+ [0, 1, 0]], dtype=np.int32)
+ y = tensor.from_numpy(y)
+ o = ffn.forward(True, x)
+ ffn.loss.forward(True, o, y)
+ g = ffn.loss.backward()
+ for pname, pvalue, pgrad in ffn.backward(g):
+ self.assertEqual(len(pvalue), len(pgrad))
+ for p, g in zip(pvalue, pgrad):
+ self.assertEqual(p.size(), g.size())
+
if __name__ == '__main__':
unittest.main()
[4/5] incubator-singa git commit: SINGA-316 Add SigmoidCrossEntropy
Posted by zh...@apache.org.
SINGA-316 Add SigmoidCrossEntropy
Apply CrossEntropy loss for prediction (probability) generated from Sigmoid
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/be093f19
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/be093f19
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/be093f19
Branch: refs/heads/master
Commit: be093f1916da3e0f7bf55f0780b5616565d22f53
Parents: ea078dc
Author: wangwei <wa...@comp.nus.edu.sg>
Authored: Tue May 23 14:27:42 2017 +0800
Committer: Wei Wang <wa...@comp.nus.edu.sg>
Committed: Wed May 24 16:41:01 2017 +0800
----------------------------------------------------------------------
python/singa/loss.py | 54 ++++++++++++++++++++++++++++++++++++++++++
python/singa/net.py | 2 +-
test/python/test_loss.py | 55 +++++++++++++++++++++++++++++++++++++++++++
3 files changed, 110 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/be093f19/python/singa/loss.py
----------------------------------------------------------------------
diff --git a/python/singa/loss.py b/python/singa/loss.py
index 60835fc..7fd3d77 100644
--- a/python/singa/loss.py
+++ b/python/singa/loss.py
@@ -39,6 +39,7 @@ Example usage::
from . import singa_wrap as singa
from proto import model_pb2
import tensor
+import numpy as np
class Loss(object):
@@ -116,6 +117,59 @@ class SoftmaxCrossEntropy(Loss):
self.swig_loss = singa.SoftmaxCrossEntropy()
+class SigmoidCrossEntropy(Loss):
+ '''This loss evaluates the cross-entropy loss between the prediction and the
+ truth values with the prediction probability generated from Sigmoid.
+ '''
+ def __init__(self, epsilon=1e-8):
+ super(SigmoidCrossEntropy, self).__init__()
+ self.truth = None
+ self.prob = None
+ self.epsilon = epsilon # to avoid log(x) with x being too small
+
+ def forward(self, flag, x, y):
+ '''loss is -yi * log pi - (1-yi) log (1-pi), where pi=sigmoid(xi)
+
+ Args:
+ flag (bool): true for training; false for evaluation
+ x (Tensor): the prediction Tensor
+ y (Tensor): the truth Tensor, a binary array value per sample
+
+ Returns:
+ a Tensor with one error value per sample
+ '''
+ p = tensor.sigmoid(x)
+ if flag:
+ self.truth = y
+ self.prob = p
+ np = 1 - p
+ p += (p < self.epsilon) * self.epsilon
+ np += (np < self.epsilon) * self.epsilon
+ l = (y-1) * tensor.log(np) - y * tensor.log(p)
+ # TODO(wangwei): add unary operation -Tensor
+ return tensor.average(l, axis=1)
+
+ def backward(self):
+ ''' Compute the gradient of loss w.r.t to x.
+
+ Returns:
+ dx = pi - yi.
+ '''
+ assert self.truth is not None, 'must call forward in a prior'
+ dx = self.prob - self.truth
+ self.truth = None
+ return dx
+
+ def evaluate(self, flag, x, y):
+ '''Compuate the averaged error.
+
+ Returns:
+ a float value as the averaged error
+ '''
+ l = self.forward(False, x, y)
+ return l.l1()
+
+
class SquaredError(Loss):
'''This loss evaluates the squared error between the prediction and the
truth values.
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/be093f19/python/singa/net.py
----------------------------------------------------------------------
diff --git a/python/singa/net.py b/python/singa/net.py
index 96a9c79..e4c1de9 100644
--- a/python/singa/net.py
+++ b/python/singa/net.py
@@ -399,7 +399,7 @@ class FeedForwardNet(object):
if cur.name in output:
ret[cur.name] = outs
# ret.update(output_of_layer)
- yield (cur.param_names(), cur.param_values(), pgrads)
+ yield (cur.param_names(), cur.param_values(), pgrads, ret)
def save(self, f, buffer_size=10, use_pickle=False):
'''Save model parameters using io/snapshot.
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/be093f19/test/python/test_loss.py
----------------------------------------------------------------------
diff --git a/test/python/test_loss.py b/test/python/test_loss.py
new file mode 100644
index 0000000..78356f2
--- /dev/null
+++ b/test/python/test_loss.py
@@ -0,0 +1,55 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import unittest
+
+import numpy as np
+
+from singa import loss
+from singa import tensor
+
+
+class TestLoss(unittest.TestCase):
+ def setUp(self):
+ self.x_np = np.asarray([[0.9, 0.2, 0.1],
+ [0.1, 0.4, 0.5],
+ [0.2, 0.4, 0.4]],
+ dtype=np.float32)
+
+ self.y_np = np.asarray([[1, 0, 1],
+ [0, 1, 1],
+ [1, 0, 0]],
+ dtype=np.float32)
+
+ self.x = tensor.from_numpy(self.x_np)
+ self.y = tensor.from_numpy(self.y_np)
+
+ def test_sigmoid_cross_entropy(self):
+ sig = loss.SigmoidCrossEntropy()
+ l1 = sig.forward(True, self.x, self.y)
+ sig.backward()
+ l2 = sig.evaluate(True, self.x, self.y)
+
+ p = 1.0 / (1 + np.exp(-self.x_np))
+ l = - (self.y_np * np.log(p) + (1-self.y_np) * np.log(1-p))
+ self.assertAlmostEqual(l1.l1(), l2)
+ self.assertAlmostEqual(l1.l1(), np.average(l))
+
+
+if __name__ == '__main__':
+ unittest.main()
[2/5] incubator-singa git commit: SINGA-312 Rename layer parameters
Posted by zh...@apache.org.
SINGA-312 Rename layer parameters
Layer parameters are now renamed to PREFIX/SUFFIX, where PREFIX could be layer name and SUFFIX could be 'weight' or 'bias' or 'mean', etc.
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/fa4f6314
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/fa4f6314
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/fa4f6314
Branch: refs/heads/master
Commit: fa4f631465ae033e9286b86f102d86554bea03d3
Parents: 0815391
Author: wangwei <wa...@comp.nus.edu.sg>
Authored: Mon May 22 16:33:50 2017 +0800
Committer: wangwei <wa...@comp.nus.edu.sg>
Committed: Mon May 22 16:33:50 2017 +0800
----------------------------------------------------------------------
CMakeLists.txt | 14 +++---
include/singa/core/tensor.h | 5 +++
python/singa/layer.py | 18 ++++----
python/singa/net.py | 93 +++++++++++++++++++++++++---------------
python/singa/snapshot.py | 8 ++--
test/python/test_net.py | 24 +++++++++--
6 files changed, 104 insertions(+), 58 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/fa4f6314/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 7dd1b57..c9e47a3 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -19,10 +19,10 @@
CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
PROJECT(singa)
-SET(PACKAGE_VERSION "1.1.0")
+SET(PACKAGE_VERSION "1.1.1")
SET(SINGA_MAJOR_VERSION 1) # 0 -
SET(SINGA_MINOR_VERSION 1) # 0 - 9
-SET(SINGA_PATCH_VERSION 0) # 0 - 99
+SET(SINGA_PATCH_VERSION 1) # 0 - 99
MATH(EXPR SINGA_VERSION "${SINGA_MAJOR_VERSION} * 1000 + ${SINGA_MINOR_VERSION} * 100 + ${SINGA_PATCH_VERSION}")
LIST(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake/Thirdparty)
@@ -174,18 +174,18 @@ INSTALL(CODE "execute_process(COMMAND python setup.py install --prefix=${CMAKE_I
# CPack
IF(PACKAGE)
IF(USE_PYTHON)
- INSTALL(FILES ${CMAKE_BINARY_DIR}/python/setup.py
+ INSTALL(FILES ${CMAKE_BINARY_DIR}/python/setup.py
DESTINATION ${CMAKE_INSTALL_PREFIX}/lib/singa/python)
INSTALL(FILES ${CMAKE_BINARY_DIR}/python/singa/singa_wrap.py
DESTINATION ${CMAKE_INSTALL_PREFIX}/lib/singa/python/singa)
INSTALL(FILES ${CMAKE_BINARY_DIR}/python/singa/_singa_wrap.so
DESTINATION ${CMAKE_INSTALL_PREFIX}/lib/singa/python/singa)
- INSTALL(DIRECTORY ${PROJECT_SOURCE_DIR}/python
- DESTINATION ${CMAKE_INSTALL_PREFIX}/lib/singa)
+ INSTALL(DIRECTORY ${PROJECT_SOURCE_DIR}/python
+ DESTINATION ${CMAKE_INSTALL_PREFIX}/lib/singa)
INSTALL(DIRECTORY ${CMAKE_BINARY_DIR}/python/singa/proto
- DESTINATION ${CMAKE_INSTALL_PREFIX}/lib/singa/python/singa)
+ DESTINATION ${CMAKE_INSTALL_PREFIX}/lib/singa/python/singa)
ENDIF()
-
+
IF (USE_MODULES)
SET(CORE_DEPENDENCIES "libgoogle-glog-dev, libstdc++6, libc6")
ELSE()
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/fa4f6314/include/singa/core/tensor.h
----------------------------------------------------------------------
diff --git a/include/singa/core/tensor.h b/include/singa/core/tensor.h
index c89fa83..6621fa0 100644
--- a/include/singa/core/tensor.h
+++ b/include/singa/core/tensor.h
@@ -106,6 +106,11 @@ class Tensor {
bool transpose() const { return transpose_; }
+ /// return true if the content of the tensor is initialized
+ bool initailized() const {
+ return block_ != nullptr && block_->initialized();
+ }
+
/// return number of total elements
size_t Size() const {
if (block_ == nullptr) return 0u;
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/fa4f6314/python/singa/layer.py
----------------------------------------------------------------------
diff --git a/python/singa/layer.py b/python/singa/layer.py
index 00b4763..4fe9983 100644
--- a/python/singa/layer.py
+++ b/python/singa/layer.py
@@ -347,7 +347,7 @@ class Conv2D(Layer):
if W_specs is None:
W_specs = {'init': 'xavier'}
if 'name' not in W_specs:
- W_specs['name'] = name + '_weight'
+ W_specs['name'] = name + '/weight'
wspecs = _construct_param_specs_from_dict(W_specs)
self.conf.param.extend([wspecs])
self.param_specs.append(wspecs)
@@ -355,7 +355,7 @@ class Conv2D(Layer):
if b_specs is None:
b_specs = {'init': 'constant'}
if 'name' not in b_specs:
- b_specs['name'] = name + '_bias'
+ b_specs['name'] = name + '/bias'
bspecs = _construct_param_specs_from_dict(b_specs)
self.conf.param.extend([bspecs])
self.param_specs.append(bspecs)
@@ -524,11 +524,11 @@ class BatchNormalization(Layer):
if gamma_specs is None:
gamma_specs = {'init': 'Xavier'}
if 'name' not in beta_specs:
- beta_specs['name'] = name + '_beta'
+ beta_specs['name'] = name + '/beta'
if 'name' not in gamma_specs:
- gamma_specs['name'] = name + '_gamma'
- mean_specs = {'init': 'constant', 'value': 0, 'name': name + '_mean'}
- var_specs = {'init': 'constant', 'value': 1, 'name': name + '_var'}
+ gamma_specs['name'] = name + '/gamma'
+ mean_specs = {'init': 'constant', 'value': 0, 'name': name + '/mean'}
+ var_specs = {'init': 'constant', 'value': 1, 'name': name + '/var'}
self.conf.param.extend([_construct_param_specs_from_dict(gamma_specs)])
self.conf.param.extend([_construct_param_specs_from_dict(beta_specs)])
self.conf.param.extend([_construct_param_specs_from_dict(mean_specs)])
@@ -656,7 +656,7 @@ class Dense(Layer):
if W_specs is None:
W_specs = {'init': 'xavier'}
if 'name' not in W_specs:
- W_specs['name'] = name + '_weight'
+ W_specs['name'] = name + '/weight'
wspecs = _construct_param_specs_from_dict(W_specs)
self.conf.param.extend([wspecs])
self.param_specs.append(wspecs)
@@ -664,7 +664,7 @@ class Dense(Layer):
if b_specs is None:
b_specs = {'init': 'constant', 'value': 0}
if 'name' not in b_specs:
- b_specs['name'] = name + '_bias'
+ b_specs['name'] = name + '/bias'
bspecs = _construct_param_specs_from_dict(b_specs)
self.conf.param.extend([bspecs])
self.param_specs.append(bspecs)
@@ -1020,7 +1020,7 @@ class RNN(Layer):
# currently only has rnn layer implemented using cudnn
_check_engine(engine, ['cudnn'])
if param_specs is None:
- param_specs = {'name': name + '-weight',
+ param_specs = {'name': name + '/weight',
'init': 'uniform', 'low': 0, 'high': 1}
self.conf.param.extend([_construct_param_specs_from_dict(param_specs)])
self.param_specs.append(_construct_param_specs_from_dict(param_specs))
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/fa4f6314/python/singa/net.py
----------------------------------------------------------------------
diff --git a/python/singa/net.py b/python/singa/net.py
index 2be8d95..0226864 100644
--- a/python/singa/net.py
+++ b/python/singa/net.py
@@ -55,11 +55,14 @@ Example usages::
"""
from .proto.model_pb2 import kTrain, kEval
+from __init__ import __version__
import tensor
import layer
import snapshot
import cPickle as pickle
+import os
+
'''For display training information, e.g L1 value of layer data'''
verbose = False
@@ -134,7 +137,7 @@ class FeedForwardNet(object):
else:
self.out_sample_shape_of_layer[lyr.name] = [out_shape]
self.layers.append(lyr)
- print lyr.name, out_shape
+ print(lyr.name, out_shape)
return lyr
def param_values(self):
@@ -181,7 +184,7 @@ class FeedForwardNet(object):
m = self.metric.evaluate(out, y)
return self.backward(), (l.l1(), m)
else:
- return self.backward(), (l.l1(),None)
+ return self.backward(), (l.l1(), None)
def evaluate(self, x, y):
'''Evaluate the loss and metric of the given data.
@@ -303,10 +306,10 @@ class FeedForwardNet(object):
disp_src = '+'.join([src.name for src in srcs])
disp_src += '-->' + cur.name
if type(out) is list:
- print '%s: %s' % (disp_src,
- ' '.join([str(o.l1()) for o in out]))
+ print('%s: %s' % (disp_src,
+ ' '.join([str(o.l1()) for o in out])))
else:
- print '%s: %f' % (disp_src, out.l1())
+ print('%s: %f' % (disp_src, out.l1()))
output_of_layer[cur.name] = out
if cur.name in output:
ret[cur.name] = out
@@ -360,10 +363,10 @@ class FeedForwardNet(object):
[dst.name for dst in self.dst_of_layer[cur.name]])
disp_src += '-->' + cur.name
if type(outs) is list:
- print '%s: %s' % (disp_src,
- ' '.join([str(o.l1()) for o in outs]))
+ print('%s: %s' % (disp_src,
+ ' '.join([str(o.l1()) for o in outs])))
else:
- print '%s: %f' % (disp_src, outs.l1())
+ print('%s: %f' % (disp_src, outs.l1()))
if type(outs) is list:
output_of_layer[cur.name] = outs[::-1]
else:
@@ -388,12 +391,18 @@ class FeedForwardNet(object):
'''
if use_pickle:
params = {}
+ # since SINGA>=1.1.1
+ params['SINGA_VERSION'] = __version__
for (name, val) in zip(self.param_names(), self.param_values()):
val.to_host()
params[name] = tensor.to_numpy(val)
- with open(f, 'wb') as fd:
- pickle.dump(params, fd)
+ if not f.endswith('.pickle'):
+ f = f + '.pickle'
+ with open(f, 'wb') as fd:
+ pickle.dump(params, fd)
else:
+ if f.endswith('.bin'):
+ f = f[0:-4]
sp = snapshot.Snapshot(f, True, buffer_size)
for (name, val) in zip(self.param_names(), self.param_values()):
val.to_host()
@@ -404,35 +413,49 @@ class FeedForwardNet(object):
Please refer to the argument description in save().
'''
+ version = 0
+
+ def get_name(name):
+ if version < 1011:
+ idx = name.rfind('/')
+ assert idx > 0, '/ must be in the parameter name'
+ name = name[:idx-1] + '_' + name[idx:]
+ return name
+
if use_pickle:
- print 'NOTE: If your model was saved using Snapshot, '\
- 'then set use_pickle=False for loading it'
+ print('NOTE: If your model was saved using Snapshot, '
+ 'then set use_pickle=False for loading it')
+ if not os.path.exists(f):
+ # guess the correct path
+ if f.endswith('.pickle'):
+ f = f[0:-7]
+ else:
+ f = f + '.pickle'
+ assert os.path.exists(f), 'file not exists %s w/o .pickle' % f
with open(f, 'rb') as fd:
params = pickle.load(fd)
- for name, val in zip(self.param_names(), self.param_values()):
- if name not in params:
- print 'Param: %s missing in the checkpoint file' % name
- continue
- try:
- val.copy_from_numpy(params[name])
- except AssertionError as err:
- print 'Error from copying values for param: %s' % name
- print 'shape of param vs checkpoint', \
- val.shape, params[name].shape
- raise err
else:
- print 'NOTE: If your model was saved using pickle, '\
- 'then set use_pickle=True for loading it'
+ print('NOTE: If your model was saved using pickle, '
+ 'then set use_pickle=True for loading it')
+ if f.endswith('.bin'):
+ f = f[0:-4]
sp = snapshot.Snapshot(f, False, buffer_size)
params = sp.read()
- for (name, val) in zip(self.param_names(), self.param_values()):
- if name not in params:
- print 'Param: %s missing in the checkpoint file' % name
- continue
- try:
+ if 'SINGA_VERSION' in params:
+ # for SINGA >= 1.1.1
+ version = params['SINGA_VERSION']
+ for name, val in zip(self.param_names(), self.param_values()):
+ name = get_name(name)
+ if name not in params:
+ print('Param: %s missing in the checkpoint file' % name)
+ continue
+ try:
+ if isinstance(params[name], tensor.Tensor):
val.copy_data(params[name])
- except AssertionError as err:
- print 'Error from copying values for param: %s' % name
- print 'shape of param vs checkpoint', \
- val.shape, params[name].shape
- raise err
+ else:
+ val.copy_from_numpy(params[name])
+ except AssertionError as err:
+ print('Error from copying values for param: %s' % name)
+ print('shape of param vs checkpoint',
+ val.shape, params[name].shape)
+ raise err
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/fa4f6314/python/singa/snapshot.py
----------------------------------------------------------------------
diff --git a/python/singa/snapshot.py b/python/singa/snapshot.py
index bd8918e..3e1298f 100644
--- a/python/singa/snapshot.py
+++ b/python/singa/snapshot.py
@@ -33,11 +33,12 @@ Example usages::
from . import singa_wrap as singa
import tensor
+
class Snapshot(object):
''' Class and member functions for singa::Snapshot.
'''
- def __init__(self, f, mode, buffer_size = 10):
+ def __init__(self, f, mode, buffer_size=10):
'''Snapshot constructor given file name and R/W mode.
Args:
@@ -55,6 +56,7 @@ class Snapshot(object):
param_val (Tensor): value tensor of the parameter
'''
self.snapshot.Write(str(param_name), param_val.singa_tensor)
+
def read(self):
'''Call read method to load all (param_name, param_val)
@@ -62,8 +64,8 @@ class Snapshot(object):
a dict of (parameter name, parameter Tensor)
'''
params = {}
- p = self.snapshot.Read();
+ p = self.snapshot.Read()
for (param_name, param_val) in p:
- print param_name
+ # print(param_name)
params[param_name] = tensor.from_raw_tensor(param_val)
return params
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/fa4f6314/test/python/test_net.py
----------------------------------------------------------------------
diff --git a/test/python/test_net.py b/test/python/test_net.py
index 53a4f24..50b976c 100644
--- a/test/python/test_net.py
+++ b/test/python/test_net.py
@@ -28,6 +28,7 @@ from singa import loss
layer.engine = 'singacpp'
# net.verbose = True
+
class TestFeedForwardNet(unittest.TestCase):
def test_single_input_output(self):
@@ -40,8 +41,9 @@ class TestFeedForwardNet(unittest.TestCase):
y.set_value(0)
out, _ = ffn.evaluate(x, y)
self.assertAlmostEqual(out * 3,
- - math.log(1.0/(1+math.exp(1))) - math.log(0.5) -math.log(0.5),
- 5);
+ - math.log(1.0/(1+math.exp(1))) -
+ math.log(0.5) - math.log(0.5),
+ 5)
def test_mult_inputs(self):
ffn = net.FeedForwardNet(loss.SoftmaxCrossEntropy())
@@ -52,7 +54,7 @@ class TestFeedForwardNet(unittest.TestCase):
x1.set_value(1.1)
x2 = tensor.Tensor((2, 2))
x2.set_value(0.9)
- out = ffn.forward(False, {'relu1':x1, 'relu2':x2})
+ out = ffn.forward(False, {'relu1': x1, 'relu2': x2})
out = tensor.to_numpy(out)
self.assertAlmostEqual(np.average(out), 2)
@@ -68,10 +70,24 @@ class TestFeedForwardNet(unittest.TestCase):
x1.set_value(1.1)
x2 = tensor.Tensor((2, 2))
x2.set_value(0.9)
- out = ffn.forward(False, {'relu1':x1, 'relu2':x2})
+ out = ffn.forward(False, {'relu1': x1, 'relu2': x2})
out = tensor.to_numpy(out['split1'])
self.assertAlmostEqual(np.average(out), 2)
+ def test_save(self):
+ ffn = net.FeedForwardNet(loss.SoftmaxCrossEntropy())
+ ffn.add(layer.Conv2D('conv', 4, 3, input_sample_shape=(3, 12, 12)))
+ ffn.add(layer.Flatten('flat'))
+ # ffn.add(layer.BatchNorm('bn'))
+ ffn.add(layer.Dense('dense', num_output=4))
+ for pname, pval in zip(ffn.param_names(), ffn.param_values()):
+ pval.set_value(0.1)
+ ffn.save('test_snaphost')
+ ffn.save('test_pickle', use_pickle=True)
+
+ ffn.load('test_snaphost')
+ ffn.load('test_pickle', use_pickle=True)
+
if __name__ == '__main__':
unittest.main()
[5/5] incubator-singa git commit: SINGA-317 Extend ImageBatchIter to
read labels in general format
Posted by zh...@apache.org.
SINGA-317 Extend ImageBatchIter to read labels in general format
To enable the image list file to include more general information than
label index.
Now the second part of each line (separated by user defined delimiter)
could be label strings, variable-length label indexs or a single label
index.
If it is a single label index, we return a numpy array of length =
batchsize for the label indexs. Otherwise, we return a list of length =
batchsize for the meta information of each image. Users have to parse
the information in their code.
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/3415099a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/3415099a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/3415099a
Branch: refs/heads/master
Commit: 3415099a96a9b8c319fe36dd06bf28a7eea3ee92
Parents: be093f1
Author: Wei Wang <wa...@comp.nus.edu.sg>
Authored: Wed May 24 19:46:05 2017 +0800
Committer: Wei Wang <wa...@comp.nus.edu.sg>
Committed: Wed May 24 19:46:05 2017 +0800
----------------------------------------------------------------------
python/singa/data.py | 34 +++++++++++++++++++++++-----------
1 file changed, 23 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3415099a/python/singa/data.py
----------------------------------------------------------------------
diff --git a/python/singa/data.py b/python/singa/data.py
index 3a99ad3..ec4aa97 100644
--- a/python/singa/data.py
+++ b/python/singa/data.py
@@ -61,7 +61,12 @@ class ImageBatchIter:
Args:
img_list_file(str): name of the file containing image meta data; each
- line consists of image_path_suffix delimiter label
+ line consists of image_path_suffix delimiter meta_info,
+ where meta info could be label index or label strings, etc.
+ meta_info should not contain the delimiter. If the meta_info
+ of each image is just the label index, then we will parse the
+ label index into a numpy array with length=batchsize
+ (for compatibility); otherwise, we return a list of meta_info
batch_size(int): num of samples in one mini-batch
image_transform: a function for image augmentation; it accepts the full
image path and outputs a list of augmented images.
@@ -106,21 +111,21 @@ class ImageBatchIter:
def run(self):
img_list = []
+ is_labelindex = True
for line in open(self.img_list_file, 'r'):
- item = line.split(self.delimiter)
- img_path = item[0]
- img_label = int(item[1])
- img_list.append((img_label, img_path))
+ item = line.strip('\n').split(self.delimiter)
+ if not item[1].strip().isdigit(): # the meta info is not label index
+ is_labelindex = False
+ img_list.append((item[0].strip(), item[1].strip()))
index = 0 # index for the image
if self.shuffle:
random.shuffle(img_list)
while not self.stop:
if not self.queue.full():
- x = []
- y = np.empty(self.batch_size, dtype=np.int32)
+ x, y = [], []
i = 0
while i < self.batch_size:
- img_label, img_path = img_list[index]
+ img_path, img_meta = img_list[index]
aug_images = self.image_transform(
os.path.join(self.image_folder, img_path))
assert i + len(aug_images) <= self.batch_size, \
@@ -129,7 +134,10 @@ class ImageBatchIter:
for img in aug_images:
ary = np.asarray(img.convert('RGB'), dtype=np.float32)
x.append(ary.transpose(2, 0, 1))
- y[i] = img_label
+ if is_labelindex:
+ y.append(int(img_meta))
+ else:
+ y.append(img_meta)
i += 1
index += 1
if index == self.num_samples:
@@ -137,7 +145,10 @@ class ImageBatchIter:
if self.shuffle:
random.shuffle(img_list)
# enqueue one mini-batch
- self.queue.put((np.asarray(x), y))
+ if is_labelindex:
+ self.queue.put((np.asarray(x), np.asarray(y, dtype=np.int32)))
+ else:
+ self.queue.put((np.asarray(x), y))
else:
time.sleep(0.1)
return
@@ -155,11 +166,12 @@ if __name__ == '__main__':
(96, 96)).flip().get()
data = ImageBatchIter('train.txt', 3,
- image_transform, shuffle=True, delimiter=',',
+ image_transform, shuffle=False, delimiter=',',
image_folder='images/',
capacity=10)
data.start()
imgs, labels = data.next()
+ print labels
for idx in range(imgs.shape[0]):
img = Image.fromarray(imgs[idx].astype(np.uint8).transpose(1, 2, 0),
'RGB')