You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@singa.apache.org by zh...@apache.org on 2016/11/22 08:49:29 UTC
[1/3] incubator-singa git commit: SINGA-271 Add Concat and Slice
layers
Repository: incubator-singa
Updated Branches:
refs/heads/master f35d217c9 -> 5afd81c84
SINGA-271 Add Concat and Slice layers
Add concat and slice layers
Pass unit tests for Slice and Concat layers;
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/16f3bf64
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/16f3bf64
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/16f3bf64
Branch: refs/heads/master
Commit: 16f3bf649b9c1fc474304df0d89a515c57b0abac
Parents: f35d217
Author: wangwei <wa...@comp.nus.edu.sg>
Authored: Sun Nov 20 00:10:05 2016 +0800
Committer: wang wei <wa...@comp.nus.edu.sg>
Committed: Tue Nov 22 06:25:00 2016 +0000
----------------------------------------------------------------------
include/singa/core/tensor.h | 11 ++-
src/core/tensor/tensor.cc | 36 ++++---
src/model/layer/concat.cc | 70 ++++++++++++++
src/model/layer/concat.h | 54 +++++++++++
src/model/layer/cudnn_rnn.cc | 20 ++++
src/model/layer/slice.cc | 72 ++++++++++++++
src/model/layer/slice.h | 54 +++++++++++
src/model/layer/split.cc | 3 +
test/singa/test_concat.cc | 193 ++++++++++++++++++++++++++++++++++++++
9 files changed, 490 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/16f3bf64/include/singa/core/tensor.h
----------------------------------------------------------------------
diff --git a/include/singa/core/tensor.h b/include/singa/core/tensor.h
index a41afbc..a39217b 100644
--- a/include/singa/core/tensor.h
+++ b/include/singa/core/tensor.h
@@ -448,21 +448,26 @@ void ComputeCrossEntropy(const Tensor &p, const Tensor &t, Tensor *loss);
/// or 2-d matrix. 'grad' has the same shape as 'p'. dx is computed into p.
void SoftmaxCrossEntropyBwd(const Tensor &t, Tensor *p);
-/// Return a tensor consisting of rows ([start, end)) from 'in'. It shares the
-/// memory with 'in'. 'in' is a 1D or 2D Tensor.
-Tensor SliceRows(const Tensor &in, const size_t start, const size_t end);
/// Return a tensor consisting of rows ([start, end)) from 'in'. It copies the
/// values from 'in'. 'in' ia a 2D Tensor.
Tensor CopyRows(const Tensor &in, const size_t start, const size_t end);
+/// Alias of CopyRows
+Tensor SliceRows(const Tensor &in, const size_t start, const size_t end);
/// Return a tensor consisting of columns ([start, end)) from 'in'. It copies
/// the values from 'in'. 'in' is a 2D Tensor.
Tensor CopyColumns(const Tensor &in, const size_t start, const size_t end);
+/// Alias of CopyColumns
+Tensor SliceColumns(const Tensor &in, const size_t start, const size_t end);
/// Return a tensor which is vertically stacked from tensors in 'in'. Each
/// tensor in 'in' is a 2D tensor. Values are copied, no memory sharing.
Tensor ConcatenateRows(const vector<Tensor> &in);
+/// Alias name for function ConcatenateRows
+Tensor ConcatRows(const vector<Tensor> &in);
/// Return a tensor which is horizontally stacked from tensors in 'in'. Each
/// tensor in 'in' is a 2D tensor. Values are copied, no memory sharing.
Tensor ConcatenateColumns(const vector<Tensor> &in);
+/// Alias name for function ConcatenateColumns
+Tensor ConcatColumns(const vector<Tensor> &in);
} // namespace singa
#endif // SINGA_CORE_TENSOR_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/16f3bf64/src/core/tensor/tensor.cc
----------------------------------------------------------------------
diff --git a/src/core/tensor/tensor.cc b/src/core/tensor/tensor.cc
index 424edb2..83e1a00 100644
--- a/src/core/tensor/tensor.cc
+++ b/src/core/tensor/tensor.cc
@@ -762,7 +762,9 @@ Tensor ConcatenateRows(const vector<Tensor> &in) {
}
return out;
}
-
+Tensor ConcatRows(const vector<Tensor> &in) {
+ return ConcatenateRows(in);
+}
// TODO(wangwei) add a copypatch function for improve the efficiency on GPU.
Tensor ConcatenateColumns(const vector<Tensor> &in) {
size_t nrow = 0, ncol = 0;
@@ -788,10 +790,13 @@ Tensor ConcatenateColumns(const vector<Tensor> &in) {
}
return out;
}
+Tensor ConcatColumns(const vector<Tensor> &in) {
+ return ConcatenateColumns(in);
+}
Tensor CopyRows(const Tensor &in, const size_t start, const size_t end) {
CHECK_LT(start, end);
- CHECK_GE(in.shape(0), end);
+ CHECK_GE(in.shape(0), end) << "Tensor size must >= end";
Shape s = in.shape();
s[0] = end - start;
size_t sample_size = in.Size() / in.shape(0);
@@ -800,6 +805,10 @@ Tensor CopyRows(const Tensor &in, const size_t start, const size_t end) {
return out;
}
+Tensor SliceRows(const Tensor &in, const size_t start, const size_t end) {
+ return CopyRows(in, start, end);
+}
+
Tensor CopyColumns(const Tensor &in, const size_t start, const size_t end) {
CHECK_EQ(in.nDim(), 2u);
CHECK_LT(start, end);
@@ -814,6 +823,11 @@ Tensor CopyColumns(const Tensor &in, const size_t start, const size_t end) {
return out;
}
+Tensor SliceColumns(const Tensor &in, const size_t start, const size_t end) {
+ return CopyColumns(in, start, end);
+}
+
+
/// Divide row 'v' by each row of matrix M; write results into 'out'
void DivRow(const Tensor &v, Tensor *M) {
Tensor inv;
@@ -851,24 +865,6 @@ void MultRow(const Tensor &v, Tensor *M) {
});
}
-Tensor SliceRows(const Tensor &in, const size_t start, const size_t end) {
- LOG(FATAL) << "Tensor::SliceRows is not implemented";
- Tensor ret;
- /*
- CHECK_LE(in.nDim(), 2);
- CHECK_LT(start, end);
- CHECK_LE(in.shape(0), end);
- Shape s;
- if (in.nDim() == 2)
- s = Shape{end - start, in.shape(1)};
- else
- s = Shape{end - start};
- Tensor out(s, in.device(), in.data_type());
- Block *b = out.block();
- */
- return ret;
-}
-
void SubColumn(const Tensor &v, Tensor *M) { AddColumn(-1, 1, v, M); }
void SubRow(const Tensor &v, Tensor *M) { AddRow(-1, 1, v, M); }
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/16f3bf64/src/model/layer/concat.cc
----------------------------------------------------------------------
diff --git a/src/model/layer/concat.cc b/src/model/layer/concat.cc
new file mode 100644
index 0000000..b1c0b11
--- /dev/null
+++ b/src/model/layer/concat.cc
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "singa/model/layer.h"
+#include "./concat.h"
+namespace singa {
+
+RegisterLayerClass(singa_concat, Concat);
+RegisterLayerClass(singacpp_concat, Concat);
+RegisterLayerClass(singacuda_concat, Concat);
+RegisterLayerClass(singacl_concat, Concat);
+
+void Concat::Setup(const vector<Shape>& in_shapes, const LayerConf& conf) {
+ Layer::Setup(in_shapes, conf);
+ dim_size_.clear();
+ axis_ = conf.concat_conf().axis();
+ out_sample_shape_ = {0, 0};
+ out_sample_shape_[1 - axis_] = in_shapes[0][1 - axis_];
+ for (auto& s: in_shapes) {
+ out_sample_shape_[axis_] += s[axis_];
+ dim_size_.push_back(s[axis_]);
+ // LOG(ERROR) << s[axis_];
+ }
+}
+
+const vector<Tensor> Concat::Forward(int flag, const vector<Tensor>& inputs) {
+ vector<Tensor> outputs;
+ if (inputs.size() == 1u) {
+ outputs = inputs;
+ } else {
+ if(axis_ == 0)
+ outputs.push_back(ConcatRows(inputs));
+ else
+ outputs.push_back(ConcatColumns(inputs));
+ }
+ return outputs;
+}
+
+const std::pair<vector<Tensor>, vector<Tensor>> Concat::Backward(
+ int flag, const vector<Tensor>& grads) {
+ vector<Tensor> input_grad, param_grad;
+ CHECK_EQ(grads.size(), 1u) << "Concat layer only have one output tensor.";
+ for (size_t i = 0, offset = 0; i < dim_size_.size(); i++) {
+ if (axis_ == 0)
+ input_grad.push_back(SliceRows(grads.at(0), offset,
+ offset + dim_size_[i]));
+ else
+ input_grad.push_back(SliceColumns(grads.at(0), offset,
+ offset + dim_size_[i]));
+ offset += dim_size_[i];
+ }
+ return std::make_pair(input_grad, param_grad);
+}
+
+} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/16f3bf64/src/model/layer/concat.h
----------------------------------------------------------------------
diff --git a/src/model/layer/concat.h b/src/model/layer/concat.h
new file mode 100644
index 0000000..59293d7
--- /dev/null
+++ b/src/model/layer/concat.h
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef SINGA_MODEL_LAYER_CONCAT_H_
+#define SINGA_MODEL_LAYER_CONCAT_H_
+#include <utility>
+#include <string>
+#include <vector>
+#include "singa/model/layer.h"
+
+namespace singa {
+class Concat : public Layer {
+ public:
+ /// \copydoc Layer::layer_type()
+ // const std::string layer_type() const override { return "Concat"; }
+
+ /// \copydoc Layer::Setup(const LayerConf&);
+ void Setup(const vector<Shape>& in_shapes, const LayerConf& conf);
+ const Shape GetOutputSampleShape() const override {
+ CHECK(out_sample_shape_.size()) << "You may haven't call Setup()";
+ return out_sample_shape_;
+ }
+
+ /// \copydoc Layer::Forward(int flag, const Tensor&)
+ const vector<Tensor> Forward(int flag, const vector<Tensor>& input) override;
+
+ /// \copydoc Layer::Backward(int, const Tensor&, const Tensor&);
+ const std::pair<vector<Tensor>, vector<Tensor>> Backward(int flag,
+ const vector<Tensor>& grad) override;
+
+ protected:
+ /// 0 for concat rows; 1 for concat cols
+ int axis_ = 0;
+ /// dim_size_[i] the size of the i-th source tensor on the concat dim
+ vector<int> dim_size_;
+ Shape out_sample_shape_;
+};
+
+} // namespace singa
+#endif // SINGA_MODEL_LAYER_CONCAT_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/16f3bf64/src/model/layer/cudnn_rnn.cc
----------------------------------------------------------------------
diff --git a/src/model/layer/cudnn_rnn.cc b/src/model/layer/cudnn_rnn.cc
index 0788801..583dcda 100644
--- a/src/model/layer/cudnn_rnn.cc
+++ b/src/model/layer/cudnn_rnn.cc
@@ -55,6 +55,7 @@ void CudnnRNN::ToDevice(std::shared_ptr<Device> device) {
RNN::ToDevice(device);
workspace_.ToDevice(device);
reserve_space_.ToDevice(device);
+ dropout_state_.ToDevice(device);
}
void CudnnRNN::DestroyIODescriptors() {
@@ -281,6 +282,23 @@ const vector<Tensor> CudnnRNN::Forward(int flag, const vector<Tensor> &inputs) {
cy.ResetLike(hy);
}
+ int did = input.device()->id();
+ CHECK_EQ(did, output.device()->id());
+ if (hx.Size()) {
+ CHECK_EQ(did, hx.device()->id());
+ CHECK_EQ(hx.device()->lang(), kCuda);
+ }
+ if (cx.Size()) {
+ CHECK_EQ(did, cx.device()->id());
+ CHECK_EQ(cx.device()->lang(), kCuda);
+ }
+ CHECK_EQ(did, weight_.device()->id());
+ CHECK_EQ(did, workspace_.device()->id());
+ CHECK_EQ(input.device()->lang(), kCuda);
+ CHECK_EQ(output.device()->lang(), kCuda);
+ CHECK_EQ(weight_.device()->lang(), kCuda);
+ CHECK_EQ(workspace_.device()->lang(), kCuda);
+
// LOG(INFO) << "hidden size " << hy.Size();
// LOG(INFO) << "weight size " << weight_.Size() << " value " << weight_.L1();
Block *inb = input.block(), *outb = output.block(),
@@ -289,6 +307,8 @@ const vector<Tensor> CudnnRNN::Forward(int flag, const vector<Tensor> &inputs) {
*wspace = this->workspace_.block(),
*rspace = this->reserve_space_.block();
if (flag & kTrain) {
+ CHECK_EQ(reserve_space_.device()->lang(), kCuda);
+ CHECK_EQ(did, reserve_space_.device()->id());
dev->Exec(
[inb, outb, wb, hxb, cxb, hyb, cyb, wspace, rspace, this](Context *ctx) {
// clang-format off
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/16f3bf64/src/model/layer/slice.cc
----------------------------------------------------------------------
diff --git a/src/model/layer/slice.cc b/src/model/layer/slice.cc
new file mode 100644
index 0000000..690a03e
--- /dev/null
+++ b/src/model/layer/slice.cc
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "singa/model/layer.h"
+#include "./slice.h"
+namespace singa {
+
+RegisterLayerClass(singa_slice, Slice);
+RegisterLayerClass(singacpp_slice, Slice);
+RegisterLayerClass(singacuda_slice, Slice);
+RegisterLayerClass(singacl_slice, Slice);
+
+void Slice::Setup(const Shape& in_sample, const LayerConf& conf) {
+ Layer::Setup(in_sample, conf);
+ out_sample_shapes_.clear();
+ axis_ = conf.slice_conf().axis();
+ int offset = 0;
+ // #slice point = # out tensors - 1
+ for (size_t p : conf.slice_conf().slice_point()) {
+ Shape s{0, 0};
+ s[1 - axis_] = in_sample[1 - axis_];
+ s[axis_] = p - offset;
+ offset = p;
+ out_sample_shapes_.push_back(s);
+ }
+ Shape s{0, 0};
+ s[1 - axis_] = in_sample[1 - axis_];
+ s[axis_] = in_sample[axis_] - offset;
+ out_sample_shapes_.push_back(s);
+}
+
+const vector<Tensor> Slice::Forward(int flag, const vector<Tensor>& inputs) {
+ vector<Tensor> outputs;
+ CHECK_EQ(inputs.size(), 1u) << "Split layer only have one input tensor.";
+ size_t offset = 0;
+ for (auto& s : out_sample_shapes_) {
+ if (axis_ == 0)
+ outputs.push_back(SliceRows(inputs.at(0), offset, offset + s[axis_]));
+ else
+ outputs.push_back(SliceColumns(inputs.at(0), offset, offset + s[axis_]));
+ offset += s[axis_];
+ }
+ return outputs;
+}
+
+const std::pair<vector<Tensor>, vector<Tensor>> Slice::Backward(
+ int flag, const vector<Tensor>& grads) {
+ vector<Tensor> input_grad, param_grad;
+ CHECK_EQ(grads.size(), out_sample_shapes_.size());
+ if (axis_ == 0)
+ input_grad.push_back(ConcatRows(grads));
+ else
+ input_grad.push_back(ConcatColumns(grads));
+ return std::make_pair(input_grad, param_grad);
+}
+
+} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/16f3bf64/src/model/layer/slice.h
----------------------------------------------------------------------
diff --git a/src/model/layer/slice.h b/src/model/layer/slice.h
new file mode 100644
index 0000000..99ce468
--- /dev/null
+++ b/src/model/layer/slice.h
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef SINGA_MODEL_LAYER_SLICE_H_
+#define SINGA_MODEL_LAYER_SLICE_H_
+#include <utility>
+#include <string>
+#include <vector>
+#include "singa/model/layer.h"
+
+namespace singa {
+class Slice : public Layer {
+ public:
+ /// \copydoc Layer::layer_type()
+ // const std::string layer_type() const override { return "Slice"; }
+
+ /// \copydoc Layer::Setup(const LayerConf&);
+ void Setup(const Shape& in_sample, const LayerConf& conf) override;
+ /// the i-th subshape is the shape of the i-th output tensor
+ const Shape GetOutputSampleShape(int k) override {
+ CHECK(out_sample_shapes_.size()) << "You may haven't call Setup()";
+ return out_sample_shapes_[k];
+ }
+
+ /// \copydoc Layer::Forward(int flag, const Tensor&)
+ const vector<Tensor> Forward(int flag, const vector<Tensor>& input) override;
+
+ /// \copydoc Layer::Backward(int, const Tensor&, const Tensor&);
+ const std::pair<vector<Tensor>, vector<Tensor>> Backward(int flag,
+ const vector<Tensor>& grad) override;
+
+ protected:
+ /// 0 for slice rows; 1 for slice cols
+ int axis_ = 0;
+ /// out_sample_shapes_[i] is the shape of the i-th output tensor
+ vector<Shape> out_sample_shapes_;
+};
+
+} // namespace singa
+#endif // SINGA_MODEL_LAYER_CONCAT_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/16f3bf64/src/model/layer/split.cc
----------------------------------------------------------------------
diff --git a/src/model/layer/split.cc b/src/model/layer/split.cc
index 6b38a2b..b0e35e6 100644
--- a/src/model/layer/split.cc
+++ b/src/model/layer/split.cc
@@ -21,6 +21,9 @@
namespace singa {
RegisterLayerClass(singa_split, Split);
+RegisterLayerClass(singacpp_split, Split);
+RegisterLayerClass(singacuda_split, Split);
+RegisterLayerClass(singacl_split, Split);
void Split::Setup(const Shape& in_sample, const LayerConf& conf) {
Layer::Setup(in_sample, conf);
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/16f3bf64/test/singa/test_concat.cc
----------------------------------------------------------------------
diff --git a/test/singa/test_concat.cc b/test/singa/test_concat.cc
new file mode 100644
index 0000000..80183a7
--- /dev/null
+++ b/test/singa/test_concat.cc
@@ -0,0 +1,193 @@
+/************************************************************
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+*************************************************************/
+
+#include "../src/model/layer/concat.h"
+#include "gtest/gtest.h"
+
+using singa::Shape;
+
+TEST(Concat, Setup) {
+ Shape s1 {2u, 3u};
+ Shape s2 {1u, 3u};
+ singa::LayerConf conf;
+ conf.set_type("singa_concat");
+ conf.mutable_concat_conf()->set_axis(0);
+ singa::Concat layer;
+ layer.Setup({s1, s2}, conf);
+ auto s = layer.GetOutputSampleShape();
+ EXPECT_EQ(s[0], 3u);
+ EXPECT_EQ(s[1], 3u);
+}
+
+void ForwardConcatRowTest(std::shared_ptr<singa::Device> dev) {
+ size_t a = 2u, b = 1u, c = 3u;
+ singa::Tensor t1({a, c}, dev);
+ singa::Tensor t2({b, c}, dev);
+ singa::LayerConf conf;
+ conf.set_type("singa_concat");
+ conf.mutable_concat_conf()->set_axis(0);
+ singa::Concat layer;
+ layer.Setup({t1.shape(), t2.shape()}, conf);
+ layer.ToDevice(dev);
+
+ t1.SetValue(1.0f);
+ t2.SetValue(2.0f);
+ auto out = layer.Forward(singa::kTrain, {t1, t2});
+ EXPECT_EQ(out.size(), 1u);
+
+ out[0].ToHost();
+ const float * outptr = out[0].data<float>();
+ for (size_t i = 0; i < a; i++) {
+ for (size_t j = 0; j < c; j++)
+ EXPECT_FLOAT_EQ(outptr[i * c + j], 1.0f);
+ }
+ for (size_t i = a; i < a + b; i++) {
+ for (size_t j = 0; j < c; j++)
+ EXPECT_FLOAT_EQ(outptr[i * c + j], 2.0f);
+ }
+
+}
+
+void ForwardConcatColumnTest(std::shared_ptr<singa::Device> dev) {
+ size_t a = 2u, b = 1u, c = 3u;
+ singa::Tensor t1({c, a}, dev);
+ singa::Tensor t2({c, b}, dev);
+ singa::LayerConf conf;
+ conf.set_type("singa_concat");
+ conf.mutable_concat_conf()->set_axis(1);
+ singa::Concat layer;
+ layer.Setup({t1.shape(), t2.shape()}, conf);
+ layer.ToDevice(dev);
+
+ t1.SetValue(1.0f);
+ t2.SetValue(2.0f);
+ auto out = layer.Forward(singa::kTrain, {t1, t2});
+ EXPECT_EQ(out.size(), 1u);
+ out[0].ToHost();
+ const float * outptr = out[0].data<float>();
+ for (size_t i = 0; i < c; i++) {
+ for (size_t j = 0; j < a; j++)
+ EXPECT_FLOAT_EQ(outptr[i * (a + b) + j], 1.0f);
+ }
+ for (size_t i = 0; i < c; i++) {
+ for (size_t j = a; j < a + b; j++)
+ EXPECT_FLOAT_EQ(outptr[i * (a + b) + j], 2.0f);
+ }
+
+}
+TEST(Concat, ForwardConcatRowCpp) {
+ ForwardConcatRowTest(singa::defaultDevice);
+}
+
+TEST(Concat, ForwardConcatColumnCpp) {
+ ForwardConcatColumnTest(singa::defaultDevice);
+}
+
+
+#ifdef USE_CUDA
+TEST(Concat, ForwardConcatRowCuda) {
+ ForwardConcatRowTest(std::make_shared<singa::CudaGPU>());
+}
+
+TEST(Concat, ForwardConcatColumnCuda) {
+ ForwardConcatColumnTest(std::make_shared<singa::CudaGPU>());
+}
+#endif // USE_CUDA
+
+
+void BackwardConcatRowTest(std::shared_ptr<singa::Device> dev) {
+ size_t a = 2u, b = 1u, c = 3u;
+ singa::LayerConf conf;
+ conf.set_type("singa_concat");
+ conf.mutable_concat_conf()->set_axis(0);
+ singa::Concat layer;
+ layer.Setup({{a, c}, {b, c}}, conf);
+ layer.ToDevice(dev);
+
+ singa::Tensor t({a + b, c}, dev);
+ singa::Uniform(-1.f, 1.f, &t);
+ auto out = layer.Backward(singa::kTrain, {t});
+ auto grads = out.first;
+ EXPECT_EQ(grads.size(), 2u);
+
+ t.ToHost();
+ const float* tptr = t.data<float>();
+
+ grads[0].ToHost();
+ const float * outa = grads[0].data<float>();
+ for (size_t i = 0; i < a; i++)
+ for (size_t j = 0; j < c; j++)
+ EXPECT_FLOAT_EQ(outa[i * c + j], tptr[i * c + j]);
+ grads[1].ToHost();
+ const float * outb = grads[1].data<float>();
+ for (size_t i = 0; i < b; i++)
+ for (size_t j = 0; j < c; j++)
+ EXPECT_FLOAT_EQ(outb[i * c + j], tptr[(i + a) * c + j]);
+}
+
+void BackwardConcatColumnTest(std::shared_ptr<singa::Device> dev) {
+ size_t a = 2u, b = 1u, c = 3u;
+ singa::LayerConf conf;
+ conf.set_type("singa_concat");
+ conf.mutable_concat_conf()->set_axis(1);
+ singa::Concat layer;
+ layer.Setup({{c, a}, {c, b}}, conf);
+ layer.ToDevice(dev);
+
+ singa::Tensor t({c, a + b}, dev);
+ singa::Uniform(-1.f, 1.f, &t);
+ auto out = layer.Backward(singa::kTrain, {t});
+ auto grads = out.first;
+ EXPECT_EQ(grads.size(), 2u);
+
+ t.ToHost();
+ const float* tptr = t.data<float>();
+
+ grads[0].ToHost();
+ const float * outa = grads[0].data<float>();
+ for (size_t i = 0; i < c; i++)
+ for (size_t j = 0; j < a; j++)
+ EXPECT_FLOAT_EQ(outa[i * a + j], tptr[i * (a + b) + j]);
+ grads[1].ToHost();
+ const float * outb = grads[1].data<float>();
+ for (size_t i = 0; i < c; i++)
+ for (size_t j = 0; j < b; j++)
+ EXPECT_FLOAT_EQ(outb[i * b + j], tptr[i * (a + b) + a + j]);
+}
+
+TEST(Concat, BackwardConcatRowCpp) {
+ BackwardConcatRowTest(singa::defaultDevice);
+}
+
+TEST(Concat, BackwardConcatColumn) {
+ BackwardConcatColumnTest(singa::defaultDevice);
+}
+
+
+#ifdef USE_CUDA
+TEST(Concat, BackwardConcatRowCuda) {
+ BackwardConcatRowTest(std::make_shared<singa::CudaGPU>());
+}
+
+TEST(Concat, BackwardConcatColumnCuda) {
+ BackwardConcatColumnTest(std::make_shared<singa::CudaGPU>());
+}
+#endif // USE_CUDA
[3/3] incubator-singa git commit: SINGA-271 Add Concat and Slice
layers
Posted by zh...@apache.org.
SINGA-271 Add Concat and Slice layers
Rebased onto the master.
Add the unit test for slice layer.
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/5afd81c8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/5afd81c8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/5afd81c8
Branch: refs/heads/master
Commit: 5afd81c84b9970b2f5b33acf5a9cbb156b4f8ac4
Parents: d84af80
Author: wang wei <wa...@comp.nus.edu.sg>
Authored: Tue Nov 22 06:47:55 2016 +0000
Committer: wang wei <wa...@comp.nus.edu.sg>
Committed: Tue Nov 22 06:47:55 2016 +0000
----------------------------------------------------------------------
python/singa/layer.py | 21 ++---
test/singa/test_slice.cc | 204 ++++++++++++++++++++++++++++++++++++++++++
2 files changed, 212 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5afd81c8/python/singa/layer.py
----------------------------------------------------------------------
diff --git a/python/singa/layer.py b/python/singa/layer.py
index 964ec17..f0024c4 100644
--- a/python/singa/layer.py
+++ b/python/singa/layer.py
@@ -191,21 +191,18 @@ class Layer(object):
tensors if the layer is connected to multiple layers;
'''
assert self.has_setup, 'Must call setup() before forward()'
- if type(x) == list:
- xs = []
- for t in x:
- xs.append(t.singa_tensor)
- y = self.layer.ForwardWithMultInputs(flag, xs)
- else:
- assert isinstance(x, tensor.Tensor), \
- 'input must be a Tensor or a list of Tensor'
- xs = x.singa_tensor
if type(flag) is bool:
if flag:
flag = model_pb2.kTrain
else:
flag = model_pb2.kEval
- y = self.layer.Forward(flag, xs)
+ if type(x) is list:
+ xs = [t.singa_tensor for t in x]
+ y = self.layer.ForwardWithMultInputs(flag, xs)
+ else:
+ assert isinstance(x, tensor.Tensor), \
+ 'input must be a Tensor or a list of Tensor'
+ y = self.layer.Forward(flag, x.singa_tensor)
if type(y) is tuple:
return tensor.from_raw_tensors(y)
else:
@@ -223,9 +220,7 @@ class Layer(object):
, dpi is the gradient of the i-th parameter
'''
if type(dy) == list:
- dys = []
- for t in dy:
- dys.append(t.singa_tensor)
+ dys = [t.singa_tensor for t in dy]
ret = self.layer.BackwardWithMultInputs(flag, dys)
else:
assert isinstance(dy, tensor.Tensor), \
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5afd81c8/test/singa/test_slice.cc
----------------------------------------------------------------------
diff --git a/test/singa/test_slice.cc b/test/singa/test_slice.cc
new file mode 100644
index 0000000..6039c47
--- /dev/null
+++ b/test/singa/test_slice.cc
@@ -0,0 +1,204 @@
+/************************************************************
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+*************************************************************/
+
+#include "../src/model/layer/slice.h"
+#include "gtest/gtest.h"
+
+using singa::Shape;
+TEST(Slice, Setup) {
+ Shape s{2u, 3u};
+ singa::LayerConf conf;
+ conf.set_type("singa_slice");
+ auto slice_conf = conf.mutable_slice_conf();
+ slice_conf->set_axis(1);
+ slice_conf->add_slice_point(2);
+ singa::Slice layer;
+ layer.Setup(s, conf);
+ auto s1 = layer.GetOutputSampleShape(0);
+ EXPECT_EQ(s1[0], 2u);
+ EXPECT_EQ(s1[1], 2u);
+ auto s2 = layer.GetOutputSampleShape(1);
+ EXPECT_EQ(s2[0], 2u);
+ EXPECT_EQ(s2[1], 1u);
+}
+
+void ForwardSliceRowTest(std::shared_ptr<singa::Device> dev) {
+ size_t a = 2u, b = 1u, c = 3u;
+ singa::LayerConf conf;
+ conf.set_type("singa_slice");
+ auto slice_conf = conf.mutable_slice_conf();
+ slice_conf->set_axis(0);
+ slice_conf->add_slice_point(2);
+ singa::Slice layer;
+ layer.Setup({a + b ,c}, conf);
+ layer.ToDevice(dev);
+
+ singa::Tensor t({a + b, c}, dev);
+ singa::Uniform(-1.f, 1.f, &t);
+ auto grads = layer.Forward(singa::kTrain, {t});
+ EXPECT_EQ(grads.size(), 2u);
+
+ t.ToHost();
+ const float* tptr = t.data<float>();
+
+ grads[0].ToHost();
+ const float * outa = grads[0].data<float>();
+ for (size_t i = 0; i < a; i++)
+ for (size_t j = 0; j < c; j++)
+ EXPECT_FLOAT_EQ(outa[i * c + j], tptr[i * c + j]);
+ grads[1].ToHost();
+ const float * outb = grads[1].data<float>();
+ for (size_t i = 0; i < b; i++)
+ for (size_t j = 0; j < c; j++)
+ EXPECT_FLOAT_EQ(outb[i * c + j], tptr[(i + a) * c + j]);
+}
+
+void ForwardSliceColumnTest(std::shared_ptr<singa::Device> dev) {
+ size_t a = 2u, b = 1u, c = 3u;
+ singa::LayerConf conf;
+ conf.set_type("singa_slice");
+ auto slice_conf = conf.mutable_slice_conf();
+ slice_conf->set_axis(1);
+ slice_conf->add_slice_point(2);
+ singa::Slice layer;
+ layer.Setup({c, a + b}, conf);
+ layer.ToDevice(dev);
+
+ singa::Tensor t({c, a + b}, dev);
+ singa::Uniform(-1.f, 1.f, &t);
+ auto out = layer.Forward(singa::kTrain, {t});
+ EXPECT_EQ(out.size(), 2u);
+
+ t.ToHost();
+ const float* tptr = t.data<float>();
+
+ out[0].ToHost();
+ const float * outa = out[0].data<float>();
+ for (size_t i = 0; i < c; i++)
+ for (size_t j = 0; j < a; j++)
+ EXPECT_FLOAT_EQ(outa[i * a + j], tptr[i * (a + b) + j]);
+ out[1].ToHost();
+ const float * outb = out[1].data<float>();
+ for (size_t i = 0; i < c; i++)
+ for (size_t j = 0; j < b; j++)
+ EXPECT_FLOAT_EQ(outb[i * b + j], tptr[i * (a + b) + a + j]);
+}
+
+
+TEST(Slice, ForwardSliceRowCpp) {
+ ForwardSliceRowTest(singa::defaultDevice);
+}
+
+TEST(Slice, ForwardSliceColumn) {
+ ForwardSliceColumnTest(singa::defaultDevice);
+}
+
+
+#ifdef USE_CUDA
+TEST(Slice, ForwardSliceRowCuda) {
+ ForwardSliceRowTest(std::make_shared<singa::CudaGPU>());
+}
+
+TEST(Slice, ForwardSliceColumnCuda) {
+ ForwardSliceColumnTest(std::make_shared<singa::CudaGPU>());
+}
+#endif // USE_CUDA
+
+
+
+void BackwardSliceRowTest(std::shared_ptr<singa::Device> dev) {
+ size_t a = 2u, b = 1u, c = 3u;
+ singa::LayerConf conf;
+ conf.set_type("singa_slice");
+ auto slice_conf = conf.mutable_slice_conf();
+ slice_conf->set_axis(0);
+ slice_conf->add_slice_point(2);
+ singa::Slice layer;
+ layer.Setup({a + b ,c}, conf);
+ layer.ToDevice(dev);
+
+ singa::Tensor t1({a, c}, dev);
+ singa::Tensor t2({b, c}, dev);
+ t1.SetValue(1.0f);
+ t2.SetValue(2.0f);
+ auto out = layer.Backward(singa::kTrain, {t1, t2});
+ auto grad = out.first[0];
+
+ grad.ToHost();
+ const float * outptr = grad.data<float>();
+ for (size_t i = 0; i < a; i++) {
+ for (size_t j = 0; j < c; j++)
+ EXPECT_FLOAT_EQ(outptr[i * c + j], 1.0f);
+ }
+ for (size_t i = a; i < a + b; i++) {
+ for (size_t j = 0; j < c; j++)
+ EXPECT_FLOAT_EQ(outptr[i * c + j], 2.0f);
+ }
+}
+
+void BackwardSliceColumnTest(std::shared_ptr<singa::Device> dev) {
+ size_t a = 2u, b = 1u, c = 3u;
+ singa::LayerConf conf;
+ conf.set_type("singa_slice");
+ auto slice_conf = conf.mutable_slice_conf();
+ slice_conf->set_axis(1);
+ slice_conf->add_slice_point(2);
+ singa::Slice layer;
+ layer.Setup({c , a + b}, conf);
+ layer.ToDevice(dev);
+
+ singa::Tensor t1({c, a}, dev);
+ singa::Tensor t2({c, b}, dev);
+ t1.SetValue(1.0f);
+ t2.SetValue(2.0f);
+ auto out = layer.Backward(singa::kTrain, {t1, t2});
+ auto grad = out.first[0];
+ grad.ToHost();
+ const float * outptr = grad.data<float>();
+ for (size_t i = 0; i < c; i++) {
+ for (size_t j = 0; j < a; j++)
+ EXPECT_FLOAT_EQ(outptr[i * (a + b) + j], 1.0f);
+ }
+ for (size_t i = 0; i < c; i++) {
+ for (size_t j = a; j < a + b; j++)
+ EXPECT_FLOAT_EQ(outptr[i * (a + b) + j], 2.0f);
+ }
+}
+
+
+TEST(Slice, BackwardSliceRowCpp) {
+ BackwardSliceRowTest(singa::defaultDevice);
+}
+
+TEST(Slice, BackwardSliceColumn) {
+ BackwardSliceColumnTest(singa::defaultDevice);
+}
+
+
+#ifdef USE_CUDA
+TEST(Slice, BackwardSliceRowCuda) {
+ BackwardSliceRowTest(std::make_shared<singa::CudaGPU>());
+}
+
+TEST(Slice, BackwardSliceColumnCuda) {
+ BackwardSliceColumnTest(std::make_shared<singa::CudaGPU>());
+}
+#endif // USE_CUDA
[2/3] incubator-singa git commit: SINGA-271 Add Concat and Slice
layers
Posted by zh...@apache.org.
SINGA-271 Add Concat and Slice layers
Export c++ slice and concat layers to python
Pass python unit tests.
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/d84af801
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/d84af801
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/d84af801
Branch: refs/heads/master
Commit: d84af80172cab4094ffeb28293d8cb0820d75cbd
Parents: 16f3bf6
Author: wang wei <wa...@comp.nus.edu.sg>
Authored: Sun Nov 20 15:47:11 2016 +0000
Committer: wang wei <wa...@comp.nus.edu.sg>
Committed: Tue Nov 22 06:33:32 2016 +0000
----------------------------------------------------------------------
include/singa/model/layer.h | 3 +-
python/singa/layer.py | 79 +++++++++++++++++++++++++++++++++-------
python/singa/net.py | 32 ++++++++++++----
src/api/model_layer.i | 37 ++++++++++++-------
test/python/test_layer.py | 35 +++++++++++++-----
5 files changed, 140 insertions(+), 46 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/d84af801/include/singa/model/layer.h
----------------------------------------------------------------------
diff --git a/include/singa/model/layer.h b/include/singa/model/layer.h
index e67fcc5..ca07a19 100644
--- a/include/singa/model/layer.h
+++ b/include/singa/model/layer.h
@@ -75,8 +75,7 @@ class Layer {
}
/// Used for layers that have multiple input tensors, e.g., concatenate layer.
- virtual void Setup(const vector<Shape>& in_samples,
- const LayerConf& conf) {
+ virtual void Setup(const vector<Shape>& in_samples, const LayerConf& conf) {
name_ = conf.name();
// TODO(wangwei) load param values from checkpoint files.
}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/d84af801/python/singa/layer.py
----------------------------------------------------------------------
diff --git a/python/singa/layer.py b/python/singa/layer.py
index 730bea0..964ec17 100644
--- a/python/singa/layer.py
+++ b/python/singa/layer.py
@@ -94,20 +94,19 @@ class Layer(object):
# case1: parameters of conv and dense layers
# case2: type of activation layers
if (conf.type == 'Convolution' or conf.type == 4) or \
- (conf.type == 'InnerProduct' or conf.type == 14):
+ (conf.type == 'InnerProduct' or conf.type == 14):
w, b = _construct_param_specs_from_caffe_proto(conf)
del conf.param[:]
conf.param.extend([w, b])
self.param_specs.append(w)
self.param_specs.append(b)
- #print 'conf:\n', conf
+ # print 'conf:\n', conf
if conf.type == 'Pooling':
conf.pooling_conf.ceil = True
- #print 'conf:\n', conf
-
- elif (conf.type == 'ReLU' or conf.type == 18) or \
- (conf.type == 'Sigmoid' or conf.type == 19) or \
- (conf.type == 'TanH' or conf.type == 23):
+ # print 'conf:\n', conf
+ elif (conf.type == 'ReLU' or conf.type == 18 or
+ conf.type == 'Sigmoid' or conf.type == 19 or
+ conf.type == 'TanH' or conf.type == 23):
conf.type = (engine + '_' + conf.type).lower()
self.conf = conf
@@ -123,7 +122,6 @@ class Layer(object):
else:
self.layer = _create_layer(engine, str(self.conf.type))
-
def param_names(self):
'''
Returns:
@@ -145,8 +143,11 @@ class Layer(object):
'''
if self.has_setup:
return
- self.layer.Setup(list(in_shapes),
- self.conf.SerializeToString())
+ if type(in_shapes[0]) is tuple:
+ self.layer.SetupWithMultInputs([list(s) for s in in_shapes],
+ self.conf.SerializeToString())
+ else:
+ self.layer.Setup(list(in_shapes), self.conf.SerializeToString())
self.has_setup = True
def get_output_sample_shape(self):
@@ -194,6 +195,7 @@ class Layer(object):
xs = []
for t in x:
xs.append(t.singa_tensor)
+ y = self.layer.ForwardWithMultInputs(flag, xs)
else:
assert isinstance(x, tensor.Tensor), \
'input must be a Tensor or a list of Tensor'
@@ -204,7 +206,7 @@ class Layer(object):
else:
flag = model_pb2.kEval
y = self.layer.Forward(flag, xs)
- if type(y) == list:
+ if type(y) is tuple:
return tensor.from_raw_tensors(y)
else:
return tensor.from_raw_tensor(y)
@@ -224,12 +226,13 @@ class Layer(object):
dys = []
for t in dy:
dys.append(t.singa_tensor)
+ ret = self.layer.BackwardWithMultInputs(flag, dys)
else:
assert isinstance(dy, tensor.Tensor), \
'the input must be a Tensor or a set of Tensor'
dys = dy.singa_tensor
- ret = self.layer.Backward(flag, dys)
- if type(ret[0]) == list:
+ ret = self.layer.Backward(flag, dys)
+ if type(ret[0]) is tuple:
dxs = tensor.from_raw_tensors(ret[0])
else:
dxs = tensor.from_raw_tensor(ret[0])
@@ -275,6 +278,7 @@ class Dummy(Layer):
def backward(self, falg, dy):
return dy
+
class Conv2D(Layer):
"""Construct a layer for 2D convolution.
@@ -763,7 +767,7 @@ class Split(Layer):
self.has_setup = True
def get_output_sample_shape(self):
- return self.in_shape
+ return [self.in_shape] * self.num_output
def forward(self, flag, input):
'''Replicate the input tensor into mutiple tensors.
@@ -789,6 +793,53 @@ class Split(Layer):
return dx, []
+class Concat(Layer):
+ '''Concatenate tensors vertically (axis = 0) or horizontally (axis = 1).
+
+ Currently, only support tensors with 2 dimensions.
+
+ Args:
+ axis(int): 0 for concat row; 1 for concat columns;
+ input_sample_shapes: a list of shape tuples, one per input tensor
+ '''
+
+ def __init__(self, name, axis, input_sample_shapes=None):
+ super(Concat, self).__init__(name)
+ self.in_shapes = input_sample_shapes
+ self.axis = axis
+ self.conf.concat_conf.axis = axis
+ self.layer = _create_layer(engine, 'Concat')
+ if input_sample_shapes is not None:
+ self.setup(input_sample_shapes)
+
+
+class Slice(Layer):
+ '''Slice the input tensor into multiple sub-tensors vertially (axis=0) or
+ horizontally (axis=1).
+
+ Args:
+ axis (int): 0 for slice rows; 1 for slice columns;
+ slice_point(list): positions along the axis to do slice; there are n-1
+ points for n sub-tensors;
+ input_sample_shape: input tensor shape
+ '''
+
+ def __init__(self, name, axis, slice_point, input_sample_shape=None):
+ super(Slice, self).__init__(name)
+ self.in_shape = input_sample_shape
+ self.axis = axis
+ self.conf.slice_conf.axis = axis
+ self.conf.slice_conf.slice_point.extend(slice_point)
+ self.layer = _create_layer(engine, 'Slice')
+ if input_sample_shape is not None:
+ self.setup(input_sample_shape)
+
+ def get_output_sample_shape(self):
+ out = []
+ for i in range(len(self.conf.slice_conf.slice_point) + 1):
+ out.append(self.layer.GetOutputSampleShape(i))
+
+
class RNN(Layer):
'''Recurrent layer with 4 types of units, namely lstm, gru, tanh and relu.
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/d84af801/python/singa/net.py
----------------------------------------------------------------------
diff --git a/python/singa/net.py b/python/singa/net.py
index 293e97c..d34afbc 100644
--- a/python/singa/net.py
+++ b/python/singa/net.py
@@ -39,6 +39,7 @@ class FeedForwardNet(object):
self.src_of_layer = {}
self.dst_of_layer = None
self.ordered_layers = None
+ self.out_sample_shape_of_layer = {}
def to_device(self, dev):
for lyr in self.layers:
@@ -47,9 +48,11 @@ class FeedForwardNet(object):
def add(self, lyr, src=None):
"""Append a layer into the layer list.
- This function will get the sample shape from the last layer to setup
- the newly added layer. For the first layer, it is setup outside.
- The calling function should ensure the correctness of the layer order.
+ This function will get the sample shape from the src layers to setup the
+ newly added layer. For the first layer, it is setup outside. The calling
+ function should ensure the correctness of the layer order. If src is
+ None, the last layer is the src layer. If there are multiple src layers,
+ the src is a list of the src layers.
Args:
lyr (Layer): the layer to be added
@@ -70,11 +73,24 @@ class FeedForwardNet(object):
else:
self.src_of_layer[lyr.name] = []
if lyr.has_setup is False:
- # print shape
- in_shape = self.src_of_layer[lyr.name][0].get_output_sample_shape()
- lyr.setup(in_shape)
- print lyr.name, lyr.get_output_sample_shape()
+ in_shape = []
+ for src in self.src_of_layer[lyr.name]:
+ shapes = self.out_sample_shape_of_layer[src.name]
+ assert len(shapes) > 0, \
+ 'Cannot get output shape of layer %s' % lyr.name
+ in_shape.append(shapes[0])
+ shapes.pop(0)
+ if len(in_shape) == 1:
+ lyr.setup(in_shape[0])
+ else:
+ lyr.setup(in_shape)
+ out_shape = lyr.get_output_sample_shape()
+ if type(out_shape[0]) is tuple:
+ self.out_sample_shape_of_layer[lyr.name] = out_shape
+ else:
+ self.out_sample_shape_of_layer[lyr.name] = [out_shape]
self.layers.append(lyr)
+ print lyr.name, out_shape
return lyr
def param_values(self):
@@ -239,7 +255,7 @@ class FeedForwardNet(object):
disp_src += '-->' + cur.name
if type(out) is list:
print '%s: %s' % (disp_src,
- ' '.join([str(o.l1()) for o in out]))
+ ' '.join([str(o.l1()) for o in out]))
else:
print '%s: %f' % (disp_src, out.l1())
output_of_layer[cur.name] = out
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/d84af801/src/api/model_layer.i
----------------------------------------------------------------------
diff --git a/src/api/model_layer.i b/src/api/model_layer.i
index 3878873..7f582e7 100644
--- a/src/api/model_layer.i
+++ b/src/api/model_layer.i
@@ -40,6 +40,7 @@ using singa::ParamSpec;
using singa::DataType;
using singa::Device;
using singa::LayerConf;
+using singa::Shape;
%}
%shared_ptr(singa::Layer)
@@ -52,26 +53,36 @@ namespace std {
%template(VecStr) vector<string>;
%template(VecParamSpec) vector<singa::ParamSpec>;
%template(VecTensor) vector<singa::Tensor>;
+ %template(VecVecSize) vector<vector<size_t>>;
%template(PairTensorVecTensor) pair<singa::Tensor, vector<singa::Tensor>>;
%template(PairVecTensor) pair<vector<singa::Tensor>, vector<singa::Tensor>>;
}
-
namespace singa {
class Layer {
- public:
- Layer();
-// virtual void Setup(const std::vector<vector<size_t>>&, const string&);
- void Setup(const std::vector<size_t>& in_sample_shape,
- const std::string& proto_str);
- virtual const std::vector<Tensor> param_values();
- virtual const std::vector<size_t> GetOutputSampleShape() const;
- virtual void ToDevice(std::shared_ptr<Device> device);
- virtual void AsType(DataType dtype);
- virtual const Tensor Forward(int flag, const Tensor& input);
- virtual const std::pair<Tensor, std::vector<Tensor>> Backward(
- int flag, const Tensor& grad);
+ public:
+ Layer();
+ void Setup(const std::vector<size_t>&, const std::string& );
+ %rename(SetupWithMultInputs) Setup(const std::vector<std::vector<size_t>>&,
+ const std::string&);
+ void Setup(const std::vector<std::vector<size_t>>&, const std::string&);
+
+ virtual const std::vector<Tensor> param_values();
+ virtual const std::vector<size_t> GetOutputSampleShape() const;
+ %rename(GetOutputSampleShapeAt) GetOutputSampleShape(int k);
+ virtual const std::vector<size_t> GetOutputSampleShape(int k);
+ virtual void ToDevice(std::shared_ptr<Device> device);
+ virtual void AsType(DataType dtype);
+ virtual const Tensor Forward(int flag, const Tensor& input);
+ %rename(ForwardWithMultInputs) Forward(int flag, const std::vector<Tensor>&);
+ virtual const std::vector<Tensor> Forward(
+ int flag, const std::vector<Tensor>& inputs);
+ virtual const std::pair<Tensor, std::vector<Tensor>> Backward(
+ int flag, const Tensor& grad);
+ %rename(BackwardWithMultInputs) Backward(int, const vector<Tensor>&);
+ virtual const std::pair<std::vector<Tensor>, std::vector<Tensor>>
+ Backward(int flag, const vector<Tensor>& grads);
};
std::shared_ptr<Layer> CreateLayer(const std::string& type);
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/d84af801/test/python/test_layer.py
----------------------------------------------------------------------
diff --git a/test/python/test_layer.py b/test/python/test_layer.py
index 141cf56..d22207f 100644
--- a/test/python/test_layer.py
+++ b/test/python/test_layer.py
@@ -1,4 +1,4 @@
-#
+#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -6,25 +6,21 @@
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
-#
+#
# http://www.apache.org/licenses/LICENSE-2.0
-#
+#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-#
+#
-import sys
-import os
import unittest
import numpy as np
-#sys.path.append(os.path.join(os.path.dirname(__file__), '../../build/python'))
from singa import layer
-from singa import device
from singa import tensor
from singa.proto import model_pb2
@@ -43,7 +39,7 @@ class TestPythonLayer(unittest.TestCase):
)
def setUp(self):
- layer.engine='singacpp'
+ layer.engine = 'singacpp'
self.w = {'init': 'Xavier', 'regularizer': 1e-4}
self.b = {'init': 'Constant', 'value': 0}
self.sample_shape = None
@@ -208,6 +204,27 @@ class TestPythonLayer(unittest.TestCase):
out_sample_shape = flatten.get_output_sample_shape()
self.check_shape(out_sample_shape, (12,))
+ def test_concat(self):
+ t1 = tensor.Tensor((2, 3))
+ t2 = tensor.Tensor((1, 3))
+ t1.set_value(1)
+ t2.set_value(2)
+ lyr = layer.Concat('concat', 0, [t1.shape, t2.shape])
+ t = lyr.forward(model_pb2.kTrain, [t1, t2])
+ tnp = tensor.to_numpy(t[0])
+ self.assertEquals(np.sum(tnp), 12)
+
+ def test_slice(self):
+ t = np.zeros((3, 3))
+ t[:, :2] = float(2)
+ t[:, 2] = float(1)
+ lyr = layer.Slice('slice', 1, [2], t.shape)
+ out = lyr.forward(model_pb2.kTrain, [tensor.from_numpy(t)])
+ t1 = tensor.to_numpy(out[0])
+ t2 = tensor.to_numpy(out[1])
+ self.assertEquals(np.average(t1), 2)
+ self.assertEquals(np.average(t2), 1)
+
if __name__ == '__main__':
unittest.main()