You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@singa.apache.org by wa...@apache.org on 2015/12/10 14:12:16 UTC
[1/4] incubator-singa git commit: SINGA-111 Add slice,
concate and split layers
Repository: incubator-singa
Updated Branches:
refs/heads/master f16b1be6f -> 7cdb22f68
SINGA-111 Add slice, concate and split layers
Add slice layer implementation and test case
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/28e48a6f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/28e48a6f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/28e48a6f
Branch: refs/heads/master
Commit: 28e48a6f5abddfa17dd473ef912ace9fefe7ddaf
Parents: f16b1be
Author: WANG Sheng <wa...@gmail.com>
Authored: Tue Dec 8 18:23:02 2015 +0800
Committer: WANG Sheng <wa...@gmail.com>
Committed: Thu Dec 10 16:22:43 2015 +0800
----------------------------------------------------------------------
.../singa/neuralnet/connection_layer/slice.h | 4 +
src/neuralnet/connection_layer/slice.cc | 108 ++++++++-----
src/proto/job.proto | 1 +
src/test/test_connection_layers.cc | 156 +++++++++++++++++--
4 files changed, 212 insertions(+), 57 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/28e48a6f/include/singa/neuralnet/connection_layer/slice.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/connection_layer/slice.h b/include/singa/neuralnet/connection_layer/slice.h
index 615685b..a2f715c 100644
--- a/include/singa/neuralnet/connection_layer/slice.h
+++ b/include/singa/neuralnet/connection_layer/slice.h
@@ -37,6 +37,10 @@ class SliceLayer : public ConnectionLayer {
void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
+ const Blob<float>& data(const Layer* from) const override;
+ const Blob<float>& grad(const Layer* from) const override;
+ Blob<float>* mutable_data(const Layer* from) override;
+ Blob<float>* mutable_grad(const Layer* from) override;
private:
std::vector<Blob<float>> datavec_;
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/28e48a6f/src/neuralnet/connection_layer/slice.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/connection_layer/slice.cc b/src/neuralnet/connection_layer/slice.cc
index 8acbf94..66d3578 100644
--- a/src/neuralnet/connection_layer/slice.cc
+++ b/src/neuralnet/connection_layer/slice.cc
@@ -26,61 +26,89 @@ namespace singa {
using std::vector;
void SliceLayer::Setup(const LayerProto& conf,
- const vector<Layer*>& srclayers) {
- /*
- Layer::Setup(conf, npartitions);
+ const vector<Layer*>& srclayers) {
+ CHECK_EQ(srclayers.size(), 1);
+ Layer::Setup(conf, srclayers);
slice_dim_ = conf.slice_conf().slice_dim();
- slice_num_ = npartitions;
+ slice_num_ = conf.slice_conf().slice_num();
+ vector<int> shape = srclayers[0]->data(this).shape();
CHECK_GE(slice_dim_, 0);
- CHECK_EQ(slice_num_, dstlayers_.size());
- data_.Reshape(srclayers[0]->data(this).shape());
- grad_.ReshapeLike(data_);
+ CHECK_LT(slice_dim_, shape.size());
+ CHECK_GT(slice_num_, 0);
datavec_.resize(slice_num_);
gradvec_.resize(slice_num_);
- CHECK_EQ(data_.count() % slice_num_, 0); // restrict equal slicing
- // LOG(ERROR)<<"slice dim "<<slice_dim<<" slice num "<<slice_num;
- for (int i = 0; i < slice_num_; i++) {
- vector<int> newshape(data_.shape());
- newshape[slice_dim_] = newshape[slice_dim_] / slice_num_ +
- ((i == slice_num_ - 1) ? newshape[slice_dim_] % slice_num_ : 0);
- datavec_[i].Reshape(newshape);
- gradvec_[i].Reshape(newshape);
- // LOG(ERROR)<<"slice "<<IntVecToString(newshape);
+ // TODO(wangsh): remove equal-size restrict later
+ CHECK_EQ(shape[slice_dim_] % slice_num_, 0);
+ shape[slice_dim_] /= slice_num_;
+ for (int i = 0; i < slice_num_; ++i) {
+ // if (i == slice_num - 1) shape[slice_dim_] += remain;
+ datavec_[i].Reshape(shape);
+ gradvec_[i].Reshape(shape);
}
- */
- LOG(FATAL) << "Not implemented";
}
void SliceLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
- /*
CHECK_EQ(srclayers.size(), 1);
- if (slice_dim_ == 0) {
- const auto& blob = srclayers.at(0)->data(this);
- int size = blob.count() / slice_num_;
- for (int i = 0; i < slice_num_; i++) {
- float* dst = datavec_[i].mutable_cpu_data();
- const float* src = blob.cpu_data() + i * size;
- memcpy(dst, src, size*sizeof(float));
+ const Blob<float>& blob = srclayers[0]->data(this);
+ // calculate step for each memcpy
+ int step = datavec_[0].shape()[slice_dim_];
+ for (unsigned i = slice_dim_ + 1; i < datavec_[0].shape().size(); ++i)
+ step *= datavec_[0].shape()[i];
+ int srclayer_offset = 0;
+ int slice_offset = 0;
+ while (srclayer_offset < blob.count()) {
+ for (int i = 0; i < slice_num_; ++i) {
+ const float* src = blob.cpu_data() + srclayer_offset;
+ float* dst = datavec_[i].mutable_cpu_data() + slice_offset;
+ memcpy(dst, src, step * sizeof(float));
+ srclayer_offset += step;
}
+ slice_offset += step;
}
- */
- LOG(FATAL) << "Not implemented";
}
void SliceLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers) {
- LOG(FATAL) << "Not implemented";
+ CHECK_EQ(srclayers.size(), 1);
+ Blob<float>* blob = srclayers[0]->mutable_grad(this);
+ // calculate step for each memcpy
+ int step = gradvec_[0].shape()[slice_dim_];
+ for (size_t i = slice_dim_ + 1; i < gradvec_[0].shape().size(); ++i)
+ step *= gradvec_[0].shape()[i];
+ int srclayer_offset = 0;
+ int slice_offset = 0;
+ while (srclayer_offset < blob->count()) {
+ for (int i = 0; i < slice_num_; ++i) {
+ const float* src = gradvec_[i].cpu_data() + slice_offset;
+ float* dst = blob->mutable_cpu_data() + srclayer_offset;
+ memcpy(dst, src, step * sizeof(float));
+ srclayer_offset += step;
+ }
+ slice_offset += step;
+ }
}
-/*
-int SliceLayer::SliceID(const Layer* layer) const {
- CHECK(layer != nullptr);
- for (size_t i = 0; i < datavec_.size(); i++) {
- // LOG(ERROR)<<"get slice "<<IntVecToString(shapes_[i]);
- if (dstlayers_[i] == layer)
- return i;
- }
- CHECK(false);
- return -1;
-}*/
+const Blob<float>& SliceLayer::data(const Layer* from) const {
+ CHECK(from);
+ CHECK_LT(from->partition_id(), datavec_.size());
+ return datavec_[from->partition_id()];
+}
+
+const Blob<float>& SliceLayer::grad(const Layer* from) const {
+ CHECK(from);
+ CHECK_LT(from->partition_id(), gradvec_.size());
+ return gradvec_[from->partition_id()];
+}
+
+Blob<float>* SliceLayer::mutable_data(const Layer* from) {
+ CHECK(from);
+ CHECK_LT(from->partition_id(), datavec_.size());
+ return &datavec_[from->partition_id()];
+}
+
+Blob<float>* SliceLayer::mutable_grad(const Layer* from) {
+ CHECK(from);
+ CHECK_LT(from->partition_id(), gradvec_.size());
+ return &gradvec_[from->partition_id()];
+}
} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/28e48a6f/src/proto/job.proto
----------------------------------------------------------------------
diff --git a/src/proto/job.proto b/src/proto/job.proto
index d3f51b2..ef8f23f 100644
--- a/src/proto/job.proto
+++ b/src/proto/job.proto
@@ -453,6 +453,7 @@ message PoolingProto {
message SliceProto {
required int32 slice_dim = 1;
+ required int32 slice_num = 2;
}
message ReLUProto {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/28e48a6f/src/test/test_connection_layers.cc
----------------------------------------------------------------------
diff --git a/src/test/test_connection_layers.cc b/src/test/test_connection_layers.cc
index 837a941..a2cb1d3 100644
--- a/src/test/test_connection_layers.cc
+++ b/src/test/test_connection_layers.cc
@@ -26,23 +26,28 @@
#include "singa/comm/msg.h"
#include "singa/comm/socket.h"
#include "singa/neuralnet/connection_layer/bridge.h"
+#include "singa/neuralnet/connection_layer/slice.h"
#include "singa/neuralnet/neuron_layer/dummy.h"
#include "singa/proto/job.pb.h"
using namespace singa;
+const int N = 10; // size of dim 0
+const int M = 20; // size of dim 1
+const int K = 5; // size of partitions
+
TEST(ConnectionLayerTest, DummyTest) {
// use dummy as input layer
vector<Layer*> src_in;
LayerProto proto_in;
proto_in.set_name("dummy_input");
proto_in.mutable_dummy_conf()->set_input(true);
- proto_in.mutable_dummy_conf()->add_shape(10);
- proto_in.mutable_dummy_conf()->add_shape(20);
+ proto_in.mutable_dummy_conf()->add_shape(N);
+ proto_in.mutable_dummy_conf()->add_shape(M);
DummyLayer in;
in.Setup(proto_in, src_in);
- ASSERT_EQ(in.data(nullptr).shape(0), 10);
- ASSERT_EQ(in.data(nullptr).shape(1), 20);
+ ASSERT_EQ(in.data(nullptr).shape(0), N);
+ ASSERT_EQ(in.data(nullptr).shape(1), M);
in.ComputeFeature(0, src_in);
// use dummy as neuron layer
@@ -53,8 +58,8 @@ TEST(ConnectionLayerTest, DummyTest) {
proto_neu.mutable_dummy_conf();
DummyLayer neu;
neu.Setup(proto_neu, src_neu);
- ASSERT_EQ(neu.data(nullptr).shape(0), 10);
- ASSERT_EQ(neu.data(nullptr).shape(1), 20);
+ ASSERT_EQ(neu.data(nullptr).shape(0), N);
+ ASSERT_EQ(neu.data(nullptr).shape(1), M);
neu.ComputeFeature(0, src_neu);
ASSERT_EQ(in.data(nullptr).count(), neu.data(nullptr).count());
for (int i = 0; i < in.data(nullptr).count(); ++i)
@@ -68,8 +73,8 @@ TEST(ConnectionLayerTest, DummyTest) {
proto_out.mutable_dummy_conf()->set_output(true);
DummyLayer out;
out.Setup(proto_out, src_out);
- ASSERT_EQ(out.data(nullptr).shape(0), 10);
- ASSERT_EQ(out.data(nullptr).shape(1), 20);
+ ASSERT_EQ(out.data(nullptr).shape(0), N);
+ ASSERT_EQ(out.data(nullptr).shape(1), M);
out.ComputeFeature(0, src_out);
ASSERT_EQ(in.data(nullptr).count(), out.data(nullptr).count());
for (int i = 0; i < in.data(nullptr).count(); ++i)
@@ -83,15 +88,14 @@ TEST(ConnectionLayerTest, DummyTest) {
ASSERT_EQ(in.grad(nullptr).cpu_data()[i], out.grad(nullptr).cpu_data()[i]);
}
-
TEST(ConnectionLayerTest, BridgeTest) {
// use dummy as input layer
vector<Layer*> src_in;
LayerProto proto_in;
proto_in.set_name("dummy_input");
proto_in.mutable_dummy_conf()->set_input(true);
- proto_in.mutable_dummy_conf()->add_shape(10);
- proto_in.mutable_dummy_conf()->add_shape(20);
+ proto_in.mutable_dummy_conf()->add_shape(N);
+ proto_in.mutable_dummy_conf()->add_shape(M);
DummyLayer in;
in.Setup(proto_in, src_in);
@@ -99,11 +103,11 @@ TEST(ConnectionLayerTest, BridgeTest) {
vector<Layer*> src_src;
src_src.push_back(static_cast<Layer*>(&in));
LayerProto proto_src;
- proto_in.set_name("bridge_src");
+ proto_src.set_name("bridge_src");
BridgeSrcLayer src;
src.Setup(proto_src, src_src);
- ASSERT_EQ(src.data(nullptr).shape(0), 10);
- ASSERT_EQ(src.data(nullptr).shape(1), 20);
+ ASSERT_EQ(src.data(nullptr).shape(0), N);
+ ASSERT_EQ(src.data(nullptr).shape(1), M);
// add dst bridge layer
vector<Layer*> src_dst;
@@ -112,11 +116,11 @@ TEST(ConnectionLayerTest, BridgeTest) {
proto_dst.set_name("bridge_dst");
BridgeDstLayer dst;
dst.Setup(proto_dst, src_dst);
- ASSERT_EQ(dst.data(nullptr).shape(0), 10);
- ASSERT_EQ(dst.data(nullptr).shape(1), 20);
+ ASSERT_EQ(dst.data(nullptr).shape(0), N);
+ ASSERT_EQ(dst.data(nullptr).shape(1), M);
// bind bridges to socket
- Router router(10);
+ Router router(N);
router.Bind("inproc://router");
Dealer dealer(0);
dealer.Connect("inproc://router");
@@ -155,3 +159,121 @@ TEST(ConnectionLayerTest, BridgeTest) {
for (int i = 0; i < in.grad(nullptr).count(); ++i)
ASSERT_EQ(in.grad(nullptr).cpu_data()[i], out.grad(nullptr).cpu_data()[i]);
}
+
+TEST(ConnectionLayerTest, DataSliceTest) {
+ // use dummy as input layer
+ vector<Layer*> src_in;
+ LayerProto proto_in;
+ proto_in.set_name("dummy_input");
+ proto_in.mutable_dummy_conf()->set_input(true);
+ proto_in.mutable_dummy_conf()->add_shape(N);
+ proto_in.mutable_dummy_conf()->add_shape(M);
+ DummyLayer in;
+ in.Setup(proto_in, src_in);
+
+ // add slice layer
+ vector<Layer*> src_slice;
+ src_slice.push_back(static_cast<Layer*>(&in));
+ LayerProto proto_slice;
+ proto_slice.set_name("slice");
+ proto_slice.mutable_slice_conf()->set_slice_dim(0);
+ proto_slice.mutable_slice_conf()->set_slice_num(K);
+ SliceLayer slice;
+ slice.Setup(proto_slice, src_slice);
+ ASSERT_EQ(slice.data(static_cast<Layer*>(&slice)).shape(0), N / K);
+ ASSERT_EQ(slice.data(static_cast<Layer*>(&slice)).shape(1), M);
+
+ // use dummy as output layers
+ LayerProto proto_out[K];
+ vector<Layer*> src_out[K];
+ DummyLayer out[K];
+ for (int i = 0; i < K; ++i) {
+ src_out[i].push_back(static_cast<Layer*>(&slice));
+ proto_out[i].set_name("dummy_output_"+std::to_string(i));
+ proto_out[i].set_partition_id(i);
+ proto_out[i].mutable_dummy_conf()->set_output(true);
+ out[i].Setup(proto_out[i], src_out[i]);
+ }
+
+ // test for computing feature
+ in.ComputeFeature(0, src_in);
+ slice.ComputeFeature(0, src_slice);
+ for (int i = 0; i < K; ++i)
+ out[i].ComputeFeature(0, src_out[i]);
+ int step = (N * M) / K;
+ for (int i = 0; i < in.data(nullptr).count(); ++i) {
+ ASSERT_EQ(in.data(nullptr).cpu_data()[i],
+ out[i / step].data(nullptr).cpu_data()[i % step]);
+ }
+
+ // test for computing gradient
+ for (int i = 0; i < K; ++i)
+ out[i].ComputeGradient(0, src_out[i]);
+ slice.ComputeGradient(0, src_slice);
+ in.ComputeGradient(0, src_in);
+ for (int i = 0; i < in.grad(nullptr).count(); ++i) {
+ ASSERT_EQ(in.grad(nullptr).cpu_data()[i],
+ out[i / step].grad(nullptr).cpu_data()[i % step]);
+ }
+}
+
+TEST(ConnectionLayerTest, ModelSliceTest) {
+ // use dummy as input layer
+ vector<Layer*> src_in;
+ LayerProto proto_in;
+ proto_in.set_name("dummy_input");
+ proto_in.mutable_dummy_conf()->set_input(true);
+ proto_in.mutable_dummy_conf()->add_shape(N);
+ proto_in.mutable_dummy_conf()->add_shape(M);
+ DummyLayer in;
+ in.Setup(proto_in, src_in);
+
+ // add slice layer
+ vector<Layer*> src_slice;
+ src_slice.push_back(static_cast<Layer*>(&in));
+ LayerProto proto_slice;
+ proto_slice.set_name("slice");
+ proto_slice.mutable_slice_conf()->set_slice_dim(1);
+ proto_slice.mutable_slice_conf()->set_slice_num(K);
+ SliceLayer slice;
+ slice.Setup(proto_slice, src_slice);
+ ASSERT_EQ(slice.data(static_cast<Layer*>(&slice)).shape(0), N);
+ ASSERT_EQ(slice.data(static_cast<Layer*>(&slice)).shape(1), M / K);
+
+ // use dummy as output layers
+ LayerProto proto_out[K];
+ vector<Layer*> src_out[K];
+ DummyLayer out[K];
+ for (int i = 0; i < K; ++i) {
+ src_out[i].push_back(static_cast<Layer*>(&slice));
+ proto_out[i].set_name("dummy_output_"+std::to_string(i));
+ proto_out[i].set_partition_id(i);
+ proto_out[i].mutable_dummy_conf()->set_output(true);
+ out[i].Setup(proto_out[i], src_out[i]);
+ }
+
+ // test for computing feature
+ in.ComputeFeature(0, src_in);
+ slice.ComputeFeature(0, src_slice);
+ for (int i = 0; i < K; ++i)
+ out[i].ComputeFeature(0, src_out[i]);
+ int step = M / K;
+ int offset = 0;
+ for (int i = 0; i < in.data(nullptr).count(); ++i) {
+ if (i && i % M == 0) offset += step;
+ ASSERT_EQ(in.data(nullptr).cpu_data()[i],
+ out[(i / step) % K].data(nullptr).cpu_data()[offset + i % step]);
+ }
+
+ // test for computing gradient
+ for (int i = 0; i < K; ++i)
+ out[i].ComputeGradient(0, src_out[i]);
+ slice.ComputeGradient(0, src_slice);
+ in.ComputeGradient(0, src_in);
+ offset = 0;
+ for (int i = 0; i < in.grad(nullptr).count(); ++i) {
+ if (i && i % M == 0) offset += step;
+ ASSERT_EQ(in.grad(nullptr).cpu_data()[i],
+ out[(i / step) % K].grad(nullptr).cpu_data()[offset + i % step]);
+ }
+}
[3/4] incubator-singa git commit: SINGA-111 Add slice,
concate and split layers
Posted by wa...@apache.org.
SINGA-111 Add slice, concate and split layers
Add split layer implementation and test case
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/7e414e5e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/7e414e5e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/7e414e5e
Branch: refs/heads/master
Commit: 7e414e5e3c468eb4c225e09448d1f03a345a3152
Parents: 3ea1eb6
Author: WANG Sheng <wa...@gmail.com>
Authored: Tue Dec 8 22:05:50 2015 +0800
Committer: WANG Sheng <wa...@gmail.com>
Committed: Thu Dec 10 16:33:50 2015 +0800
----------------------------------------------------------------------
.../singa/neuralnet/connection_layer/split.h | 7 ++-
src/neuralnet/connection_layer/split.cc | 33 +++++++++--
src/proto/job.proto | 2 +-
src/test/test_connection_layers.cc | 58 ++++++++++++++++++++
4 files changed, 92 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7e414e5e/include/singa/neuralnet/connection_layer/split.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/connection_layer/split.h b/include/singa/neuralnet/connection_layer/split.h
index 7557766..f4de238 100644
--- a/include/singa/neuralnet/connection_layer/split.h
+++ b/include/singa/neuralnet/connection_layer/split.h
@@ -38,9 +38,12 @@ class SplitLayer : public ConnectionLayer {
void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
+ const Blob<float>& grad(const Layer* from) const override;
+ Blob<float>* mutable_grad(const Layer* from) override;
- protected:
- Blob<float> grads_;
+ private:
+ std::vector<Blob<float>> grads_;
+ int split_num_;
};
} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7e414e5e/src/neuralnet/connection_layer/split.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/connection_layer/split.cc b/src/neuralnet/connection_layer/split.cc
index beb5619..a9fd291 100644
--- a/src/neuralnet/connection_layer/split.cc
+++ b/src/neuralnet/connection_layer/split.cc
@@ -26,19 +26,42 @@ namespace singa {
using std::vector;
void SplitLayer::Setup(const LayerProto& conf,
- const vector<Layer*>& srclayers) {
- Layer::Setup(conf, srclayers);
+ const vector<Layer*>& srclayers) {
CHECK_EQ(srclayers.size(), 1);
+ Layer::Setup(conf, srclayers);
+ split_num_ = conf.split_conf().split_num();
data_.Reshape(srclayers[0]->data(this).shape());
- grad_.Reshape(srclayers[0]->data(this).shape());
+ data_.ShareData(srclayers[0]->data(this));
+ grads_.resize(split_num_);
+ for (int i = 0; i < split_num_; ++i)
+ grads_[i].Reshape(srclayers[0]->data(this).shape());
}
void SplitLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
- LOG(FATAL) << "Not implemented";
+ // data is shared from its source,
+ // nothing to do in compute feature phase
}
void SplitLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers) {
- LOG(FATAL) << "Not implemented";
+ CHECK_EQ(srclayers.size(), 1);
+ // aggregate all gradients to grad_[0]
+ for (int i = 1; i < split_num_; ++i)
+ for (int j = 0; j < grads_[0].count(); ++j)
+ grads_[0].mutable_cpu_data()[j] += grads_[i].cpu_data()[j];
+ // copy grad_[0] to srclayer's grad
+ srclayers[0]->mutable_grad(this)->CopyFrom(grads_[0]);
+}
+
+const Blob<float>& SplitLayer::grad(const Layer* from) const {
+ CHECK(from);
+ CHECK_LT(from->partition_id(), grads_.size());
+ return grads_[from->partition_id()];
+}
+
+Blob<float>* SplitLayer::mutable_grad(const Layer* from) {
+ CHECK(from);
+ CHECK_LT(from->partition_id(), grads_.size());
+ return &grads_[from->partition_id()];
}
} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7e414e5e/src/proto/job.proto
----------------------------------------------------------------------
diff --git a/src/proto/job.proto b/src/proto/job.proto
index ef8f23f..7a30b73 100644
--- a/src/proto/job.proto
+++ b/src/proto/job.proto
@@ -320,7 +320,7 @@ message PrefetchProto {
}
message SplitProto {
- optional int32 num_splits = 1 [default = 1];
+ optional int32 split_num = 1 [default = 1];
}
message StoreProto {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7e414e5e/src/test/test_connection_layers.cc
----------------------------------------------------------------------
diff --git a/src/test/test_connection_layers.cc b/src/test/test_connection_layers.cc
index e072114..4d8c984 100644
--- a/src/test/test_connection_layers.cc
+++ b/src/test/test_connection_layers.cc
@@ -28,6 +28,7 @@
#include "singa/neuralnet/connection_layer/bridge.h"
#include "singa/neuralnet/connection_layer/concate.h"
#include "singa/neuralnet/connection_layer/slice.h"
+#include "singa/neuralnet/connection_layer/split.h"
#include "singa/neuralnet/neuron_layer/dummy.h"
#include "singa/proto/job.pb.h"
@@ -396,3 +397,60 @@ TEST(ConnectionLayerTest, ModelConcateTest) {
out.grad(nullptr).cpu_data()[i]);
}
}
+
+TEST(ConnectionLayerTest, SplitTest) {
+ // use dummy as input layer
+ vector<Layer*> src_in;
+ LayerProto proto_in;
+ proto_in.set_name("dummy_input");
+ proto_in.mutable_dummy_conf()->set_input(true);
+ proto_in.mutable_dummy_conf()->add_shape(N);
+ proto_in.mutable_dummy_conf()->add_shape(M);
+ DummyLayer in;
+ in.Setup(proto_in, src_in);
+
+ // add split layer
+ vector<Layer*> src_split;
+ src_split.push_back(static_cast<Layer*>(&in));
+ LayerProto proto_split;
+ proto_split.set_name("split");
+ proto_split.mutable_split_conf()->set_split_num(K);
+ SplitLayer split;
+ split.Setup(proto_split, src_split);
+ ASSERT_EQ(split.data(static_cast<Layer*>(&split)).shape(0), N);
+ ASSERT_EQ(split.data(static_cast<Layer*>(&split)).shape(1), M);
+
+ // use dummy as output layers
+ LayerProto proto_out[K];
+ vector<Layer*> src_out[K];
+ DummyLayer out[K];
+ for (int i = 0; i < K; ++i) {
+ src_out[i].push_back(static_cast<Layer*>(&split));
+ proto_out[i].set_name("dummy_output_"+std::to_string(i));
+ proto_out[i].set_partition_id(i);
+ proto_out[i].mutable_dummy_conf()->set_output(true);
+ out[i].Setup(proto_out[i], src_out[i]);
+ }
+
+ // test for computing feature
+ in.ComputeFeature(0, src_in);
+ split.ComputeFeature(0, src_split);
+ for (int i = 0; i < K; ++i)
+ out[i].ComputeFeature(0, src_out[i]);
+ for (int i = 0; i < in.data(nullptr).count(); ++i) {
+ for (int k = 0; k < K; ++k)
+ ASSERT_EQ(in.data(nullptr).cpu_data()[i],
+ out[k].data(nullptr).cpu_data()[i]);
+ }
+
+ // test for computing gradient
+ for (int i = 0; i < K; ++i)
+ out[i].ComputeGradient(0, src_out[i]);
+ split.ComputeGradient(0, src_split);
+ in.ComputeGradient(0, src_in);
+ for (int i = 0; i < in.grad(nullptr).count(); ++i) {
+ float grad = 0;
+ for (int k = 0; k < K; ++k) grad += out[k].grad(nullptr).cpu_data()[i];
+ ASSERT_EQ(in.grad(nullptr).cpu_data()[i], grad);
+ }
+}
[4/4] incubator-singa git commit: SINGA-111 Add slice,
concate and split layers
Posted by wa...@apache.org.
SINGA-111 Add slice, concate and split layers
remove dedicated conf fields (slice_conf, concate_conf, split_conf)
instead, use partition_dim, num_partitions in LayerProto for configuration
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/7cdb22f6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/7cdb22f6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/7cdb22f6
Branch: refs/heads/master
Commit: 7cdb22f687b3a1c68e3ca2e28329e8eceba20017
Parents: 7e414e5
Author: WANG Sheng <wa...@gmail.com>
Authored: Thu Dec 10 16:19:11 2015 +0800
Committer: WANG Sheng <wa...@gmail.com>
Committed: Thu Dec 10 16:34:02 2015 +0800
----------------------------------------------------------------------
.../singa/neuralnet/connection_layer/concate.h | 3 -
.../singa/neuralnet/connection_layer/slice.h | 7 +-
.../singa/neuralnet/connection_layer/split.h | 5 +-
include/singa/neuralnet/layer.h | 4 +-
src/neuralnet/connection_layer/concate.cc | 18 ++---
src/neuralnet/connection_layer/slice.cc | 70 +++++++++++---------
src/neuralnet/connection_layer/split.cc | 33 +++++----
src/neuralnet/neuralnet.cc | 8 +--
src/proto/job.proto | 20 ------
src/test/test_connection_layers.cc | 16 +++--
10 files changed, 86 insertions(+), 98 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7cdb22f6/include/singa/neuralnet/connection_layer/concate.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/connection_layer/concate.h b/include/singa/neuralnet/connection_layer/concate.h
index 6e40040..5875835 100644
--- a/include/singa/neuralnet/connection_layer/concate.h
+++ b/include/singa/neuralnet/connection_layer/concate.h
@@ -37,9 +37,6 @@ class ConcateLayer : public ConnectionLayer {
void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
-
- private:
- int concate_dim_;
};
} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7cdb22f6/include/singa/neuralnet/connection_layer/slice.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/connection_layer/slice.h b/include/singa/neuralnet/connection_layer/slice.h
index a2f715c..023ebc1 100644
--- a/include/singa/neuralnet/connection_layer/slice.h
+++ b/include/singa/neuralnet/connection_layer/slice.h
@@ -34,6 +34,7 @@ namespace singa {
*/
class SliceLayer : public ConnectionLayer {
public:
+ ~SliceLayer();
void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
@@ -41,12 +42,6 @@ class SliceLayer : public ConnectionLayer {
const Blob<float>& grad(const Layer* from) const override;
Blob<float>* mutable_data(const Layer* from) override;
Blob<float>* mutable_grad(const Layer* from) override;
-
- private:
- std::vector<Blob<float>> datavec_;
- std::vector<Blob<float>> gradvec_;
- int slice_dim_;
- int slice_num_;
};
} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7cdb22f6/include/singa/neuralnet/connection_layer/split.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/connection_layer/split.h b/include/singa/neuralnet/connection_layer/split.h
index f4de238..959d1a3 100644
--- a/include/singa/neuralnet/connection_layer/split.h
+++ b/include/singa/neuralnet/connection_layer/split.h
@@ -35,15 +35,12 @@ namespace singa {
*/
class SplitLayer : public ConnectionLayer {
public:
+ ~SplitLayer();
void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
const Blob<float>& grad(const Layer* from) const override;
Blob<float>* mutable_grad(const Layer* from) override;
-
- private:
- std::vector<Blob<float>> grads_;
- int split_num_;
};
} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7cdb22f6/include/singa/neuralnet/layer.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/layer.h b/include/singa/neuralnet/layer.h
index a78deb7..7a92ced 100644
--- a/include/singa/neuralnet/layer.h
+++ b/include/singa/neuralnet/layer.h
@@ -284,11 +284,11 @@ class InputLayer : virtual public Layer {
void ComputeGradient(int flag, const vector<Layer*>& srclayers) override {}
ConnectionType dst_layer_connection() const override { return kOneToMany; }
Blob<float>* mutable_grad(const Layer* layer) override {
- // LOG(FATAL) << "Input layer has no gradient blob";
+ LOG(FATAL) << "Input layer has no gradient blob";
return nullptr;
}
const Blob<float>& grad(const Layer* from) const override {
- // LOG(FATAL) << "Input layer has no gradient blob";
+ LOG(FATAL) << "Input layer has no gradient blob";
return grad_;
}
};
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7cdb22f6/src/neuralnet/connection_layer/concate.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/connection_layer/concate.cc b/src/neuralnet/connection_layer/concate.cc
index 8f519d2..f9d6416 100644
--- a/src/neuralnet/connection_layer/concate.cc
+++ b/src/neuralnet/connection_layer/concate.cc
@@ -29,14 +29,14 @@ void ConcateLayer::Setup(const LayerProto& conf,
const vector<Layer*>& srclayers) {
CHECK_GT(srclayers.size(), 1);
Layer::Setup(conf, srclayers);
- concate_dim_ = conf.concate_conf().concate_dim();
vector<int> shape = srclayers[0]->data(this).shape();
- CHECK_GE(concate_dim_, 0);
- CHECK_LT(concate_dim_, shape.size());
+ CHECK_GE(partition_dim(), 0);
+ CHECK_LT(partition_dim(), shape.size());
+ CHECK_EQ(num_partitions(), srclayers.size());
for (size_t i = 1; i < srclayers.size(); i++) {
const vector<int>& src_shape = srclayers[i]->data(this).shape();
for (size_t j = 0; j < shape.size(); j++)
- if (static_cast<int>(j) == concate_dim_)
+ if (static_cast<int>(j) == partition_dim())
shape[j] += src_shape[j];
else
CHECK_EQ(shape[j], src_shape[j]);
@@ -47,9 +47,10 @@ void ConcateLayer::Setup(const LayerProto& conf,
void ConcateLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
CHECK_GT(srclayers.size(), 1);
+ CHECK_EQ(num_partitions(), srclayers.size());
// calculate step for each memcpy
- int step = srclayers[0]->data(this).shape()[concate_dim_];
- for (unsigned i = concate_dim_ + 1; i < data_.shape().size(); ++i)
+ int step = srclayers[0]->data(this).shape()[partition_dim()];
+ for (unsigned i = partition_dim() + 1; i < data_.shape().size(); ++i)
step *= data_.shape()[i];
int srclayer_offset = 0;
int concate_offset = 0;
@@ -66,9 +67,10 @@ void ConcateLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
void ConcateLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers) {
CHECK_GT(srclayers.size(), 1);
+ CHECK_EQ(num_partitions(), srclayers.size());
// calculate step for each memcpy
- int step = srclayers[0]->grad(this).shape()[concate_dim_];
- for (unsigned i = concate_dim_ + 1; i < grad_.shape().size(); ++i)
+ int step = srclayers[0]->grad(this).shape()[partition_dim()];
+ for (unsigned i = partition_dim() + 1; i < grad_.shape().size(); ++i)
step *= grad_.shape()[i];
int srclayer_offset = 0;
int concate_offset = 0;
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7cdb22f6/src/neuralnet/connection_layer/slice.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/connection_layer/slice.cc b/src/neuralnet/connection_layer/slice.cc
index 66d3578..c69f797 100644
--- a/src/neuralnet/connection_layer/slice.cc
+++ b/src/neuralnet/connection_layer/slice.cc
@@ -25,25 +25,33 @@ namespace singa {
using std::vector;
+SliceLayer::~SliceLayer() {
+ for (size_t i = 1; i < datavec_.size(); ++i) {
+ if (datavec_[i] != nullptr) delete datavec_[i];
+ if (gradvec_[i] != nullptr) delete gradvec_[i];
+ }
+}
+
void SliceLayer::Setup(const LayerProto& conf,
const vector<Layer*>& srclayers) {
CHECK_EQ(srclayers.size(), 1);
Layer::Setup(conf, srclayers);
- slice_dim_ = conf.slice_conf().slice_dim();
- slice_num_ = conf.slice_conf().slice_num();
vector<int> shape = srclayers[0]->data(this).shape();
- CHECK_GE(slice_dim_, 0);
- CHECK_LT(slice_dim_, shape.size());
- CHECK_GT(slice_num_, 0);
- datavec_.resize(slice_num_);
- gradvec_.resize(slice_num_);
+ CHECK_GE(partition_dim(), 0);
+ CHECK_LT(partition_dim(), shape.size());
+ CHECK_GT(num_partitions(), 0);
+ // add num_partitions()-1 more blobs
+ for (int i = 1; i < num_partitions(); ++i) {
+ datavec_.push_back(new Blob<float>());
+ gradvec_.push_back(new Blob<float>());
+ }
// TODO(wangsh): remove equal-size restrict later
- CHECK_EQ(shape[slice_dim_] % slice_num_, 0);
- shape[slice_dim_] /= slice_num_;
- for (int i = 0; i < slice_num_; ++i) {
- // if (i == slice_num - 1) shape[slice_dim_] += remain;
- datavec_[i].Reshape(shape);
- gradvec_[i].Reshape(shape);
+ CHECK_EQ(shape[partition_dim()] % num_partitions(), 0);
+ shape[partition_dim()] /= num_partitions();
+ for (int i = 0; i < num_partitions(); ++i) {
+ // if (i == slice_num - 1) shape[partition_dim()] += remain;
+ datavec_[i]->Reshape(shape);
+ gradvec_[i]->Reshape(shape);
}
}
@@ -51,15 +59,15 @@ void SliceLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
CHECK_EQ(srclayers.size(), 1);
const Blob<float>& blob = srclayers[0]->data(this);
// calculate step for each memcpy
- int step = datavec_[0].shape()[slice_dim_];
- for (unsigned i = slice_dim_ + 1; i < datavec_[0].shape().size(); ++i)
- step *= datavec_[0].shape()[i];
+ int step = datavec_[0]->shape()[partition_dim()];
+ for (unsigned i = partition_dim() + 1; i < datavec_[0]->shape().size(); ++i)
+ step *= datavec_[0]->shape()[i];
int srclayer_offset = 0;
int slice_offset = 0;
while (srclayer_offset < blob.count()) {
- for (int i = 0; i < slice_num_; ++i) {
+ for (int i = 0; i < num_partitions(); ++i) {
const float* src = blob.cpu_data() + srclayer_offset;
- float* dst = datavec_[i].mutable_cpu_data() + slice_offset;
+ float* dst = datavec_[i]->mutable_cpu_data() + slice_offset;
memcpy(dst, src, step * sizeof(float));
srclayer_offset += step;
}
@@ -71,14 +79,14 @@ void SliceLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers) {
CHECK_EQ(srclayers.size(), 1);
Blob<float>* blob = srclayers[0]->mutable_grad(this);
// calculate step for each memcpy
- int step = gradvec_[0].shape()[slice_dim_];
- for (size_t i = slice_dim_ + 1; i < gradvec_[0].shape().size(); ++i)
- step *= gradvec_[0].shape()[i];
+ int step = gradvec_[0]->shape()[partition_dim()];
+ for (size_t i = partition_dim() + 1; i < gradvec_[0]->shape().size(); ++i)
+ step *= gradvec_[0]->shape()[i];
int srclayer_offset = 0;
int slice_offset = 0;
while (srclayer_offset < blob->count()) {
- for (int i = 0; i < slice_num_; ++i) {
- const float* src = gradvec_[i].cpu_data() + slice_offset;
+ for (int i = 0; i < num_partitions(); ++i) {
+ const float* src = gradvec_[i]->cpu_data() + slice_offset;
float* dst = blob->mutable_cpu_data() + srclayer_offset;
memcpy(dst, src, step * sizeof(float));
srclayer_offset += step;
@@ -89,26 +97,26 @@ void SliceLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers) {
const Blob<float>& SliceLayer::data(const Layer* from) const {
CHECK(from);
- CHECK_LT(from->partition_id(), datavec_.size());
- return datavec_[from->partition_id()];
+ CHECK_LT(from->partition_id(), num_partitions());
+ return *datavec_[from->partition_id()];
}
const Blob<float>& SliceLayer::grad(const Layer* from) const {
CHECK(from);
- CHECK_LT(from->partition_id(), gradvec_.size());
- return gradvec_[from->partition_id()];
+ CHECK_LT(from->partition_id(), num_partitions());
+ return *gradvec_[from->partition_id()];
}
Blob<float>* SliceLayer::mutable_data(const Layer* from) {
CHECK(from);
- CHECK_LT(from->partition_id(), datavec_.size());
- return &datavec_[from->partition_id()];
+ CHECK_LT(from->partition_id(), num_partitions());
+ return datavec_[from->partition_id()];
}
Blob<float>* SliceLayer::mutable_grad(const Layer* from) {
CHECK(from);
- CHECK_LT(from->partition_id(), gradvec_.size());
- return &gradvec_[from->partition_id()];
+ CHECK_LT(from->partition_id(), num_partitions());
+ return gradvec_[from->partition_id()];
}
} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7cdb22f6/src/neuralnet/connection_layer/split.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/connection_layer/split.cc b/src/neuralnet/connection_layer/split.cc
index a9fd291..36b391c 100644
--- a/src/neuralnet/connection_layer/split.cc
+++ b/src/neuralnet/connection_layer/split.cc
@@ -25,16 +25,25 @@ namespace singa {
using std::vector;
+SplitLayer::~SplitLayer() {
+ for (size_t i = 1; i < gradvec_.size(); ++i) {
+ if (gradvec_[i] != nullptr) delete gradvec_[i];
+ }
+}
+
void SplitLayer::Setup(const LayerProto& conf,
const vector<Layer*>& srclayers) {
CHECK_EQ(srclayers.size(), 1);
Layer::Setup(conf, srclayers);
- split_num_ = conf.split_conf().split_num();
data_.Reshape(srclayers[0]->data(this).shape());
data_.ShareData(srclayers[0]->data(this));
- grads_.resize(split_num_);
- for (int i = 0; i < split_num_; ++i)
- grads_[i].Reshape(srclayers[0]->data(this).shape());
+ CHECK_GT(num_partitions(), 0);
+ // add num_partitions()-1 more grad blobs
+ for (int i = 1; i < num_partitions(); ++i) {
+ gradvec_.push_back(new Blob<float>());
+ }
+ for (int i = 0; i < num_partitions(); ++i)
+ gradvec_[i]->Reshape(srclayers[0]->data(this).shape());
}
void SplitLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
@@ -45,23 +54,23 @@ void SplitLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
void SplitLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers) {
CHECK_EQ(srclayers.size(), 1);
// aggregate all gradients to grad_[0]
- for (int i = 1; i < split_num_; ++i)
- for (int j = 0; j < grads_[0].count(); ++j)
- grads_[0].mutable_cpu_data()[j] += grads_[i].cpu_data()[j];
+ for (int i = 1; i < num_partitions(); ++i)
+ for (int j = 0; j < gradvec_[0]->count(); ++j)
+ gradvec_[0]->mutable_cpu_data()[j] += gradvec_[i]->cpu_data()[j];
// copy grad_[0] to srclayer's grad
- srclayers[0]->mutable_grad(this)->CopyFrom(grads_[0]);
+ srclayers[0]->mutable_grad(this)->CopyFrom(*gradvec_[0]);
}
const Blob<float>& SplitLayer::grad(const Layer* from) const {
CHECK(from);
- CHECK_LT(from->partition_id(), grads_.size());
- return grads_[from->partition_id()];
+ CHECK_LT(from->partition_id(), num_partitions());
+ return *gradvec_[from->partition_id()];
}
Blob<float>* SplitLayer::mutable_grad(const Layer* from) {
CHECK(from);
- CHECK_LT(from->partition_id(), grads_.size());
- return &grads_[from->partition_id()];
+ CHECK_LT(from->partition_id(), num_partitions());
+ return gradvec_[from->partition_id()];
}
} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7cdb22f6/src/neuralnet/neuralnet.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuralnet.cc b/src/neuralnet/neuralnet.cc
index 8ae1805..b7944e7 100644
--- a/src/neuralnet/neuralnet.cc
+++ b/src/neuralnet/neuralnet.cc
@@ -178,9 +178,8 @@ Node* SliceNode(Graph* graph, Node* srcnode,
proto->set_type(LayerType::kSlice);
proto->set_partition_id(
static_cast<LayerProto*>(srcnode->proto)->partition_id());
- auto conf = proto->mutable_slice_conf();
- conf->set_slice_dim(
- static_cast<LayerProto*>(dstnodes[0]->proto)->partition_dim());
+ proto->set_partition_dim(
+ static_cast<LayerProto*>(srcnode->proto)->partition_dim());
Node* node = new Node(name, "##" + name, proto->partition_id(), proto);
graph->AddNode(node);
graph->AddEdge(srcnode, node);
@@ -198,8 +197,7 @@ Node* ConcateNodes(Graph* graph, const vector<Node*>& srcnodes, Node* dstnode) {
proto->set_type(LayerType::kConcate);
proto->set_partition_id(
static_cast<LayerProto*>(dstnode->proto)->partition_id());
- auto conf = proto->mutable_concate_conf();
- conf->set_concate_dim(
+ proto->set_partition_dim(
static_cast<LayerProto*>(srcnodes[0]->proto)->partition_dim());
Node* node = new Node(name, "##" + name, proto->partition_id(), proto);
graph->AddNode(node);
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7cdb22f6/src/proto/job.proto
----------------------------------------------------------------------
diff --git a/src/proto/job.proto b/src/proto/job.proto
index 7a30b73..ae461f8 100644
--- a/src/proto/job.proto
+++ b/src/proto/job.proto
@@ -192,8 +192,6 @@ message LayerProto {
optional ArgSortProto argsort_conf = 52;
// configuration for convolution layer
optional ConvolutionProto convolution_conf = 30;
- // configuration for concatenation layer
- optional ConcateProto concate_conf = 31;
// configuration for dummy layer
optional DummyProto dummy_conf = 53;
// configuration for dropout layer
@@ -218,12 +216,8 @@ message LayerProto {
optional RGBImageProto rgbimage_conf = 39;
// configuration for data layer
optional DataProto sharddata_conf = 32;
- // configuration for slice layer
- optional SliceProto slice_conf = 41;
// configuration for softmax loss layer
optional SoftmaxLossProto softmaxloss_conf = 40;
- // configuration for split layer
- optional SplitProto split_conf = 42;
// configuration for store input layers
optional StoreProto store_conf = 51;
@@ -319,10 +313,6 @@ message PrefetchProto {
repeated LayerProto sublayers = 1;
}
-message SplitProto {
- optional int32 split_num = 1 [default = 1];
-}
-
message StoreProto {
optional string backend = 1;
optional string path = 2;
@@ -363,11 +353,6 @@ message ConvolutionProto {
optional bool bias_term = 32 [default = true];
}
-message ConcateProto {
- // on which dimension, starts from 0
- required int32 concate_dim = 1;
-}
-
message DataProto {
// path to the data file/folder, absolute or relative to the workspace
required string path = 2;
@@ -451,11 +436,6 @@ message PoolingProto {
optional uint32 stride = 32 [default = 1];
}
-message SliceProto {
- required int32 slice_dim = 1;
- required int32 slice_num = 2;
-}
-
message ReLUProto {
// Ref. Maas, A. L., Hannun, A. Y., & Ng, A. Y. (2013).
// Rectifier nonlinearities improve neural network acoustic models.
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7cdb22f6/src/test/test_connection_layers.cc
----------------------------------------------------------------------
diff --git a/src/test/test_connection_layers.cc b/src/test/test_connection_layers.cc
index 4d8c984..2415fcd 100644
--- a/src/test/test_connection_layers.cc
+++ b/src/test/test_connection_layers.cc
@@ -178,8 +178,8 @@ TEST(ConnectionLayerTest, DataSliceTest) {
src_slice.push_back(static_cast<Layer*>(&in));
LayerProto proto_slice;
proto_slice.set_name("slice");
- proto_slice.mutable_slice_conf()->set_slice_dim(0);
- proto_slice.mutable_slice_conf()->set_slice_num(K);
+ proto_slice.set_partition_dim(0);
+ proto_slice.set_num_partitions(K);
SliceLayer slice;
slice.Setup(proto_slice, src_slice);
ASSERT_EQ(slice.data(static_cast<Layer*>(&slice)).shape(0), N / K);
@@ -235,8 +235,8 @@ TEST(ConnectionLayerTest, ModelSliceTest) {
src_slice.push_back(static_cast<Layer*>(&in));
LayerProto proto_slice;
proto_slice.set_name("slice");
- proto_slice.mutable_slice_conf()->set_slice_dim(1);
- proto_slice.mutable_slice_conf()->set_slice_num(K);
+ proto_slice.set_partition_dim(1);
+ proto_slice.set_num_partitions(K);
SliceLayer slice;
slice.Setup(proto_slice, src_slice);
ASSERT_EQ(slice.data(static_cast<Layer*>(&slice)).shape(0), N);
@@ -300,7 +300,8 @@ TEST(ConnectionLayerTest, DataConcateTest) {
src_concate.push_back(static_cast<Layer*>(&in[i]));
LayerProto proto_concate;
proto_concate.set_name("concate");
- proto_concate.mutable_concate_conf()->set_concate_dim(0);
+ proto_concate.set_partition_dim(0);
+ proto_concate.set_num_partitions(K);
ConcateLayer concate;
concate.Setup(proto_concate, src_concate);
ASSERT_EQ(concate.data(static_cast<Layer*>(&concate)).shape(0), N);
@@ -357,7 +358,8 @@ TEST(ConnectionLayerTest, ModelConcateTest) {
src_concate.push_back(static_cast<Layer*>(&in[i]));
LayerProto proto_concate;
proto_concate.set_name("concate");
- proto_concate.mutable_concate_conf()->set_concate_dim(1);
+ proto_concate.set_partition_dim(1);
+ proto_concate.set_num_partitions(K);
ConcateLayer concate;
concate.Setup(proto_concate, src_concate);
ASSERT_EQ(concate.data(static_cast<Layer*>(&concate)).shape(0), N);
@@ -414,7 +416,7 @@ TEST(ConnectionLayerTest, SplitTest) {
src_split.push_back(static_cast<Layer*>(&in));
LayerProto proto_split;
proto_split.set_name("split");
- proto_split.mutable_split_conf()->set_split_num(K);
+ proto_split.set_num_partitions(K);
SplitLayer split;
split.Setup(proto_split, src_split);
ASSERT_EQ(split.data(static_cast<Layer*>(&split)).shape(0), N);
[2/4] incubator-singa git commit: SINGA-111 Add slice,
concate and split layers
Posted by wa...@apache.org.
SINGA-111 Add slice, concate and split layers
Add concate layer implementation and test case
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/3ea1eb69
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/3ea1eb69
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/3ea1eb69
Branch: refs/heads/master
Commit: 3ea1eb691708b39b4a786f6494f56f6dbfe8e1df
Parents: 28e48a6
Author: WANG Sheng <wa...@gmail.com>
Authored: Tue Dec 8 21:28:30 2015 +0800
Committer: WANG Sheng <wa...@gmail.com>
Committed: Thu Dec 10 16:22:59 2015 +0800
----------------------------------------------------------------------
.../singa/neuralnet/connection_layer/concate.h | 3 +
src/neuralnet/connection_layer/concate.cc | 52 ++++++--
src/test/test_connection_layers.cc | 119 +++++++++++++++++++
3 files changed, 164 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3ea1eb69/include/singa/neuralnet/connection_layer/concate.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/connection_layer/concate.h b/include/singa/neuralnet/connection_layer/concate.h
index 5875835..6e40040 100644
--- a/include/singa/neuralnet/connection_layer/concate.h
+++ b/include/singa/neuralnet/connection_layer/concate.h
@@ -37,6 +37,9 @@ class ConcateLayer : public ConnectionLayer {
void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
+
+ private:
+ int concate_dim_;
};
} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3ea1eb69/src/neuralnet/connection_layer/concate.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/connection_layer/concate.cc b/src/neuralnet/connection_layer/concate.cc
index fd2fb24..8f519d2 100644
--- a/src/neuralnet/connection_layer/concate.cc
+++ b/src/neuralnet/connection_layer/concate.cc
@@ -26,30 +26,62 @@ namespace singa {
using std::vector;
void ConcateLayer::Setup(const LayerProto& conf,
- const vector<Layer*>& srclayers) {
- Layer::Setup(conf, srclayers);
- size_t concate_dim = conf.concate_conf().concate_dim();
- CHECK_GE(concate_dim, 0);
+ const vector<Layer*>& srclayers) {
CHECK_GT(srclayers.size(), 1);
+ Layer::Setup(conf, srclayers);
+ concate_dim_ = conf.concate_conf().concate_dim();
vector<int> shape = srclayers[0]->data(this).shape();
+ CHECK_GE(concate_dim_, 0);
+ CHECK_LT(concate_dim_, shape.size());
for (size_t i = 1; i < srclayers.size(); i++) {
- const vector<int>& srcshape = srclayers[i]->data(this).shape();
+ const vector<int>& src_shape = srclayers[i]->data(this).shape();
for (size_t j = 0; j < shape.size(); j++)
- if (j == concate_dim)
- shape[j] += srcshape[j];
+ if (static_cast<int>(j) == concate_dim_)
+ shape[j] += src_shape[j];
else
- CHECK_EQ(shape[j], srcshape[j]);
+ CHECK_EQ(shape[j], src_shape[j]);
}
data_.Reshape(shape);
grad_.Reshape(shape);
}
void ConcateLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
- LOG(FATAL) << "Not implemented for Concate Layer";
+ CHECK_GT(srclayers.size(), 1);
+ // calculate step for each memcpy
+ int step = srclayers[0]->data(this).shape()[concate_dim_];
+ for (unsigned i = concate_dim_ + 1; i < data_.shape().size(); ++i)
+ step *= data_.shape()[i];
+ int srclayer_offset = 0;
+ int concate_offset = 0;
+ while (concate_offset < data_.count()) {
+ for (size_t i = 0; i < srclayers.size(); ++i) {
+ const float* src = srclayers[i]->data(this).cpu_data() + srclayer_offset;
+ float* dst = data_.mutable_cpu_data() + concate_offset;
+ memcpy(dst, src, step * sizeof(float));
+ concate_offset += step;
+ }
+ srclayer_offset += step;
+ }
}
void ConcateLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers) {
- LOG(FATAL) << "Not implemented for Concate Layer";
+ CHECK_GT(srclayers.size(), 1);
+ // calculate step for each memcpy
+ int step = srclayers[0]->grad(this).shape()[concate_dim_];
+ for (unsigned i = concate_dim_ + 1; i < grad_.shape().size(); ++i)
+ step *= grad_.shape()[i];
+ int srclayer_offset = 0;
+ int concate_offset = 0;
+ while (concate_offset < grad_.count()) {
+ for (size_t i = 0; i < srclayers.size(); ++i) {
+ const float* src = grad_.cpu_data() + concate_offset;
+ float* dst = srclayers[i]->mutable_grad(this)->mutable_cpu_data()
+ + srclayer_offset;
+ memcpy(dst, src, step * sizeof(float));
+ concate_offset += step;
+ }
+ srclayer_offset += step;
+ }
}
} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/3ea1eb69/src/test/test_connection_layers.cc
----------------------------------------------------------------------
diff --git a/src/test/test_connection_layers.cc b/src/test/test_connection_layers.cc
index a2cb1d3..e072114 100644
--- a/src/test/test_connection_layers.cc
+++ b/src/test/test_connection_layers.cc
@@ -26,6 +26,7 @@
#include "singa/comm/msg.h"
#include "singa/comm/socket.h"
#include "singa/neuralnet/connection_layer/bridge.h"
+#include "singa/neuralnet/connection_layer/concate.h"
#include "singa/neuralnet/connection_layer/slice.h"
#include "singa/neuralnet/neuron_layer/dummy.h"
#include "singa/proto/job.pb.h"
@@ -277,3 +278,121 @@ TEST(ConnectionLayerTest, ModelSliceTest) {
out[(i / step) % K].grad(nullptr).cpu_data()[offset + i % step]);
}
}
+
+TEST(ConnectionLayerTest, DataConcateTest) {
+ // use dummy as input layers
+ LayerProto proto_in[K];
+ vector<Layer*> src_in[K];
+ DummyLayer in[K];
+ for (int i = 0; i < K; ++i) {
+ proto_in[i].set_name("dummy_input_"+std::to_string(i));
+ proto_in[i].set_partition_id(i);
+ proto_in[i].mutable_dummy_conf()->set_input(true);
+ proto_in[i].mutable_dummy_conf()->add_shape(N / K);
+ proto_in[i].mutable_dummy_conf()->add_shape(M);
+ in[i].Setup(proto_in[i], src_in[i]);
+ }
+
+ // add concate layer
+ vector<Layer*> src_concate;
+ for (int i = 0; i < K; ++i)
+ src_concate.push_back(static_cast<Layer*>(&in[i]));
+ LayerProto proto_concate;
+ proto_concate.set_name("concate");
+ proto_concate.mutable_concate_conf()->set_concate_dim(0);
+ ConcateLayer concate;
+ concate.Setup(proto_concate, src_concate);
+ ASSERT_EQ(concate.data(static_cast<Layer*>(&concate)).shape(0), N);
+ ASSERT_EQ(concate.data(static_cast<Layer*>(&concate)).shape(1), M);
+
+ // use dummy as output layer
+ vector<Layer*> src_out;
+ src_out.push_back(static_cast<Layer*>(&concate));
+ LayerProto proto_out;
+ proto_out.set_name("dummy_output");
+ proto_out.mutable_dummy_conf()->set_output(true);
+ DummyLayer out;
+ out.Setup(proto_out, src_out);
+
+ // test for computing feature
+ for (int i = 0; i < K; ++i)
+ in[i].ComputeFeature(0, src_in[i]);
+ concate.ComputeFeature(0, src_concate);
+ out.ComputeFeature(0, src_out);
+ int step = (N * M) / K;
+ for (int i = 0; i < out.data(nullptr).count(); ++i) {
+ ASSERT_EQ(in[i / step].data(nullptr).cpu_data()[i % step],
+ out.data(nullptr).cpu_data()[i]);
+ }
+
+ // test for computing gradient
+ out.ComputeGradient(0, src_out);
+ concate.ComputeGradient(0, src_concate);
+ for (int i = 0; i < K; ++i)
+ in[i].ComputeGradient(0, src_in[i]);
+ for (int i = 0; i < out.grad(nullptr).count(); ++i) {
+ ASSERT_EQ(in[i / step].grad(nullptr).cpu_data()[i % step],
+ out.grad(nullptr).cpu_data()[i]);
+ }
+}
+
+TEST(ConnectionLayerTest, ModelConcateTest) {
+ // use dummy as input layers
+ LayerProto proto_in[K];
+ vector<Layer*> src_in[K];
+ DummyLayer in[K];
+ for (int i = 0; i < K; ++i) {
+ proto_in[i].set_name("dummy_input_"+std::to_string(i));
+ proto_in[i].set_partition_id(i);
+ proto_in[i].mutable_dummy_conf()->set_input(true);
+ proto_in[i].mutable_dummy_conf()->add_shape(N);
+ proto_in[i].mutable_dummy_conf()->add_shape(M / K);
+ in[i].Setup(proto_in[i], src_in[i]);
+ }
+
+ // add concate layer
+ vector<Layer*> src_concate;
+ for (int i = 0; i < K; ++i)
+ src_concate.push_back(static_cast<Layer*>(&in[i]));
+ LayerProto proto_concate;
+ proto_concate.set_name("concate");
+ proto_concate.mutable_concate_conf()->set_concate_dim(1);
+ ConcateLayer concate;
+ concate.Setup(proto_concate, src_concate);
+ ASSERT_EQ(concate.data(static_cast<Layer*>(&concate)).shape(0), N);
+ ASSERT_EQ(concate.data(static_cast<Layer*>(&concate)).shape(1), M);
+
+ // use dummy as output layer
+ vector<Layer*> src_out;
+ src_out.push_back(static_cast<Layer*>(&concate));
+ LayerProto proto_out;
+ proto_out.set_name("dummy_output");
+ proto_out.mutable_dummy_conf()->set_output(true);
+ DummyLayer out;
+ out.Setup(proto_out, src_out);
+
+ // test for computing feature
+ for (int i = 0; i < K; ++i)
+ in[i].ComputeFeature(0, src_in[i]);
+ concate.ComputeFeature(0, src_concate);
+ out.ComputeFeature(0, src_out);
+ int step = M / K;
+ int offset = 0;
+ for (int i = 0; i < out.grad(nullptr).count(); ++i) {
+ if (i && i % M == 0) offset += step;
+ ASSERT_EQ(in[(i / step) % K].data(nullptr).cpu_data()[offset + i % step],
+ out.data(nullptr).cpu_data()[i]);
+ }
+
+ // test for computing gradient
+ out.ComputeGradient(0, src_out);
+ concate.ComputeGradient(0, src_concate);
+ for (int i = 0; i < K; ++i)
+ in[i].ComputeGradient(0, src_in[i]);
+ offset = 0;
+ for (int i = 0; i < out.grad(nullptr).count(); ++i) {
+ if (i && i % M == 0) offset += step;
+ ASSERT_EQ(in[(i / step) % K].grad(nullptr).cpu_data()[offset + i % step],
+ out.grad(nullptr).cpu_data()[i]);
+ }
+}