You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@singa.apache.org by wa...@apache.org on 2015/12/28 04:31:27 UTC
[1/5] incubator-singa git commit: SINGA-113 Model/Hybrid Partition
Support
Repository: incubator-singa
Updated Branches:
refs/heads/master 138599fd1 -> 4a0db51f5
SINGA-113 Model/Hybrid Partition Support
NeuralNet how will automatically add connection layers for distributed training.
The partition is transparent to users.
User just need to configure partition_dim field in NetProto:
partition_dim = 0 : data partition
partition_dim = 1 : model partition
User can also overwrite partition_dim for a specific layer in LayerProto.
This will result in hybrid partition.
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/2d38cb30
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/2d38cb30
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/2d38cb30
Branch: refs/heads/master
Commit: 2d38cb307466c966e19acd5f888ba0954d60aa19
Parents: 138599f
Author: WANG Sheng <wa...@gmail.com>
Authored: Sat Dec 12 19:10:57 2015 +0800
Committer: Wei Wang <wa...@comp.nus.edu.sg>
Committed: Fri Dec 25 20:25:27 2015 +0800
----------------------------------------------------------------------
configure.ac | 52 ++--
include/singa/neuralnet/neuralnet.h | 12 +-
include/singa/utils/graph.h | 6 +
src/driver.cc | 9 +-
src/neuralnet/connection_layer/slice.cc | 23 +-
src/neuralnet/connection_layer/split.cc | 23 +-
src/neuralnet/neuralnet.cc | 408 +++++++++++++++------------
src/proto/job.proto | 7 +
src/test/test_connection_layers.cc | 2 +-
src/test/test_neuralnet.cc | 96 ++++++-
src/utils/blob.cc | 5 +-
src/utils/common.cc | 2 +-
src/utils/graph.cc | 72 +----
src/worker.cc | 24 +-
14 files changed, 410 insertions(+), 331 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/2d38cb30/configure.ac
----------------------------------------------------------------------
diff --git a/configure.ac b/configure.ac
index 07dbdc3..bde1d8e 100644
--- a/configure.ac
+++ b/configure.ac
@@ -38,7 +38,7 @@ AC_PROG_LIBTOOL
# Setup custom CUDA paths
AC_ARG_ENABLE(cuda,
- [AS_HELP_STRING(--enable-cuda,enable CUDA support)],
+ [AS_HELP_STRING(--enable-cuda,enable CUDA support)],
cudaval="yes",
cudaval="no")
AM_CONDITIONAL(DCUDA, [test "$cudaval" = "yes"])
@@ -71,17 +71,17 @@ if test x"$cudaval" = x"yes"; then
LIBS="$LIBS $CUDA_LIBS"
LDFLAGS="$LDFLAGS $CUDA_LDFLAGS"
NVCC="nvcc"
- DEBUG="-DUSE_GPU"
+ DEBUG="-DUSE_GPU"
AC_DEFINE(DCUDA,[1],[Defined if CUDA should be used])
- AC_CHECK_LIB([cuda], [main], [], [
- AC_MSG_ERROR([unable to find cuda library])
- ])
- AC_CHECK_LIB([cudart], [main], [], [
- AC_MSG_ERROR([unable to find cudart library])
- ])
- AC_CHECK_LIB([curand], [main], [], [
- AC_MSG_ERROR([unable to find curand library])
- ])
+ AC_CHECK_LIB([cuda], [main], [], [
+ AC_MSG_ERROR([unable to find cuda library])
+ ])
+ AC_CHECK_LIB([cudart], [main], [], [
+ AC_MSG_ERROR([unable to find cudart library])
+ ])
+ AC_CHECK_LIB([curand], [main], [], [
+ AC_MSG_ERROR([unable to find curand library])
+ ])
else
CUDA_CFLAGS=""
CUDA_LDFLAGS=""
@@ -149,35 +149,35 @@ AC_CHECK_LIB([protobuf], [main], [], [
AC_ARG_ENABLE(lmdb,
AS_HELP_STRING([--enable-lmdb],[enable debug option]),
- [enable_lmdb=yes],[enable_lmdb=no])
+ [enable_lmdb=yes],[enable_lmdb=no])
AM_CONDITIONAL(LMDB, test "$enable_lmdb" = yes)
if test x"$enable_lmdb" = x"yes"; then
- AC_SEARCH_LIBS([mdb_env_create], [lmdb], [], [
- AC_MSG_ERROR([unable to find mdb_env_create() function])
- ])
- AC_DEFINE(LMDB, 1, [Enable Option layer])
+ AC_SEARCH_LIBS([mdb_env_create], [lmdb], [], [
+ AC_MSG_ERROR([unable to find mdb_env_create() function])
+ ])
+ AC_DEFINE(LMDB, 1, [Enable Option layer])
fi
AC_ARG_ENABLE(test,
- AS_HELP_STRING([--enable-test],[enable singa test]),
- [enable_test=yes],[enable_test=no])
+ AS_HELP_STRING([--enable-test],[enable singa test]),
+ [enable_test=yes],[enable_test=no])
AM_CONDITIONAL(SINGATEST, test "$enable_test" = yes)
if test x"$enable_test" != x"no"; then
- PROGS='singatest test '
- LTLIBS='libgtest.la '
+ PROGS='singatest test '
+ LTLIBS='libgtest.la '
else
- PROGS=''
- LTLIBS=''
+ PROGS=''
+ LTLIBS=''
fi
AC_ARG_ENABLE(debug,
- AS_HELP_STRING([--enable-debug],[enable debug mode]),
- [enable_debug=yes],[enable_debug=no])
+ AS_HELP_STRING([--enable-debug],[enable debug mode]),
+ [enable_debug=yes],[enable_debug=no])
AM_CONDITIONAL(DEBUG, test "$enable_debug" = yes)
if test x"$enable_debug" != x"no"; then
- DEBUG+=' -g'
+ DEBUG+=' -g'
else
- DEBUG+=' -O2'
+ DEBUG+=' -O2'
fi
AC_ARG_ENABLE(python,
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/2d38cb30/include/singa/neuralnet/neuralnet.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/neuralnet.h b/include/singa/neuralnet/neuralnet.h
index b4105d3..bc1a7d8 100644
--- a/include/singa/neuralnet/neuralnet.h
+++ b/include/singa/neuralnet/neuralnet.h
@@ -131,11 +131,21 @@ class NeuralNet {
/**
* Create neural net from graph, one layer per node.
*/
- void CreateNetFromGraph(Graph* graph, int num_partitions);
+ void CreateNetFromGraph(Graph* graph);
/**
* prepare data structures, e.g., params_, layers_, etc.
*/
void PrepareDataStructures();
+ /**
+ * add split layers, due to connections to multiple dst-layers
+ */
+ NetProto AddModelSplitLayers(const NetProto& netproto);
+ /**
+ * add connection layers, due to partition of the whole nerualnet
+ * this should be done after AddModelSplitLayers()
+ */
+ NetProto AddPartitionConnectionLayers(const NetProto& netproto,
+ int npartitions);
protected:
std::vector<Layer*> layers_;
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/2d38cb30/include/singa/utils/graph.h
----------------------------------------------------------------------
diff --git a/include/singa/utils/graph.h b/include/singa/utils/graph.h
index d26a8ee..2462808 100644
--- a/include/singa/utils/graph.h
+++ b/include/singa/utils/graph.h
@@ -122,6 +122,12 @@ class Graph {
*/
Node* AddNode(const string& name, const std::map<string, string>& attrs);
/**
+ * @deprecated {remove layer related info from node attrs}
+ * Add a node with given name and other info.
+ */
+ Node* AddNode(const std::string& name, const std::string& origin, int id,
+ void* proto);
+ /**
* Add an edge connecting the two given nodes.
*/
void AddEdge(Node* srcnode, Node* dstnode);
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/2d38cb30/src/driver.cc
----------------------------------------------------------------------
diff --git a/src/driver.cc b/src/driver.cc
index 9389fde..4eb474a 100644
--- a/src/driver.cc
+++ b/src/driver.cc
@@ -69,15 +69,18 @@ void Driver::Init(int argc, char **argv) {
RegisterLayer<RecordOutputLayer, int>(kRecordOutput);
RegisterLayer<CSVOutputLayer, int>(kCSVOutput);
+ // connection layers
RegisterLayer<BridgeDstLayer, int>(kBridgeDst);
RegisterLayer<BridgeSrcLayer, int>(kBridgeSrc);
+ RegisterLayer<ConcateLayer, int>(kConcate);
+ RegisterLayer<SliceLayer, int>(kSlice);
+ RegisterLayer<SplitLayer, int>(kSplit);
RegisterLayer<AccuracyLayer, int>(kAccuracy);
RegisterLayer<ArgSortLayer, int>(kArgSort);
RegisterLayer<ConvolutionLayer, int>(kConvolution);
RegisterLayer<CConvolutionLayer, int>(kCConvolution);
RegisterLayer<CPoolingLayer, int>(kCPooling);
- RegisterLayer<ConcateLayer, int>(kConcate);
#ifdef USE_CUDNN
RegisterLayer<CudnnActivationLayer, int>(kCudnnActivation);
@@ -88,6 +91,8 @@ void Driver::Init(int argc, char **argv) {
RegisterLayer<CudnnSoftmaxLossLayer, int>(kCudnnSoftmaxLoss);
#endif
+ RegisterLayer<DropoutLayer, int>(kDropout);
+ RegisterLayer<DummyLayer, int>(kDummy);
RegisterLayer<EuclideanLossLayer, int>(kEuclideanLoss);
RegisterLayer<InnerProductLayer, int>(kInnerProduct);
RegisterLayer<LabelLayer, int>(kLabel);
@@ -101,9 +106,7 @@ void Driver::Init(int argc, char **argv) {
RegisterLayer<ReLULayer, int>(kReLU);
RegisterLayer<ShardDataLayer, int>(kShardData);
RegisterLayer<SigmoidLayer, int>(kSigmoid);
- RegisterLayer<SliceLayer, int>(kSlice);
RegisterLayer<SoftmaxLossLayer, int>(kSoftmaxLoss);
- RegisterLayer<SplitLayer, int>(kSplit);
RegisterLayer<STanhLayer, int>(kSTanh);
RegisterLayer<SoftmaxLayer, int>(kSoftmax);
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/2d38cb30/src/neuralnet/connection_layer/slice.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/connection_layer/slice.cc b/src/neuralnet/connection_layer/slice.cc
index b28d4c4..ee635e0 100644
--- a/src/neuralnet/connection_layer/slice.cc
+++ b/src/neuralnet/connection_layer/slice.cc
@@ -97,27 +97,28 @@ void SliceLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers) {
}
const Blob<float>& SliceLayer::data(const Layer* from) const {
- CHECK(from);
- CHECK_LT(from->partition_id(), num_partitions());
- return *datavec_[from->partition_id()];
+ int id = from ? from->partition_id() : 0;
+ CHECK_LT(id, num_partitions());
+ return *datavec_[id];
}
const Blob<float>& SliceLayer::grad(const Layer* from) const {
- CHECK(from);
- CHECK_LT(from->partition_id(), num_partitions());
- return *gradvec_[from->partition_id()];
+ int id = from ? from->partition_id() : 0;
+ CHECK_LT(id, num_partitions());
+ return *gradvec_[id];
}
Blob<float>* SliceLayer::mutable_data(const Layer* from) {
+ int id = from ? from->partition_id() : 0;
CHECK(from);
- CHECK_LT(from->partition_id(), num_partitions());
- return datavec_[from->partition_id()];
+ CHECK_LT(id, num_partitions());
+ return datavec_[id];
}
Blob<float>* SliceLayer::mutable_grad(const Layer* from) {
- CHECK(from);
- CHECK_LT(from->partition_id(), num_partitions());
- return gradvec_[from->partition_id()];
+ int id = from ? from->partition_id() : 0;
+ CHECK_LT(id, num_partitions());
+ return gradvec_[id];
}
const std::string SliceLayer::ToString(bool debug, int flag) {
if (!debug)
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/2d38cb30/src/neuralnet/connection_layer/split.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/connection_layer/split.cc b/src/neuralnet/connection_layer/split.cc
index d2af59d..0ebadc2 100644
--- a/src/neuralnet/connection_layer/split.cc
+++ b/src/neuralnet/connection_layer/split.cc
@@ -37,13 +37,14 @@ void SplitLayer::Setup(const LayerProto& conf,
CHECK_EQ(srclayers.size(), 1);
Layer::Setup(conf, srclayers);
data_.Reshape(srclayers[0]->data(this).shape());
- data_.ShareData(srclayers[0]->mutable_data(this));
- CHECK_GT(num_partitions(), 0);
- // add num_partitions()-1 more grad blobs
- for (int i = 1; i < num_partitions(); ++i) {
+ data_.ShareData(srclayers[0]->data(this), false);
+ int num_splits = conf.split_conf().num_splits();
+ CHECK_GT(num_splits, 0);
+ // add num_splits-1 more grad blobs
+ for (int i = 1; i < num_splits; ++i) {
gradvec_.push_back(new Blob<float>());
}
- for (int i = 0; i < num_partitions(); ++i)
+ for (int i = 0; i < num_splits; ++i)
gradvec_[i]->Reshape(srclayers[0]->data(this).shape());
}
@@ -55,22 +56,24 @@ void SplitLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
void SplitLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers) {
CHECK_EQ(srclayers.size(), 1);
// aggregate all gradients to grad_[0]
- for (int i = 1; i < num_partitions(); ++i)
- for (int j = 0; j < gradvec_[0]->count(); ++j)
- gradvec_[0]->mutable_cpu_data()[j] += gradvec_[i]->cpu_data()[j];
+ for (int i = 1; i < num_splits; ++i)
+ AXPY<float>(cpu, 1.0, *gradvec_[i], gradvec_[0]);
+// for (int i = 1; i < num_splits; ++i)
+// for (int j = 0; j < gradvec_[0]->count(); ++j)
+// gradvec_[0]->mutable_cpu_data()[j] += gradvec_[i]->cpu_data()[j];
// copy grad_[0] to srclayer's grad
srclayers[0]->mutable_grad(this)->CopyFrom(*gradvec_[0]);
}
const Blob<float>& SplitLayer::grad(const Layer* from) const {
CHECK(from);
- CHECK_LT(from->partition_id(), num_partitions());
+ CHECK_LT(from->partition_id(), num_splits);
return *gradvec_[from->partition_id()];
}
Blob<float>* SplitLayer::mutable_grad(const Layer* from) {
CHECK(from);
- CHECK_LT(from->partition_id(), num_partitions());
+ CHECK_LT(from->partition_id(), num_splits);
return gradvec_[from->partition_id()];
}
const std::string SplitLayer::ToString(bool debug, int flag) {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/2d38cb30/src/neuralnet/neuralnet.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuralnet.cc b/src/neuralnet/neuralnet.cc
index acfc9ea..238bd3f 100644
--- a/src/neuralnet/neuralnet.cc
+++ b/src/neuralnet/neuralnet.cc
@@ -99,20 +99,20 @@ NeuralNet* NeuralNet::Create(const NetProto& net_conf, Phase phase,
param->set_name(name);
param->set_share_from(from);
}
- LOG(INFO) << "NeuralNet config is\n" << conf.DebugString();
+ LOG(INFO) << "Initial NeuralNet Config is\n" << conf.DebugString();
// TODO(wangwei) create net based on net type, e.g., directed, undirected, etc
return new NeuralNet(conf, npartitions);
}
NeuralNet::NeuralNet(NetProto netproto, int npartitions) {
- LOG(INFO) << "Constructing Neural Net...";
+ LOG(INFO) << "Constructing NeuralNet...";
auto graph = CreateGraph(netproto, npartitions);
- CreateNetFromGraph(graph, npartitions);
+ CreateNetFromGraph(graph);
PrepareDataStructures();
for (Node* node : graph->nodes())
delete static_cast<LayerProto*>(node->proto);
delete graph;
- LOG(INFO) << "Neural net constructed";
+ LOG(INFO) << "NeuralNet Constructed";
}
NeuralNet::~NeuralNet() {
@@ -142,20 +142,7 @@ void NeuralNet::Load(const vector<string>& paths,
}
}
-/*
-std::string NeuralNet::ToAdjacency() {
- string disp = "";
- for (auto& layer : layers_) {
- disp += layer->name()+": ";
- for (const auto& dst : layer->dstlayers())
- disp += dst->name()+", ";
- disp += "\n";
- }
- return disp;
-}
-*/
-
-void NeuralNet::ShareParamsFrom(NeuralNet* other, bool cpu_only) {
+void NeuralNet::ShareParamsFrom(NeuralNet* other) {
for (auto& layer : layers_) {
auto otherlayer = other->name2layer(layer->name());
if (otherlayer != nullptr) {
@@ -169,189 +156,241 @@ void NeuralNet::ShareParamsFrom(NeuralNet* other, bool cpu_only) {
}
}
-// add a node for SliceLayer between srcnode and dstnodes
-Node* SliceNode(Graph* graph, Node* srcnode,
- const vector<Node*>& dstnodes, bool connect_dst) {
- string name = srcnode->name + "<";
- LayerProto *proto = new LayerProto();
- proto->set_name(name);
- proto->set_type(LayerType::kSlice);
- proto->set_partition_id(
- static_cast<LayerProto*>(srcnode->proto)->partition_id());
- proto->set_partition_dim(
- static_cast<LayerProto*>(srcnode->proto)->partition_dim());
- Node* node = new Node(name, "##" + name, proto->partition_id(), proto);
- graph->AddNode(node);
- graph->AddEdge(srcnode, node);
- if (connect_dst)
- for (Node* dst : dstnodes)
- graph->AddEdge(node, dst);
- return node;
+// name of connection layers
+string splitName(const string& layer) { return "split("+layer+")"; }
+string sliceName(const string& layer) { return "slice("+layer+")"; }
+string concateName(const string& layer) { return "concate("+layer+")"; }
+string bridgeName(const string& src, const string& dst) { return src+"->"+dst; }
+string bridgeSrcName(const string& src, const string& dst) {
+ return "bridge_src("+bridgeName(src, dst)+")";
+}
+string bridgeDstName(const string& src, const string& dst) {
+ return "bridge_dst("+bridgeName(src, dst)+")";
}
-// add a node for ConcateLayer between srcnodes and dstnode
-Node* ConcateNodes(Graph* graph, const vector<Node*>& srcnodes, Node* dstnode) {
- string name = ">" + dstnode->name;
- LayerProto *proto = new LayerProto();
- proto->set_name(name);
- proto->set_type(LayerType::kConcate);
- proto->set_partition_id(
- static_cast<LayerProto*>(dstnode->proto)->partition_id());
- proto->set_partition_dim(
- static_cast<LayerProto*>(srcnodes[0]->proto)->partition_dim());
- Node* node = new Node(name, "##" + name, proto->partition_id(), proto);
- graph->AddNode(node);
- graph->AddEdge(node, dstnode);
- for (Node* src : srcnodes)
- graph->AddEdge(src, node);
- return node;
+ConnectionType dstLayerConnection(const LayerProto& proto) {
+ auto layer = Layer::Create(proto);
+ auto ret = layer->dst_layer_connection();
+ delete layer;
+ return ret;
}
-// add a node for SplitLayer between srcnode and dstnodes
-Node* SplitNode(Graph* graph, Node* srcnode, const vector<Node*>& dstnodes) {
- string name = srcnode->name + "+";
- LayerProto *proto = new LayerProto();
- proto->set_name(name);
- proto->set_type(LayerType::kSplit);
- proto->set_partition_id(
- static_cast<LayerProto*>(srcnode->proto)->partition_id());
- Node* node = new Node(name, "##" + name, proto->partition_id(), proto);
- graph->AddNode(node);
- graph->AddEdge(srcnode, node);
- for (Node* dst : dstnodes)
- graph->AddEdge(node, dst);
- return node;
+ConnectionType srcNeuronConnection(const LayerProto& proto) {
+ auto layer = Layer::Create(proto);
+ auto ret = layer->src_neuron_connection(0);
+ delete layer;
+ return ret;
}
-// add a pair of nodes for BridgeSrcLayer and BridgeDstLayer between srcnode
-// and dstnode
-void BridgeNodes(Graph* graph, Node* srcnode, Node* dstnode) {
- string sname = srcnode->name + ":-";
- LayerProto *sproto = new LayerProto();
- sproto->set_name(sname);
- sproto->set_type(LayerType::kBridgeSrc);
- sproto->set_partition_id(
- static_cast<LayerProto*>(srcnode->proto)->partition_id());
- auto sbridge = new Node(sname, "##" + sname, sproto->partition_id(), sproto);
- string dname = "-:" + dstnode->name;
- LayerProto *dproto = new LayerProto();
- dproto->set_name(dname);
- dproto->set_type(LayerType::kBridgeDst);
- dproto->set_partition_id(
- static_cast<LayerProto*>(dstnode->proto)->partition_id());
- auto dbridge = new Node(dname, "##" + dname, dproto->partition_id(), dproto);
- graph->AddNode(sbridge);
- graph->AddNode(dbridge);
- graph->AddEdge(srcnode, sbridge);
- graph->AddEdge(sbridge, dbridge);
- graph->AddEdge(dbridge, dstnode);
+NetProto NeuralNet::AddModelSplitLayers(const NetProto& netproto) {
+ NetProto net_w_split;
+ net_w_split.CopyFrom(netproto);
+ net_w_split.clear_layer();
+ // calculate number of dst-layers for each layer
+ map<string, int> dst_count;
+ for (const LayerProto& layer : netproto.layer())
+ for (const string& src_name : layer.srclayers())
+ ++dst_count[src_name];
+ // tag to add split layer if:
+ // dst_count[] > 1 && dst_layer_connection() = OneToOne
+ for (const LayerProto& layer : netproto.layer())
+ if ((dst_count[layer.name()] > 1 && dstLayerConnection(layer) == kOneToOne))
+ dst_count[layer.name()] = -dst_count[layer.name()];
+ // add orginal layers and adjust srclayers
+ for (const LayerProto& layer : netproto.layer()) {
+ LayerProto* proto = net_w_split.add_layer();
+ proto->CopyFrom(layer);
+ proto->clear_srclayers();
+ for (const string& src_name : layer.srclayers())
+ if (dst_count[src_name] < 0)
+ proto->add_srclayers(splitName(src_name));
+ else
+ proto->add_srclayers(src_name);
+ }
+ // add split layers
+ for (const LayerProto& layer : netproto.layer()) {
+ if (dst_count[layer.name()] < 0) {
+ LayerProto* split_proto = net_w_split.add_layer();
+ split_proto->set_name(splitName(layer.name()));
+ split_proto->set_type(kSplit);
+ split_proto->set_partition_dim(layer.partition_dim());
+ split_proto->add_srclayers(layer.name());
+ split_proto->mutable_split_conf()
+ ->set_num_splits(-dst_count[layer.name()]);
+ }
+ }
+ // LOG(INFO) << "NeuralNet Config After Model Split is\n"
+ // << net_w_split.DebugString();
+ return net_w_split;
+}
+
+NetProto NeuralNet::AddPartitionConnectionLayers(const NetProto& netproto,
+ int npartitions) {
+ CHECK_GT(npartitions, 0);
+ NetProto net_w_connection;
+ net_w_connection.CopyFrom(netproto);
+ // if npartitions is 1, no need to add connection layers
+ if (npartitions == 1) return net_w_connection;
+ // add original layers, but remove all edges first
+ net_w_connection.clear_layer();
+ map<string, LayerProto*> name2proto;
+ for (const LayerProto& layer : netproto.layer()) {
+ LayerProto* layer_proto = net_w_connection.add_layer();
+ layer_proto->CopyFrom(layer);
+ layer_proto->clear_srclayers();
+ name2proto[layer_proto->name()] = layer_proto;
+ }
+ /*
+ * Add Slice, Concate, Split Layers for Model Partition
+ *
+ * All cases are as follows:
+ * src_pdim | dst_pdim | connection_type | Action
+ * 0 | 0 | OneToOne | Direct Connection
+ * 1 | 1 | OneToOne | Direct Connection
+ * 0 | 0 | OneToAll | Direct Connection
+ * 1 | 0 | OneToOne | Slice -> Concate
+ * 0 | 1 | OneToOne | Slice -> Concate
+ * 1 | 0 | OneToAll | Slice -> Concate
+ * 0 | 1 | OneToAll | Split -> Concate
+ * 1 | 1 | OneToAll | Split -> Concate
+ *
+ * Logic:
+ * dst_pdim = 1 && OneToAll ?
+ * (YES) Split -> Concate
+ * (NO) src_pdim = dst_pdim ?
+ * (YES) Direct Connection
+ * (NO) Slice -> Concate
+ */
+ for (const LayerProto& origin_layer : netproto.layer()) {
+ LayerProto* dst_layer = name2proto[origin_layer.name()];
+ int dst_pdim = dst_layer->partition_dim();
+ ConnectionType connection = srcNeuronConnection(*dst_layer);
+ for (const string& src_name : origin_layer.srclayers()) {
+ LayerProto* src_layer = name2proto[src_name];
+ int src_pdim = src_layer->partition_dim();
+ // dst_pdim = 1 && OneToAll ?
+ if (dst_pdim == 1 && connection == kOneToAll) {
+ // add split layer
+ LayerProto* split_layer = net_w_connection.add_layer();
+ split_layer->set_name(splitName(src_layer->name()));
+ split_layer->set_type(kSplit);
+ split_layer->set_partition_dim(src_layer->partition_dim());
+ split_layer->add_srclayers(src_layer->name());
+ split_layer->mutable_split_conf()->set_num_splits(npartitions);
+ // add concate layer
+ LayerProto* concate_layer = net_w_connection.add_layer();
+ concate_layer->set_name(concateName(split_layer->name()));
+ concate_layer->set_type(kConcate);
+ // concate on src_pdim
+ concate_layer->set_partition_dim(split_layer->partition_dim());
+ concate_layer->add_srclayers(split_layer->name());
+ // connect dst_layer to concate layer
+ dst_layer->add_srclayers(concate_layer->name());
+ } else {
+ // src_pdim = dst_pdim ?
+ if (dst_pdim == src_pdim) {
+ // direct connection
+ dst_layer->add_srclayers(src_layer->name());
+ } else {
+ // add slice layer
+ LayerProto* slice_layer = net_w_connection.add_layer();
+ slice_layer->set_name(sliceName(src_layer->name()));
+ slice_layer->set_type(kSlice);
+ // slice on dst_pdim
+ slice_layer->set_partition_dim(dst_layer->partition_dim());
+ slice_layer->add_srclayers(src_layer->name());
+ // add concate layer
+ LayerProto* concate_layer = net_w_connection.add_layer();
+ concate_layer->set_name(concateName(slice_layer->name()));
+ concate_layer->set_type(kConcate);
+ // concate on src_pdim
+ concate_layer->set_partition_dim(src_layer->partition_dim());
+ concate_layer->add_srclayers(slice_layer->name());
+ // connect dst_layer to concate layer
+ dst_layer->add_srclayers(concate_layer->name());
+ }
+ }
+ }
+ }
+ LOG(INFO) << "NeuralNet Config After Adding Connection Layers is\n"
+ << net_w_connection.DebugString();
+ return net_w_connection;
}
Graph* NeuralNet::CreateGraph(const NetProto& netproto, int npartitions) {
- Graph *graph = new Graph();
- // from name of original layer to nodes
+ NetProto net_w_split = AddModelSplitLayers(netproto);
+ NetProto net_w_connection =
+ AddPartitionConnectionLayers(net_w_split, npartitions);
+ // for each original layer proto, create #npartitions of nodes
+ Graph* graph = new Graph();
map<string, vector<Node*>> name2nodes;
map<string, const LayerProto*> name2proto;
- for (const auto& layer : netproto.layer()) {
+ for (const LayerProto& layer : net_w_connection.layer()) {
vector<Node*> nodes;
- int pdim = layer.partition_dim();
- if (pdim == 0 || pdim == 1) {
- char suffix[4];
- for (int i = 0; i < npartitions; i++) {
- LayerProto *proto = new LayerProto(layer);
- snprintf(suffix, sizeof(suffix), "%02d", i);
- // differentiate partitions
- string nodename = layer.name() + "@" + string(suffix);
- proto->set_partition_id(i);
- proto->set_num_partitions(npartitions);
- proto->set_name(nodename);
- auto node = new Node(nodename, layer.name(), i, proto);
- graph->AddNode(node);
- nodes.push_back(node);
- }
- } else if (pdim == -1) {
+ char suffix[4];
+ for (int i = 0; i < npartitions; i++) {
LayerProto *proto = new LayerProto(layer);
- auto node = new Node(layer.name(), layer.name(), 0, proto);
- graph->AddNode(node);
+ snprintf(suffix, sizeof(suffix), "%02d", i);
+ // differentiate partitions
+ string nodename = layer.name() + "@" + string(suffix);
+ proto->set_name(nodename);
+ proto->set_type(layer.type());
+ proto->set_partition_dim(layer.partition_dim());
+ proto->set_partition_id(i);
+ proto->set_num_partitions(npartitions);
+ Node* node = graph->AddNode(nodename, layer.name(), i, proto);
nodes.push_back(node);
- } else {
- LOG(FATAL) << "Cannot partition layer (" << layer.name() <<") on dim: "
- << layer.partition_dim();
}
name2nodes[layer.name()] = nodes;
name2proto[layer.name()] = &layer;
}
-
- // connect nodes, nodes for ConcateLayer, SliceLayer and SplitLayer are added.
- for (const auto& layerproto : netproto.layer()) {
- string name = layerproto.name();
- int pdim = layerproto.partition_dim();
- const vector<Node*>& nodes = name2nodes.at(name);
- for (auto srcname : layerproto.srclayers()) {
- const vector<Node*>& srcnodes = name2nodes.at(srcname);
- // TODO(wangwei): consider the type of each connection
- Layer *layer = Layer::Create(layerproto);
- ConnectionType connection = layer->src_neuron_connection(0);
- delete layer;
- int src_pdim = name2proto[srcname]->partition_dim();
- // no partition of src layer
- if (src_pdim == -1) {
- Node* srcnode = srcnodes[0];
- if (pdim == 0 || (pdim == 1 && connection == kOneToOne))
- SliceNode(graph, srcnode, nodes, true);
- else if (pdim == -1)
- graph->AddEdge(srcnode, nodes[0]);
- else // type==kLayerPartition&&connection==kOneToAll
- SplitNode(graph, srcnode, nodes);
- } else if ((pdim == -1 && (src_pdim == 0 || src_pdim == 1))
- ||(pdim == 1 && connection == kOneToAll && src_pdim == 0)) {
- // copy/concate the whole srclayer for every dst partition
- for (Node* node : nodes)
- ConcateNodes(graph, srcnodes, node);
- } else if ((src_pdim == 1 && pdim == 0) || (src_pdim == 0 && pdim == 1)) {
- // TODO(wangwei) rewrite the whole src-dst construction in a clear way
- LOG(FATAL) << "not implemented";
- // the most complext scenario
- // vector<Node*> nodes;
- // for (Node* srcnode : srcnodes)
- // nodes.push_back(SliceNode(graph, srcnode, nodes, false));
- // for (Node* node : nodes)
- // ConcateNodes(graph, nodes, node);
- } else if ((src_pdim == 0 && pdim == 0)||
- (src_pdim == 1 && pdim == 1 && connection == kOneToOne)) {
- CHECK_EQ(srcnodes.size(), nodes.size());
- for (size_t i = 0; i < srcnodes.size(); i++)
- graph->AddEdge(srcnodes[i], nodes[i]);
+ // connect layers, add bridge layers if partition id is different
+ for (const LayerProto& origin_layer : net_w_connection.layer()) {
+ vector<Node*> dst_nodes = name2nodes[origin_layer.name()];
+ for (const string& src_name : origin_layer.srclayers()) {
+ vector<Node*> src_nodes = name2nodes[src_name];
+ if (origin_layer.type() != kConcate) {
+ for (size_t i = 0; i < src_nodes.size(); ++i) {
+ CHECK_EQ(src_nodes[i]->partition_id, i);
+ CHECK_EQ(dst_nodes[i]->partition_id, i);
+ graph->AddEdge(src_nodes[i], dst_nodes[i]);
+ }
} else {
- LOG(FATAL) << "in wrong branch, not implemented";
- }
- }
- }
- // must do topology sort, because we have added new nodes.
- graph->Sort();
- // add nodes for SplitLayer
- vector<Node*> oldnodes = graph->nodes();
- for (Node* node : oldnodes) {
- auto layer = Layer::Create(*static_cast<LayerProto*>(node->proto));
- if (node->dstnodes.size() > 1
- && layer->dst_layer_connection() == kOneToOne) {
- vector<Node*> dstnodes = node->dstnodes;
- for (Node* dst : dstnodes)
- graph->RemoveEdge(node, dst);
- SplitNode(graph, node, dstnodes);
- }
- delete layer;
- }
- // add nodes for bridge layers
- for (Node* node : oldnodes) {
- vector<Node*> dstnodes = node->dstnodes;
- auto pid1 = static_cast<LayerProto*>(node->proto)->partition_id();
- for (size_t i = 0; i < dstnodes.size(); i++) {
- Node* dstnode = dstnodes.at(i);
- auto pid2 = static_cast<LayerProto*>(node->proto)->partition_id();
- if (pid1 != pid2) {
- graph->RemoveEdge(node, dstnode);
- BridgeNodes(graph, node, dstnode);
+ // need to add bridge layers
+ for (size_t i = 0; i < src_nodes.size(); ++i) {
+ CHECK_EQ(src_nodes[i]->partition_id, i);
+ for (size_t j = 0; j < dst_nodes.size(); ++j) {
+ CHECK_EQ(dst_nodes[j]->partition_id, j);
+ if (i == j) { // in same partition, no bridge needed
+ graph->AddEdge(src_nodes[i], dst_nodes[j]);
+ } else { // add bridges
+ // bridge src && dst layer
+ LayerProto *proto_bsrc = new LayerProto();
+ LayerProto *proto_bdst = new LayerProto();
+ string bsrc_name = bridgeSrcName(src_nodes[i]->name,
+ dst_nodes[j]->name);
+ string bdst_name = bridgeDstName(src_nodes[i]->name,
+ dst_nodes[j]->name);
+ proto_bsrc->set_name(bsrc_name);
+ proto_bdst->set_name(bdst_name);
+ proto_bsrc->set_type(kBridgeSrc);
+ proto_bdst->set_type(kBridgeDst);
+ proto_bsrc->set_partition_dim(origin_layer.partition_dim());
+ proto_bdst->set_partition_dim(origin_layer.partition_dim());
+ proto_bsrc->set_partition_id(src_nodes[i]->partition_id);
+ proto_bdst->set_partition_id(dst_nodes[j]->partition_id);
+ proto_bsrc->set_num_partitions(npartitions);
+ proto_bdst->set_num_partitions(npartitions);
+ Node* bsrc_node = graph->AddNode(bsrc_name, bsrc_name, i,
+ proto_bsrc);
+ Node* bdst_node = graph->AddNode(bdst_name, bdst_name, j,
+ proto_bdst);
+ graph->AddEdge(src_nodes[i], bsrc_node);
+ graph->AddEdge(bsrc_node, bdst_node);
+ graph->AddEdge(bdst_node, dst_nodes[j]);
+ }
+ }
+ }
}
}
}
@@ -360,7 +399,7 @@ Graph* NeuralNet::CreateGraph(const NetProto& netproto, int npartitions) {
return graph;
}
-void NeuralNet::CreateNetFromGraph(Graph* graph, int npartitions) {
+void NeuralNet::CreateNetFromGraph(Graph* graph) {
// create one layer per node
for (Node* node : graph->nodes()) {
auto proto_ptr = static_cast<LayerProto*>(node->proto);
@@ -375,12 +414,12 @@ void NeuralNet::CreateNetFromGraph(Graph* graph, int npartitions) {
for (Node* src : node->srcnodes)
src_map_[layer].push_back(name2layer(src->name));
}
-
// setup layers
int paramid = 0;
map<string, string> layerinfo;
map<string, vector<Layer*>> share_param_layers;
for (Node* node : graph->nodes()) {
+ LOG(INFO) << "constructing graph: " << node->name;
auto layer = name2layer(node->name);
layer->Setup(*(static_cast<LayerProto*>(node->proto)), srclayers(layer));
DLOG(INFO) << "constructing graph: " << layer->name();
@@ -397,7 +436,6 @@ void NeuralNet::CreateNetFromGraph(Graph* graph, int npartitions) {
if (layer->partition_dim() == 0)
share_param_layers[node->origin].push_back(layer);
}
- // LOG(INFO) << "Neural net structure\n" << graph->ToJson(layerinfo);
// create map from param name to param ptr
std::unordered_map<string, Param*> name2param;
for (auto layer : layers_) {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/2d38cb30/src/proto/job.proto
----------------------------------------------------------------------
diff --git a/src/proto/job.proto b/src/proto/job.proto
index 80752c3..035f2c1 100644
--- a/src/proto/job.proto
+++ b/src/proto/job.proto
@@ -227,6 +227,8 @@ message LayerProto {
optional SoftmaxProto softmax_conf = 53;
// configuration for softmax loss layer
optional SoftmaxLossProto softmaxloss_conf = 40;
+ // configuration for split layer
+ optional SplitProto split_conf = 42;
// configuration for store input layers
optional StoreProto store_conf = 51;
@@ -334,6 +336,10 @@ message PrefetchProto {
repeated LayerProto sublayers = 1;
}
+message SplitProto {
+ optional int32 num_splits = 1 [default = 1];
+}
+
message StoreProto {
optional string backend = 1;
optional string path = 2;
@@ -614,6 +620,7 @@ enum LayerType {
kCudnnActivation = 54;
kCudnnSoftmaxLoss = 55;
kDropout = 4;
+ kDummy = 20;
kInnerProduct = 5;
kLRN = 6;
kPooling = 8;
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/2d38cb30/src/test/test_connection_layers.cc
----------------------------------------------------------------------
diff --git a/src/test/test_connection_layers.cc b/src/test/test_connection_layers.cc
index 28ae341..eab349a 100644
--- a/src/test/test_connection_layers.cc
+++ b/src/test/test_connection_layers.cc
@@ -413,7 +413,7 @@ TEST(ConnectionLayerTest, SplitTest) {
src_split.push_back(static_cast<Layer*>(&in));
LayerProto proto_split;
proto_split.set_name("split");
- proto_split.set_num_partitions(K);
+ proto_split.mutable_split_conf()->set_num_splits(K);
SplitLayer split;
split.Setup(proto_split, src_split);
ASSERT_EQ(split.data(static_cast<Layer*>(&split)).shape(0), N);
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/2d38cb30/src/test/test_neuralnet.cc
----------------------------------------------------------------------
diff --git a/src/test/test_neuralnet.cc b/src/test/test_neuralnet.cc
index 842dabf..e2b8dd7 100644
--- a/src/test/test_neuralnet.cc
+++ b/src/test/test_neuralnet.cc
@@ -20,11 +20,99 @@
*************************************************************/
#include "gtest/gtest.h"
+#include "singa/driver.h"
#include "singa/neuralnet/neuralnet.h"
+#include "singa/neuralnet/neuron_layer/dummy.h"
+#include "singa/neuralnet/connection_layer/slice.h"
+#include "singa/neuralnet/connection_layer/split.h"
+#include "singa/neuralnet/connection_layer/concate.h"
+
using namespace singa;
-TEST(NeuralNet, ParamShareFrom) {
- NetProto conf;
- // add net.conf file into test folder, e.g., conf/net.conf
- // add data shard example in test folder, e.g., data/test.shard
+const int N = 10; // size of dim 0
+const int M = 20; // size of dim 1
+const int K = 2; // size of partitions
+
+TEST(NeuralNet, RegisterLayers) {
+ Driver driver;
+ driver.RegisterLayer<DummyLayer, int>(kDummy);
+ driver.RegisterLayer<SliceLayer, int>(kSlice);
+ driver.RegisterLayer<SplitLayer, int>(kSplit);
+ driver.RegisterLayer<ConcateLayer, int>(kConcate);
+ driver.RegisterLayer<BridgeSrcLayer, int>(kBridgeSrc);
+ driver.RegisterLayer<BridgeDstLayer, int>(kBridgeDst);
+}
+
+TEST(NeuralNet, AddModelSplitLayers) {
+ NetProto proto;
+ // use dummy as input layer
+ LayerProto* proto_in = proto.add_layer();
+ proto_in->set_name("dummy_input");
+ proto_in->set_type(kDummy);
+ proto_in->mutable_dummy_conf()->set_input(true);
+ proto_in->mutable_dummy_conf()->add_shape(N);
+ proto_in->mutable_dummy_conf()->add_shape(M);
+ // use 2 dummy neuron layers
+ for (int i = 0; i < 2; ++i) {
+ LayerProto* proto_neuron = proto.add_layer();
+ proto_neuron->set_name("dummy_neuron_" + std::to_string(i));
+ proto_neuron->set_type(kDummy);
+ proto_neuron->add_srclayers("dummy_input");
+ }
+ // use dummy as output layer
+ for (int i = 0; i < 2; ++i) {
+ LayerProto* proto_out = proto.add_layer();
+ proto_out->set_name("dummy_output" + std::to_string(i));
+ proto_out->set_type(kDummy);
+ proto_out->mutable_dummy_conf()->set_output(true);
+ proto_out->add_srclayers("dummy_neuron_" + std::to_string(i));
+ }
+ NeuralNet::Create(proto, kTrain, K);
+}
+
+TEST(NeuralNet, DirectConnection) {
+ NetProto proto;
+ // use dummy as input layer
+ LayerProto* proto_in = proto.add_layer();
+ proto_in->set_name("dummy_input");
+ proto_in->set_type(kDummy);
+ proto_in->mutable_dummy_conf()->set_input(true);
+ proto_in->mutable_dummy_conf()->add_shape(N);
+ proto_in->mutable_dummy_conf()->add_shape(M);
+ // use dummy neuron layer
+ LayerProto* proto_neuron = proto.add_layer();
+ proto_neuron->set_name("dummy_neuron");
+ proto_neuron->set_type(kDummy);
+ proto_neuron->add_srclayers("dummy_input");
+ // use dummy as output layer
+ LayerProto* proto_out = proto.add_layer();
+ proto_out->set_name("dummy_output");
+ proto_out->set_type(kDummy);
+ proto_out->mutable_dummy_conf()->set_output(true);
+ proto_out->add_srclayers("dummy_neuron");
+ NeuralNet::Create(proto, kTrain, K);
+}
+
+TEST(NeuralNet, SliceConcate) {
+ NetProto proto;
+ // use dummy as input layer
+ LayerProto* proto_in = proto.add_layer();
+ proto_in->set_name("dummy_input");
+ proto_in->set_type(kDummy);
+ proto_in->mutable_dummy_conf()->set_input(true);
+ proto_in->mutable_dummy_conf()->add_shape(N);
+ proto_in->mutable_dummy_conf()->add_shape(M);
+ // use dummy neuron layer
+ LayerProto* proto_neuron = proto.add_layer();
+ proto_neuron->set_name("dummy_neuron");
+ proto_neuron->set_type(kDummy);
+ proto_neuron->add_srclayers("dummy_input");
+ // use dummy as output layer
+ LayerProto* proto_out = proto.add_layer();
+ proto_out->set_name("dummy_output");
+ proto_out->set_type(kDummy);
+ proto_out->set_partition_dim(1);
+ proto_out->mutable_dummy_conf()->set_output(true);
+ proto_out->add_srclayers("dummy_neuron");
+ NeuralNet::Create(proto, kTrain, K);
}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/2d38cb30/src/utils/blob.cc
----------------------------------------------------------------------
diff --git a/src/utils/blob.cc b/src/utils/blob.cc
index a49d169..91542e6 100644
--- a/src/utils/blob.cc
+++ b/src/utils/blob.cc
@@ -187,15 +187,12 @@ void SyncedMemory::to_gpu() {
template <typename Dtype>
void Blob<Dtype>::Reshape(const std::vector<int>& shape) {
- int count = count_;
- count_ = 1;
shape_ = shape;
+ count_ = shape.size() ? 1 : 0;
for (size_t i = 0; i < shape.size(); ++i) {
CHECK(shape[i]);
count_ *= shape[i];
}
- if (count > 0)
- CHECK_EQ(count, count_);
if (count_ > capacity_) {
capacity_ = count_;
data_.reset(new SyncedMemory(capacity_ * sizeof(Dtype)));
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/2d38cb30/src/utils/common.cc
----------------------------------------------------------------------
diff --git a/src/utils/common.cc b/src/utils/common.cc
index 73d1f69..928d4bb 100644
--- a/src/utils/common.cc
+++ b/src/utils/common.cc
@@ -242,7 +242,7 @@ string GetHostIP() {
close(fd);
string ip(inet_ntoa(((struct sockaddr_in *)&ifr.ifr_addr)->sin_addr));
/* display result */
- LOG(INFO) << "Host IP= " << ip;
+ LOG(INFO) << "Host IP = " << ip;
return ip;
}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/2d38cb30/src/utils/graph.cc
----------------------------------------------------------------------
diff --git a/src/utils/graph.cc b/src/utils/graph.cc
index f38ea53..0211e5a 100644
--- a/src/utils/graph.cc
+++ b/src/utils/graph.cc
@@ -84,16 +84,13 @@ Graph::~Graph() {
delete node;
}
-void Graph::AddNode(Node* node) {
+Node* Graph::AddNode(const string& name, const string& origin, int id,
+ void* proto) {
+ Node* node = new Node(name, origin, id, proto);
nodes_.push_back(node);
CHECK(name2node_.find(node->name) == name2node_.end())
<< "node " << node->name << " already exists";
name2node_[node->name] = node;
-}
-
-Node* Graph::AddNode(const string& name) {
- Node* node = new Node(name);
- AddNode(node);
return node;
}
@@ -140,64 +137,6 @@ void Graph::RemoveEdge(const string &src, const string& dst) {
RemoveEdge(srcnode->second, dstnode->second);
}
-string Graph::ToJson() const {
- map<string, string> label;
- return ToJson(label);
-}
-
-/*
-string Graph::ToJson(const map<string, string>& info) const {
- map<string, int> nodeid;
- string disp = "{\"directed\":1,\n";
-
- // add nodes
- disp += "\"nodes\":[\n";
- bool first = true;
- vector<string> colors = {"red", "blue", "black", "green"};
-
- // see for more shapes at http://www.graphviz.org/doc/info/shapes.html
- // vector<string> shapes = {"box", "ellipse"};
- int id = 0;
- for (auto node : nodes_) {
- char str[1024];
- string name = node->name;
- string color = colors[(node->partition_id)%colors.size()];
- string shape;
- string origin = node->origin;
- snprintf(str, sizeof(str),
- "{\"id\":\"%s%s\", \"color\":\"%s\",\"shape\":\"%s\"}\n", name.c_str(),
- info.find(name) != info.end() ? info.at(name).c_str() : "",
- color.c_str(), "box");
- if (!first)
- disp += ",";
- else
- first = false;
- disp += string(str);
- nodeid[name] = id++;
- }
- disp += "]\n,";
-
- // add edges
- disp += "\"links\":[\n";
- first = true;
- for (auto src : nodes_) {
- for (auto dst : src->dstnodes) {
- char str[1024];
- snprintf(str, sizeof(str),
- "{\"source\":%d, \"target\":%d, \"color\":\"%s\"}\n",
- nodeid[src->name], nodeid[dst->name], "black");
- if (!first)
- disp += ",";
- else
- first = false;
- disp += string(str);
- }
- }
- disp += "]\n";
- return disp+"}";
-}
-*/
-
// sort to make `bottom' nodes be placed in the front positions
void Graph::Sort() {
// nodes to be visited
@@ -271,6 +210,11 @@ const Graph Graph::Reverse() const {
}
return g;
}
+string Graph::ToJson() const {
+ map<string, string> label;
+ return ToJson(label);
+}
+
string Graph::ToJson(const map<string, string>& label) const {
string disp = "{\"directed\":1,\n";
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/2d38cb30/src/worker.cc
----------------------------------------------------------------------
diff --git a/src/worker.cc b/src/worker.cc
index 23868da..c240e84 100644
--- a/src/worker.cc
+++ b/src/worker.cc
@@ -130,8 +130,8 @@ void Worker::InitSockets(const NeuralNet* net) {
ConnectStub(grp_id_, id_, dealer_, kWorkerParam);
for (auto layer : net->layers()) {
if (layer->partition_id() == id_) {
- if (typeid(layer) == typeid(BridgeDstLayer)
- || typeid(layer) == typeid(BridgeSrcLayer)) {
+ if (typeid(*layer) == typeid(BridgeDstLayer)
+ || typeid(*layer) == typeid(BridgeSrcLayer)) {
// TODO(wangsh): provide a unique socket id from cluster
bridge_dealer_ = new Dealer(1);
ConnectStub(grp_id_, id_, bridge_dealer_, kWorkerLayer);
@@ -142,7 +142,7 @@ void Worker::InitSockets(const NeuralNet* net) {
// bind dealer to bridge layers
if (bridge_dealer_ != nullptr) {
for (auto dst : net->layers()) {
- if (typeid(dst) == typeid(BridgeDstLayer)) {
+ if (typeid(*dst) == typeid(BridgeDstLayer)) {
auto src = net->srclayers(dst)[0];
name2bridge_[src->name()] = src;
name2bridge_[dst->name()] = dst;
@@ -338,10 +338,6 @@ void BPWorker::Forward(int step, Phase phase, NeuralNet* net) {
map<string, string> label;
for (auto& layer : net->layers()) {
if (layer->partition_id() == id_) {
- // TODO(wangwei): enable this for model partition
- // recv data from other workers
- // if (typeid(*layer) == typeid(BridgeDstLayer))
- // ReceiveBlobs(true, false, dynamic_cast<BridgeLayer*>(layer), net);
if (phase == kTrain) {
// wait until param is updated
for (Param* p : layer->GetParams()) {
@@ -352,11 +348,6 @@ void BPWorker::Forward(int step, Phase phase, NeuralNet* net) {
layer->ComputeFeature(phase | kForward, net->srclayers(layer));
if (job_conf_.debug() && grp_id_ == 0)
label[layer->name()] = layer->ToString(true, phase | kForward);
-
- // TODO(wangwei): enable this for model partition
- // send data to other workers
- // if (typeid(*layer) == typeid(BridgeSrcLayer))
- // SendBlobs(true, false, dynamic_cast<BridgeLayer*>(layer), net);
}
}
if (label.size()) {
@@ -372,20 +363,11 @@ void BPWorker::Backward(int step, NeuralNet* net) {
for (auto it = layers.rbegin(); it != layers.rend(); it++) {
Layer* layer = *it;
if (layer->partition_id() == id_) {
- // TODO(wangwei): enable this for model partition
- // send data to other workers
- // if (typeid(layer) == typeid(BridgeSrcLayer))
- // ReceiveBlobs(false, true, layer, net);
- // LOG(ERROR) << layer->name() << " backward";
layer->ComputeGradient(kTrain | kBackward, net->srclayers(layer));
if (job_conf_.debug() && grp_id_ == 0)
label[layer->name()] = layer->ToString(true, kTrain | kBackward);
for (Param* p : layer->GetParams())
Update(step, p);
- // TODO(wangwei): enable this for model partition
- // recv data from other workers
- // if (typeid(layer) == typeid(BridgeDstLayer))
- // SendBlobs(false, true, dynamic_cast<BridgeDstLayer*>(layer), net);
}
}
if (label.size()) {
[3/5] incubator-singa git commit: SINGA-113 Model/Hybrid Partition
Support
Posted by wa...@apache.org.
SINGA-113 Model/Hybrid Partition Support
Fix a bug caused by the mutable_grad(Layer*) function of SplitLayer and SlicLayer.
They return the grad blob based on dest layer's partition_id.
The bug we encountered:
the slice layer is connected to a BridgeSrcLayer and a ConcateLayer, both with
partition id = 0 (the partition id of the BridgeDstLayer connected from BridgeSrcLayer is 1);
during backward propagation, both BridgeSrcLayer and ConcateLayer get
the set same grad blob of the SliceLayer, hence this blob is overwritten and the other grad Blob is empty.
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/82563f69
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/82563f69
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/82563f69
Branch: refs/heads/master
Commit: 82563f6905fa1efc45a2a5dfbd85d10c96693643
Parents: 8af565c
Author: Wei Wang <wa...@comp.nus.edu.sg>
Authored: Fri Dec 25 13:40:18 2015 +0800
Committer: Wei Wang <wa...@comp.nus.edu.sg>
Committed: Fri Dec 25 20:27:51 2015 +0800
----------------------------------------------------------------------
examples/cifar10/hybrid.conf | 292 ++++++++++++++++++++++++
include/singa/neuralnet/connection_layer.h | 28 ++-
include/singa/neuralnet/layer.h | 29 ++-
src/neuralnet/connection_layer/bridge.cc | 4 +-
src/neuralnet/connection_layer/slice.cc | 29 +--
src/neuralnet/connection_layer/split.cc | 12 +-
6 files changed, 357 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/82563f69/examples/cifar10/hybrid.conf
----------------------------------------------------------------------
diff --git a/examples/cifar10/hybrid.conf b/examples/cifar10/hybrid.conf
new file mode 100644
index 0000000..ec3da0c
--- /dev/null
+++ b/examples/cifar10/hybrid.conf
@@ -0,0 +1,292 @@
+name: "cifar10-convnet"
+train_steps: 1000
+test_steps: 0
+test_freq: 200
+#validate_steps: 100
+#validate_freq: 300
+disp_freq: 30
+#debug: true
+#checkpoint_path: "examples/cifar10/checkpoint/step1000-worker0"
+train_one_batch {
+ alg: kBP
+}
+updater{
+ type: kSGD
+ weight_decay:0.004
+ momentum:0.9
+ learning_rate {
+ type: kFixedStep
+ fixedstep_conf:{
+ step:0
+ step:60000
+ step:65000
+ step_lr:0.001
+ step_lr:0.0001
+ step_lr:0.00001
+ }
+ }
+}
+neuralnet {
+ layer{
+ name: "data"
+ type: kRecordInput
+ store_conf {
+ backend: "kvfile"
+ path: "examples/cifar10/train_data.bin"
+ mean_file: "examples/cifar10/image_mean.bin"
+ batchsize: 100
+ #random_skip: 5000
+ shape: 3
+ shape: 32
+ shape: 32
+ }
+ include: kTrain
+ }
+# layer{
+# name: "data"
+# type: kRecordInput
+# store_conf {
+# backend: "kvfile"
+# path: "examples/cifar10/val_data.bin"
+# mean_file: "examples/cifar10/image_mean.bin"
+# batchsize: 64
+# random_skip: 5000
+# shape: 3
+# shape: 32
+# shape: 32
+# }
+# include: kVal
+# }
+ layer{
+ name: "data"
+ type: kRecordInput
+ store_conf {
+ backend: "kvfile"
+ path: "examples/cifar10/test_data.bin"
+ mean_file: "examples/cifar10/image_mean.bin"
+ batchsize: 100
+ shape: 3
+ shape: 32
+ shape: 32
+ }
+ include: kTest
+ }
+
+ layer {
+ partition_dim: 0
+ name: "conv1"
+ type: kCConvolution
+ srclayers: "data"
+ convolution_conf {
+ num_filters: 32
+ kernel: 5
+ stride: 1
+ pad:2
+ }
+ param {
+ name: "w1"
+ init {
+ type:kGaussian
+ std:0.0001
+ }
+ }
+ param {
+ name: "b1"
+ lr_scale:2.0
+ init {
+ type: kConstant
+ value:0
+ }
+ }
+ }
+
+ layer {
+ partition_dim: 0
+ name: "pool1"
+ type: kCPooling
+ srclayers: "conv1"
+ pooling_conf {
+ pool: MAX
+ kernel: 3
+ stride: 2
+ }
+ }
+ layer {
+ partition_dim: 0
+ name: "relu1"
+ type: kReLU
+ srclayers:"pool1"
+ }
+ layer {
+ partition_dim: 0
+ name: "norm1"
+ type: kLRN
+ lrn_conf {
+ local_size: 3
+ alpha: 5e-05
+ beta: 0.75
+ }
+ srclayers:"relu1"
+ }
+ layer {
+ partition_dim: 0
+ name: "conv2"
+ type: kCConvolution
+ srclayers: "norm1"
+ convolution_conf {
+ num_filters: 32
+ kernel: 5
+ stride: 1
+ pad:2
+ }
+ param {
+ name: "w2"
+ init {
+ type:kGaussian
+ std:0.01
+ }
+ }
+ param {
+ name: "b2"
+ lr_scale:2.0
+ init {
+ type: kConstant
+ value:0
+ }
+ }
+ }
+ layer {
+ partition_dim: 0
+ name: "relu2"
+ type: kReLU
+ srclayers:"conv2"
+ }
+ layer {
+ partition_dim: 0
+ name: "pool2"
+ type: kCPooling
+ srclayers: "relu2"
+ pooling_conf {
+ pool: AVG
+ kernel: 3
+ stride: 2
+ }
+ }
+ layer {
+ partition_dim: 0
+ name: "norm2"
+ type: kLRN
+ lrn_conf {
+ local_size: 3
+ alpha: 5e-05
+ beta: 0.75
+ }
+ srclayers:"pool2"
+ }
+ layer {
+ partition_dim: 0
+ name: "conv3"
+ type: kCConvolution
+ srclayers: "norm2"
+ convolution_conf {
+ num_filters: 64
+ kernel: 5
+ stride: 1
+ pad:2
+ }
+ param {
+ name: "w3"
+ init {
+ type:kGaussian
+ std:0.01
+ }
+ }
+ param {
+ name: "b3"
+ init {
+ type: kConstant
+ value:0
+ }
+ }
+ }
+ layer {
+ partition_dim: 0
+ name: "relu3"
+ type: kReLU
+ srclayers:"conv3"
+ }
+ layer {
+ partition_dim: 0
+ name: "pool3"
+ type: kCPooling
+ srclayers: "relu3"
+ pooling_conf {
+ pool: AVG
+ kernel: 3
+ stride: 2
+ }
+ }
+ layer {
+ partition_dim: 1
+ name: "ip1"
+ type: kInnerProduct
+ srclayers:"pool3"
+ innerproduct_conf {
+ num_output: 10
+ }
+ param {
+ name: "w4"
+ wd_scale:250
+ init {
+ type:kGaussian
+ std:0.01
+ }
+ }
+ param {
+ name: "b4"
+ lr_scale:2.0
+ wd_scale:0
+ init {
+ type: kConstant
+ value:0
+ }
+ }
+ }
+# layer {
+# name : "softmax"
+# type: kSoftmax
+# srclayers: "ip1"
+# }
+#
+# layer {
+# name : "argsort"
+# type: kArgSort
+# srclayers: "softmax"
+# }
+ layer{
+ name: "loss"
+ type: kSoftmaxLoss
+ softmaxloss_conf{
+ topk:1
+ }
+ srclayers:"ip1"
+ srclayers: "data"
+ }
+# uncomment "softmax", "argsort", "output" layer and comment "loss" layer
+# to extract features from argsort
+# layer {
+# name : "output"
+# type: kCSVOutput
+# srclayers: "argsort"
+# store_conf {
+# path: "examples/cifar10/out.csv"
+# }
+# }
+}
+cluster {
+ nworker_groups: 1
+ nserver_groups: 1
+ nworkers_per_group: 2
+ nworkers_per_procs: 2
+ workspace: "examples/cifar10"
+}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/82563f69/include/singa/neuralnet/connection_layer.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/connection_layer.h b/include/singa/neuralnet/connection_layer.h
index ae918ce..14e5092 100644
--- a/include/singa/neuralnet/connection_layer.h
+++ b/include/singa/neuralnet/connection_layer.h
@@ -29,6 +29,24 @@
#include "singa/neuralnet/layer.h"
namespace singa {
+/**
+ * Used inside SplitLayer and SliceLayer to locate the out-going connection
+ * index given the Layer pointer.
+ */
+class Layer2Index {
+ public:
+ int Get(const Layer* layer) {
+ if (layer2idx_.find(layer) == layer2idx_.end()) {
+ int idx = layer2idx_.size();
+ layer2idx_[layer] = idx;
+ }
+ return layer2idx_[layer];
+ }
+
+ private:
+ std::unordered_map<const Layer*, int> layer2idx_;
+};
+
class BridgeLayer : public ConnectionLayer {
public:
@@ -102,14 +120,15 @@ class SliceLayer : public ConnectionLayer {
void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
const std::string ToString(bool debug, int flag) override;
- const Blob<float>& data(const Layer* from) const override;
- const Blob<float>& grad(const Layer* from) const override;
+ const Blob<float>& data(const Layer* from) override;
+ const Blob<float>& grad(const Layer* from) override;
Blob<float>* mutable_data(const Layer* from) override;
Blob<float>* mutable_grad(const Layer* from) override;
-
+
private:
int num_slices = 0;
int slice_dim = 0;
+ Layer2Index layer_idx_;
};
/**
@@ -126,11 +145,12 @@ class SplitLayer : public ConnectionLayer {
void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
const std::string ToString(bool debug, int flag) override;
- const Blob<float>& grad(const Layer* from) const override;
+ const Blob<float>& grad(const Layer* from) override;
Blob<float>* mutable_grad(const Layer* from) override;
private:
int num_splits = 0;
+ Layer2Index layer_idx_;
};
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/82563f69/include/singa/neuralnet/layer.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/layer.h b/include/singa/neuralnet/layer.h
index 3dbb3fc..28ab92e 100644
--- a/include/singa/neuralnet/layer.h
+++ b/include/singa/neuralnet/layer.h
@@ -174,6 +174,13 @@ class Layer {
*/
inline const std::string& name() const { return layer_conf_.name(); }
/**
+ * @return a const ref for Blob vector storing feature values of this layer.
+ */
+ virtual const vector<Blob<float>*>& data() const {
+ return datavec_;
+ }
+
+ /**
* @param[in] from pointer to one of the dst layer. For some layers, they have
* more than one data Blob. In this case, this argument identifies the layer
* that is requesting the data Blob.
@@ -182,21 +189,19 @@ class Layer {
* virtual const vector<Blob<float>>& data() const or
* virtual const Blob<float>& data(int k) const instead}.
*/
- virtual const Blob<float>& data(const Layer* from) const {
+ virtual const Blob<float>& data(const Layer* from) {
return data_;
}
/**
- * @return a const ref for Blob vector storing feature values of this layer.
- */
- virtual const vector<Blob<float>*>& data() const {
- return datavec_;
- }
- /**
* @return a const ref for the kth Blob.
+ * TODO(wangwei) if make this function const, there will be a warning
+ * indicating that data(const Layer*) and this function are ambiguous for
+ * data(0).
*/
- virtual const Blob<float>& data(int k) const {
+ virtual const Blob<float>& data(int k) {
return *datavec_.at(k);
}
+
/**
* @see data().
* @return the pointer to the Blob storing feature values of this layer.
@@ -226,7 +231,7 @@ class Layer {
* virtual const vector<Blob<float>>& grad() const or
* virtual const Blob<float>& grad(int k) const instead}.
*/
- virtual const Blob<float>& grad(const Layer* from) const {
+ virtual const Blob<float>& grad(const Layer* from) {
return grad_;
}
/**
@@ -286,7 +291,7 @@ class InputLayer : virtual public Layer {
return nullptr;
// LOG(FATAL) << "Input layer has no gradient blob";
}
- const Blob<float>& grad(const Layer* from) const override {
+ const Blob<float>& grad(const Layer* from) override {
return grad_;
// LOG(FATAL) << "Input layer has no gradient blob";
}
@@ -312,7 +317,7 @@ class LossLayer : virtual public Layer {
return nullptr;
// LOG(FATAL) << "Loss layer has no gradient blob";
}
- const Blob<float>& grad(const Layer* from) const override {
+ const Blob<float>& grad(const Layer* from) override {
return grad_;
// LOG(FATAL) << "Loss layer has no gradient blob";
}
@@ -328,7 +333,7 @@ class OutputLayer : virtual public Layer {
return nullptr;
// LOG(FATAL) << "Output layer has no gradient blob";
}
- const Blob<float>& grad(const Layer* from) const override {
+ const Blob<float>& grad(const Layer* from) override {
return grad_;
// LOG(FATAL) << "Output layer has no gradient blob";
}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/82563f69/src/neuralnet/connection_layer/bridge.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/connection_layer/bridge.cc b/src/neuralnet/connection_layer/bridge.cc
index a2302ab..2cfd55a 100644
--- a/src/neuralnet/connection_layer/bridge.cc
+++ b/src/neuralnet/connection_layer/bridge.cc
@@ -69,8 +69,8 @@ void BridgeSrcLayer::Setup(const LayerProto& conf,
Layer::Setup(conf, srclayers);
data_.Reshape(srclayers[0]->data(this).shape());
grad_.ReshapeLike(data_);
- data_.ShareData(srclayers[0]->mutable_data(this));
- grad_.ShareData(srclayers[0]->mutable_grad(this));
+ data_.ShareData(srclayers[0]->mutable_data(this), false);
+ grad_.ShareData(srclayers[0]->mutable_grad(this), false);
}
void BridgeSrcLayer::ComputeFeature(int flag, const vector<Layer*>& srcs) {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/82563f69/src/neuralnet/connection_layer/slice.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/connection_layer/slice.cc b/src/neuralnet/connection_layer/slice.cc
index db83f1b..b625c66 100644
--- a/src/neuralnet/connection_layer/slice.cc
+++ b/src/neuralnet/connection_layer/slice.cc
@@ -98,29 +98,30 @@ void SliceLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers) {
}
}
-const Blob<float>& SliceLayer::data(const Layer* from) const {
- int id = from ? from->partition_id() : 0;
- CHECK_LT(id, num_slices);
- return *datavec_[id];
+const Blob<float>& SliceLayer::data(const Layer* from) {
+ int idx = from ? layer_idx_.Get(from) : 0;
+ CHECK_LT(idx, num_slices);
+ return *datavec_[idx];
}
-const Blob<float>& SliceLayer::grad(const Layer* from) const {
- int id = from ? from->partition_id() : 0;
- CHECK_LT(id, num_slices);
- return *gradvec_[id];
+const Blob<float>& SliceLayer::grad(const Layer* from) {
+ int idx = from ? layer_idx_.Get(from) : 0;
+ CHECK_LT(idx, num_slices);
+ return *gradvec_[idx];
}
Blob<float>* SliceLayer::mutable_data(const Layer* from) {
- int id = from ? from->partition_id() : 0;
CHECK(from);
- CHECK_LT(id, num_slices);
- return datavec_[id];
+ int idx = layer_idx_.Get(from);
+ CHECK_LT(idx, num_slices);
+ return datavec_[idx];
}
Blob<float>* SliceLayer::mutable_grad(const Layer* from) {
- int id = from ? from->partition_id() : 0;
- CHECK_LT(id, num_slices);
- return gradvec_[id];
+ CHECK(from);
+ int idx = layer_idx_.Get(from);
+ CHECK_LT(idx, num_slices);
+ return gradvec_[idx];
}
const std::string SliceLayer::ToString(bool debug, int flag) {
if (!debug)
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/82563f69/src/neuralnet/connection_layer/split.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/connection_layer/split.cc b/src/neuralnet/connection_layer/split.cc
index 9bbe59e..a9270fb 100644
--- a/src/neuralnet/connection_layer/split.cc
+++ b/src/neuralnet/connection_layer/split.cc
@@ -62,16 +62,18 @@ void SplitLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers) {
Copy(*gradvec_[0], srclayers[0]->mutable_grad(this));
}
-const Blob<float>& SplitLayer::grad(const Layer* from) const {
+const Blob<float>& SplitLayer::grad(const Layer* from) {
CHECK(from);
- CHECK_LT(from->partition_id(), num_splits);
- return *gradvec_[from->partition_id()];
+ int idx = layer_idx_.Get(from);
+ CHECK_LT(idx, num_splits);
+ return *gradvec_[idx];
}
Blob<float>* SplitLayer::mutable_grad(const Layer* from) {
CHECK(from);
- CHECK_LT(from->partition_id(), num_splits);
- return gradvec_[from->partition_id()];
+ int idx = layer_idx_.Get(from);
+ CHECK_LT(idx, num_splits);
+ return gradvec_[idx];
}
const std::string SplitLayer::ToString(bool debug, int flag) {
if (!debug)
[5/5] incubator-singa git commit: SINGA-113 Model/Hybrid Partition
Support
Posted by wa...@apache.org.
SINGA-113 Model/Hybrid Partition Support
check with cpplint
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/4a0db51f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/4a0db51f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/4a0db51f
Branch: refs/heads/master
Commit: 4a0db51f56103a928c5904289abccc3ce5bb1e13
Parents: bf4cd3b
Author: WANG Sheng <wa...@gmail.com>
Authored: Mon Dec 28 11:21:40 2015 +0800
Committer: WANG Sheng <wa...@gmail.com>
Committed: Mon Dec 28 11:21:40 2015 +0800
----------------------------------------------------------------------
src/neuralnet/neuralnet.cc | 122 ++++++++++++++++++++--------------------
src/test/test_neuralnet.cc | 2 +-
2 files changed, 62 insertions(+), 62 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/4a0db51f/src/neuralnet/neuralnet.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuralnet.cc b/src/neuralnet/neuralnet.cc
index dfd477e..9cdaff4 100644
--- a/src/neuralnet/neuralnet.cc
+++ b/src/neuralnet/neuralnet.cc
@@ -260,67 +260,67 @@ NetProto NeuralNet::AddPartitionConnectionLayers(const NetProto& netproto,
* (NO) src_pdim = dst_pdim ?
* (YES) Direct Connection
* (NO) Slice -> Concate
- */
- for (const LayerProto& origin_layer : netproto.layer()) {
- LayerProto* dst_layer = name2proto[origin_layer.name()];
- int dst_pdim = dst_layer->partition_dim();
- ConnectionType connection = srcNeuronConnection(*dst_layer);
- for (const string& src_name : origin_layer.srclayers()) {
- LayerProto* src_layer = name2proto[src_name];
- int src_pdim = src_layer->partition_dim();
- // dst_pdim = 1 && OneToAll ?
- if (dst_pdim == 1 && connection == kOneToAll) {
- // add split layer
- LayerProto* split_layer = net_w_connection.add_layer();
- split_layer->set_name(splitName(src_layer->name()));
- split_layer->set_type(kSplit);
- split_layer->set_partition_dim(src_layer->partition_dim());
- split_layer->add_srclayers(src_layer->name());
- split_layer->mutable_split_conf()->set_num_splits(npartitions);
- // add concate layer
- LayerProto* concate_layer = net_w_connection.add_layer();
- concate_layer->set_name(concateName(split_layer->name()));
- concate_layer->set_type(kConcate);
- concate_layer->set_partition_dim(dst_layer->partition_dim());
- // concate on src_pdim
- concate_layer->mutable_concate_conf()
- ->set_concate_dim(src_layer->partition_dim());
- concate_layer->mutable_concate_conf()->set_num_concates(npartitions);
- concate_layer->add_srclayers(split_layer->name());
- // connect dst_layer to concate layer
- dst_layer->add_srclayers(concate_layer->name());
- } else {
- // src_pdim = dst_pdim ?
- if (dst_pdim == src_pdim) {
- // direct connection
- dst_layer->add_srclayers(src_layer->name());
- } else {
- // add slice layer
- LayerProto* slice_layer = net_w_connection.add_layer();
- slice_layer->set_name(sliceName(src_layer->name()));
- slice_layer->set_type(kSlice);
- slice_layer->set_partition_dim(src_layer->partition_dim());
- // slice on dst_pdim
- slice_layer->mutable_slice_conf()
- ->set_slice_dim(dst_layer->partition_dim());
- slice_layer->mutable_slice_conf()->set_num_slices(npartitions);
- slice_layer->add_srclayers(src_layer->name());
- // add concate layer
- LayerProto* concate_layer = net_w_connection.add_layer();
- concate_layer->set_name(concateName(slice_layer->name()));
- concate_layer->set_type(kConcate);
- concate_layer->set_partition_dim(dst_layer->partition_dim());
- // concate on src_pdim
- concate_layer->mutable_concate_conf()
- ->set_concate_dim(src_layer->partition_dim());
- concate_layer->mutable_concate_conf()->set_num_concates(npartitions);
- concate_layer->add_srclayers(slice_layer->name());
- // connect dst_layer to concate layer
- dst_layer->add_srclayers(concate_layer->name());
- }
- }
- }
- }
+ */
+ for (const LayerProto& origin_layer : netproto.layer()) {
+ LayerProto* dst_layer = name2proto[origin_layer.name()];
+ int dst_pdim = dst_layer->partition_dim();
+ ConnectionType connection = srcNeuronConnection(*dst_layer);
+ for (const string& src_name : origin_layer.srclayers()) {
+ LayerProto* src_layer = name2proto[src_name];
+ int src_pdim = src_layer->partition_dim();
+ // dst_pdim = 1 && OneToAll ?
+ if (dst_pdim == 1 && connection == kOneToAll) {
+ // add split layer
+ LayerProto* split_layer = net_w_connection.add_layer();
+ split_layer->set_name(splitName(src_layer->name()));
+ split_layer->set_type(kSplit);
+ split_layer->set_partition_dim(src_layer->partition_dim());
+ split_layer->add_srclayers(src_layer->name());
+ split_layer->mutable_split_conf()->set_num_splits(npartitions);
+ // add concate layer
+ LayerProto* concate_layer = net_w_connection.add_layer();
+ concate_layer->set_name(concateName(split_layer->name()));
+ concate_layer->set_type(kConcate);
+ concate_layer->set_partition_dim(dst_layer->partition_dim());
+ // concate on src_pdim
+ concate_layer->mutable_concate_conf()
+ ->set_concate_dim(src_layer->partition_dim());
+ concate_layer->mutable_concate_conf()->set_num_concates(npartitions);
+ concate_layer->add_srclayers(split_layer->name());
+ // connect dst_layer to concate layer
+ dst_layer->add_srclayers(concate_layer->name());
+ } else {
+ // src_pdim = dst_pdim ?
+ if (dst_pdim == src_pdim) {
+ // direct connection
+ dst_layer->add_srclayers(src_layer->name());
+ } else {
+ // add slice layer
+ LayerProto* slice_layer = net_w_connection.add_layer();
+ slice_layer->set_name(sliceName(src_layer->name()));
+ slice_layer->set_type(kSlice);
+ slice_layer->set_partition_dim(src_layer->partition_dim());
+ // slice on dst_pdim
+ slice_layer->mutable_slice_conf()
+ ->set_slice_dim(dst_layer->partition_dim());
+ slice_layer->mutable_slice_conf()->set_num_slices(npartitions);
+ slice_layer->add_srclayers(src_layer->name());
+ // add concate layer
+ LayerProto* concate_layer = net_w_connection.add_layer();
+ concate_layer->set_name(concateName(slice_layer->name()));
+ concate_layer->set_type(kConcate);
+ concate_layer->set_partition_dim(dst_layer->partition_dim());
+ // concate on src_pdim
+ concate_layer->mutable_concate_conf()
+ ->set_concate_dim(src_layer->partition_dim());
+ concate_layer->mutable_concate_conf()->set_num_concates(npartitions);
+ concate_layer->add_srclayers(slice_layer->name());
+ // connect dst_layer to concate layer
+ dst_layer->add_srclayers(concate_layer->name());
+ }
+ }
+ }
+ }
LOG(INFO) << "NeuralNet Config After Adding Connection Layers is\n"
<< net_w_connection.DebugString();
return net_w_connection;
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/4a0db51f/src/test/test_neuralnet.cc
----------------------------------------------------------------------
diff --git a/src/test/test_neuralnet.cc b/src/test/test_neuralnet.cc
index 198ee0d..3ab197b 100644
--- a/src/test/test_neuralnet.cc
+++ b/src/test/test_neuralnet.cc
@@ -58,7 +58,7 @@ TEST(NeuralNet, AddModelSplitLayers) {
proto_neuron->add_srclayers("dummy_input");
}
// use dummy as output layer
- for (int i = 0; i < 2; ++i) {
+ for (int i = 0; i < 2; ++i) {
LayerProto* proto_out = proto.add_layer();
proto_out->set_name("dummy_output" + std::to_string(i));
proto_out->set_type(kDummy);
[2/5] incubator-singa git commit: SINGA-113 Model/Hybrid Partition
Support
Posted by wa...@apache.org.
SINGA-113 Model/Hybrid Partition Support
minor change after rebase to master branch
* pass compilation
* change Blob A.CopyFrom(B) to Copy(A, B)
* make LayerProto.partition_dim stand for own partition dimension only
* change InnerProduct layer connection to OneToAll
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/8af565cb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/8af565cb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/8af565cb
Branch: refs/heads/master
Commit: 8af565cb7c120ee97b5722b8356c27006c0ebbf7
Parents: 2d38cb3
Author: WANG Sheng <wa...@gmail.com>
Authored: Wed Dec 16 21:11:56 2015 +0800
Committer: Wei Wang <wa...@comp.nus.edu.sg>
Committed: Fri Dec 25 20:26:28 2015 +0800
----------------------------------------------------------------------
include/singa/neuralnet/connection_layer.h | 11 +++++++
include/singa/neuralnet/neuron_layer.h | 3 ++
src/neuralnet/connection_layer/concate.cc | 22 +++++++-------
src/neuralnet/connection_layer/slice.cc | 40 +++++++++++++------------
src/neuralnet/connection_layer/split.cc | 7 ++---
src/neuralnet/neuralnet.cc | 17 ++++++++---
src/neuralnet/neuron_layer/dummy.cc | 8 ++---
src/proto/job.proto | 10 ++++++-
src/test/test_connection_layers.cc | 16 +++++-----
src/test/test_neuralnet.cc | 6 ++--
10 files changed, 85 insertions(+), 55 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8af565cb/include/singa/neuralnet/connection_layer.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/connection_layer.h b/include/singa/neuralnet/connection_layer.h
index c9b0caf..ae918ce 100644
--- a/include/singa/neuralnet/connection_layer.h
+++ b/include/singa/neuralnet/connection_layer.h
@@ -83,6 +83,10 @@ class ConcateLayer : public ConnectionLayer {
void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
+
+ private:
+ int num_concates = 0;
+ int concate_dim = 0;
};
/**
@@ -102,6 +106,10 @@ class SliceLayer : public ConnectionLayer {
const Blob<float>& grad(const Layer* from) const override;
Blob<float>* mutable_data(const Layer* from) override;
Blob<float>* mutable_grad(const Layer* from) override;
+
+ private:
+ int num_slices = 0;
+ int slice_dim = 0;
};
/**
@@ -120,6 +128,9 @@ class SplitLayer : public ConnectionLayer {
const std::string ToString(bool debug, int flag) override;
const Blob<float>& grad(const Layer* from) const override;
Blob<float>* mutable_grad(const Layer* from) override;
+
+ private:
+ int num_splits = 0;
};
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8af565cb/include/singa/neuralnet/neuron_layer.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/neuron_layer.h b/include/singa/neuralnet/neuron_layer.h
index 0b936c4..b02aa7d 100644
--- a/include/singa/neuralnet/neuron_layer.h
+++ b/include/singa/neuralnet/neuron_layer.h
@@ -141,6 +141,9 @@ class InnerProductLayer : public NeuronLayer {
void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
+ ConnectionType src_neuron_connection(int k) const override {
+ return kOneToAll;
+ }
const std::vector<Param*> GetParams() const override {
std::vector<Param*> params{weight_, bias_};
return params;
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8af565cb/src/neuralnet/connection_layer/concate.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/connection_layer/concate.cc b/src/neuralnet/connection_layer/concate.cc
index 13a7ea7..adf701e 100644
--- a/src/neuralnet/connection_layer/concate.cc
+++ b/src/neuralnet/connection_layer/concate.cc
@@ -28,13 +28,15 @@ void ConcateLayer::Setup(const LayerProto& conf,
CHECK_GT(srclayers.size(), 1);
Layer::Setup(conf, srclayers);
vector<int> shape = srclayers[0]->data(this).shape();
- CHECK_GE(partition_dim(), 0);
- CHECK_LT(partition_dim(), shape.size());
- CHECK_EQ(num_partitions(), srclayers.size());
+ concate_dim = conf.concate_conf().concate_dim();
+ num_concates = conf.concate_conf().num_concates();
+ CHECK_GE(concate_dim, 0);
+ CHECK_LT(concate_dim, shape.size());
+ CHECK_EQ(num_concates, srclayers.size());
for (size_t i = 1; i < srclayers.size(); i++) {
const vector<int>& src_shape = srclayers[i]->data(this).shape();
for (size_t j = 0; j < shape.size(); j++)
- if (static_cast<int>(j) == partition_dim())
+ if (static_cast<int>(j) == concate_dim)
shape[j] += src_shape[j];
else
CHECK_EQ(shape[j], src_shape[j]);
@@ -45,10 +47,10 @@ void ConcateLayer::Setup(const LayerProto& conf,
void ConcateLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
CHECK_GT(srclayers.size(), 1);
- CHECK_EQ(num_partitions(), srclayers.size());
+ CHECK_EQ(num_concates, srclayers.size());
// calculate step for each memcpy
- int step = srclayers[0]->data(this).shape()[partition_dim()];
- for (unsigned i = partition_dim() + 1; i < data_.shape().size(); ++i)
+ int step = srclayers[0]->data(this).shape()[concate_dim];
+ for (unsigned i = concate_dim + 1; i < data_.shape().size(); ++i)
step *= data_.shape()[i];
int srclayer_offset = 0;
int concate_offset = 0;
@@ -65,10 +67,10 @@ void ConcateLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
void ConcateLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers) {
CHECK_GT(srclayers.size(), 1);
- CHECK_EQ(num_partitions(), srclayers.size());
+ CHECK_EQ(num_concates, srclayers.size());
// calculate step for each memcpy
- int step = srclayers[0]->grad(this).shape()[partition_dim()];
- for (unsigned i = partition_dim() + 1; i < grad_.shape().size(); ++i)
+ int step = srclayers[0]->grad(this).shape()[concate_dim];
+ for (unsigned i = concate_dim + 1; i < grad_.shape().size(); ++i)
step *= grad_.shape()[i];
int srclayer_offset = 0;
int concate_offset = 0;
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8af565cb/src/neuralnet/connection_layer/slice.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/connection_layer/slice.cc b/src/neuralnet/connection_layer/slice.cc
index ee635e0..db83f1b 100644
--- a/src/neuralnet/connection_layer/slice.cc
+++ b/src/neuralnet/connection_layer/slice.cc
@@ -38,19 +38,21 @@ void SliceLayer::Setup(const LayerProto& conf,
CHECK_EQ(srclayers.size(), 1);
Layer::Setup(conf, srclayers);
vector<int> shape = srclayers[0]->data(this).shape();
- CHECK_GE(partition_dim(), 0);
- CHECK_LT(partition_dim(), shape.size());
- CHECK_GT(num_partitions(), 0);
- // add num_partitions()-1 more blobs
- for (int i = 1; i < num_partitions(); ++i) {
+ slice_dim = conf.slice_conf().slice_dim();
+ num_slices = conf.slice_conf().num_slices();
+ CHECK_GE(slice_dim, 0);
+ CHECK_LT(slice_dim, shape.size());
+ CHECK_GT(num_slices, 0);
+ // add num_slices-1 more blobs
+ for (int i = 1; i < num_slices; ++i) {
datavec_.push_back(new Blob<float>());
gradvec_.push_back(new Blob<float>());
}
// TODO(wangsh): remove equal-size restrict later
- CHECK_EQ(shape[partition_dim()] % num_partitions(), 0);
- shape[partition_dim()] /= num_partitions();
- for (int i = 0; i < num_partitions(); ++i) {
- // if (i == slice_num - 1) shape[partition_dim()] += remain;
+ CHECK_EQ(shape[slice_dim] % num_slices, 0);
+ shape[slice_dim] /= num_slices;
+ for (int i = 0; i < num_slices; ++i) {
+ // if (i == slice_num - 1) shape[slice_dim] += remain;
datavec_[i]->Reshape(shape);
gradvec_[i]->Reshape(shape);
}
@@ -60,13 +62,13 @@ void SliceLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
CHECK_EQ(srclayers.size(), 1);
const Blob<float>& blob = srclayers[0]->data(this);
// calculate step for each memcpy
- int step = datavec_[0]->shape()[partition_dim()];
- for (unsigned i = partition_dim() + 1; i < datavec_[0]->shape().size(); ++i)
+ int step = datavec_[0]->shape()[slice_dim];
+ for (unsigned i = slice_dim + 1; i < datavec_[0]->shape().size(); ++i)
step *= datavec_[0]->shape()[i];
int srclayer_offset = 0;
int slice_offset = 0;
while (srclayer_offset < blob.count()) {
- for (int i = 0; i < num_partitions(); ++i) {
+ for (int i = 0; i < num_slices; ++i) {
const float* src = blob.cpu_data() + srclayer_offset;
float* dst = datavec_[i]->mutable_cpu_data() + slice_offset;
memcpy(dst, src, step * sizeof(float));
@@ -80,13 +82,13 @@ void SliceLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers) {
CHECK_EQ(srclayers.size(), 1);
Blob<float>* blob = srclayers[0]->mutable_grad(this);
// calculate step for each memcpy
- int step = gradvec_[0]->shape()[partition_dim()];
- for (size_t i = partition_dim() + 1; i < gradvec_[0]->shape().size(); ++i)
+ int step = gradvec_[0]->shape()[slice_dim];
+ for (size_t i = slice_dim + 1; i < gradvec_[0]->shape().size(); ++i)
step *= gradvec_[0]->shape()[i];
int srclayer_offset = 0;
int slice_offset = 0;
while (srclayer_offset < blob->count()) {
- for (int i = 0; i < num_partitions(); ++i) {
+ for (int i = 0; i < num_slices; ++i) {
const float* src = gradvec_[i]->cpu_data() + slice_offset;
float* dst = blob->mutable_cpu_data() + srclayer_offset;
memcpy(dst, src, step * sizeof(float));
@@ -98,26 +100,26 @@ void SliceLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers) {
const Blob<float>& SliceLayer::data(const Layer* from) const {
int id = from ? from->partition_id() : 0;
- CHECK_LT(id, num_partitions());
+ CHECK_LT(id, num_slices);
return *datavec_[id];
}
const Blob<float>& SliceLayer::grad(const Layer* from) const {
int id = from ? from->partition_id() : 0;
- CHECK_LT(id, num_partitions());
+ CHECK_LT(id, num_slices);
return *gradvec_[id];
}
Blob<float>* SliceLayer::mutable_data(const Layer* from) {
int id = from ? from->partition_id() : 0;
CHECK(from);
- CHECK_LT(id, num_partitions());
+ CHECK_LT(id, num_slices);
return datavec_[id];
}
Blob<float>* SliceLayer::mutable_grad(const Layer* from) {
int id = from ? from->partition_id() : 0;
- CHECK_LT(id, num_partitions());
+ CHECK_LT(id, num_slices);
return gradvec_[id];
}
const std::string SliceLayer::ToString(bool debug, int flag) {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8af565cb/src/neuralnet/connection_layer/split.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/connection_layer/split.cc b/src/neuralnet/connection_layer/split.cc
index 0ebadc2..9bbe59e 100644
--- a/src/neuralnet/connection_layer/split.cc
+++ b/src/neuralnet/connection_layer/split.cc
@@ -57,12 +57,9 @@ void SplitLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers) {
CHECK_EQ(srclayers.size(), 1);
// aggregate all gradients to grad_[0]
for (int i = 1; i < num_splits; ++i)
- AXPY<float>(cpu, 1.0, *gradvec_[i], gradvec_[0]);
-// for (int i = 1; i < num_splits; ++i)
-// for (int j = 0; j < gradvec_[0]->count(); ++j)
-// gradvec_[0]->mutable_cpu_data()[j] += gradvec_[i]->cpu_data()[j];
+ AXPY<float>(1.0, *gradvec_[i], gradvec_[0]);
// copy grad_[0] to srclayer's grad
- srclayers[0]->mutable_grad(this)->CopyFrom(*gradvec_[0]);
+ Copy(*gradvec_[0], srclayers[0]->mutable_grad(this));
}
const Blob<float>& SplitLayer::grad(const Layer* from) const {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8af565cb/src/neuralnet/neuralnet.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuralnet.cc b/src/neuralnet/neuralnet.cc
index 238bd3f..dfd477e 100644
--- a/src/neuralnet/neuralnet.cc
+++ b/src/neuralnet/neuralnet.cc
@@ -142,7 +142,7 @@ void NeuralNet::Load(const vector<string>& paths,
}
}
-void NeuralNet::ShareParamsFrom(NeuralNet* other) {
+void NeuralNet::ShareParamsFrom(NeuralNet* other, bool cpu_only) {
for (auto& layer : layers_) {
auto otherlayer = other->name2layer(layer->name());
if (otherlayer != nullptr) {
@@ -281,8 +281,11 @@ NetProto NeuralNet::AddPartitionConnectionLayers(const NetProto& netproto,
LayerProto* concate_layer = net_w_connection.add_layer();
concate_layer->set_name(concateName(split_layer->name()));
concate_layer->set_type(kConcate);
+ concate_layer->set_partition_dim(dst_layer->partition_dim());
// concate on src_pdim
- concate_layer->set_partition_dim(split_layer->partition_dim());
+ concate_layer->mutable_concate_conf()
+ ->set_concate_dim(src_layer->partition_dim());
+ concate_layer->mutable_concate_conf()->set_num_concates(npartitions);
concate_layer->add_srclayers(split_layer->name());
// connect dst_layer to concate layer
dst_layer->add_srclayers(concate_layer->name());
@@ -296,15 +299,21 @@ NetProto NeuralNet::AddPartitionConnectionLayers(const NetProto& netproto,
LayerProto* slice_layer = net_w_connection.add_layer();
slice_layer->set_name(sliceName(src_layer->name()));
slice_layer->set_type(kSlice);
+ slice_layer->set_partition_dim(src_layer->partition_dim());
// slice on dst_pdim
- slice_layer->set_partition_dim(dst_layer->partition_dim());
+ slice_layer->mutable_slice_conf()
+ ->set_slice_dim(dst_layer->partition_dim());
+ slice_layer->mutable_slice_conf()->set_num_slices(npartitions);
slice_layer->add_srclayers(src_layer->name());
// add concate layer
LayerProto* concate_layer = net_w_connection.add_layer();
concate_layer->set_name(concateName(slice_layer->name()));
concate_layer->set_type(kConcate);
+ concate_layer->set_partition_dim(dst_layer->partition_dim());
// concate on src_pdim
- concate_layer->set_partition_dim(src_layer->partition_dim());
+ concate_layer->mutable_concate_conf()
+ ->set_concate_dim(src_layer->partition_dim());
+ concate_layer->mutable_concate_conf()->set_num_concates(npartitions);
concate_layer->add_srclayers(slice_layer->name());
// connect dst_layer to concate layer
dst_layer->add_srclayers(concate_layer->name());
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8af565cb/src/neuralnet/neuron_layer/dummy.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/dummy.cc b/src/neuralnet/neuron_layer/dummy.cc
index a242f75..11ce3ec 100644
--- a/src/neuralnet/neuron_layer/dummy.cc
+++ b/src/neuralnet/neuron_layer/dummy.cc
@@ -21,6 +21,7 @@
#include <glog/logging.h>
#include "singa/neuralnet/neuron_layer.h"
+#include "singa/utils/math_blob.h"
namespace singa {
@@ -55,7 +56,7 @@ void DummyLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
data_.mutable_cpu_data()[i] = dis(gen);
}
if (srclayers.size() > 0)
- data_.CopyFrom(srclayers[0]->data(this));
+ Copy(srclayers[0]->data(this), &data_);
}
void DummyLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers) {
@@ -64,9 +65,8 @@ void DummyLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers) {
for (int i = 0; i < data_.count(); ++i)
grad_.mutable_cpu_data()[i] = dis(gen);
}
- if (srclayers.size() > 0) {
- srclayers[0]->mutable_grad(this)->CopyFrom(grad_);
- }
+ if (srclayers.size() > 0)
+ Copy(grad_, srclayers[0]->mutable_grad(this));
}
} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8af565cb/src/proto/job.proto
----------------------------------------------------------------------
diff --git a/src/proto/job.proto b/src/proto/job.proto
index 035f2c1..f6c8c73 100644
--- a/src/proto/job.proto
+++ b/src/proto/job.proto
@@ -195,6 +195,8 @@ message LayerProto {
optional ActivationProto activation_conf = 54;
// configuration for argsort layer
optional ArgSortProto argsort_conf = 52;
+ // configuration for concate layer
+ optional ConcateProto concate_conf = 46;
// configuration for convolution layer
optional ConvolutionProto convolution_conf = 30;
// configuration for dummy layer
@@ -367,6 +369,11 @@ message ArgSortProto {
optional int32 topk = 1 [default = 1];
}
+message ConcateProto {
+ optional int32 concate_dim = 1 [default = 0];
+ optional int32 num_concates = 2 [default = 1];
+}
+
message ConvolutionProto {
// The number of outputs for the layer
optional int32 num_filters = 1;
@@ -491,7 +498,8 @@ message ReLUProto {
}
message SliceProto {
- required int32 slice_dim = 1;
+ optional int32 slice_dim = 1 [default = 0];
+ optional int32 num_slices = 2 [default = 1];
}
message SoftmaxProto {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8af565cb/src/test/test_connection_layers.cc
----------------------------------------------------------------------
diff --git a/src/test/test_connection_layers.cc b/src/test/test_connection_layers.cc
index eab349a..5517bde 100644
--- a/src/test/test_connection_layers.cc
+++ b/src/test/test_connection_layers.cc
@@ -175,8 +175,8 @@ TEST(ConnectionLayerTest, DataSliceTest) {
src_slice.push_back(static_cast<Layer*>(&in));
LayerProto proto_slice;
proto_slice.set_name("slice");
- proto_slice.set_partition_dim(0);
- proto_slice.set_num_partitions(K);
+ proto_slice.mutable_slice_conf()->set_slice_dim(0);
+ proto_slice.mutable_slice_conf()->set_num_slices(K);
SliceLayer slice;
slice.Setup(proto_slice, src_slice);
ASSERT_EQ(slice.data(static_cast<Layer*>(&slice)).shape(0), N / K);
@@ -232,8 +232,8 @@ TEST(ConnectionLayerTest, ModelSliceTest) {
src_slice.push_back(static_cast<Layer*>(&in));
LayerProto proto_slice;
proto_slice.set_name("slice");
- proto_slice.set_partition_dim(1);
- proto_slice.set_num_partitions(K);
+ proto_slice.mutable_slice_conf()->set_slice_dim(1);
+ proto_slice.mutable_slice_conf()->set_num_slices(K);
SliceLayer slice;
slice.Setup(proto_slice, src_slice);
ASSERT_EQ(slice.data(static_cast<Layer*>(&slice)).shape(0), N);
@@ -297,8 +297,8 @@ TEST(ConnectionLayerTest, DataConcateTest) {
src_concate.push_back(static_cast<Layer*>(&in[i]));
LayerProto proto_concate;
proto_concate.set_name("concate");
- proto_concate.set_partition_dim(0);
- proto_concate.set_num_partitions(K);
+ proto_concate.mutable_concate_conf()->set_concate_dim(0);
+ proto_concate.mutable_concate_conf()->set_num_concates(K);
ConcateLayer concate;
concate.Setup(proto_concate, src_concate);
ASSERT_EQ(concate.data(static_cast<Layer*>(&concate)).shape(0), N);
@@ -355,8 +355,8 @@ TEST(ConnectionLayerTest, ModelConcateTest) {
src_concate.push_back(static_cast<Layer*>(&in[i]));
LayerProto proto_concate;
proto_concate.set_name("concate");
- proto_concate.set_partition_dim(1);
- proto_concate.set_num_partitions(K);
+ proto_concate.mutable_concate_conf()->set_concate_dim(1);
+ proto_concate.mutable_concate_conf()->set_num_concates(K);
ConcateLayer concate;
concate.Setup(proto_concate, src_concate);
ASSERT_EQ(concate.data(static_cast<Layer*>(&concate)).shape(0), N);
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/8af565cb/src/test/test_neuralnet.cc
----------------------------------------------------------------------
diff --git a/src/test/test_neuralnet.cc b/src/test/test_neuralnet.cc
index e2b8dd7..198ee0d 100644
--- a/src/test/test_neuralnet.cc
+++ b/src/test/test_neuralnet.cc
@@ -21,11 +21,9 @@
#include "gtest/gtest.h"
#include "singa/driver.h"
+#include "singa/neuralnet/connection_layer.h"
#include "singa/neuralnet/neuralnet.h"
-#include "singa/neuralnet/neuron_layer/dummy.h"
-#include "singa/neuralnet/connection_layer/slice.h"
-#include "singa/neuralnet/connection_layer/split.h"
-#include "singa/neuralnet/connection_layer/concate.h"
+#include "singa/neuralnet/neuron_layer.h"
using namespace singa;
[4/5] incubator-singa git commit: SINGA-113 Model/Hybrid Partition
Support
Posted by wa...@apache.org.
SINGA-113 Model/Hybrid Partition Support
Update memory copy in SliceLayer and ConcateLayer considering cpu (memcpy) and gpu (cudaMemcpy).
Test hybrid partition running on GPU (CUDNN) using cudnn_hybrid.conf
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/bf4cd3bc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/bf4cd3bc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/bf4cd3bc
Branch: refs/heads/master
Commit: bf4cd3bc9473d802d97327226c636ddc442cda0f
Parents: 82563f6
Author: Wei Wang <wa...@comp.nus.edu.sg>
Authored: Fri Dec 25 17:47:13 2015 +0800
Committer: Wei Wang <wa...@comp.nus.edu.sg>
Committed: Fri Dec 25 21:27:42 2015 +0800
----------------------------------------------------------------------
examples/cifar10/cudnn_hybrid.conf | 306 ++++++++++++++++++++++++
include/singa/neuralnet/connection_layer.h | 10 +-
src/neuralnet/connection_layer/concate.cc | 67 ++++--
src/neuralnet/connection_layer/slice.cc | 76 ++++--
src/neuralnet/connection_layer/split.cc | 16 +-
src/utils/graph.cc | 5 +-
6 files changed, 422 insertions(+), 58 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/bf4cd3bc/examples/cifar10/cudnn_hybrid.conf
----------------------------------------------------------------------
diff --git a/examples/cifar10/cudnn_hybrid.conf b/examples/cifar10/cudnn_hybrid.conf
new file mode 100644
index 0000000..a11145c
--- /dev/null
+++ b/examples/cifar10/cudnn_hybrid.conf
@@ -0,0 +1,306 @@
+name: "cifar10-convnet"
+train_steps: 10000
+test_steps: 0
+test_freq: 200
+#validate_steps: 100
+#validate_freq: 300
+disp_freq: 200
+gpu: 0
+gpu: 1
+#debug: true
+#checkpoint_path: "examples/cifar10/checkpoint/step1000-worker0"
+train_one_batch {
+ alg: kBP
+}
+updater{
+ type: kSGD
+ weight_decay:0.004
+ momentum:0.9
+ learning_rate {
+ type: kFixedStep
+ fixedstep_conf:{
+ step:0
+ step:60000
+ step:65000
+ step_lr:0.001
+ step_lr:0.0001
+ step_lr:0.00001
+ }
+ }
+}
+neuralnet {
+ layer{
+ name: "data"
+ type: kRecordInput
+ store_conf {
+ backend: "kvfile"
+ path: "examples/cifar10/train_data.bin"
+ mean_file: "examples/cifar10/image_mean.bin"
+ batchsize: 100
+ #random_skip: 5000
+ shape: 3
+ shape: 32
+ shape: 32
+ }
+ include: kTrain
+ partition_dim: 0
+ }
+# layer{
+# name: "data"
+# type: kRecordInput
+# store_conf {
+# backend: "kvfile"
+# path: "examples/cifar10/val_data.bin"
+# mean_file: "examples/cifar10/image_mean.bin"
+# batchsize: 64
+# random_skip: 5000
+# shape: 3
+# shape: 32
+# shape: 32
+# }
+# include: kVal
+# }
+ layer{
+ name: "data"
+ type: kRecordInput
+ store_conf {
+ backend: "kvfile"
+ path: "examples/cifar10/test_data.bin"
+ mean_file: "examples/cifar10/image_mean.bin"
+ batchsize: 100
+ shape: 3
+ shape: 32
+ shape: 32
+ }
+ include: kTest
+ partition_dim: 0
+ }
+
+ layer {
+ partition_dim: 0
+ name: "conv1"
+ type: kCudnnConv
+ srclayers: "data"
+ convolution_conf {
+ num_filters: 32
+ kernel: 5
+ stride: 1
+ pad:2
+ }
+ param {
+ name: "w1"
+ init {
+ type:kGaussian
+ std:0.0001
+ }
+ }
+ param {
+ name: "b1"
+ lr_scale:2.0
+ init {
+ type: kConstant
+ value:0
+ }
+ }
+ }
+
+ layer {
+ partition_dim: 0
+ name: "pool1"
+ type: kCudnnPool
+ srclayers: "conv1"
+ pooling_conf {
+ pool: MAX
+ kernel: 3
+ stride: 2
+ }
+ }
+ layer {
+ partition_dim: 0
+ name: "relu1"
+ type: kCudnnActivation
+ activation_conf {
+ type: RELU
+ }
+ srclayers:"pool1"
+ }
+ layer {
+ partition_dim: 0
+ name: "norm1"
+ type: kCudnnLRN
+ lrn_conf {
+ local_size: 3
+ alpha: 5e-05
+ beta: 0.75
+ }
+ srclayers:"relu1"
+ }
+ layer {
+ partition_dim: 0
+ name: "conv2"
+ type: kCudnnConv
+ srclayers: "norm1"
+ convolution_conf {
+ num_filters: 32
+ kernel: 5
+ stride: 1
+ pad:2
+ }
+ param {
+ name: "w2"
+ init {
+ type:kGaussian
+ std:0.01
+ }
+ }
+ param {
+ name: "b2"
+ lr_scale:2.0
+ init {
+ type: kConstant
+ value:0
+ }
+ }
+ }
+ layer {
+ partition_dim: 0
+ name: "relu2"
+ type: kCudnnActivation
+ activation_conf {
+ type: RELU
+ }
+ srclayers:"conv2"
+ }
+ layer {
+ partition_dim: 0
+ name: "pool2"
+ type: kCudnnPool
+ srclayers: "relu2"
+ pooling_conf {
+ pool: AVG
+ kernel: 3
+ stride: 2
+ }
+ }
+ layer {
+ partition_dim: 0
+ name: "norm2"
+ type: kCudnnLRN
+ lrn_conf {
+ local_size: 3
+ alpha: 5e-05
+ beta: 0.75
+ }
+ srclayers:"pool2"
+ }
+ layer {
+ partition_dim: 0
+ name: "conv3"
+ type: kCudnnConv
+ srclayers: "norm2"
+ convolution_conf {
+ num_filters: 64
+ kernel: 5
+ stride: 1
+ pad:2
+ }
+ param {
+ name: "w3"
+ init {
+ type:kGaussian
+ std:0.01
+ }
+ }
+ param {
+ name: "b3"
+ init {
+ type: kConstant
+ value:0
+ }
+ }
+ }
+ layer {
+ partition_dim: 0
+ name: "relu3"
+ type: kCudnnActivation
+ activation_conf {
+ type: RELU
+ }
+ srclayers:"conv3"
+ }
+ layer {
+ partition_dim: 0
+ name: "pool3"
+ type: kCudnnPool
+ srclayers: "relu3"
+ pooling_conf {
+ pool: AVG
+ kernel: 3
+ stride: 2
+ }
+ }
+ layer {
+ partition_dim: 1
+ name: "ip1"
+ type: kInnerProduct
+ srclayers:"pool3"
+ innerproduct_conf {
+ num_output: 10
+ }
+ param {
+ name: "w4"
+ wd_scale:250
+ init {
+ type:kGaussian
+ std:0.01
+ }
+ }
+ param {
+ name: "b4"
+ lr_scale:2.0
+ wd_scale:0
+ init {
+ type: kConstant
+ value:0
+ }
+ }
+ }
+# layer {
+# name : "softmax"
+# type: kSoftmax
+# srclayers: "ip1"
+# }
+#
+# layer {
+# name : "argsort"
+# type: kArgSort
+# srclayers: "softmax"
+# }
+ layer{
+ partition_dim: 0
+ name: "loss"
+ type: kSoftmaxLoss
+ softmaxloss_conf{
+ topk:1
+ }
+ srclayers:"ip1"
+ srclayers: "data"
+ }
+# uncomment "softmax", "argsort", "output" layer and comment "loss" layer
+# to extract features from argsort
+# layer {
+# name : "output"
+# type: kCSVOutput
+# srclayers: "argsort"
+# store_conf {
+# path: "examples/cifar10/out.csv"
+# }
+# }
+}
+cluster {
+ nworker_groups: 1
+ nserver_groups: 1
+ nworkers_per_group: 2
+ nworkers_per_procs: 2
+ workspace: "examples/cifar10"
+}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/bf4cd3bc/include/singa/neuralnet/connection_layer.h
----------------------------------------------------------------------
diff --git a/include/singa/neuralnet/connection_layer.h b/include/singa/neuralnet/connection_layer.h
index 14e5092..a18f458 100644
--- a/include/singa/neuralnet/connection_layer.h
+++ b/include/singa/neuralnet/connection_layer.h
@@ -103,8 +103,8 @@ class ConcateLayer : public ConnectionLayer {
void ComputeGradient(int flag, const vector<Layer*>& srclayers) override;
private:
- int num_concates = 0;
- int concate_dim = 0;
+ int num_concates_ = 0;
+ int concate_dim_ = 0;
};
/**
@@ -126,8 +126,8 @@ class SliceLayer : public ConnectionLayer {
Blob<float>* mutable_grad(const Layer* from) override;
private:
- int num_slices = 0;
- int slice_dim = 0;
+ int num_slices_ = 0;
+ int slice_dim_ = 0;
Layer2Index layer_idx_;
};
@@ -149,7 +149,7 @@ class SplitLayer : public ConnectionLayer {
Blob<float>* mutable_grad(const Layer* from) override;
private:
- int num_splits = 0;
+ int num_splits_ = 0;
Layer2Index layer_idx_;
};
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/bf4cd3bc/src/neuralnet/connection_layer/concate.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/connection_layer/concate.cc b/src/neuralnet/connection_layer/concate.cc
index adf701e..0cdd812 100644
--- a/src/neuralnet/connection_layer/concate.cc
+++ b/src/neuralnet/connection_layer/concate.cc
@@ -20,6 +20,8 @@
*************************************************************/
#include "singa/neuralnet/connection_layer.h"
+#include "singa/utils/singleton.h"
+#include "singa/utils/context.h"
namespace singa {
@@ -28,15 +30,15 @@ void ConcateLayer::Setup(const LayerProto& conf,
CHECK_GT(srclayers.size(), 1);
Layer::Setup(conf, srclayers);
vector<int> shape = srclayers[0]->data(this).shape();
- concate_dim = conf.concate_conf().concate_dim();
- num_concates = conf.concate_conf().num_concates();
- CHECK_GE(concate_dim, 0);
- CHECK_LT(concate_dim, shape.size());
- CHECK_EQ(num_concates, srclayers.size());
+ concate_dim_ = conf.concate_conf().concate_dim();
+ num_concates_ = conf.concate_conf().num_concates();
+ CHECK_GE(concate_dim_, 0);
+ CHECK_LT(concate_dim_, shape.size());
+ CHECK_EQ(num_concates_, srclayers.size());
for (size_t i = 1; i < srclayers.size(); i++) {
const vector<int>& src_shape = srclayers[i]->data(this).shape();
for (size_t j = 0; j < shape.size(); j++)
- if (static_cast<int>(j) == concate_dim)
+ if (static_cast<int>(j) == concate_dim_)
shape[j] += src_shape[j];
else
CHECK_EQ(shape[j], src_shape[j]);
@@ -47,18 +49,32 @@ void ConcateLayer::Setup(const LayerProto& conf,
void ConcateLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
CHECK_GT(srclayers.size(), 1);
- CHECK_EQ(num_concates, srclayers.size());
+ CHECK_EQ(num_concates_, srclayers.size());
// calculate step for each memcpy
- int step = srclayers[0]->data(this).shape()[concate_dim];
- for (unsigned i = concate_dim + 1; i < data_.shape().size(); ++i)
+ int step = srclayers[0]->data(this).shape()[concate_dim_];
+ for (unsigned i = concate_dim_ + 1; i < data_.shape().size(); ++i)
step *= data_.shape()[i];
int srclayer_offset = 0;
int concate_offset = 0;
+ auto context = Singleton<Context>::Instance();
+ int device = context->device_id(std::this_thread::get_id());
while (concate_offset < data_.count()) {
for (size_t i = 0; i < srclayers.size(); ++i) {
- const float* src = srclayers[i]->data(this).cpu_data() + srclayer_offset;
- float* dst = data_.mutable_cpu_data() + concate_offset;
- memcpy(dst, src, step * sizeof(float));
+ if (device == -1) {
+ const float* src = srclayers[i]->data(this).cpu_data()
+ + srclayer_offset;
+ float* dst = data_.mutable_cpu_data() + concate_offset;
+ memcpy(dst, src, step * sizeof(float));
+ } else {
+#ifdef USE_GPU
+ const float* src = srclayers[i]->data(this).gpu_data()
+ + srclayer_offset;
+ float* dst = data_.mutable_gpu_data() + concate_offset;
+ cudaMemcpy(dst, src, step * sizeof(float), cudaMemcpyDefault);
+#else
+ LOG(FATAL) << "GPU is supported";
+#endif
+ }
concate_offset += step;
}
srclayer_offset += step;
@@ -67,19 +83,32 @@ void ConcateLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
void ConcateLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers) {
CHECK_GT(srclayers.size(), 1);
- CHECK_EQ(num_concates, srclayers.size());
+ CHECK_EQ(num_concates_, srclayers.size());
// calculate step for each memcpy
- int step = srclayers[0]->grad(this).shape()[concate_dim];
- for (unsigned i = concate_dim + 1; i < grad_.shape().size(); ++i)
+ int step = srclayers[0]->grad(this).shape()[concate_dim_];
+ for (unsigned i = concate_dim_ + 1; i < grad_.shape().size(); ++i)
step *= grad_.shape()[i];
int srclayer_offset = 0;
int concate_offset = 0;
+ auto context = Singleton<Context>::Instance();
+ int device = context->device_id(std::this_thread::get_id());
while (concate_offset < grad_.count()) {
for (size_t i = 0; i < srclayers.size(); ++i) {
- const float* src = grad_.cpu_data() + concate_offset;
- float* dst = srclayers[i]->mutable_grad(this)->mutable_cpu_data()
- + srclayer_offset;
- memcpy(dst, src, step * sizeof(float));
+ if (device == -1) {
+ const float* src = grad_.cpu_data() + concate_offset;
+ float* dst = srclayers[i]->mutable_grad(this)->mutable_cpu_data()
+ + srclayer_offset;
+ memcpy(dst, src, step * sizeof(float));
+ } else {
+#ifdef USE_GPU
+ const float* src = grad_.gpu_data() + concate_offset;
+ float* dst = srclayers[i]->mutable_grad(this)->mutable_gpu_data()
+ + srclayer_offset;
+ cudaMemcpy(dst, src, step * sizeof(float), cudaMemcpyDefault);
+#else
+ LOG(FATAL) << "GPU is supported";
+#endif
+ }
concate_offset += step;
}
srclayer_offset += step;
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/bf4cd3bc/src/neuralnet/connection_layer/slice.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/connection_layer/slice.cc b/src/neuralnet/connection_layer/slice.cc
index b625c66..15fdc48 100644
--- a/src/neuralnet/connection_layer/slice.cc
+++ b/src/neuralnet/connection_layer/slice.cc
@@ -21,6 +21,8 @@
#include "singa/neuralnet/connection_layer.h"
#include "singa/utils/math_blob.h"
+#include "singa/utils/singleton.h"
+#include "singa/utils/context.h"
namespace singa {
@@ -38,20 +40,20 @@ void SliceLayer::Setup(const LayerProto& conf,
CHECK_EQ(srclayers.size(), 1);
Layer::Setup(conf, srclayers);
vector<int> shape = srclayers[0]->data(this).shape();
- slice_dim = conf.slice_conf().slice_dim();
- num_slices = conf.slice_conf().num_slices();
- CHECK_GE(slice_dim, 0);
- CHECK_LT(slice_dim, shape.size());
- CHECK_GT(num_slices, 0);
+ slice_dim_ = conf.slice_conf().slice_dim();
+ num_slices_ = conf.slice_conf().num_slices();
+ CHECK_GE(slice_dim_, 0);
+ CHECK_LT(slice_dim_, shape.size());
+ CHECK_GT(num_slices_, 0);
// add num_slices-1 more blobs
- for (int i = 1; i < num_slices; ++i) {
+ for (int i = 1; i < num_slices_; ++i) {
datavec_.push_back(new Blob<float>());
gradvec_.push_back(new Blob<float>());
}
// TODO(wangsh): remove equal-size restrict later
- CHECK_EQ(shape[slice_dim] % num_slices, 0);
- shape[slice_dim] /= num_slices;
- for (int i = 0; i < num_slices; ++i) {
+ CHECK_EQ(shape[slice_dim_] % num_slices_, 0);
+ shape[slice_dim_] /= num_slices_;
+ for (int i = 0; i < num_slices_; ++i) {
// if (i == slice_num - 1) shape[slice_dim] += remain;
datavec_[i]->Reshape(shape);
gradvec_[i]->Reshape(shape);
@@ -62,16 +64,28 @@ void SliceLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
CHECK_EQ(srclayers.size(), 1);
const Blob<float>& blob = srclayers[0]->data(this);
// calculate step for each memcpy
- int step = datavec_[0]->shape()[slice_dim];
- for (unsigned i = slice_dim + 1; i < datavec_[0]->shape().size(); ++i)
+ int step = datavec_[0]->shape()[slice_dim_];
+ for (unsigned i = slice_dim_ + 1; i < datavec_[0]->shape().size(); ++i)
step *= datavec_[0]->shape()[i];
int srclayer_offset = 0;
int slice_offset = 0;
+ auto context = Singleton<Context>::Instance();
+ int device = context->device_id(std::this_thread::get_id());
while (srclayer_offset < blob.count()) {
- for (int i = 0; i < num_slices; ++i) {
- const float* src = blob.cpu_data() + srclayer_offset;
- float* dst = datavec_[i]->mutable_cpu_data() + slice_offset;
- memcpy(dst, src, step * sizeof(float));
+ for (int i = 0; i < num_slices_; ++i) {
+ if (device == -1) {
+ const float* src = blob.cpu_data() + srclayer_offset;
+ float* dst = datavec_[i]->mutable_cpu_data() + slice_offset;
+ memcpy(dst, src, step * sizeof(float));
+ } else {
+#ifdef USE_GPU
+ const float* src = blob.gpu_data() + srclayer_offset;
+ float* dst = datavec_[i]->mutable_gpu_data() + slice_offset;
+ cudaMemcpy(dst, src, step * sizeof(float), cudaMemcpyDefault);
+#else
+ LOG(FATAL) << "GPU is supported";
+#endif
+ }
srclayer_offset += step;
}
slice_offset += step;
@@ -82,16 +96,28 @@ void SliceLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers) {
CHECK_EQ(srclayers.size(), 1);
Blob<float>* blob = srclayers[0]->mutable_grad(this);
// calculate step for each memcpy
- int step = gradvec_[0]->shape()[slice_dim];
- for (size_t i = slice_dim + 1; i < gradvec_[0]->shape().size(); ++i)
+ int step = gradvec_[0]->shape()[slice_dim_];
+ for (size_t i = slice_dim_ + 1; i < gradvec_[0]->shape().size(); ++i)
step *= gradvec_[0]->shape()[i];
int srclayer_offset = 0;
int slice_offset = 0;
+ auto context = Singleton<Context>::Instance();
+ int device = context->device_id(std::this_thread::get_id());
while (srclayer_offset < blob->count()) {
- for (int i = 0; i < num_slices; ++i) {
- const float* src = gradvec_[i]->cpu_data() + slice_offset;
- float* dst = blob->mutable_cpu_data() + srclayer_offset;
- memcpy(dst, src, step * sizeof(float));
+ for (int i = 0; i < num_slices_; ++i) {
+ if (device == -1) {
+ const float* src = gradvec_[i]->cpu_data() + slice_offset;
+ float* dst = blob->mutable_cpu_data() + srclayer_offset;
+ memcpy(dst, src, step * sizeof(float));
+ } else {
+#ifdef USE_GPU
+ const float* src = gradvec_[i]->gpu_data() + slice_offset;
+ float* dst = blob->mutable_gpu_data() + srclayer_offset;
+ cudaMemcpy(dst, src, step * sizeof(float), cudaMemcpyDefault);
+#else
+ LOG(FATAL) << "GPU is supported";
+#endif
+ }
srclayer_offset += step;
}
slice_offset += step;
@@ -100,27 +126,27 @@ void SliceLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers) {
const Blob<float>& SliceLayer::data(const Layer* from) {
int idx = from ? layer_idx_.Get(from) : 0;
- CHECK_LT(idx, num_slices);
+ CHECK_LT(idx, num_slices_);
return *datavec_[idx];
}
const Blob<float>& SliceLayer::grad(const Layer* from) {
int idx = from ? layer_idx_.Get(from) : 0;
- CHECK_LT(idx, num_slices);
+ CHECK_LT(idx, num_slices_);
return *gradvec_[idx];
}
Blob<float>* SliceLayer::mutable_data(const Layer* from) {
CHECK(from);
int idx = layer_idx_.Get(from);
- CHECK_LT(idx, num_slices);
+ CHECK_LT(idx, num_slices_);
return datavec_[idx];
}
Blob<float>* SliceLayer::mutable_grad(const Layer* from) {
CHECK(from);
int idx = layer_idx_.Get(from);
- CHECK_LT(idx, num_slices);
+ CHECK_LT(idx, num_slices_);
return gradvec_[idx];
}
const std::string SliceLayer::ToString(bool debug, int flag) {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/bf4cd3bc/src/neuralnet/connection_layer/split.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/connection_layer/split.cc b/src/neuralnet/connection_layer/split.cc
index a9270fb..e46b902 100644
--- a/src/neuralnet/connection_layer/split.cc
+++ b/src/neuralnet/connection_layer/split.cc
@@ -37,14 +37,14 @@ void SplitLayer::Setup(const LayerProto& conf,
CHECK_EQ(srclayers.size(), 1);
Layer::Setup(conf, srclayers);
data_.Reshape(srclayers[0]->data(this).shape());
- data_.ShareData(srclayers[0]->data(this), false);
- int num_splits = conf.split_conf().num_splits();
- CHECK_GT(num_splits, 0);
+ data_.ShareData(srclayers[0]->mutable_data(this), false);
+ num_splits_ = conf.split_conf().num_splits();
+ CHECK_GT(num_splits_, 0);
// add num_splits-1 more grad blobs
- for (int i = 1; i < num_splits; ++i) {
+ for (int i = 1; i < num_splits_; ++i) {
gradvec_.push_back(new Blob<float>());
}
- for (int i = 0; i < num_splits; ++i)
+ for (int i = 0; i < num_splits_; ++i)
gradvec_[i]->Reshape(srclayers[0]->data(this).shape());
}
@@ -56,7 +56,7 @@ void SplitLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
void SplitLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers) {
CHECK_EQ(srclayers.size(), 1);
// aggregate all gradients to grad_[0]
- for (int i = 1; i < num_splits; ++i)
+ for (int i = 1; i < num_splits_; ++i)
AXPY<float>(1.0, *gradvec_[i], gradvec_[0]);
// copy grad_[0] to srclayer's grad
Copy(*gradvec_[0], srclayers[0]->mutable_grad(this));
@@ -65,14 +65,14 @@ void SplitLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers) {
const Blob<float>& SplitLayer::grad(const Layer* from) {
CHECK(from);
int idx = layer_idx_.Get(from);
- CHECK_LT(idx, num_splits);
+ CHECK_LT(idx, num_splits_);
return *gradvec_[idx];
}
Blob<float>* SplitLayer::mutable_grad(const Layer* from) {
CHECK(from);
int idx = layer_idx_.Get(from);
- CHECK_LT(idx, num_splits);
+ CHECK_LT(idx, num_splits_);
return gradvec_[idx];
}
const std::string SplitLayer::ToString(bool debug, int flag) {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/bf4cd3bc/src/utils/graph.cc
----------------------------------------------------------------------
diff --git a/src/utils/graph.cc b/src/utils/graph.cc
index 0211e5a..4f59635 100644
--- a/src/utils/graph.cc
+++ b/src/utils/graph.cc
@@ -97,7 +97,10 @@ Node* Graph::AddNode(const string& name, const string& origin, int id,
Node* Graph::AddNode(const string& name,
const std::map<string, string>& attrs) {
Node* node = new Node(name, attrs);
- AddNode(node);
+ nodes_.push_back(node);
+ CHECK(name2node_.find(node->name) == name2node_.end())
+ << "node " << node->name << " already exists";
+ name2node_[node->name] = node;
return node;
}