You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@singa.apache.org by wa...@apache.org on 2015/09/04 14:51:13 UTC
[1/2] incubator-singa git commit: SINGA-65 Add an example of
implementing user's own layers
Repository: incubator-singa
Updated Branches:
refs/heads/tutorial [created] 7306bcffb
SINGA-65 Add an example of implementing user's own layers
The folder examples/mlp includes files for a user-defined layer, i.e.,
HiddenLayer class. The main.cc register this class. The myproto.proto defines
the configuration of this layer and extends it to the base LayerProto.
Makefile.example contains instructions for compiling and linking
user code.
job.conf specifies a simple model with only one hidden layer.
deep.conf configures a deep model with 6 hidden layers, which is slower
per iteration, but converges faster than the simple model.
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/98b49952
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/98b49952
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/98b49952
Branch: refs/heads/tutorial
Commit: 98b499527012ab6b61fd854b39225b35d7370716
Parents: 50deedd
Author: wangwei <wa...@comp.nus.edu.sg>
Authored: Thu Sep 3 09:57:03 2015 +0800
Committer: wangwei <wa...@comp.nus.edu.sg>
Committed: Fri Sep 4 20:24:45 2015 +0800
----------------------------------------------------------------------
examples/cifar10/job.conf | 6 +-
examples/mlp/Makefile.example | 8 ++
examples/mlp/deep.conf | 221 +++++++++++++++++++++++++++++++++++++
examples/mlp/hidden_layer.cc | 72 ++++++++++++
examples/mlp/hidden_layer.h | 22 ++++
examples/mlp/job.conf | 95 ++++++++++++++++
examples/mlp/main.cc | 25 +++++
examples/mlp/myproto.proto | 11 ++
src/utils/tool.cc | 43 +++++++-
9 files changed, 498 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/98b49952/examples/cifar10/job.conf
----------------------------------------------------------------------
diff --git a/examples/cifar10/job.conf b/examples/cifar10/job.conf
index 0fdd244..90705e0 100644
--- a/examples/cifar10/job.conf
+++ b/examples/cifar10/job.conf
@@ -3,10 +3,12 @@ train_steps: 1000
test_steps: 100
test_freq:300
disp_freq:30
-train_one_batch {
+train_one_batch
+
+{
alg: kBP
}
-updater{
+ updater {
type: kSGD
weight_decay:0.004
learning_rate {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/98b49952/examples/mlp/Makefile.example
----------------------------------------------------------------------
diff --git a/examples/mlp/Makefile.example b/examples/mlp/Makefile.example
new file mode 100644
index 0000000..71edd71
--- /dev/null
+++ b/examples/mlp/Makefile.example
@@ -0,0 +1,8 @@
+# refer to examples/mnist/ for data creation
+MSHADOW_FLAGS :=-DMSHADOW_USE_CUDA=0 -DMSHADOW_USE_CBLAS=1 -DMSHADOW_USE_MKL=0
+
+all:
+ protoc --proto_path=../../src/proto --proto_path=. --cpp_out=. myproto.proto
+ $(CXX) main.cc hidden_layer.cc myproto.pb.cc $(MSHADOW_FLAGS) -std=c++11 -lsinga -lglog -lprotobuf -lopenblas -I../../include\
+ -I../../include/proto/ -L../../.libs/ -L/usr/local -Wl,-unresolved-symbols=ignore-in-shared-libs -Wl,-rpath=../../.libs/\
+ -o mlp.bin
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/98b49952/examples/mlp/deep.conf
----------------------------------------------------------------------
diff --git a/examples/mlp/deep.conf b/examples/mlp/deep.conf
new file mode 100644
index 0000000..588fefa
--- /dev/null
+++ b/examples/mlp/deep.conf
@@ -0,0 +1,221 @@
+name: "mlp"
+train_steps: 1200
+test_steps:10
+test_freq:60
+disp_freq:10
+train_one_batch {
+ alg: kBP
+}
+updater{
+ type: kSGD
+ learning_rate{
+ type : kStep
+ base_lr: 0.001
+ step_conf{
+ change_freq: 60
+ gamma: 0.997
+ }
+ }
+}
+neuralnet {
+ layer {
+ name: "data"
+ type: kShardData
+ sharddata_conf {
+ path: "examples/mnist/mnist_train_shard"
+ batchsize: 1000
+ }
+ exclude: kTest
+ }
+
+ layer {
+ name: "data"
+ type: kShardData
+ sharddata_conf {
+ path: "examples/mnist/mnist_test_shard"
+ batchsize: 1000
+ }
+ exclude: kTrain
+ }
+
+ layer{
+ name:"mnist"
+ type: kMnist
+ srclayers: "data"
+ mnist_conf {
+ norm_a: 127.5
+ norm_b: 1
+ }
+ }
+ layer{
+ name: "label"
+ type: kLabel
+ srclayers: "data"
+ }
+
+ layer{
+ name: "hid1"
+ user_type: "kHidden"
+ srclayers:"mnist"
+ [singa.hidden_conf] {
+ num_output: 2500
+ }
+ param{
+ name: "w1"
+ init {
+ type: kUniform
+ low:-0.05
+ high:0.05
+ }
+ }
+ param{
+ name: "b1"
+ init {
+ type : kUniform
+ low: -0.05
+ high:0.05
+ }
+ }
+ }
+
+ layer{
+ name: "hid2"
+ user_type: "kHidden"
+ srclayers:"hid1"
+ [singa.hidden_conf] {
+ num_output: 2000
+ }
+ param{
+ name: "w2"
+ init {
+ type: kUniform
+ low:-0.05
+ high:0.05
+ }
+ }
+ param{
+ name: "b2"
+ init {
+ type : kUniform
+ low: -0.05
+ high:0.05
+ }
+ }
+ }
+
+ layer{
+ name: "hid3"
+ user_type: "kHidden"
+ srclayers:"hid2"
+ [singa.hidden_conf] {
+ num_output: 1500
+ }
+ param{
+ name: "w3"
+ init {
+ type: kUniform
+ low:-0.05
+ high:0.05
+ }
+ }
+ param{
+ name: "b3"
+ init {
+ type : kUniform
+ low: -0.05
+ high:0.05
+ }
+ }
+ }
+
+ layer{
+ name: "hid4"
+ user_type: "kHidden"
+ srclayers:"hid3"
+ [singa.hidden_conf] {
+ num_output: 1000
+ }
+ param{
+ name: "w4"
+ init {
+ type: kUniform
+ low:-0.05
+ high:0.05
+ }
+ }
+ param{
+ name: "b4"
+ init {
+ type : kUniform
+ low: -0.05
+ high:0.05
+ }
+ }
+ }
+
+
+ layer{
+ name: "hid5"
+ user_type: "kHidden"
+ srclayers:"hid4"
+ [singa.hidden_conf] {
+ num_output: 500
+ }
+ param{
+ name: "w5"
+ init {
+ type: kUniform
+ low:-0.05
+ high:0.05
+ }
+ }
+ param{
+ name: "b5"
+ init {
+ type : kUniform
+ low: -0.05
+ high:0.05
+ }
+ }
+ }
+
+
+ layer{
+ name: "hid6"
+ user_type: "kHidden"
+ srclayers:"hid5"
+ [singa.hidden_conf] {
+ num_output: 10
+ }
+ param{
+ name: "w6"
+ init {
+ type: kUniform
+ low:-0.05
+ high:0.05
+ }
+ }
+ param{
+ name: "b6"
+ init {
+ type : kUniform
+ low: -0.05
+ high:0.05
+ }
+ }
+ }
+ layer{
+ name: "loss"
+ type:kSoftmaxLoss
+ softmaxloss_conf{
+ topk:1
+ }
+ srclayers:"hid6"
+ srclayers:"label"
+ }
+}
+cluster {
+ nworker_groups: 1
+ nserver_groups: 1
+ workspace: "examples/mnist"
+}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/98b49952/examples/mlp/hidden_layer.cc
----------------------------------------------------------------------
diff --git a/examples/mlp/hidden_layer.cc b/examples/mlp/hidden_layer.cc
new file mode 100644
index 0000000..f4dacf2
--- /dev/null
+++ b/examples/mlp/hidden_layer.cc
@@ -0,0 +1,72 @@
+#include "hidden_layer.h"
+
+#include "mshadow/tensor.h"
+#include "mshadow/cxxnet_op.h"
+
+namespace singa {
+using namespace mshadow;
+using mshadow::cpu;
+using mshadow::Shape1;
+using mshadow::Shape2;
+
+inline Tensor<cpu, 2> NewTensor2(Blob<float>* blob) {
+ const vector<int>& shape = blob->shape();
+ Tensor<cpu, 2> tensor(blob->mutable_cpu_data(),
+ Shape2(shape[0], blob->count() / shape[0]));
+ return tensor;
+}
+
+inline Tensor<cpu, 1> NewTensor1(Blob<float>* blob) {
+ Tensor<cpu, 1> tensor(blob->mutable_cpu_data(), Shape1(blob->count()));
+ return tensor;
+}
+
+
+HiddenLayer::~HiddenLayer() {
+ delete weight_;
+ delete bias_;
+}
+
+void HiddenLayer::Setup(const LayerProto& proto, int npartitions) {
+ Layer::Setup(proto, npartitions);
+ CHECK_EQ(srclayers_.size(), 1);
+ const auto& src = srclayers_[0]->data(this);
+ batchsize_ = src.shape()[0];
+ vdim_ = src.count() / batchsize_;
+ hdim_ = layer_proto_.GetExtension(hidden_conf).num_output();
+ data_.Reshape(vector<int>{batchsize_, hdim_});
+ grad_.ReshapeLike(data_);
+ weight_ = Param::Create(proto.param(0));
+ bias_ = Param::Create(proto.param(1));
+ weight_->Setup(vector<int>{hdim_, vdim_});
+ bias_->Setup(vector<int>{hdim_});
+}
+
+void HiddenLayer::ComputeFeature(int flag, Metric* perf) {
+ auto data = NewTensor2(&data_);
+ auto src = NewTensor2(srclayers_[0]->mutable_data(this));
+ auto weight = NewTensor2(weight_->mutable_data());
+ auto bias = NewTensor1(bias_->mutable_data());
+ data = dot(src, weight.T());
+ // repmat: repeat bias vector into batchsize rows
+ data += expr::repmat(bias, batchsize_);
+ data = expr::F<op::stanh>(data);
+}
+
+void HiddenLayer::ComputeGradient(int flag, Metric* perf) {
+ auto data = NewTensor2(&data_);
+ auto src = NewTensor2(srclayers_[0]->mutable_data(this));
+ auto grad = NewTensor2(&grad_);
+ auto weight = NewTensor2(weight_->mutable_data());
+ auto gweight = NewTensor2(weight_->mutable_grad());
+ auto gbias = NewTensor1(bias_->mutable_grad());
+
+ grad = expr::F<op::stanh_grad>(data) * grad;
+ gbias = expr::sum_rows(grad);
+ gweight = dot(grad.T(), src);
+ if (srclayers_[0]->mutable_grad(this) != nullptr) {
+ auto gsrc = NewTensor2(srclayers_[0]->mutable_grad(this));
+ gsrc = dot(grad, weight);
+ }
+}
+}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/98b49952/examples/mlp/hidden_layer.h
----------------------------------------------------------------------
diff --git a/examples/mlp/hidden_layer.h b/examples/mlp/hidden_layer.h
new file mode 100644
index 0000000..c305261
--- /dev/null
+++ b/examples/mlp/hidden_layer.h
@@ -0,0 +1,22 @@
+#include "neuralnet/layer.h"
+#include "myproto.pb.h"
+
+namespace singa {
+class HiddenLayer : public NeuronLayer {
+ public:
+ ~HiddenLayer();
+ void Setup(const LayerProto& proto, int npartitions) override;
+ void ComputeFeature(int flag, Metric* perf) override;
+ void ComputeGradient(int flag, Metric* perf) override;
+ const std::vector<Param*> GetParams() const override {
+ std::vector<Param*> params{weight_, bias_};
+ return params;
+ }
+
+ private:
+ int batchsize_;
+ int vdim_, hdim_;
+ bool transpose_;
+ Param *weight_, *bias_;
+};
+}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/98b49952/examples/mlp/job.conf
----------------------------------------------------------------------
diff --git a/examples/mlp/job.conf b/examples/mlp/job.conf
new file mode 100644
index 0000000..163bc19
--- /dev/null
+++ b/examples/mlp/job.conf
@@ -0,0 +1,95 @@
+name: "mlp"
+train_steps: 1200
+test_steps:10
+test_freq:60
+disp_freq:10
+train_one_batch {
+ alg: kBP
+}
+updater{
+ type: kSGD
+ learning_rate{
+ type : kStep
+ base_lr: 0.001
+ step_conf{
+ change_freq: 60
+ gamma: 0.997
+ }
+ }
+}
+neuralnet {
+ layer {
+ name: "data"
+ type: kShardData
+ sharddata_conf {
+ path: "examples/mnist/mnist_train_shard"
+ batchsize: 1000
+ }
+ exclude: kTest
+ }
+
+ layer {
+ name: "data"
+ type: kShardData
+ sharddata_conf {
+ path: "examples/mnist/mnist_test_shard"
+ batchsize: 1000
+ }
+ exclude: kTrain
+ }
+
+ layer{
+ name:"mnist"
+ type: kMnist
+ srclayers: "data"
+ mnist_conf {
+ norm_a: 127.5
+ norm_b: 1
+ }
+ }
+ layer{
+ name: "label"
+ type: kLabel
+ srclayers: "data"
+ }
+
+
+ layer{
+ name: "hid1"
+ user_type: "kHidden"
+ srclayers:"mnist"
+ [singa.hidden_conf] {
+ num_output: 10
+ }
+ param{
+ name: "w1"
+ init {
+ type: kUniform
+ low:-0.05
+ high:0.05
+ }
+ }
+ param{
+ name: "b1"
+ init {
+ type : kUniform
+ low: -0.05
+ high:0.05
+ }
+ }
+ }
+ layer{
+ name: "loss"
+ type:kSoftmaxLoss
+ softmaxloss_conf{
+ topk:1
+ }
+ srclayers:"hid1"
+ srclayers:"label"
+ }
+}
+cluster {
+ nworker_groups: 1
+ nserver_groups: 1
+ workspace: "examples/mnist"
+}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/98b49952/examples/mlp/main.cc
----------------------------------------------------------------------
diff --git a/examples/mlp/main.cc b/examples/mlp/main.cc
new file mode 100644
index 0000000..c27f38d
--- /dev/null
+++ b/examples/mlp/main.cc
@@ -0,0 +1,25 @@
+#include <string>
+#include "singa.h"
+#include "hidden_layer.h"
+#include "myproto.pb.h"
+#include "utils/common.h"
+
+int main(int argc, char **argv) {
+ // must create driver at the beginning and call its Init method.
+ singa::Driver driver;
+ driver.Init(argc, argv);
+
+ // if -resume in argument list, set resume to true; otherwise false
+ int resume_pos = singa::ArgPos(argc, argv, "-resume");
+ bool resume = (resume_pos != -1);
+
+ // users can register new subclasses of layer, updater, etc.
+ driver.RegisterLayer<singa::HiddenLayer, std::string>("kHidden");
+
+ // get the job conf, and custmize it if need
+ singa::JobProto jobConf = driver.job_conf();
+
+ // submit the job
+ driver.Submit(resume, jobConf);
+ return 0;
+}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/98b49952/examples/mlp/myproto.proto
----------------------------------------------------------------------
diff --git a/examples/mlp/myproto.proto b/examples/mlp/myproto.proto
new file mode 100644
index 0000000..ba62ac2
--- /dev/null
+++ b/examples/mlp/myproto.proto
@@ -0,0 +1,11 @@
+package singa;
+import "job.proto";
+
+message HiddenProto {
+ required int32 num_output = 1;
+ optional bool use_stanh = 2 [default = true];
+}
+
+extend LayerProto {
+ optional HiddenProto hidden_conf = 102;
+}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/98b49952/src/utils/tool.cc
----------------------------------------------------------------------
diff --git a/src/utils/tool.cc b/src/utils/tool.cc
index 084ebea..ea28a6c 100644
--- a/src/utils/tool.cc
+++ b/src/utils/tool.cc
@@ -2,6 +2,7 @@
#include <algorithm>
#include <fstream>
#include <iostream>
+#include <google/protobuf/text_format.h>
#include "proto/job.pb.h"
#include "proto/singa.pb.h"
#include "utils/cluster_rt.h"
@@ -30,12 +31,48 @@ int create() {
return SUCCESS;
}
+// extract cluster configuration part from the job config file
+// TODO improve this function to make it robust
+const std::string extract_cluster(const char* jobfile) {
+ std::ifstream fin;
+ fin.open(jobfile, std::ifstream::in);
+ CHECK(fin.is_open()) << "cannot open job conf file " << jobfile;
+ std::string line;
+ std::string cluster;
+ while (std::getline(fin, line)) {
+ // end of extraction (cluster config has not nested messages)
+ if (line.find("}") != std::string::npos && cluster.length()) {
+ cluster += line.substr(0, line.find("}"));
+ break;
+ }
+ unsigned int pos = 0;
+ while (pos < line.length() && line.at(pos) == ' ' ) pos++;
+ if (line.find("cluster", pos) == pos) { // start with <whitespace> cluster
+ pos += 7;
+ do { // looking for the first '{', which may be in the next lines
+ while (pos < line.length() &&
+ (line.at(pos) == ' ' || line.at(pos) =='\t')) pos++;
+ if (pos < line.length()) {
+ CHECK_EQ(line.at(pos), '{') << "error around 'cluster' field";
+ cluster = " "; // start extraction
+ break;
+ } else
+ pos = 0;
+ }while(std::getline(fin, line));
+ } else if (cluster.length()) {
+ cluster += line + "\n";
+ }
+ }
+ return cluster;
+}
+
+
// generate a host list
int genhost(char* job_conf) {
// compute required #process from job conf
- singa::JobProto job;
- singa::ReadProtoFromTextFile(job_conf, &job);
- singa::ClusterProto cluster = job.cluster();
+ singa::ClusterProto cluster;
+ google::protobuf::TextFormat::ParseFromString(extract_cluster(job_conf),
+ &cluster);
int nworker_procs = cluster.nworker_groups() * cluster.nworkers_per_group()
/ cluster.nworkers_per_procs();
int nserver_procs = cluster.nserver_groups() * cluster.nservers_per_group()
[2/2] incubator-singa git commit: SINGA-65 Add an example of
implementing user's own layers
Posted by wa...@apache.org.
SINGA-65 Add an example of implementing user's own layers
Move the complete code into example/mlp/full foder.
Remove some code in the source files to let users fill in.
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/7306bcff
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/7306bcff
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/7306bcff
Branch: refs/heads/tutorial
Commit: 7306bcffb0a6c04144d801c1bb298859a71b9b4a
Parents: 98b4995
Author: wangwei <wa...@comp.nus.edu.sg>
Authored: Fri Sep 4 18:39:25 2015 +0800
Committer: wangwei <wa...@comp.nus.edu.sg>
Committed: Fri Sep 4 20:45:25 2015 +0800
----------------------------------------------------------------------
examples/mlp/full/hidden_layer.cc | 72 ++++++++++++++++++++++++++
examples/mlp/full/hidden_layer.h | 22 ++++++++
examples/mlp/full/job.conf | 95 ++++++++++++++++++++++++++++++++++
examples/mlp/full/main.cc | 25 +++++++++
examples/mlp/full/myproto.proto | 11 ++++
examples/mlp/hidden_layer.cc | 20 ++-----
examples/mlp/hidden_layer.h | 11 +---
examples/mlp/job.conf | 42 ++-------------
examples/mlp/main.cc | 3 +-
examples/mlp/myproto.proto | 9 +---
10 files changed, 237 insertions(+), 73 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7306bcff/examples/mlp/full/hidden_layer.cc
----------------------------------------------------------------------
diff --git a/examples/mlp/full/hidden_layer.cc b/examples/mlp/full/hidden_layer.cc
new file mode 100644
index 0000000..f4dacf2
--- /dev/null
+++ b/examples/mlp/full/hidden_layer.cc
@@ -0,0 +1,72 @@
+#include "hidden_layer.h"
+
+#include "mshadow/tensor.h"
+#include "mshadow/cxxnet_op.h"
+
+namespace singa {
+using namespace mshadow;
+using mshadow::cpu;
+using mshadow::Shape1;
+using mshadow::Shape2;
+
+inline Tensor<cpu, 2> NewTensor2(Blob<float>* blob) {
+ const vector<int>& shape = blob->shape();
+ Tensor<cpu, 2> tensor(blob->mutable_cpu_data(),
+ Shape2(shape[0], blob->count() / shape[0]));
+ return tensor;
+}
+
+inline Tensor<cpu, 1> NewTensor1(Blob<float>* blob) {
+ Tensor<cpu, 1> tensor(blob->mutable_cpu_data(), Shape1(blob->count()));
+ return tensor;
+}
+
+
+HiddenLayer::~HiddenLayer() {
+ delete weight_;
+ delete bias_;
+}
+
+void HiddenLayer::Setup(const LayerProto& proto, int npartitions) {
+ Layer::Setup(proto, npartitions);
+ CHECK_EQ(srclayers_.size(), 1);
+ const auto& src = srclayers_[0]->data(this);
+ batchsize_ = src.shape()[0];
+ vdim_ = src.count() / batchsize_;
+ hdim_ = layer_proto_.GetExtension(hidden_conf).num_output();
+ data_.Reshape(vector<int>{batchsize_, hdim_});
+ grad_.ReshapeLike(data_);
+ weight_ = Param::Create(proto.param(0));
+ bias_ = Param::Create(proto.param(1));
+ weight_->Setup(vector<int>{hdim_, vdim_});
+ bias_->Setup(vector<int>{hdim_});
+}
+
+void HiddenLayer::ComputeFeature(int flag, Metric* perf) {
+ auto data = NewTensor2(&data_);
+ auto src = NewTensor2(srclayers_[0]->mutable_data(this));
+ auto weight = NewTensor2(weight_->mutable_data());
+ auto bias = NewTensor1(bias_->mutable_data());
+ data = dot(src, weight.T());
+ // repmat: repeat bias vector into batchsize rows
+ data += expr::repmat(bias, batchsize_);
+ data = expr::F<op::stanh>(data);
+}
+
+void HiddenLayer::ComputeGradient(int flag, Metric* perf) {
+ auto data = NewTensor2(&data_);
+ auto src = NewTensor2(srclayers_[0]->mutable_data(this));
+ auto grad = NewTensor2(&grad_);
+ auto weight = NewTensor2(weight_->mutable_data());
+ auto gweight = NewTensor2(weight_->mutable_grad());
+ auto gbias = NewTensor1(bias_->mutable_grad());
+
+ grad = expr::F<op::stanh_grad>(data) * grad;
+ gbias = expr::sum_rows(grad);
+ gweight = dot(grad.T(), src);
+ if (srclayers_[0]->mutable_grad(this) != nullptr) {
+ auto gsrc = NewTensor2(srclayers_[0]->mutable_grad(this));
+ gsrc = dot(grad, weight);
+ }
+}
+}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7306bcff/examples/mlp/full/hidden_layer.h
----------------------------------------------------------------------
diff --git a/examples/mlp/full/hidden_layer.h b/examples/mlp/full/hidden_layer.h
new file mode 100644
index 0000000..c305261
--- /dev/null
+++ b/examples/mlp/full/hidden_layer.h
@@ -0,0 +1,22 @@
+#include "neuralnet/layer.h"
+#include "myproto.pb.h"
+
+namespace singa {
+class HiddenLayer : public NeuronLayer {
+ public:
+ ~HiddenLayer();
+ void Setup(const LayerProto& proto, int npartitions) override;
+ void ComputeFeature(int flag, Metric* perf) override;
+ void ComputeGradient(int flag, Metric* perf) override;
+ const std::vector<Param*> GetParams() const override {
+ std::vector<Param*> params{weight_, bias_};
+ return params;
+ }
+
+ private:
+ int batchsize_;
+ int vdim_, hdim_;
+ bool transpose_;
+ Param *weight_, *bias_;
+};
+}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7306bcff/examples/mlp/full/job.conf
----------------------------------------------------------------------
diff --git a/examples/mlp/full/job.conf b/examples/mlp/full/job.conf
new file mode 100644
index 0000000..163bc19
--- /dev/null
+++ b/examples/mlp/full/job.conf
@@ -0,0 +1,95 @@
+name: "mlp"
+train_steps: 1200
+test_steps:10
+test_freq:60
+disp_freq:10
+train_one_batch {
+ alg: kBP
+}
+updater{
+ type: kSGD
+ learning_rate{
+ type : kStep
+ base_lr: 0.001
+ step_conf{
+ change_freq: 60
+ gamma: 0.997
+ }
+ }
+}
+neuralnet {
+ layer {
+ name: "data"
+ type: kShardData
+ sharddata_conf {
+ path: "examples/mnist/mnist_train_shard"
+ batchsize: 1000
+ }
+ exclude: kTest
+ }
+
+ layer {
+ name: "data"
+ type: kShardData
+ sharddata_conf {
+ path: "examples/mnist/mnist_test_shard"
+ batchsize: 1000
+ }
+ exclude: kTrain
+ }
+
+ layer{
+ name:"mnist"
+ type: kMnist
+ srclayers: "data"
+ mnist_conf {
+ norm_a: 127.5
+ norm_b: 1
+ }
+ }
+ layer{
+ name: "label"
+ type: kLabel
+ srclayers: "data"
+ }
+
+
+ layer{
+ name: "hid1"
+ user_type: "kHidden"
+ srclayers:"mnist"
+ [singa.hidden_conf] {
+ num_output: 10
+ }
+ param{
+ name: "w1"
+ init {
+ type: kUniform
+ low:-0.05
+ high:0.05
+ }
+ }
+ param{
+ name: "b1"
+ init {
+ type : kUniform
+ low: -0.05
+ high:0.05
+ }
+ }
+ }
+ layer{
+ name: "loss"
+ type:kSoftmaxLoss
+ softmaxloss_conf{
+ topk:1
+ }
+ srclayers:"hid1"
+ srclayers:"label"
+ }
+}
+cluster {
+ nworker_groups: 1
+ nserver_groups: 1
+ workspace: "examples/mnist"
+}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7306bcff/examples/mlp/full/main.cc
----------------------------------------------------------------------
diff --git a/examples/mlp/full/main.cc b/examples/mlp/full/main.cc
new file mode 100644
index 0000000..c27f38d
--- /dev/null
+++ b/examples/mlp/full/main.cc
@@ -0,0 +1,25 @@
+#include <string>
+#include "singa.h"
+#include "hidden_layer.h"
+#include "myproto.pb.h"
+#include "utils/common.h"
+
+int main(int argc, char **argv) {
+ // must create driver at the beginning and call its Init method.
+ singa::Driver driver;
+ driver.Init(argc, argv);
+
+ // if -resume in argument list, set resume to true; otherwise false
+ int resume_pos = singa::ArgPos(argc, argv, "-resume");
+ bool resume = (resume_pos != -1);
+
+ // users can register new subclasses of layer, updater, etc.
+ driver.RegisterLayer<singa::HiddenLayer, std::string>("kHidden");
+
+ // get the job conf, and custmize it if need
+ singa::JobProto jobConf = driver.job_conf();
+
+ // submit the job
+ driver.Submit(resume, jobConf);
+ return 0;
+}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7306bcff/examples/mlp/full/myproto.proto
----------------------------------------------------------------------
diff --git a/examples/mlp/full/myproto.proto b/examples/mlp/full/myproto.proto
new file mode 100644
index 0000000..ba62ac2
--- /dev/null
+++ b/examples/mlp/full/myproto.proto
@@ -0,0 +1,11 @@
+package singa;
+import "job.proto";
+
+message HiddenProto {
+ required int32 num_output = 1;
+ optional bool use_stanh = 2 [default = true];
+}
+
+extend LayerProto {
+ optional HiddenProto hidden_conf = 102;
+}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7306bcff/examples/mlp/hidden_layer.cc
----------------------------------------------------------------------
diff --git a/examples/mlp/hidden_layer.cc b/examples/mlp/hidden_layer.cc
index f4dacf2..14eee23 100644
--- a/examples/mlp/hidden_layer.cc
+++ b/examples/mlp/hidden_layer.cc
@@ -32,14 +32,8 @@ void HiddenLayer::Setup(const LayerProto& proto, int npartitions) {
CHECK_EQ(srclayers_.size(), 1);
const auto& src = srclayers_[0]->data(this);
batchsize_ = src.shape()[0];
- vdim_ = src.count() / batchsize_;
- hdim_ = layer_proto_.GetExtension(hidden_conf).num_output();
- data_.Reshape(vector<int>{batchsize_, hdim_});
- grad_.ReshapeLike(data_);
- weight_ = Param::Create(proto.param(0));
- bias_ = Param::Create(proto.param(1));
- weight_->Setup(vector<int>{hdim_, vdim_});
- bias_->Setup(vector<int>{hdim_});
+
+ // please add code for setup this layer
}
void HiddenLayer::ComputeFeature(int flag, Metric* perf) {
@@ -47,10 +41,7 @@ void HiddenLayer::ComputeFeature(int flag, Metric* perf) {
auto src = NewTensor2(srclayers_[0]->mutable_data(this));
auto weight = NewTensor2(weight_->mutable_data());
auto bias = NewTensor1(bias_->mutable_data());
- data = dot(src, weight.T());
- // repmat: repeat bias vector into batchsize rows
- data += expr::repmat(bias, batchsize_);
- data = expr::F<op::stanh>(data);
+ // please add code for computing the hidden feature
}
void HiddenLayer::ComputeGradient(int flag, Metric* perf) {
@@ -61,9 +52,8 @@ void HiddenLayer::ComputeGradient(int flag, Metric* perf) {
auto gweight = NewTensor2(weight_->mutable_grad());
auto gbias = NewTensor1(bias_->mutable_grad());
- grad = expr::F<op::stanh_grad>(data) * grad;
- gbias = expr::sum_rows(grad);
- gweight = dot(grad.T(), src);
+ // please add code for computing the gradients
+
if (srclayers_[0]->mutable_grad(this) != nullptr) {
auto gsrc = NewTensor2(srclayers_[0]->mutable_grad(this));
gsrc = dot(grad, weight);
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7306bcff/examples/mlp/hidden_layer.h
----------------------------------------------------------------------
diff --git a/examples/mlp/hidden_layer.h b/examples/mlp/hidden_layer.h
index c305261..5dbae12 100644
--- a/examples/mlp/hidden_layer.h
+++ b/examples/mlp/hidden_layer.h
@@ -8,15 +8,6 @@ class HiddenLayer : public NeuronLayer {
void Setup(const LayerProto& proto, int npartitions) override;
void ComputeFeature(int flag, Metric* perf) override;
void ComputeGradient(int flag, Metric* perf) override;
- const std::vector<Param*> GetParams() const override {
- std::vector<Param*> params{weight_, bias_};
- return params;
- }
- private:
- int batchsize_;
- int vdim_, hdim_;
- bool transpose_;
- Param *weight_, *bias_;
-};
+// please fill HiddenLayer class declaration
}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7306bcff/examples/mlp/job.conf
----------------------------------------------------------------------
diff --git a/examples/mlp/job.conf b/examples/mlp/job.conf
index 163bc19..6f57677 100644
--- a/examples/mlp/job.conf
+++ b/examples/mlp/job.conf
@@ -3,20 +3,9 @@ train_steps: 1200
test_steps:10
test_freq:60
disp_freq:10
-train_one_batch {
- alg: kBP
-}
-updater{
- type: kSGD
- learning_rate{
- type : kStep
- base_lr: 0.001
- step_conf{
- change_freq: 60
- gamma: 0.997
- }
- }
-}
+
+# please add the config for updater and train_one_batch
+
neuralnet {
layer {
name: "data"
@@ -53,31 +42,8 @@ neuralnet {
srclayers: "data"
}
+# please add the config for the hidden layer
- layer{
- name: "hid1"
- user_type: "kHidden"
- srclayers:"mnist"
- [singa.hidden_conf] {
- num_output: 10
- }
- param{
- name: "w1"
- init {
- type: kUniform
- low:-0.05
- high:0.05
- }
- }
- param{
- name: "b1"
- init {
- type : kUniform
- low: -0.05
- high:0.05
- }
- }
- }
layer{
name: "loss"
type:kSoftmaxLoss
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7306bcff/examples/mlp/main.cc
----------------------------------------------------------------------
diff --git a/examples/mlp/main.cc b/examples/mlp/main.cc
index c27f38d..cf104e6 100644
--- a/examples/mlp/main.cc
+++ b/examples/mlp/main.cc
@@ -13,8 +13,7 @@ int main(int argc, char **argv) {
int resume_pos = singa::ArgPos(argc, argv, "-resume");
bool resume = (resume_pos != -1);
- // users can register new subclasses of layer, updater, etc.
- driver.RegisterLayer<singa::HiddenLayer, std::string>("kHidden");
+ // please register the HiddenLayer here
// get the job conf, and custmize it if need
singa::JobProto jobConf = driver.job_conf();
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/7306bcff/examples/mlp/myproto.proto
----------------------------------------------------------------------
diff --git a/examples/mlp/myproto.proto b/examples/mlp/myproto.proto
index ba62ac2..deea1d4 100644
--- a/examples/mlp/myproto.proto
+++ b/examples/mlp/myproto.proto
@@ -1,11 +1,4 @@
package singa;
import "job.proto";
-message HiddenProto {
- required int32 num_output = 1;
- optional bool use_stanh = 2 [default = true];
-}
-
-extend LayerProto {
- optional HiddenProto hidden_conf = 102;
-}
+// please add the definition of HiddenProto