You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@singa.apache.org by wa...@apache.org on 2016/06/03 07:48:14 UTC
[09/60] incubator-singa git commit: SINGA-163 - Reorganize the
project folder layout
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dd1e4afa/src/neuralnet/connection_layer/slice.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/connection_layer/slice.cc b/src/neuralnet/connection_layer/slice.cc
deleted file mode 100644
index 3cca3fd..0000000
--- a/src/neuralnet/connection_layer/slice.cc
+++ /dev/null
@@ -1,166 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-#include "singa/neuralnet/connection_layer.h"
-#include "singa/utils/math_blob.h"
-#include "singa/utils/singleton.h"
-#include "singa/utils/context.h"
-
-namespace singa {
-
-using std::vector;
-
-SliceLayer::~SliceLayer() {
- for (size_t i = 1; i < datavec_.size(); ++i) {
- if (datavec_[i] != nullptr) delete datavec_[i];
- if (gradvec_[i] != nullptr) delete gradvec_[i];
- }
-}
-
-void SliceLayer::Setup(const LayerProto& conf,
- const vector<Layer*>& srclayers) {
- CHECK_EQ(srclayers.size(), 1);
- Layer::Setup(conf, srclayers);
- vector<int> shape = srclayers[0]->data(this).shape();
- slice_dim_ = conf.slice_conf().slice_dim();
- num_slices_ = conf.slice_conf().num_slices();
- CHECK_GE(slice_dim_, 0);
- CHECK_LT(slice_dim_, shape.size());
- CHECK_GT(num_slices_, 0);
- // add num_slices-1 more blobs
- for (int i = 1; i < num_slices_; ++i) {
- datavec_.push_back(new Blob<float>());
- gradvec_.push_back(new Blob<float>());
- }
- // TODO(wangsh): remove equal-size restrict later
- CHECK_EQ(shape[slice_dim_] % num_slices_, 0);
- shape[slice_dim_] /= num_slices_;
- for (int i = 0; i < num_slices_; ++i) {
- // if (i == slice_num - 1) shape[slice_dim] += remain;
- datavec_[i]->Reshape(shape);
- gradvec_[i]->Reshape(shape);
- }
-}
-
-void SliceLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
- CHECK_EQ(srclayers.size(), 1);
- const Blob<float>& blob = srclayers[0]->data(this);
- // calculate step for each memcpy
- int step = datavec_[0]->shape()[slice_dim_];
- for (unsigned i = slice_dim_ + 1; i < datavec_[0]->shape().size(); ++i)
- step *= datavec_[0]->shape()[i];
- int srclayer_offset = 0;
- int slice_offset = 0;
- auto context = Singleton<Context>::Instance();
- int device = context->device_id(std::this_thread::get_id());
- while (srclayer_offset < blob.count()) {
- for (int i = 0; i < num_slices_; ++i) {
- if (device < 0) {
- const float* src = blob.cpu_data() + srclayer_offset;
- float* dst = datavec_[i]->mutable_cpu_data() + slice_offset;
- memcpy(dst, src, step * sizeof(float));
- } else {
-#ifdef USE_GPU
- const float* src = blob.gpu_data() + srclayer_offset;
- float* dst = datavec_[i]->mutable_gpu_data() + slice_offset;
- cudaMemcpy(dst, src, step * sizeof(float), cudaMemcpyDefault);
-#else
- LOG(FATAL) << "GPU is not supported";
-#endif
- }
- srclayer_offset += step;
- }
- slice_offset += step;
- }
-}
-
-void SliceLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers) {
- CHECK_EQ(srclayers.size(), 1);
- Blob<float>* blob = srclayers[0]->mutable_grad(this);
- // calculate step for each memcpy
- int step = gradvec_[0]->shape()[slice_dim_];
- for (size_t i = slice_dim_ + 1; i < gradvec_[0]->shape().size(); ++i)
- step *= gradvec_[0]->shape()[i];
- int srclayer_offset = 0;
- int slice_offset = 0;
- auto context = Singleton<Context>::Instance();
- int device = context->device_id(std::this_thread::get_id());
- while (srclayer_offset < blob->count()) {
- for (int i = 0; i < num_slices_; ++i) {
- if (device < 0) {
- const float* src = gradvec_[i]->cpu_data() + slice_offset;
- float* dst = blob->mutable_cpu_data() + srclayer_offset;
- memcpy(dst, src, step * sizeof(float));
- } else {
-#ifdef USE_GPU
- const float* src = gradvec_[i]->gpu_data() + slice_offset;
- float* dst = blob->mutable_gpu_data() + srclayer_offset;
- cudaMemcpy(dst, src, step * sizeof(float), cudaMemcpyDefault);
-#else
- LOG(FATAL) << "GPU is not supported";
-#endif
- }
- srclayer_offset += step;
- }
- slice_offset += step;
- }
-}
-
-const Blob<float>& SliceLayer::data(const Layer* from) {
- int idx = from ? layer_idx_.Get(from) : 0;
- CHECK_LT(idx, num_slices_);
- return *datavec_[idx];
-}
-
-const Blob<float>& SliceLayer::grad(const Layer* from) {
- int idx = from ? layer_idx_.Get(from) : 0;
- CHECK_LT(idx, num_slices_);
- return *gradvec_[idx];
-}
-
-Blob<float>* SliceLayer::mutable_data(const Layer* from) {
- CHECK(from);
- int idx = layer_idx_.Get(from);
- CHECK_LT(idx, num_slices_);
- return datavec_[idx];
-}
-
-Blob<float>* SliceLayer::mutable_grad(const Layer* from) {
- CHECK(from);
- int idx = layer_idx_.Get(from);
- CHECK_LT(idx, num_slices_);
- return gradvec_[idx];
-}
-const std::string SliceLayer::ToString(bool debug, int flag) {
- if (!debug)
- return "";
- string ret = "";
- if ((flag & kForward) == kForward && data_.count() !=0) {
- for (unsigned k = 0; k < datavec_.size(); k++)
- ret += StringPrintf("data-%u :%e ", k, Asum(*datavec_.at(k)));
- }
- if ((flag & kBackward) == kBackward && grad_.count() != 0) {
- for (unsigned k = 0; k < gradvec_.size(); k++)
- ret += StringPrintf("grad-%u:%e ", k, Asum(*gradvec_.at(k)));
- }
- return ret;
-}
-} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dd1e4afa/src/neuralnet/connection_layer/split.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/connection_layer/split.cc b/src/neuralnet/connection_layer/split.cc
deleted file mode 100644
index e46b902..0000000
--- a/src/neuralnet/connection_layer/split.cc
+++ /dev/null
@@ -1,91 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-#include "singa/neuralnet/connection_layer.h"
-#include "singa/utils/math_blob.h"
-
-namespace singa {
-
-using std::vector;
-
-SplitLayer::~SplitLayer() {
- for (size_t i = 1; i < gradvec_.size(); ++i) {
- if (gradvec_[i] != nullptr) delete gradvec_[i];
- }
-}
-
-void SplitLayer::Setup(const LayerProto& conf,
- const vector<Layer*>& srclayers) {
- CHECK_EQ(srclayers.size(), 1);
- Layer::Setup(conf, srclayers);
- data_.Reshape(srclayers[0]->data(this).shape());
- data_.ShareData(srclayers[0]->mutable_data(this), false);
- num_splits_ = conf.split_conf().num_splits();
- CHECK_GT(num_splits_, 0);
- // add num_splits-1 more grad blobs
- for (int i = 1; i < num_splits_; ++i) {
- gradvec_.push_back(new Blob<float>());
- }
- for (int i = 0; i < num_splits_; ++i)
- gradvec_[i]->Reshape(srclayers[0]->data(this).shape());
-}
-
-void SplitLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
- // data is shared from its source,
- // nothing to do in compute feature phase
-}
-
-void SplitLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers) {
- CHECK_EQ(srclayers.size(), 1);
- // aggregate all gradients to grad_[0]
- for (int i = 1; i < num_splits_; ++i)
- AXPY<float>(1.0, *gradvec_[i], gradvec_[0]);
- // copy grad_[0] to srclayer's grad
- Copy(*gradvec_[0], srclayers[0]->mutable_grad(this));
-}
-
-const Blob<float>& SplitLayer::grad(const Layer* from) {
- CHECK(from);
- int idx = layer_idx_.Get(from);
- CHECK_LT(idx, num_splits_);
- return *gradvec_[idx];
-}
-
-Blob<float>* SplitLayer::mutable_grad(const Layer* from) {
- CHECK(from);
- int idx = layer_idx_.Get(from);
- CHECK_LT(idx, num_splits_);
- return gradvec_[idx];
-}
-const std::string SplitLayer::ToString(bool debug, int flag) {
- if (!debug)
- return "";
- string ret = "";
- if ((flag & kForward) == kForward && data_.count() !=0) {
- ret += StringPrintf("data:%13.9f ", Asum(data_));
- }
- if ((flag & kBackward) == kBackward && grad_.count() != 0) {
- for (unsigned k = 0; k < gradvec_.size(); k++)
- ret += StringPrintf("grad-%u:%13.9f ", k, Asum(*gradvec_.at(k)));
- }
- return ret;
-}
-} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dd1e4afa/src/neuralnet/input_layer/char_rnn.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/input_layer/char_rnn.cc b/src/neuralnet/input_layer/char_rnn.cc
deleted file mode 100644
index 8a56711..0000000
--- a/src/neuralnet/input_layer/char_rnn.cc
+++ /dev/null
@@ -1,93 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-#include <sstream>
-#include <fstream>
-#include "singa/neuralnet/input_layer.h"
-namespace singa {
-
-void CharRNNInputLayer::Setup(const LayerProto& conf,
- const vector<Layer*>& srclayers) {
- InputLayer::Setup(conf, srclayers);
- batchsize_ = conf.char_rnn_conf().batchsize();
- path_ = conf.char_rnn_conf().path();
- vocab_path_ = conf.char_rnn_conf().vocab_path();
- unroll_len_ = conf.char_rnn_conf().unroll_len();
- datavec_.clear();
- // each unroll layer has a input blob
- for (int i = 0; i <= unroll_len_; i++) {
- datavec_.push_back(new Blob<float>(batchsize_));
- }
-}
-
-void CharRNNInputLayer::ComputeFeature(int flag,
- const vector<Layer*>& srclayers) {
- if (buf_.size() == 0) {
-
- // read the vocab
- {
- std::ifstream fin;
- fin.open(vocab_path_);
- CHECK(fin.is_open()) << "Can't open vocab_path = " << vocab_path_;
- std::stringstream stream;
- stream << fin.rdbuf();
- string vocab = stream.str();
- LOG(ERROR) << "Vocab_size = " << vocab.length();
- for (char c : vocab)
- char2index_[c] = char2index_.size() - 1;
- fin.close();
- }
-
- // read the whole text file
- {
- std::ifstream fin;
- fin.open(path_);
- CHECK(fin.is_open()) << "Can't open filepath = " << path_;
- std::stringstream stream;
- stream << fin.rdbuf();
- buf_ = stream.str();
- fin.close();
- }
-
- // decide the start pos of each instance in one mini-batch
- int max_offset = buf_.length() / batchsize_;
- CHECK_GT(max_offset, unroll_len_);
- LOG(ERROR) << "Max iteration per epoch = " << max_offset / unroll_len_;
- for (int i = 0; i < batchsize_; i ++) {
- start_.push_back(i * max_offset);
- }
- }
-
- for (int l = 0; l < unroll_len_ + 1; l++) {
- float* ptr = datavec_[l]->mutable_cpu_data();
- for (int i = 0; i < batchsize_; i++) {
- ptr[i] = static_cast<float>(char2index_.at(buf_[start_[i] + offset_ + l]));
- }
- }
- offset_ += unroll_len_;
- if (offset_ >= buf_.length() / batchsize_) {
-// unsigned seed = std::chrono::system_clock::now().time_since_epoch().count();
-// std::mt19937 g(seed);
-// std::shuffle(start_.begin(), start_.end(), g);
- offset_ = 0;
- // return -1;
- }
-}
-} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dd1e4afa/src/neuralnet/input_layer/csv.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/input_layer/csv.cc b/src/neuralnet/input_layer/csv.cc
deleted file mode 100644
index 53cabff..0000000
--- a/src/neuralnet/input_layer/csv.cc
+++ /dev/null
@@ -1,67 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-#include "singa/neuralnet/input_layer.h"
-#include "singa/utils/tokenizer.h"
-
-namespace singa {
-
-void CSVInputLayer::Setup(const LayerProto& conf,
- const vector<Layer*>& srclayers) {
- SingleLabelRecordLayer::Setup(conf, srclayers);
- sep_ = conf.store_conf().separator();
-}
-
-void CSVInputLayer::LoadRecord(const string& backend,
- const string&path, Blob<float>* to) {
- io::Store* store = io::OpenStore(backend, path, io::kRead);
- string key, val;
- CHECK(store->Read(&key, &val));
- float* ptr = to->mutable_cpu_data();
- Tokenizer t(val, sep_);
- string x;
- for (int i = 0; i< to->count(); i++) {
- t >> x;
- ptr[i] = stof(x);
- }
- CHECK(!t.Valid());
- delete store;
-}
-
-bool CSVInputLayer::Parse(int k, int flag, const string& key,
- const string& value) {
- float* ptr = data_.mutable_cpu_data() + k * data_.count() / batchsize_;
- Tokenizer t(value, sep_);
- string x;
- // parse label if not deploy phase and has_label is set.
- if ((flag & kDeploy) == 0 && layer_conf_.store_conf().has_label()) {
- t >> x;
- aux_data_[k] = stoi(x);
- }
- for (int i = 0; i< data_.count() / batchsize_; i++) {
- t >> x;
- ptr[i] = stof(x);
- }
- CHECK(!t.Valid());
- return true;
-}
-
-} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dd1e4afa/src/neuralnet/input_layer/deprecated.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/input_layer/deprecated.cc b/src/neuralnet/input_layer/deprecated.cc
deleted file mode 100644
index d2901f7..0000000
--- a/src/neuralnet/input_layer/deprecated.cc
+++ /dev/null
@@ -1,373 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-#include <random>
-#include "singa/neuralnet/input_layer.h"
-#include "singa/utils/context.h"
-#include "singa/utils/singleton.h"
-#include "mshadow/tensor.h"
-namespace singa {
-
-using namespace mshadow;
-using mshadow::cpu;
-using mshadow::Shape4;
-using mshadow::Tensor;
-
-using std::string;
-using std::vector;
-
-ShardDataLayer::~ShardDataLayer() {
- if (shard_ != nullptr)
- delete shard_;
- shard_ = nullptr;
-}
-
-void ShardDataLayer::Setup(const LayerProto& proto,
- const vector<Layer*>& srclayers) {
- Layer::Setup(proto, srclayers);
- shard_ = new DataShard(proto.sharddata_conf().path(), DataShard::kRead);
- string key;
- shard_->Next(&key, &sample_);
- delete shard_;
- shard_ = nullptr;
- batchsize_ = proto.sharddata_conf().batchsize();
- if (partition_dim() == 0)
- batchsize_ /= proto.num_partitions();
- records_.resize(batchsize_);
- random_skip_ = proto.sharddata_conf().random_skip();
-}
-
-void ShardDataLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
- if (shard_ == nullptr)
- shard_ = new DataShard(layer_conf_.sharddata_conf().path(),
- DataShard::kRead);
- if (random_skip_) {
- std::uniform_int_distribution<int> distribution(0, random_skip_);
- auto generator = Singleton<Context>::Instance()->rand_generator();
- int nskip = distribution(*generator);
- LOG(INFO) << "Random Skip " << nskip << " records, there are "
- << shard_->Count() << " records in total";
- string key;
- for (int i = 0; i < nskip; i++) {
- shard_->Next(&key, &sample_);
- }
- random_skip_ = 0;
- }
- for (auto& record : records_) {
- string key;
- if (!shard_->Next(&key, &record)) {
- shard_->SeekToFirst();
- CHECK(shard_->Next(&key, &record));
- }
- }
-}
-
-/*****************LMDB data layer*******************/
-#ifdef USE_LMDB
-LMDBDataLayer::~LMDBDataLayer() {
- mdb_cursor_close(mdb_cursor_);
- mdb_txn_abort(mdb_txn_);
- mdb_cursor_ = nullptr;
-}
-
-void LMDBDataLayer::Setup(const LayerProto& proto,
- const vector<Layer*>& srclayers) {
- Layer::Setup(proto, srclayers);
- OpenLMDB(proto.lmdbdata_conf().path());
- CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_, &mdb_value_, MDB_NEXT),
- MDB_SUCCESS);
- mdb_cursor_close(mdb_cursor_);
- mdb_txn_abort(mdb_txn_);
- mdb_cursor_ = nullptr;
- CaffeDatum datum;
- datum.ParseFromArray(mdb_value_.mv_data, mdb_value_.mv_size);
- SingleLabelImageRecord* record = sample_.mutable_image();
- ConvertCaffeDatumToRecord(datum, record);
- batchsize_ = proto.lmdbdata_conf().batchsize();
- if (partition_dim() == 0)
- batchsize_ /= proto.num_partitions();
- records_.resize(batchsize_);
- random_skip_ = proto.lmdbdata_conf().random_skip();
-}
-
-void LMDBDataLayer::OpenLMDB(const std::string& path) {
- CHECK_EQ(mdb_env_create(&mdb_env_), MDB_SUCCESS) << "mdb_env_create failed";
- CHECK_EQ(mdb_env_set_mapsize(mdb_env_, 1099511627776), MDB_SUCCESS); // 1TB
- CHECK_EQ(mdb_env_open(mdb_env_, path.c_str(),
- MDB_RDONLY, 0664), MDB_SUCCESS) << "cannot open lmdb " << path;
- CHECK_EQ(mdb_txn_begin(mdb_env_, NULL, MDB_RDONLY, &mdb_txn_), MDB_SUCCESS)
- << "mdb_txn_begin failed";
- CHECK_EQ(mdb_open(mdb_txn_, NULL, 0, &mdb_dbi_), MDB_SUCCESS)
- << "mdb_open failed";
- CHECK_EQ(mdb_cursor_open(mdb_txn_, mdb_dbi_, &mdb_cursor_), MDB_SUCCESS)
- << "mdb_cursor_open failed";
- LOG(INFO) << "Opening lmdb " << path;
- CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_, &mdb_value_, MDB_FIRST),
- MDB_SUCCESS) << "mdb_cursor_get failed";
-}
-
-void LMDBDataLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
- if (mdb_cursor_ == nullptr)
- OpenLMDB(layer_conf_.lmdbdata_conf().path());
- if (random_skip_) {
- std::uniform_int_distribution<int> distribution(0, random_skip_);
- auto generator =
- Singleton<Context>::Instance()->rand_generator(std::this_thread::get_id());
- int nskip = distribution(*generator);
-
- int n = 0;
- CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_,
- &mdb_value_, MDB_FIRST), MDB_SUCCESS);
- while (mdb_cursor_get(mdb_cursor_, &mdb_key_,
- &mdb_value_, MDB_NEXT) == MDB_SUCCESS)
- n++;
- LOG(INFO) << "Random Skip " << nskip << " records of total "
- << n << "records";
- // We have reached the end. Restart from the first.
- CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_,
- &mdb_value_, MDB_FIRST), MDB_SUCCESS);
- for (int i = 0; i < nskip; i++) {
- if (mdb_cursor_get(mdb_cursor_, &mdb_key_,
- &mdb_value_, MDB_NEXT) != MDB_SUCCESS) {
- // We have reached the end. Restart from the first.
- DLOG(INFO) << "Restarting data prefetching from start.";
- CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_,
- &mdb_value_, MDB_FIRST), MDB_SUCCESS);
- }
- }
- random_skip_ = 0;
- }
- CaffeDatum datum;
- for (auto& record : records_) {
- SingleLabelImageRecord* image = record.mutable_image();
- CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_,
- &mdb_value_, MDB_GET_CURRENT), MDB_SUCCESS);
- datum.ParseFromArray(mdb_value_.mv_data, mdb_value_.mv_size);
- ConvertCaffeDatumToRecord(datum, image);
- if (mdb_cursor_get(mdb_cursor_, &mdb_key_,
- &mdb_value_, MDB_NEXT) != MDB_SUCCESS) {
- // We have reached the end. Restart from the first.
- DLOG(INFO) << "Restarting data prefetching from start.";
- CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_,
- &mdb_value_, MDB_FIRST), MDB_SUCCESS);
- }
- }
-}
-
-void LMDBDataLayer::ConvertCaffeDatumToRecord(const CaffeDatum& datum,
- SingleLabelImageRecord* record) {
- record->set_label(datum.label());
- record->clear_shape();
- if (datum.has_channels())
- record->add_shape(datum.channels());
- if (datum.has_height())
- record->add_shape(datum.height());
- if (datum.has_width())
- record->add_shape(datum.width());
- if (datum.has_data())
- record->set_pixel(datum.data());
- if (datum.float_data_size()) {
- record->clear_data();
- for (float x : datum.float_data())
- record->add_data(x);
- }
-}
-#endif
-
-/***************Parser layer*******************/
-void ParserLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
- CHECK_EQ(srclayers.size(), 1);
- auto datalayer = dynamic_cast<DataLayer*>(*srclayers.begin());
- ParseRecords(flag, datalayer->records(), &data_);
-}
-
-/**********Mnist Layer************/
-void MnistLayer::ParseRecords(int flag, const vector<Record>& records,
- Blob<float>* blob) {
- LOG_IF(ERROR, records.size() == 0) << "Empty records to parse";
- int ndim = records.at(0).image().shape_size();
- int inputsize = records.at(0).image().shape(ndim-1);
- CHECK_EQ(inputsize, blob->shape()[2]);
-
- float* dptr = blob->mutable_cpu_data();
- for (const Record& record : records) {
- const SingleLabelImageRecord& imagerecord = record.image();
- if (imagerecord.pixel().size()) {
- string pixel = imagerecord.pixel();
- for (int i = 0, k = 0; i < inputsize; i++) {
- for (int j = 0; j < inputsize; j++) {
- // NOTE!!! must cast pixel to uint8_t then to float!!! waste a lot of
- // time to debug this
- float x = static_cast<float>(static_cast<uint8_t>(pixel[k++]));
- x = x / norm_a_-norm_b_;
- *dptr = x;
- dptr++;
- }
- }
- } else {
- for (int i = 0, k = 0; i < inputsize; i++) {
- for (int j = 0; j < inputsize; j++) {
- *dptr = imagerecord.data(k++) / norm_a_ - norm_b_;
- dptr++;
- }
- }
- }
- }
- CHECK_EQ(dptr, blob->mutable_cpu_data() + blob->count());
-}
-
-void MnistLayer::Setup(const LayerProto& proto,
- const vector<Layer*>& srclayers) {
- Layer::Setup(proto, srclayers);
- CHECK_EQ(srclayers.size(), 1);
- int batchsize = dynamic_cast<DataLayer*>(srclayers[0])->batchsize();
- Record sample = dynamic_cast<DataLayer*>(srclayers[0])->sample();
- norm_a_ = proto.mnist_conf().norm_a();
- norm_b_ = proto.mnist_conf().norm_b();
- int ndim = sample.image().shape_size();
- CHECK_GE(ndim, 2);
- int s = sample.image().shape(ndim - 1);
- CHECK_EQ(s, sample.image().shape(ndim - 2));
- data_.Reshape(vector<int>{batchsize, 1, s, s});
-}
-
-/**********RGB image layer****************/
-void RGBImageLayer::ParseRecords(int flag, const vector<Record>& records,
- Blob<float>* blob) {
- const vector<int>& s = blob->shape();
- Tensor<cpu, 4> images(data_.mutable_cpu_data(),
- Shape4(s[0], s[1], s[2], s[3]));
- const SingleLabelImageRecord& r = records.at(0).image();
- Tensor<cpu, 3> raw_image(Shape3(r.shape(0), r.shape(1), r.shape(2)));
- AllocSpace(raw_image);
- Tensor<cpu, 3> croped_image(nullptr, Shape3(s[1], s[2], s[3]));
- if (cropsize_)
- AllocSpace(croped_image);
- int rid = 0;
- const float* meandptr = mean_.cpu_data();
-
- std::uniform_int_distribution<int> distribution(0, r.shape(0) - cropsize_);
- auto generator =
- Singleton<Context>::Instance()->rand_generator(std::this_thread::get_id());
- for (const Record& record : records) {
- auto image = images[rid];
- bool do_crop = cropsize_> 0 && ((flag & kTrain) == kTrain);
- bool do_mirror = mirror_
- && (distribution(*generator) % 2)
- && ((flag & kTrain) == kTrain);
- float* dptr = nullptr;
- if (do_crop || do_mirror)
- dptr = raw_image.dptr;
- else
- dptr = image.dptr;
- if (record.image().pixel().size()) {
- string pixel = record.image().pixel();
- for (size_t i = 0; i < pixel.size(); i++)
- dptr[i] = static_cast<float>(static_cast<uint8_t>(pixel[i]));
- } else {
- memcpy(dptr, record.image().data().data(),
- sizeof(float) * record.image().data_size());
- }
- for (int i = 0; i < mean_.count(); i++)
- dptr[i] -= meandptr[i];
- if (do_crop) {
- int hoff = distribution(*generator);
- int woff = distribution(*generator);
- Shape<2> cropshape = Shape2(cropsize_, cropsize_);
- if (do_mirror) {
- croped_image = expr::crop(raw_image, cropshape, hoff, woff);
- image = expr::mirror(croped_image);
- } else {
- image = expr::crop(raw_image, cropshape, hoff, woff);
- }
- } else if (do_mirror) {
- image = expr::mirror(raw_image);
- }
- rid++;
- }
- if (scale_)
- images = images * scale_;
- FreeSpace(raw_image);
- if (cropsize_)
- FreeSpace(croped_image);
-}
-
-void RGBImageLayer::Setup(const LayerProto& proto,
- const vector<Layer*>& srclayers) {
- ParserLayer::Setup(proto, srclayers);
- CHECK_EQ(srclayers.size(), 1);
- scale_ = proto.rgbimage_conf().scale();
- cropsize_ = proto.rgbimage_conf().cropsize();
- mirror_ = proto.rgbimage_conf().mirror();
- int batchsize = dynamic_cast<DataLayer*>(srclayers[0])->batchsize();
- Record sample = dynamic_cast<DataLayer*>(srclayers[0])->sample();
- vector<int> shape;
- shape.push_back(batchsize);
- for (int x : sample.image().shape()) {
- shape.push_back(x);
- }
- CHECK_EQ(shape.size(), 4);
- if (cropsize_) {
- shape[2] = cropsize_;
- shape[3] = cropsize_;
- }
- data_.Reshape(shape);
- mean_.Reshape({shape[1], shape[2], shape[3]});
- if (proto.rgbimage_conf().has_meanfile()) {
- if (proto.rgbimage_conf().meanfile().find("binaryproto") != string::npos) {
- CaffeBlob mean;
- ReadProtoFromBinaryFile(proto.rgbimage_conf().meanfile().c_str(), &mean);
- CHECK_EQ(mean_.count(), mean.data_size());
- memcpy(mean_.mutable_cpu_data(), mean.data().data(),
- sizeof(float)*mean.data_size());
- } else {
- SingleLabelImageRecord mean;
- ReadProtoFromBinaryFile(proto.rgbimage_conf().meanfile().c_str(), &mean);
- CHECK_EQ(mean_.count(), mean.data_size());
- memcpy(mean_.mutable_cpu_data(), mean.data().data(),
- sizeof(float)*mean.data_size());
- }
- } else {
- memset(mean_.mutable_cpu_data(), 0, sizeof(float) * mean_.count());
- }
-}
-
-/*************Label layer *************/
-
-void LabelLayer::Setup(const LayerProto& proto,
- const vector<Layer*>& srclayers) {
- Layer::Setup(proto, srclayers);
- CHECK_EQ(srclayers.size(), 1);
- int batchsize = dynamic_cast<DataLayer*>(srclayers[0])->batchsize();
- data_.Reshape(vector<int>{batchsize});
-}
-
-void LabelLayer::ParseRecords(int flag, const vector<Record>& records,
- Blob<float>* blob) {
- int rid = 0;
- float *label = blob->mutable_cpu_data();
- for (const Record& record : records) {
- label[rid++] = record.image().label();
- // CHECK_LT(record.image().label(),10);
- }
- CHECK_EQ(rid, blob->shape()[0]);
-}
-} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dd1e4afa/src/neuralnet/input_layer/image_preprocess.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/input_layer/image_preprocess.cc b/src/neuralnet/input_layer/image_preprocess.cc
deleted file mode 100644
index 6f2e094..0000000
--- a/src/neuralnet/input_layer/image_preprocess.cc
+++ /dev/null
@@ -1,78 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-#include "singa/neuralnet/input_layer.h"
-#include "singa/utils/image_transform.h"
-#include "singa/utils/context.h"
-#include "singa/utils/singleton.h"
-
-namespace singa {
-
-using std::vector;
-
-void ImagePreprocessLayer::Setup(const LayerProto& conf,
- const vector<Layer*>& srclayers) {
- CHECK_EQ(srclayers.size(), 1);
- InputLayer::Setup(conf, srclayers);
- scale_ = conf.rgbimage_conf().scale();
- cropsize_ = conf.rgbimage_conf().cropsize();
- mirror_ = conf.rgbimage_conf().mirror();
- const auto& src = srclayers.at(0)->data(this);
- const auto& shape = src.shape();
- CHECK_EQ(shape.size(), 4);
- CHECK_EQ(shape.at(2), shape.at(3));
- if (cropsize_ && (cropsize_ != shape.at(2) || cropsize_ != shape.at(3))) {
- data_.Reshape(vector<int>{shape.at(0), shape.at(1), cropsize_, cropsize_});
- } else {
- data_ = src;
- }
-}
-
-void ImagePreprocessLayer::ComputeFeature(int flag,
- const vector<Layer*>& srclayers) {
- const auto& srcdata = srclayers.at(0)->data(this);
- int batchsize = srcdata.shape(0), channel = srcdata.shape(1);
- int height = srcdata.shape(2), width = srcdata.shape(3);
- int srcimage_size = channel * height * width;
- int image_size = channel * data_.shape(2) * data_.shape(3);
- std::uniform_int_distribution<int> rand1(0, height - cropsize_);
- std::uniform_int_distribution<int> rand2(0, width - cropsize_);
- auto generator = Singleton<Context>::Instance()->rand_generator();
-
- const float* srcdptr = srcdata.cpu_data();
- float* dptr = data_.mutable_cpu_data();
-
- for (int k = 0; k < batchsize; k++) {
- int h_offset = 0, w_offset = 0;
- if (cropsize_> 0 && (flag & kTrain)) {
- h_offset = rand1(*generator);
- w_offset = rand2(*generator);
- }
- bool do_mirror = mirror_
- && (rand1(*generator) % 2)
- && (flag & kTrain);
- ImageTransform(srcdptr + k * srcimage_size, nullptr, do_mirror, cropsize_,
- cropsize_, h_offset, w_offset, channel, height, width,
- scale_, dptr + k * image_size);
- }
-}
-
-} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dd1e4afa/src/neuralnet/input_layer/onehot.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/input_layer/onehot.cc b/src/neuralnet/input_layer/onehot.cc
deleted file mode 100644
index 4b83705..0000000
--- a/src/neuralnet/input_layer/onehot.cc
+++ /dev/null
@@ -1,40 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-#include "singa/neuralnet/input_layer.h"
-
-namespace singa {
-void OneHotLayer::Setup(const LayerProto& conf,
- const vector<Layer*>& srclayers) {
- InputLayer::Setup(conf, srclayers);
- batchsize_ = srclayers.at(0)->data(unroll_index()).shape(0);
- dim_ = conf.onehot_conf().vocab_size();
- data_.Reshape(batchsize_, dim_);
-}
-
-void OneHotLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
- float* ptr = data_.mutable_cpu_data();
- memset(ptr, 0, sizeof(float) * data_.count());
- const float* idx = srclayers[0]->data(unroll_index()).cpu_data();
- for (int i = 0; i < batchsize_; i++) {
- ptr[i * dim_ + static_cast<int>(idx[i])] = 1;
- }
-}
-} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dd1e4afa/src/neuralnet/input_layer/record.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/input_layer/record.cc b/src/neuralnet/input_layer/record.cc
deleted file mode 100644
index b14fc80..0000000
--- a/src/neuralnet/input_layer/record.cc
+++ /dev/null
@@ -1,73 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-#include "singa/neuralnet/input_layer.h"
-namespace singa {
-
-using std::string;
-using std::vector;
-
-void RecordInputLayer::Setup(const LayerProto& conf,
- const vector<Layer*>& srclayers) {
- SingleLabelRecordLayer::Setup(conf, srclayers);
- encoded_ = conf.store_conf().encoded();
-}
-
-void RecordInputLayer::LoadRecord(const string& backend,
- const string& path, Blob<float>* to) {
- io::Store* store = io::OpenStore(backend, path, io::kRead);
- string key, val;
- CHECK(store->Read(&key, &val));
- RecordProto image;
- image.ParseFromString(val);
- CHECK_EQ(to->count(), image.data_size());
- float* ptr = to->mutable_cpu_data();
- for (int i = 0; i< to->count(); i++)
- ptr[i] = image.data(i);
- delete store;
-}
-
-bool RecordInputLayer::Parse(int k, int flag, const string& key,
- const string& value) {
- RecordProto image;
- image.ParseFromString(value);
- int size = data_.count() / batchsize_;
- if (image.data_size()) {
- CHECK_EQ(size, image.data_size());
- float* ptr = data_.mutable_cpu_data() + k * size;
- for (int i = 0; i< size; i++)
- ptr[i] = image.data(i);
- } else if (image.pixel().size()) {
- CHECK_EQ(size, image.pixel().size());
- float* ptr = data_.mutable_cpu_data() + k * size;
- string pixel = image.pixel();
- for (int i = 0; i < size; i++)
- ptr[i] = static_cast<float>(static_cast<uint8_t>(pixel[i]));
- } else {
- LOG(ERROR) << "not pixel nor pixel";
- }
- if ((flag & kDeploy) == 0) { // deploy mode does not have label
- aux_data_.at(k) = image.label();
- }
- return true;
-}
-
-} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dd1e4afa/src/neuralnet/input_layer/rnn_label.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/input_layer/rnn_label.cc b/src/neuralnet/input_layer/rnn_label.cc
deleted file mode 100644
index 4924d87..0000000
--- a/src/neuralnet/input_layer/rnn_label.cc
+++ /dev/null
@@ -1,35 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-#include "singa/neuralnet/input_layer.h"
-namespace singa {
-void RNNLabelLayer::Setup(const LayerProto& proto,
- const vector<Layer*>& srclayers) {
- InputLayer::Setup(proto, srclayers);
- aux_data_.resize(srclayers[0]->data(unroll_index() + 1).shape(0));
-}
-void RNNLabelLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
- const float* input = srclayers[0]->data(unroll_index() + 1).cpu_data();
- for (unsigned i = 0; i < aux_data_.size(); i++) {
- aux_data_[i] = static_cast<int>(input[i]);
- }
-}
-} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dd1e4afa/src/neuralnet/input_layer/store.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/input_layer/store.cc b/src/neuralnet/input_layer/store.cc
deleted file mode 100644
index 32f1887..0000000
--- a/src/neuralnet/input_layer/store.cc
+++ /dev/null
@@ -1,162 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-#include "singa/neuralnet/input_layer.h"
-#include "singa/utils/context.h"
-#include "singa/utils/singleton.h"
-namespace singa {
-
-using std::thread;
-
-StoreInputLayer::~StoreInputLayer() {
- if (thread_ != nullptr) {
- thread_->join();
- delete thread_;
- }
- if (store_ != nullptr) {
- delete store_;
- }
-}
-
-void StoreInputLayer::Setup(const LayerProto& conf,
- const vector<Layer*>& srclayers) {
- InputLayer::Setup(conf, srclayers);
- const auto& batchsize = conf.store_conf().batchsize();
- CHECK(batchsize.size());
- if (conf.partition_dim() == 0) {
- if (batchsize.size() == 1) // equal partition
- batchsize_ = batchsize.Get(0) / conf.num_partitions();
- else // manual partition
- batchsize_ = batchsize.Get(conf.partition_id());
- } else {
- batchsize_ = conf.store_conf().batchsize(0);
- }
-
- vector<int> shape {batchsize_};
- for (int s : conf.store_conf().shape())
- shape.push_back(s);
- data_.Reshape(shape);
- aux_data_.resize(batchsize_);
-}
-
-void StoreInputLayer::fetch_data() {
- if (store_ == nullptr) {
- store_ = io::OpenStore(layer_conf_.store_conf().backend(),
- layer_conf_.store_conf().path(),
- io::kRead);
- if (layer_conf_.store_conf().random_skip() > 0) {
- std::uniform_int_distribution<int>
- distribution(0, layer_conf_.store_conf().random_skip());
- auto generator = Singleton<Context>::Instance()->rand_generator(
- std::this_thread::get_id());
- random_skip_ = distribution(*generator);
- }
-
- string key, val;
- while (random_skip_ > 0) {
- if (!store_->Read(&key, &val)) {
- store_->SeekToFirst();
- CHECK(store_->Read(&key, &val));
- }
- random_skip_--;
- }
- buf_keys_.resize(batchsize_);
- buf_vals_.resize(batchsize_);
- }
- for (int k = 0; k < batchsize_; k++) {
- if (!store_->Read(&buf_keys_[k], &buf_vals_[k])) {
- store_->SeekToFirst();
- CHECK(store_->Read(&buf_keys_[k], &buf_vals_[k]));
- }
- }
-}
-
-void StoreInputLayer::ComputeFeature(int flag,
- const vector<Layer*>& srclayers) {
-
- // if prefetching, wait for the thread to finish
- if (layer_conf_.store_conf().prefetching()) {
- if (thread_ == nullptr) {
- thread_ = new thread(&StoreInputLayer::fetch_data, this);
- }
- thread_->join();
- delete thread_;
- thread_ = nullptr;
- } else {
- fetch_data();
- }
- for (int k = 0; k < batchsize_; k++)
- Parse(k, flag, buf_keys_[k], buf_vals_[k]);
- if (layer_conf_.store_conf().prefetching())
- thread_ = new thread(&StoreInputLayer::fetch_data, this);
-}
-
-void SingleLabelRecordLayer::Setup(const LayerProto& conf,
- const vector<Layer*>& srclayers) {
- StoreInputLayer::Setup(conf, srclayers);
-}
-
-void SingleLabelRecordLayer::ComputeFeature(int flag,
- const vector<Layer*>& srclayers) {
-
- StoreInputLayer::ComputeFeature(flag, srclayers);
- auto& store_conf = layer_conf_.store_conf();
-
- if (store_conf.has_mean_file() && mean_.count() == 0) {
- mean_.Reshape(vector<int>{data_.count() / batchsize_});
- LoadRecord(store_conf.backend(), store_conf.mean_file(), &mean_);
- } else if (store_conf.has_mean_value() && mean_.count() == 0) {
- mean_.Reshape(vector<int>{data_.count() / batchsize_});
- for (int i = 0; i < data_.count() / batchsize_; i++)
- mean_.mutable_cpu_data()[i] = store_conf.mean_value();
- }
- if (store_conf.has_std_file() && std_.count() == 0) {
- std_.Reshape(vector<int>{data_.count() / batchsize_});
- LoadRecord(store_conf.backend(), store_conf.std_file(), &std_);
- // TODO(wangwei) check std[i] != 0
- } else if (store_conf.has_std_value() && std_.count() == 0) {
- std_.Reshape(vector<int>{data_.count() / batchsize_});
- CHECK_NE(store_conf.std_value(), 0);
- for (int i = 0; i < data_.count() / batchsize_; i++)
- std_.mutable_cpu_data()[i] = store_conf.std_value();
- }
-
- if (mean_.count()) {
- const float* mean = mean_.cpu_data();
- for (int k = 0; k < batchsize_; k++) {
- float* dptr = data_.mutable_cpu_data() + k * mean_.count();
- for (int i = 0; i < mean_.count(); i++) {
- dptr[i] -= mean[i];
- }
- }
- }
- if (std_.count()) {
- const float* std = std_.cpu_data();
- for (int k = 0; k < batchsize_; k++) {
- float* dptr = data_.mutable_cpu_data() + k * std_.count();
- for (int i = 0; i < std_.count(); i++) {
- dptr[i] /= std[i];
- }
- }
- }
-}
-
-} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dd1e4afa/src/neuralnet/layer.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/layer.cc b/src/neuralnet/layer.cc
deleted file mode 100644
index ef1629f..0000000
--- a/src/neuralnet/layer.cc
+++ /dev/null
@@ -1,82 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-#include "singa/worker.h"
-#include "singa/neuralnet/layer.h"
-#include "singa/neuralnet/input_layer.h"
-#include "singa/neuralnet/neuron_layer.h"
-#include "singa/neuralnet/loss_layer.h"
-
-#include <cblas.h>
-#include <glog/logging.h>
-#include <math.h>
-#include <cfloat>
-#include "singa/utils/factory.h"
-#include "singa/utils/singleton.h"
-#include "singa/utils/math_blob.h"
-
-namespace singa {
-
-using std::string;
-
-void Layer::SetupLayer(Layer* layer, const string str, const vector<Layer*>& srclayers) {
- LayerProto layer_conf;
- layer_conf.ParseFromString(str);
- layer->Setup(layer_conf, srclayers);
- for (auto param : layer->GetParams())
- param->InitValues();
-}
-
-Layer* Layer::CreateLayer(const string str) {
- LayerProto layer_conf;
- layer_conf.ParseFromString(str);
- return Layer::Create(layer_conf);
-}
-
-Layer* Layer::Create(const LayerProto& proto) {
- auto* factory = Singleton<Factory<Layer>>::Instance();
- Layer* layer = nullptr;
- if (proto.has_user_type())
- layer = factory->Create(proto.user_type());
- else
- layer = factory->Create(proto.type());
- return layer;
-}
-
-const std::string Layer::ToString(bool debug, int flag) {
- if (!debug)
- return "";
- string ret = "";
- if ((flag & kForward) == kForward && data_.count() !=0) {
- ret += StringPrintf("data:%e ", Asum(data_));
- for (Param* p : GetParams())
- ret += StringPrintf("%s:%13.9f ",
- p->name().c_str(), Asum(p->data()));
- }
- if ((flag & kBackward) == kBackward && grad_.count() != 0) {
- ret += StringPrintf("grad:%e ", Asum(grad_));
- for (Param* p : GetParams())
- ret += StringPrintf("%s:%13.9f ",
- p->name().c_str(), Asum(p->grad()));
- }
- return ret;
-}
-} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dd1e4afa/src/neuralnet/loss_layer/cudnn_softmaxloss.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/loss_layer/cudnn_softmaxloss.cc b/src/neuralnet/loss_layer/cudnn_softmaxloss.cc
deleted file mode 100644
index 0d4ba45..0000000
--- a/src/neuralnet/loss_layer/cudnn_softmaxloss.cc
+++ /dev/null
@@ -1,83 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-#include "singa/neuralnet/loss_layer.h"
-#include "singa/utils/blob.h"
-#include "singa/utils/math_blob.h"
-#include "singa/utils/math_kernel.h"
-
-namespace singa {
-void CudnnSoftmaxLossLayer::Setup(const LayerProto& conf,
- const vector<Layer*>& srclayers) {
- LossLayer::Setup(conf, srclayers);
- softmax_.Setup(conf, vector<Layer*> {srclayers.at(0)});
- data_.Reshape(softmax_.data(this).shape());
- data_.ShareData(softmax_.mutable_data(this), false);
- batchsize_ = data_.shape(0);
- dim_ = data_.count() / batchsize_;
-}
-void CudnnSoftmaxLossLayer::ComputeFeature(int flag,
- const vector<Layer*>& srclayers) {
- softmax_.ComputeFeature(flag, srclayers);
- Blob<int> label(batchsize_);
- int *labelptr = label.mutable_cpu_data();
- // aux_data: vector<int>, convert vector to int array.
- for (int i = 0; i < batchsize_; ++i) {
- labelptr[i] = srclayers[1]->aux_data(this)[i];
- }
-
- Blob<float> loss(batchsize_);
- singa_gpu_softmaxloss_forward(batchsize_, dim_, data_.gpu_data(),
- label.gpu_data(), loss.mutable_gpu_data());
- loss_ += Asum(loss);
- counter_++;
-}
-
-void CudnnSoftmaxLossLayer::ComputeGradient(int flag,
- const vector<Layer*>& srclayers) {
- Blob<float>* gsrcblob = srclayers[0]->mutable_grad(this);
- Copy(data_, gsrcblob);
- // gsrcblob->CopyFrom(data_);
- float* gsrcptr = gsrcblob->mutable_gpu_data();
-
- Blob<int> label(batchsize_);
- int *labelptr = label.mutable_cpu_data();
-
- // aux_data: vector<int>, convert vector to int array.
- for (int i = 0; i < batchsize_; ++i) {
- labelptr[i] = srclayers[1]->aux_data(this)[i];
- }
-
- singa_gpu_softmaxloss_backward(batchsize_, dim_, 1.0f, label.gpu_data(),
- gsrcptr);
- Scale(1.0f / batchsize_, gsrcblob);
-}
-
-const std::string CudnnSoftmaxLossLayer::ToString(bool debug, int flag) {
- if (debug)
- return Layer::ToString(debug, flag);
-
- string disp = "Loss = " + std::to_string(loss_ / counter_);
- counter_ = 0;
- loss_ = 0;
- return disp;
-}
-} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dd1e4afa/src/neuralnet/loss_layer/euclidean.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/loss_layer/euclidean.cc b/src/neuralnet/loss_layer/euclidean.cc
deleted file mode 100644
index 67c0cd5..0000000
--- a/src/neuralnet/loss_layer/euclidean.cc
+++ /dev/null
@@ -1,80 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-#include <glog/logging.h>
-#include "singa/neuralnet/loss_layer.h"
-#include "mshadow/tensor.h"
-
-namespace singa {
-
-using namespace mshadow;
-using mshadow::cpu;
-
-using mshadow::Shape1;
-using mshadow::Tensor;
-
-using std::vector;
-
-void EuclideanLossLayer::Setup(const LayerProto& conf,
- const vector<Layer*>& srclayers) {
- CHECK_EQ(srclayers.size(), 2);
- Layer::Setup(conf, srclayers);
-}
-
-void EuclideanLossLayer::ComputeFeature(int flag,
- const vector<Layer*>& srclayers) {
- int count = srclayers[0]->data(this).count();
- CHECK_EQ(count, srclayers[1]->data(this).count());
- const float* reconstruct_dptr = srclayers[0]->data(this).cpu_data();
- const float* input_dptr = srclayers[1]->data(this).cpu_data();
- float loss = 0;
- for (int i = 0; i < count; i++) {
- loss += (input_dptr[i] - reconstruct_dptr[i]) *
- (input_dptr[i] - reconstruct_dptr[i]);
- }
- loss_ += loss / srclayers[0]->data(this).shape()[0];
- counter_++;
-}
-
-void EuclideanLossLayer::ComputeGradient(int flag,
- const vector<Layer*>& srclayers) {
- int count = srclayers[0]->data(this).count();
- CHECK_EQ(count, srclayers[1]->data(this).count());
- const float* reconstruct_dptr = srclayers[0]->data(this).cpu_data();
- const float* input_dptr = srclayers[1]->data(this).cpu_data();
- Blob<float>* gsrcblob = srclayers[0]->mutable_grad(this);
- float* gsrcptr = gsrcblob->mutable_cpu_data();
- for (int i = 0; i < count; i++) {
- gsrcptr[i] = reconstruct_dptr[i]-input_dptr[i];
- }
- Tensor<cpu, 1> gsrc(gsrcptr, Shape1(gsrcblob->count()));
- gsrc /= srclayers[0]->data(this).shape()[0];
-}
-const std::string EuclideanLossLayer::ToString(bool debug, int flag) {
- if (debug)
- return Layer::ToString(debug, flag);
-
- string disp = "Loss = " + std::to_string(loss_ / counter_);
- counter_ = 0;
- loss_ = 0;
- return disp;
-}
-} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dd1e4afa/src/neuralnet/loss_layer/softmax.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/loss_layer/softmax.cc b/src/neuralnet/loss_layer/softmax.cc
deleted file mode 100644
index 9d0cb1d..0000000
--- a/src/neuralnet/loss_layer/softmax.cc
+++ /dev/null
@@ -1,112 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-
-#include <glog/logging.h>
-#include <algorithm>
-#include "singa/neuralnet/loss_layer.h"
-#include "mshadow/tensor.h"
-#include "singa/utils/math_blob.h"
-
-namespace singa {
-
-using namespace mshadow;
-using mshadow::cpu;
-
-using mshadow::Shape;
-using mshadow::Shape1;
-using mshadow::Shape2;
-using mshadow::Tensor;
-
-using std::vector;
-
-void SoftmaxLossLayer::Setup(const LayerProto& proto,
- const vector<Layer*>& srclayers) {
- CHECK_EQ(srclayers.size(), 2);
- LossLayer::Setup(proto, srclayers);
- data_.Reshape(srclayers[0]->data(this).shape());
- batchsize_ = data_.shape()[0];
- dim_ = data_.count() / batchsize_;
- topk_ = proto.softmaxloss_conf().topk();
- scale_ = proto.softmaxloss_conf().scale();
-}
-
-void SoftmaxLossLayer::ComputeFeature(int flag,
- const vector<Layer*>& srclayers) {
- Shape<2> s = Shape2(batchsize_, dim_);
- Tensor<cpu, 2> prob(data_.mutable_cpu_data(), s);
- Tensor<cpu, 2> src(srclayers[0]->mutable_data(this)->mutable_cpu_data(), s);
- Softmax(prob, src);
- const auto& label = srclayers[1]->aux_data(this);
- const float* probptr = prob.dptr;
- float loss = 0, precision = 0;
- for (int n = 0; n < batchsize_; n++) {
- int ilabel = static_cast<int>(label[n]);
- // CHECK_LT(ilabel,10);
- CHECK_GE(ilabel, 0);
- float prob_of_truth = probptr[ilabel];
- loss -= log(std::max(prob_of_truth, FLT_MIN));
- vector<std::pair<float, int> > probvec;
- for (int j = 0; j < dim_; ++j) {
- probvec.push_back(std::make_pair(probptr[j], j));
- }
- std::partial_sort(probvec.begin(), probvec.begin() + topk_, probvec.end(),
- std::greater<std::pair<float, int> >());
- // check if true label is in top k predictions
- for (int k = 0; k < topk_; k++) {
- if (probvec[k].second == static_cast<int>(label[n])) {
- precision++;
- break;
- }
- }
- probptr += dim_;
- }
- CHECK_EQ(probptr, prob.dptr + prob.shape.Size());
- loss_ += loss * scale_ / (1.0f * batchsize_);
- accuracy_ += precision * scale_ / (1.0f * batchsize_);
- counter_++;
-}
-
-void SoftmaxLossLayer::ComputeGradient(int flag,
- const vector<Layer*>& srclayers) {
- const auto& label = srclayers[1]->aux_data();
- Blob<float>* gsrcblob = srclayers[0]->mutable_grad(this);
- Copy(data_, gsrcblob);
-// gsrcblob->CopyFrom(data_);
- float* gsrcptr = gsrcblob->mutable_cpu_data();
- for (int n = 0; n < batchsize_; n++) {
- gsrcptr[n*dim_ + static_cast<int>(label[n])] -= 1.0f;
- }
- Tensor<cpu, 1> gsrc(gsrcptr, Shape1(gsrcblob->count()));
- gsrc *= scale_ / (1.0f * batchsize_);
-}
-
-const std::string SoftmaxLossLayer::ToString(bool debug, int flag) {
- if (debug)
- return Layer::ToString(debug, flag);
-
- string disp = "Loss = " + std::to_string(loss_ / counter_)
- + ", accuracy = " + std::to_string(accuracy_ / counter_);
- counter_ = 0;
- loss_ = accuracy_ = 0;
- return disp;
-}
-} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dd1e4afa/src/neuralnet/neuralnet.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuralnet.cc b/src/neuralnet/neuralnet.cc
deleted file mode 100644
index b045e06..0000000
--- a/src/neuralnet/neuralnet.cc
+++ /dev/null
@@ -1,644 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-
-#include "singa/neuralnet/neuralnet.h"
-#include <unordered_map>
-#include <algorithm>
-#include <queue>
-#include "singa/utils/singleton.h"
-
-namespace singa {
-
-using std::map;
-using std::string;
-using std::vector;
-
-/**
- * Check user defined net config and make some preprocessing, e.g., assing names
- * to params.
- * TODO(wnagwei) implement the following functions.
- * 1. layer and paramname should not include '@', '+' and '#'. '@<suffix>'
- * is used for identifying layer location/partition. '<prefix>#' is used for
- * identifying the unrolled Param in RNN models.
- * 2. assign names to unnamed Param, e.g., p<param_id>+<layer_name>.
- */
-const NetProto NetConfPreprocess(const NetProto& conf) {
- /*
- string param_name = "$";
- // if user does not name the param, then name it based on layer name.
- if (param->name() == "") {
- param->set_name(layer->name() + param_name);
- param_name += "$";
- }
- */
- NetProto proto = conf;
- for (int i = 0; i < proto.layer_size(); i++) {
- if (!proto.layer(i).has_unroll_len())
- proto.mutable_layer(i)->set_unroll_len(proto.unroll_len());
- }
- return proto;
-}
-
-NeuralNet* NeuralNet::Create(const NetProto& net_conf, Phase phase,
- int npartitions) {
- const NetProto& full_net_conf = NetConfPreprocess(net_conf);
- NetProto conf = full_net_conf;
- conf.clear_layer();
- // flag=0: neither exclude nor include field appears
- // flag=1: exclude field appears
- // flag=2: include field appears
- int flag = 0;
- // exclude layers according to phase
- // exclude field is deprecated
- // please use include field instead
- for (const auto& layer : full_net_conf.layer()) {
- bool include = true;
- for (auto p : layer.exclude()) {
- // check whether both exclude and include field
- // appear in the same .conf file
- CHECK(flag == 0 || flag == 1) << "Don't use include and exclude together";
- if (p == phase)
- include = false;
- flag = 1;
- }
- // neural net only include the specified layer in the include field
- for (auto p : layer.include()) {
- CHECK(flag == 0 || flag == 2) << "Don't use include and exclude together";
- if (p == phase) {
- include = true;
- break;
- }
- include = false;
- flag = 2;
- }
- if (include == false) continue;
- LayerProto* layer_conf = conf.add_layer();
- layer_conf->CopyFrom(layer);
- // using net partition if layer partition is not set
- if (!layer_conf->has_partition_dim())
- layer_conf->set_partition_dim(net_conf.partition_dim());
- }
- // LOG(INFO) << "Before unrolling: \n" << conf.DebugString();
- conf = Unrolling(conf);
-
- // Copy shared parameters for sharing param conf
- std::vector<ParamProto*> shares;
- std::unordered_map<string, ParamProto*> name2param;
- for (int index = 0; index < conf.layer_size(); index++) {
- LayerProto* layer = conf.mutable_layer(index);
- for (int i = 0; i < layer->param_size(); i++) {
- ParamProto* param = layer->mutable_param(i);
- CHECK(name2param.find(param->name()) == name2param.end())
- << "Repeated param = " << param->name();
- name2param[param->name()] = param;
- if (param->has_share_from() && param->share_from() != "")
- shares.push_back(param);
- }
- }
- for (auto param : shares) {
- const std::string from = param->share_from();
- const std::string name = param->name();
- CHECK(name2param.find(from) != name2param.end())
- << "can't find share_from = " << from;
- // CopyFrom will overwrite the name and share_from fields
- param->CopyFrom(*name2param.at(from));
- param->set_name(name);
- param->set_share_from(from);
- }
- LOG(INFO) << "Initial NeuralNet Config is\n" << conf.DebugString();
- // TODO(wangwei) create net based on net type, e.g., directed, undirected.
- return new NeuralNet(conf, npartitions);
-}
-
-const NetProto NeuralNet::Unrolling(const NetProto& net_conf) {
- // Step 1: Unroll each layer & set parameter sharing
- NetProto conf;
-
- std::vector<std::vector<int>> layer_groups;
- std::unordered_map<string, int> org_layer_names;
- for (int index = 0; index < net_conf.layer_size(); index ++) {
- const LayerProto& org_layer = net_conf.layer(index);
- org_layer_names[org_layer.name()] = index; // layer_name -> index
-
- std::vector<int> layer_group;
- for (int i = 0; i < org_layer.unroll_len(); i ++) { // unroll
- LayerProto* unroll_layer = conf.add_layer();
- unroll_layer->CopyFrom(org_layer); // create a new layer conf
- // update layer names
- std::stringstream sstm;
- sstm << i << '#' << unroll_layer->name();
- unroll_layer->set_name(sstm.str());
- unroll_layer->set_unroll_index(i);
- // update layer parameter sharing
- for (int j = 0; j < unroll_layer->param_size(); j ++) {
- ParamProto* param = unroll_layer->mutable_param(j);
- if (i > 0) {
- param->set_share_from("0#" + param->name());
- }
- std::stringstream sstm1;
- sstm1 << i << '#' << param->name();
- param->set_name(sstm1.str());
- }
- // clear unrolling related fields
- unroll_layer->clear_unroll_len();
- unroll_layer->clear_unroll_conn_type();
- unroll_layer->clear_shift();
- unroll_layer->clear_srclayers();
-
- layer_group.push_back(conf.layer_size() - 1);
- // LOG(ERROR) << "unrolling layer " << unroll_layer->name();
- }
- layer_groups.push_back(layer_group);
- }
- // Step 2: Connect unrolled layers by setting `srclayers`
- for (int index = 0; index < net_conf.layer_size(); index ++) {
- const LayerProto& org_layer = net_conf.layer(index);
- if (org_layer.srclayers_size() == 0)
- continue; // no src layer
- for (int i = 0; i < org_layer.srclayers_size(); i ++) {
- const string& org_layer_src = org_layer.srclayers(i);
- singa::UnrollConnType unroll_conn_type = kUnrollOneToOne;
- if (i < org_layer.unroll_conn_type_size())
- unroll_conn_type = org_layer.unroll_conn_type(i);
- unsigned int shift = 0;
- if (i < org_layer.shift_size())
- shift = org_layer.shift(i);
-
- const std::vector<int> unroll_layer_srcs
- = layer_groups[org_layer_names[org_layer_src]];
-
- for (unsigned int j = 0; j < layer_groups[index].size(); j ++) {
- LayerProto* unroll_layer = conf.mutable_layer(layer_groups[index][j]);
- // Update src layers of `unroll_layer` by considering the types
- if (unroll_conn_type == kUnrollOneToAll) {
- for (int unroll_layer_src : unroll_layer_srcs) {
- unroll_layer->add_srclayers(conf.layer(unroll_layer_src).name());
- }
- } else if (unroll_conn_type == kUnrollOneToOne) {
- if (j < shift) continue; // no need to connect with the src
- int unroll_layer_src = unroll_layer_srcs[j - shift];
- unroll_layer->add_srclayers(conf.layer(unroll_layer_src).name());
- } else if (unroll_conn_type == kUnrollFirstToLast) {
- if (j > 0) break;
- int unroll_layer_src =
- unroll_layer_srcs[unroll_layer_srcs.size() - 1];
- unroll_layer->add_srclayers(conf.layer(unroll_layer_src).name());
- }
- }
- }
-
- // TODO(fanju): add LSTM when it is ready
- if (org_layer.type() == kGRU) { // connect GRU layers
- for (unsigned int j = 1; j < layer_groups[index].size(); j ++) {
- LayerProto* unroll_layer = conf.mutable_layer(layer_groups[index][j]);
- string srcname = conf.layer(layer_groups[index][j-1]).name();
- unroll_layer->add_srclayers(srcname);
- }
- }
- }
- return conf;
-}
-
-
-NeuralNet::NeuralNet(NetProto netproto, int npartitions) {
- LOG(INFO) << "Constructing NeuralNet...";
- auto graph = CreateGraph(netproto, npartitions);
- CreateNetFromGraph(graph);
- PrepareDataStructures();
-
- for (Node* node : graph->nodes())
- delete static_cast<LayerProto*>(node->proto);
- delete graph;
- LOG(INFO) << "NeuralNet Constructed";
- unroll_len_ = netproto.unroll_len();
-}
-
-NeuralNet::~NeuralNet() {
- for (auto layer : layers_)
- delete layer;
-}
-void NeuralNet::Load(const vector<string>& paths) {
- unordered_map<string, Param*> params;
- for (auto p : params_) {
- params[p->name()] = p;
- }
- Load(paths, params);
-}
-void NeuralNet::Load(const vector<string>& paths,
- const unordered_map<string, Param*>& params) {
- for (const auto path : paths) {
- LOG(ERROR) << "Load from checkpoint file " << path;
- BlobProtos bps;
- // TODO(wangwei) extend to read checkpoint from HDFS
- ReadProtoFromBinaryFile(path.c_str(), &bps);
- for (int i = 0; i < bps.name_size(); i++) {
- if (params.find(bps.name(i)) != params.end()) {
- // LOG(ERROR) << "Loading param = " << bps.name(i);
- params.at(bps.name(i))->FromProto(bps.blob(i));
- params.at(bps.name(i))->set_version(bps.version(i));
- }
- }
- }
-}
-
-void NeuralNet::ShareParamsFrom(NeuralNet* other, bool cpu_only) {
- for (auto& layer : layers_) {
- auto otherlayer = other->name2layer(layer->name());
- if (otherlayer != nullptr) {
- const auto& otherparams = otherlayer->GetParams();
- const auto& params = layer->GetParams();
- CHECK_EQ(params.size(), otherparams.size());
- for (size_t i = 0; i < params.size(); i++) {
- params[i]->ShareDataFrom(otherparams[i], cpu_only);
- }
- }
- }
-}
-
-// name of connection layers
-string splitName(const string& layer) { return "split("+layer+")"; }
-string sliceName(const string& layer) { return "slice("+layer+")"; }
-string concateName(const string& layer) { return "concate("+layer+")"; }
-string bridgeName(const string& src, const string& dst) { return src+"->"+dst; }
-string bridgeSrcName(const string& src, const string& dst) {
- return "bridge_src("+bridgeName(src, dst)+")";
-}
-string bridgeDstName(const string& src, const string& dst) {
- return "bridge_dst("+bridgeName(src, dst)+")";
-}
-
-ConnectionType dstLayerConnection(const LayerProto& proto) {
- auto layer = Layer::Create(proto);
- auto ret = layer->dst_layer_connection();
- delete layer;
- return ret;
-}
-
-ConnectionType srcNeuronConnection(const LayerProto& proto) {
- auto layer = Layer::Create(proto);
- auto ret = layer->src_neuron_connection(0);
- delete layer;
- return ret;
-}
-
-NetProto NeuralNet::AddModelSplitLayers(const NetProto& netproto) {
- NetProto net_w_split;
- net_w_split.CopyFrom(netproto);
- net_w_split.clear_layer();
- // calculate number of dst-layers for each layer
- map<string, int> dst_count;
- for (const LayerProto& layer : netproto.layer())
- for (const string& src_name : layer.srclayers())
- ++dst_count[src_name];
- // tag to add split layer if:
- // dst_count[] > 1 && dst_layer_connection() = OneToOne
- for (const LayerProto& layer : netproto.layer())
- if ((dst_count[layer.name()] > 1 && dstLayerConnection(layer) == kOneToOne))
- dst_count[layer.name()] = -dst_count[layer.name()];
- // add orginal layers and adjust srclayers
- for (const LayerProto& layer : netproto.layer()) {
- LayerProto* proto = net_w_split.add_layer();
- proto->CopyFrom(layer);
- proto->clear_srclayers();
- for (const string& src_name : layer.srclayers())
- if (dst_count[src_name] < 0)
- proto->add_srclayers(splitName(src_name));
- else
- proto->add_srclayers(src_name);
- }
- // add split layers
- for (const LayerProto& layer : netproto.layer()) {
- if (dst_count[layer.name()] < 0) {
- LayerProto* split_proto = net_w_split.add_layer();
- split_proto->set_name(splitName(layer.name()));
- split_proto->set_type(kSplit);
- split_proto->set_partition_dim(layer.partition_dim());
- split_proto->add_srclayers(layer.name());
- split_proto->mutable_split_conf()
- ->set_num_splits(-dst_count[layer.name()]);
- }
- }
- // LOG(INFO) << "NeuralNet Config After Model Split is\n"
- // << net_w_split.DebugString();
- return net_w_split;
-}
-
-NetProto NeuralNet::AddPartitionConnectionLayers(const NetProto& netproto,
- int npartitions) {
- CHECK_GT(npartitions, 0);
- NetProto net_w_connection;
- net_w_connection.CopyFrom(netproto);
- // if npartitions is 1, no need to add connection layers
- if (npartitions == 1) return net_w_connection;
- // add original layers, but remove all edges first
- net_w_connection.clear_layer();
- map<string, LayerProto*> name2proto;
- for (const LayerProto& layer : netproto.layer()) {
- LayerProto* layer_proto = net_w_connection.add_layer();
- layer_proto->CopyFrom(layer);
- layer_proto->clear_srclayers();
- name2proto[layer_proto->name()] = layer_proto;
- }
- /*
- * Add Slice, Concate, Split Layers for Model Partition
- *
- * All cases are as follows:
- * src_pdim | dst_pdim | connection_type | Action
- * 0 | 0 | OneToOne | Direct Connection
- * 1 | 1 | OneToOne | Direct Connection
- * 0 | 0 | OneToAll | Direct Connection
- * 1 | 0 | OneToOne | Slice -> Concate
- * 0 | 1 | OneToOne | Slice -> Concate
- * 1 | 0 | OneToAll | Slice -> Concate
- * 0 | 1 | OneToAll | Split -> Concate
- * 1 | 1 | OneToAll | Split -> Concate
- *
- * Logic:
- * dst_pdim = 1 && OneToAll ?
- * (YES) Split -> Concate
- * (NO) src_pdim = dst_pdim ?
- * (YES) Direct Connection
- * (NO) Slice -> Concate
- */
- for (const LayerProto& origin_layer : netproto.layer()) {
- LayerProto* dst_layer = name2proto[origin_layer.name()];
- int dst_pdim = dst_layer->partition_dim();
- ConnectionType connection = srcNeuronConnection(*dst_layer);
- for (const string& src_name : origin_layer.srclayers()) {
- LayerProto* src_layer = name2proto[src_name];
- int src_pdim = src_layer->partition_dim();
- // dst_pdim = 1 && OneToAll ?
- if (dst_pdim == 1 && connection == kOneToAll) {
- // add split layer
- LayerProto* split_layer = net_w_connection.add_layer();
- split_layer->set_name(splitName(src_layer->name()));
- split_layer->set_type(kSplit);
- split_layer->set_partition_dim(src_layer->partition_dim());
- split_layer->add_srclayers(src_layer->name());
- split_layer->mutable_split_conf()->set_num_splits(npartitions);
- // add concate layer
- LayerProto* concate_layer = net_w_connection.add_layer();
- concate_layer->set_name(concateName(split_layer->name()));
- concate_layer->set_type(kConcate);
- concate_layer->set_partition_dim(dst_layer->partition_dim());
- // concate on src_pdim
- concate_layer->mutable_concate_conf()
- ->set_concate_dim(src_layer->partition_dim());
- concate_layer->mutable_concate_conf()->set_num_concates(npartitions);
- concate_layer->add_srclayers(split_layer->name());
- // connect dst_layer to concate layer
- dst_layer->add_srclayers(concate_layer->name());
- } else {
- // src_pdim = dst_pdim ?
- if (dst_pdim == src_pdim) {
- // direct connection
- dst_layer->add_srclayers(src_layer->name());
- } else {
- // add slice layer
- LayerProto* slice_layer = net_w_connection.add_layer();
- slice_layer->set_name(sliceName(src_layer->name()));
- slice_layer->set_type(kSlice);
- slice_layer->set_partition_dim(src_layer->partition_dim());
- // slice on dst_pdim
- slice_layer->mutable_slice_conf()
- ->set_slice_dim(dst_layer->partition_dim());
- slice_layer->mutable_slice_conf()->set_num_slices(npartitions);
- slice_layer->add_srclayers(src_layer->name());
- // add concate layer
- LayerProto* concate_layer = net_w_connection.add_layer();
- concate_layer->set_name(concateName(slice_layer->name()));
- concate_layer->set_type(kConcate);
- concate_layer->set_partition_dim(dst_layer->partition_dim());
- // concate on src_pdim
- concate_layer->mutable_concate_conf()
- ->set_concate_dim(src_layer->partition_dim());
- concate_layer->mutable_concate_conf()->set_num_concates(npartitions);
- concate_layer->add_srclayers(slice_layer->name());
- // connect dst_layer to concate layer
- dst_layer->add_srclayers(concate_layer->name());
- }
- }
- }
- }
- LOG(INFO) << "NeuralNet Config After Adding Connection Layers is\n"
- << net_w_connection.DebugString();
- return net_w_connection;
-}
-
-Graph* NeuralNet::CreateGraph(const NetProto& netproto, int npartitions) {
- NetProto net_w_split = AddModelSplitLayers(netproto);
- NetProto net_w_connection =
- AddPartitionConnectionLayers(net_w_split, npartitions);
- // for each original layer proto, create #npartitions of nodes
- Graph* graph = new Graph();
- map<string, vector<Node*>> name2nodes;
- map<string, const LayerProto*> name2proto;
- for (const LayerProto& layer : net_w_connection.layer()) {
- vector<Node*> nodes;
- for (int i = 0; i < npartitions; i++) {
- LayerProto *proto = new LayerProto(layer);
- // differentiate partitions
- string nodename = layer.name() + "@" + std::to_string(i);
- proto->set_name(nodename);
- proto->set_type(layer.type());
- proto->set_partition_dim(layer.partition_dim());
- proto->set_partition_id(i);
- proto->set_num_partitions(npartitions);
- Node* node = graph->AddNode(nodename, layer.name(), i, proto);
- nodes.push_back(node);
- // TODO(wangwei) update param name
- }
- name2nodes[layer.name()] = nodes;
- name2proto[layer.name()] = &layer;
- }
- // connect layers, add bridge layers if partition id is different
- for (const LayerProto& origin_layer : net_w_connection.layer()) {
- vector<Node*> dst_nodes = name2nodes[origin_layer.name()];
- for (const string& src_name : origin_layer.srclayers()) {
- vector<Node*> src_nodes = name2nodes[src_name];
- if (origin_layer.type() != kConcate) {
- for (size_t i = 0; i < src_nodes.size(); ++i) {
- CHECK_EQ(src_nodes[i]->partition_id, i);
- CHECK_EQ(dst_nodes[i]->partition_id, i);
- graph->AddEdge(src_nodes[i], dst_nodes[i]);
- }
- } else {
- // need to add bridge layers
- for (size_t i = 0; i < src_nodes.size(); ++i) {
- CHECK_EQ(src_nodes[i]->partition_id, i);
- for (size_t j = 0; j < dst_nodes.size(); ++j) {
- CHECK_EQ(dst_nodes[j]->partition_id, j);
- if (i == j) { // in same partition, no bridge needed
- graph->AddEdge(src_nodes[i], dst_nodes[j]);
- } else { // add bridges
- // bridge src && dst layer
- LayerProto *proto_bsrc = new LayerProto();
- LayerProto *proto_bdst = new LayerProto();
- string bsrc_name = bridgeSrcName(src_nodes[i]->name,
- dst_nodes[j]->name);
- string bdst_name = bridgeDstName(src_nodes[i]->name,
- dst_nodes[j]->name);
- proto_bsrc->set_name(bsrc_name);
- proto_bdst->set_name(bdst_name);
- proto_bsrc->set_type(kBridgeSrc);
- proto_bdst->set_type(kBridgeDst);
- proto_bsrc->set_partition_dim(origin_layer.partition_dim());
- proto_bdst->set_partition_dim(origin_layer.partition_dim());
- proto_bsrc->set_partition_id(src_nodes[i]->partition_id);
- proto_bdst->set_partition_id(dst_nodes[j]->partition_id);
- proto_bsrc->set_num_partitions(npartitions);
- proto_bdst->set_num_partitions(npartitions);
- Node* bsrc_node = graph->AddNode(bsrc_name, bsrc_name, i,
- proto_bsrc);
- Node* bdst_node = graph->AddNode(bdst_name, bdst_name, j,
- proto_bdst);
- graph->AddEdge(src_nodes[i], bsrc_node);
- graph->AddEdge(bsrc_node, bdst_node);
- graph->AddEdge(bdst_node, dst_nodes[j]);
- }
- }
- }
- }
- }
- }
- graph->Sort();
- // DLOG(INFO) << "Pure graph structure\n" << graph->ToJson();
- return graph;
-}
-
-void NeuralNet::CreateNetFromGraph(Graph* graph) {
- // create one layer per node
- for (Node* node : graph->nodes()) {
- auto proto_ptr = static_cast<LayerProto*>(node->proto);
- auto layer = Layer::Create(*proto_ptr);
- layers_.push_back(layer);
- name2layer_[node->name] = layer;
- }
- // connect layers
- for (Node* node : graph->nodes()) {
- auto layer = name2layer(node->name);
- src_map_[layer] = vector<Layer*>{};
- for (Node* src : node->srcnodes)
- src_map_[layer].push_back(name2layer(src->name));
- }
- // setup layers
- int paramid = 0;
- map<string, string> layerinfo;
- map<string, vector<Layer*>> share_param_layers;
- for (Node* node : graph->nodes()) {
- LOG(INFO) << "constructing graph: " << node->name;
- auto layer = name2layer(node->name);
- layer->Setup(*(static_cast<LayerProto*>(node->proto)), srclayers(layer));
- DLOG(INFO) << "constructing graph: " << layer->name();
- layerinfo[layer->name()] = IntVecToString(layer->data(nullptr).shape());
- for (auto param : layer->GetParams()) {
- param->set_id(paramid++);
- }
- if (layer->partition_dim() == 0)
- share_param_layers[node->origin].push_back(layer);
- }
- // create map from param name to param ptr
- std::unordered_map<string, Param*> name2param;
- for (auto layer : layers_) {
- for (auto param : layer->GetParams()) {
- name2param[param->name()] = param;
- }
- }
- for (auto & entry : share_param_layers) {
- // overwrite entries for replicated params due to layer partition (dim 0).
- for (auto *param : entry.second.front()->GetParams())
- name2param.at(param->name()) = param;
- }
- // share params based on share_from field
- for (auto & entry : name2param) {
- Param* param = entry.second;
- const string share_from = param->share_from();
- if (param->share_from() != "") {
- if (name2param.find(share_from) != name2param.end()) {
- param->ShareDataFrom(name2param.at(param->share_from()), false);
- } else {
- LOG(FATAL) << "No param with the name (share_from) " << share_from;
- }
- }
- }
-
- // share params due to laye unrolling
- for (auto & entry : name2param) {
- Param* param = entry.second;
- auto pos = param->name().find("#");
- if (pos != std::string::npos && param->owner() != param->id()) {
- string from = "0" + param->name().substr(pos);
- CHECK(name2param.find(from) != name2param.end())
- << "Can't find owner = " << from << " for param = " << param->name();
- Param* owner = name2param.at(from);
- param->ShareFrom(owner);
- }
- }
- // share Params for layers generated (partitioned) from the same origin layer
- for (auto & entry : share_param_layers) {
- const auto& owner = entry.second.begin();
- const auto& owner_params = (*owner)->GetParams();
- for (auto it = owner + 1; it != entry.second.end(); it++) {
- auto params = (*it)->GetParams();
- CHECK_EQ(params.size(), owner_params.size());
- for (size_t i = 0; i < params.size(); i++)
- params.at(i)->ShareDataFrom(owner_params.at(i), true);
- }
- }
-}
-
-void NeuralNet::PrepareDataStructures() {
- params_.clear();
- paramid2param_.clear();
- name2layer_.clear();
- for (auto& layer : layers_) {
- name2layer_[layer->name()] = layer;
- for (Param* p : layer->GetParams()) {
- paramid2param_[p->id()] = p;
- params_.push_back(p);
- }
- }
-}
-
-const Graph NeuralNet::ToGraph(bool include_shape) const {
- Graph g;
- map<string, string> attrs;
- attrs["shape"] = "box";
- vector<string> colors {"black", "red", "yellow", "blue"};
- for (auto layer : layers_) {
- LOG_IF(WARNING, layer->partition_id() >= static_cast<int>(colors.size()))
- << "Too many partitions for displaying";
- attrs["color"] = colors[layer->partition_id() % colors.size()];
- if (include_shape) {
- attrs["label"] = "shape: ";
- for (const auto& x : layer->data(nullptr).shape())
- attrs["label"] += std::to_string(x) + " ";
- }
- g.AddNode(layer->name(), attrs);
- }
-
- for (auto layer : layers_)
- for (auto src : src_map_.at(layer))
- g.AddEdge(src->name(), layer->name());
- return g;
-}
-} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dd1e4afa/src/neuralnet/neuron_layer/activation.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuron_layer/activation.cc b/src/neuralnet/neuron_layer/activation.cc
deleted file mode 100644
index f75961e..0000000
--- a/src/neuralnet/neuron_layer/activation.cc
+++ /dev/null
@@ -1,87 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-#include "singa/neuralnet/neuron_layer.h"
-#include "singa/utils/math_blob.h"
-#include "singa/utils/singa_op.h"
-#include "singa/proto/job.pb.h"
-namespace singa {
-
-void ActivationLayer::Setup(const LayerProto& conf,
- const vector<Layer*>& srclayers) {
- NeuronLayer::Setup(conf, srclayers);
- data_.ReshapeLike(srclayers[0]->data(this));
- grad_.ReshapeLike(data_);
- if (conf.share_src_blobs()) {
- data_.ShareData(srclayers[0]->mutable_data(this), false);
- grad_.ShareData(srclayers[0]->mutable_grad(this), false);
- }
-}
-void
-ActivationLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
- switch (layer_conf_.activation_conf().type()) {
- case RELU:
- Map<op::Relu<float>, float>(srclayers[0]->data(this), &data_);
- break;
- case SIGMOID:
- Map<op::Sigmoid<float>, float>(srclayers[0]->data(this), &data_);
- break;
- case TANH:
- Map<op::Tanh<float>, float>(srclayers[0]->data(this), &data_);
- break;
- /*
- case ActivationType_STANH:
- Map<op::STanh<float>, float>(srclayers[0]->data(this), &data_);
- break;
- */
- default:
- LOG(ERROR) << "Unknow activation type " <<
- layer_conf_.activation_conf().type();
- }
-}
-void
-ActivationLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers) {
- Blob<float> * gsrc = srclayers[0]->mutable_grad(this);
- switch (layer_conf_.activation_conf().type()) {
- case RELU:
- Map<op::ReluGrad<float>, float>(data_, gsrc);
- Mult(*gsrc, grad_, gsrc);
- break;
- case SIGMOID:
- Map<op::SigmoidGrad<float>, float>(data_, gsrc);
- Mult(*gsrc, grad_, gsrc);
- break;
- case TANH:
- Map<op::TanhGrad<float>, float>(data_, gsrc);
- Mult(*gsrc, grad_, gsrc);
- break;
- /*
- case ActivationType_STANH:
- Map<op::STanhGrad<float>, float>(data_, gsrc);
- Mult(*gsrc, grad_, gsrc);
- break;
- */
- default:
- LOG(ERROR) << "Unknow activation type " <<
- layer_conf_.activation_conf().type();
- }
-}
-
-} // namespace singa