You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@singa.apache.org by wa...@apache.org on 2015/10/07 09:26:11 UTC
[4/5] incubator-singa git commit: SINGA-82 Refactor input layers
using data store abstraction
SINGA-82 Refactor input layers using data store abstraction
add header guard for textfile_store.h, image_transform.h
format code
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/dc7f1996
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/dc7f1996
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/dc7f1996
Branch: refs/heads/master
Commit: dc7f1996df26687612f61945bbb58ccbe0db65f4
Parents: 4afc7b8
Author: wang sheng <wa...@gmail.com>
Authored: Wed Oct 7 15:09:21 2015 +0800
Committer: wang sheng <wa...@gmail.com>
Committed: Wed Oct 7 15:19:59 2015 +0800
----------------------------------------------------------------------
examples/mnist/create_data.cc | 5 +++--
examples/rnnlm/rnnlm.cc | 5 +++--
include/io/kvfile.h | 6 +++---
include/io/kvfile_store.h | 9 +++++----
include/io/store.h | 10 +++++++---
include/io/textfile_store.h | 12 +++++++++---
include/neuralnet/input_layer.h | 2 +-
include/singa.h | 2 +-
include/utils/image_transform.h | 8 +++++++-
include/utils/tokenizer.h | 25 +++++++++++++++----------
src/io/kvfile.cc | 7 ++++---
src/io/kvfile_store.cc | 10 +++++-----
src/io/textfile_store.cc | 12 ++++++------
src/neuralnet/input_layer.cc | 8 ++++----
src/test/test_csv_record_layer.cc | 2 +-
src/test/test_proto_record_layer.cc | 2 +-
src/utils/image_transform.cc | 3 ++-
17 files changed, 77 insertions(+), 51 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dc7f1996/examples/mnist/create_data.cc
----------------------------------------------------------------------
diff --git a/examples/mnist/create_data.cc b/examples/mnist/create_data.cc
index aad1f56..14b279f 100644
--- a/examples/mnist/create_data.cc
+++ b/examples/mnist/create_data.cc
@@ -105,9 +105,10 @@ void create_data(const char* image_filename, const char* label_filename,
int main(int argc, char** argv) {
if (argc != 4) {
- std::cout<<"This program create a DataShard for a MNIST dataset\n"
+ std::cout << "This program create a DataShard for a MNIST dataset\n"
"Usage:\n"
- " create_shard.bin input_image_file input_label_file output_db_file\n"
+ " create_shard.bin input_image_file input_label_file"
+ " output_db_file\n"
"The MNIST dataset could be downloaded at\n"
" http://yann.lecun.com/exdb/mnist/\n"
"You should gunzip them after downloading.";
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dc7f1996/examples/rnnlm/rnnlm.cc
----------------------------------------------------------------------
diff --git a/examples/rnnlm/rnnlm.cc b/examples/rnnlm/rnnlm.cc
index a6b65f7..641b465 100644
--- a/examples/rnnlm/rnnlm.cc
+++ b/examples/rnnlm/rnnlm.cc
@@ -18,12 +18,13 @@
* under the License.
*
*************************************************************/
+#include "./rnnlm.h"
+
#include <string>
#include <algorithm>
#include "mshadow/tensor.h"
#include "mshadow/tensor_expr.h"
#include "mshadow/cxxnet_op.h"
-#include "./rnnlm.h"
#include "./rnnlm.pb.h"
namespace rnnlm {
@@ -64,7 +65,7 @@ void DataLayer::Setup(const LayerProto& conf, const vector<Layer*>& srclayers) {
window_ = 0;
}
-void SetInst(int k, WordRecord& word, Blob<float>* to) {
+void SetInst(int k, const WordRecord& word, Blob<float>* to) {
float* dptr = to->mutable_cpu_data() + k * 4;
dptr[0] = static_cast<float>(word.word_index());
dptr[1] = static_cast<float>(word.class_index());
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dc7f1996/include/io/kvfile.h
----------------------------------------------------------------------
diff --git a/include/io/kvfile.h b/include/io/kvfile.h
index d70f198..27dd35e 100644
--- a/include/io/kvfile.h
+++ b/include/io/kvfile.h
@@ -32,8 +32,8 @@
#include <google/protobuf/message.h>
#endif
-namespace singa { namespace io {
-
+namespace singa {
+namespace io {
/**
* KVFile stores training/validation/test tuples.
@@ -171,7 +171,7 @@ class KVFile {
//!< bytes in buf_, used in reading
int bufsize_ = 0;
};
-} /* io */
+} // namespace io
/**
* @deprecated {ShardData is deprecated! Use KVFile}.
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dc7f1996/include/io/kvfile_store.h
----------------------------------------------------------------------
diff --git a/include/io/kvfile_store.h b/include/io/kvfile_store.h
index c3fd868..bcd70eb 100644
--- a/include/io/kvfile_store.h
+++ b/include/io/kvfile_store.h
@@ -19,7 +19,6 @@
*
*************************************************************/
-
#ifndef SINGA_IO_KVFILE_STORE_H_
#define SINGA_IO_KVFILE_STORE_H_
@@ -27,7 +26,8 @@
#include "io/store.h"
#include "io/kvfile.h"
-namespace singa { namespace io {
+namespace singa {
+namespace io {
/**
* Use the KVFile as the data storage.
@@ -49,6 +49,7 @@ class KVFileStore : public Store {
Mode mode_;
};
-} /* io */
-} /* singa */
+} // namespace io
+} // namespace singa
+
#endif // SINGA_IO_KVFILE_STORE_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dc7f1996/include/io/store.h
----------------------------------------------------------------------
diff --git a/include/io/store.h b/include/io/store.h
index f3cc282..15afb6a 100644
--- a/include/io/store.h
+++ b/include/io/store.h
@@ -24,7 +24,9 @@
#include <string>
-namespace singa { namespace io {
+namespace singa {
+namespace io {
+
using std::string;
enum Mode { kCreate, kRead, kAppend };
@@ -96,6 +98,8 @@ Store* CreateStore(const string& backend);
* @param[in] mode kRead or kCreate or kAppend
*/
Store* OpenStore(const string& backend, const string& path, Mode mode);
-} // namespace io
-} /* singa */
+
+} // namespace io
+} // namespace singa
+
#endif // SINGA_IO_STORE_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dc7f1996/include/io/textfile_store.h
----------------------------------------------------------------------
diff --git a/include/io/textfile_store.h b/include/io/textfile_store.h
index 788dc20..5450f00 100644
--- a/include/io/textfile_store.h
+++ b/include/io/textfile_store.h
@@ -19,11 +19,15 @@
*
*************************************************************/
+#ifndef SINGA_IO_TEXTFILE_STORE_H_
+#define SINGA_IO_TEXTFILE_STORE_H_
#include <fstream>
+#include <string>
#include "io/store.h"
-namespace singa { namespace io {
+namespace singa {
+namespace io {
/**
* Use text file as the data storage, one line per tuple.
*
@@ -45,6 +49,8 @@ class TextFileStore : public Store {
std::fstream* fs_ = nullptr;
Mode mode_;
};
-} /* io */
-} /* singa */
+} // namespace io
+} // namespace singa
+
+#endif // SINGA_IO_TEXTFILE_STORE_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dc7f1996/include/neuralnet/input_layer.h
----------------------------------------------------------------------
diff --git a/include/neuralnet/input_layer.h b/include/neuralnet/input_layer.h
index 65e42cb..2e8725e 100644
--- a/include/neuralnet/input_layer.h
+++ b/include/neuralnet/input_layer.h
@@ -24,9 +24,9 @@
#include <string>
#include <vector>
+#include "io/store.h"
#include "neuralnet/layer.h"
#include "utils/data_shard.h"
-#include "io/store.h"
/**
* \file this file includes the declarations of input layers that inherit the
* base InputLayer to load input features.
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dc7f1996/include/singa.h
----------------------------------------------------------------------
diff --git a/include/singa.h b/include/singa.h
index 63acb0a..ddfec0e 100644
--- a/include/singa.h
+++ b/include/singa.h
@@ -23,6 +23,7 @@
#define SINGA_SINGA_H_
#include "comm/socket.h"
+#include "io/store.h"
#include "neuralnet/neuralnet.h"
#include "neuralnet/layer.h"
#include "proto/job.pb.h"
@@ -31,7 +32,6 @@
#include "utils/param.h"
#include "utils/singleton.h"
#include "utils/factory.h"
-#include "io/store.h"
#include "./driver.h"
#endif // SINGA_SINGA_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dc7f1996/include/utils/image_transform.h
----------------------------------------------------------------------
diff --git a/include/utils/image_transform.h b/include/utils/image_transform.h
index 24fe75d..2867ad2 100644
--- a/include/utils/image_transform.h
+++ b/include/utils/image_transform.h
@@ -18,6 +18,10 @@
* under the License.
*
*************************************************************/
+
+#ifndef SINGA_UTILS_IMAGE_TRANSFORM_H_
+#define SINGA_UTILS_IMAGE_TRANSFORM_H_
+
#include <glog/logging.h>
// TODO(wangwei) provide image transformation API, the implementation can be
// done by opencv, manual transform, or mshadow.
@@ -26,4 +30,6 @@ namespace singa {
void ImageTransform(const float* in, const float* mean, bool mirror, int h_crop,
int w_crop, int h_offset, int w_offset, int channel, int height, int width,
float scale, float* out);
-} /* singa */
+} // namespace singa
+
+#endif // SINGA_UTILS_IMAGE_TRANSFORM_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dc7f1996/include/utils/tokenizer.h
----------------------------------------------------------------------
diff --git a/include/utils/tokenizer.h b/include/utils/tokenizer.h
index fc6ba8a..c66e0af 100644
--- a/include/utils/tokenizer.h
+++ b/include/utils/tokenizer.h
@@ -18,10 +18,13 @@
* under the License.
*
*************************************************************/
-#ifndef SINGA_UTILS_TOKENIER_H_
-#define SINGA_UTILS_TOKENIER_H_
-#include <string>
+
+#ifndef SINGA_UTILS_TOKENIZER_H_
+#define SINGA_UTILS_TOKENIZER_H_
+
#include <glog/logging.h>
+#include <string>
+
namespace singa {
/**
* Tokenize a string.
@@ -48,12 +51,14 @@ class Tokenizer {
out = buf_.substr(start, pos);
return *this;
}
-
bool Valid() { return start_ < buf_.length(); }
- private:
- unsigned start_;
- std::string sep_;
- const std::string& buf_;
+
+ private:
+ unsigned start_;
+ std::string sep_;
+ const std::string& buf_;
};
-} /* singa */
-#endif // SINGA_UTILS_TOKENIER_H_
+
+} // namespace singa
+
+#endif // SINGA_UTILS_TOKENIZER_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dc7f1996/src/io/kvfile.cc
----------------------------------------------------------------------
diff --git a/src/io/kvfile.cc b/src/io/kvfile.cc
index aa52150..465d3ee 100644
--- a/src/io/kvfile.cc
+++ b/src/io/kvfile.cc
@@ -22,9 +22,10 @@
#include "io/kvfile.h"
#include <glog/logging.h>
-#include <string>
-namespace singa { namespace io {
+namespace singa {
+namespace io {
+
KVFile::KVFile(const std::string& path, Mode mode, int capacity) :
path_(path), mode_(mode), capacity_(capacity) {
buf_ = new char[capacity];
@@ -213,5 +214,5 @@ bool KVFile::PrepareNextField(int size) {
return true;
}
-} /* io */
+} // namespace io
} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dc7f1996/src/io/kvfile_store.cc
----------------------------------------------------------------------
diff --git a/src/io/kvfile_store.cc b/src/io/kvfile_store.cc
index 11609bf..4d251f9 100644
--- a/src/io/kvfile_store.cc
+++ b/src/io/kvfile_store.cc
@@ -19,11 +19,12 @@
*
*************************************************************/
-
#include "io/kvfile_store.h"
+
#include <glog/logging.h>
-namespace singa { namespace io {
+namespace singa {
+namespace io {
bool KVFileStore::Open(const std::string& source, Mode mode) {
CHECK(file_ == nullptr);
@@ -66,6 +67,5 @@ void KVFileStore::Flush() {
file_->Flush();
}
-} /* io */
-
-} /* singa */
+} // namespace io
+} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dc7f1996/src/io/textfile_store.cc
----------------------------------------------------------------------
diff --git a/src/io/textfile_store.cc b/src/io/textfile_store.cc
index 77694a0..9371de7 100644
--- a/src/io/textfile_store.cc
+++ b/src/io/textfile_store.cc
@@ -23,13 +23,14 @@
#include "io/textfile_store.h"
#include <glog/logging.h>
-namespace singa { namespace io {
+namespace singa {
+namespace io {
+
bool TextFileStore::Open(const std::string& source, Mode mode) {
if (mode == kRead)
fs_ = new std::fstream(source, std::fstream::in);
- else if (mode == kCreate) {
+ else if (mode == kCreate)
fs_ = new std::fstream(source, std::fstream::out);
- }
mode_ = mode;
return fs_->is_open();
}
@@ -79,6 +80,5 @@ void TextFileStore::Flush() {
fs_->flush();
}
-} /* io */
-
-} /* singa */
+} // namespace io
+} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dc7f1996/src/neuralnet/input_layer.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/input_layer.cc b/src/neuralnet/input_layer.cc
index 1599def..d4bef70 100644
--- a/src/neuralnet/input_layer.cc
+++ b/src/neuralnet/input_layer.cc
@@ -104,10 +104,10 @@ void StoreInputLayer::ComputeFeature(int flag,
store_->SeekToFirst();
CHECK(store_->Read(&key, &val));
}
- random_skip_ --;
+ random_skip_--;
}
}
- for (int k = 0; k < batchsize_; k++){
+ for (int k = 0; k < batchsize_; k++) {
if (!store_->Read(&key, &val)) {
store_->SeekToFirst();
CHECK(store_->Read(&key, &val));
@@ -153,7 +153,7 @@ void SingleLabelRecordLayer::ComputeFeature(int flag,
if (mean_.count()) {
const float* mean = mean_.cpu_data();
- for (int k = 0; k < batchsize_; k++){
+ for (int k = 0; k < batchsize_; k++) {
float* dptr = data_.mutable_cpu_data() + k * mean_.count();
for (int i = 0; i < mean_.count(); i++) {
dptr[i] -= mean[i];
@@ -162,7 +162,7 @@ void SingleLabelRecordLayer::ComputeFeature(int flag,
}
if (std_.count()) {
const float* std = std_.cpu_data();
- for (int k = 0; k < batchsize_; k++){
+ for (int k = 0; k < batchsize_; k++) {
float* dptr = data_.mutable_cpu_data() + k * std_.count();
for (int i = 0; i < std_.count(); i++) {
dptr[i] /= std[i];
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dc7f1996/src/test/test_csv_record_layer.cc
----------------------------------------------------------------------
diff --git a/src/test/test_csv_record_layer.cc b/src/test/test_csv_record_layer.cc
index 4517698..2839db9 100644
--- a/src/test/test_csv_record_layer.cc
+++ b/src/test/test_csv_record_layer.cc
@@ -48,7 +48,7 @@ class CSVRecordLayerTest : public ::testing::Test {
TEST_F(CSVRecordLayerTest, Setup) {
singa::CSVRecordLayer layer;
layer.Setup(csv_conf, std::vector<singa::Layer*>{});
- EXPECT_EQ(2, layer.aux_data().size());
+ EXPECT_EQ(2, static_cast<int>(layer.aux_data().size()));
EXPECT_EQ(6, layer.data(nullptr).count());
}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dc7f1996/src/test/test_proto_record_layer.cc
----------------------------------------------------------------------
diff --git a/src/test/test_proto_record_layer.cc b/src/test/test_proto_record_layer.cc
index 0a39c13..03c7230 100644
--- a/src/test/test_proto_record_layer.cc
+++ b/src/test/test_proto_record_layer.cc
@@ -79,7 +79,7 @@ class ProtoRecordLayerTest : public ::testing::Test {
TEST_F(ProtoRecordLayerTest, Setup) {
singa::ProtoRecordLayer layer;
layer.Setup(image_conf, std::vector<singa::Layer*>{});
- EXPECT_EQ(2, layer.aux_data().size());
+ EXPECT_EQ(2, static_cast<int>(layer.aux_data().size()));
EXPECT_EQ(6, layer.data(nullptr).count());
}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dc7f1996/src/utils/image_transform.cc
----------------------------------------------------------------------
diff --git a/src/utils/image_transform.cc b/src/utils/image_transform.cc
index 4851334..1e18e6b 100644
--- a/src/utils/image_transform.cc
+++ b/src/utils/image_transform.cc
@@ -53,4 +53,5 @@ void ImageTransform(const float* in, const float* mean, bool mirror, int h_crop,
}
}
}
-} /* singa */
+
+} // namespace singa