You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@singa.apache.org by wa...@apache.org on 2015/10/07 09:26:08 UTC
[1/5] incubator-singa git commit: SINGA-82 Refactor input layers
using data store abstraction
Repository: incubator-singa
Updated Branches:
refs/heads/master 958dfebbe -> dc7f1996d
SINGA-82 Refactor input layers using data store abstraction
Add Store abstraction for read (writing data). Implemented two backend,
1. KVFile, which was named DataShard. It is a binary file, each tuple
has a unique key.
2. TextFile, which is a plain text file with each line be the value
field of a tuple (the key is the line No.).
TODO, implment HDFS and image folder as the backend.
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/d99b24cb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/d99b24cb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/d99b24cb
Branch: refs/heads/master
Commit: d99b24cb75def9fdbdc59273c4297abb75813c36
Parents: 958dfeb
Author: Wei Wang <wa...@comp.nus.edu.sg>
Authored: Mon Oct 5 19:26:45 2015 +0800
Committer: wang sheng <wa...@gmail.com>
Committed: Wed Oct 7 15:19:58 2015 +0800
----------------------------------------------------------------------
Makefile.am | 13 ++-
include/io/hdfs_store.h | 22 ++++
include/io/imagefolder_store.h | 21 ++++
include/io/kvfile.h | 182 ++++++++++++++++++++++++++++++
include/io/kvfile_store.h | 53 +++++++++
include/io/store.h | 79 +++++++++++++
include/io/textfile_store.h | 49 ++++++++
src/io/kvfile.cc | 217 ++++++++++++++++++++++++++++++++++++
src/io/kvfile_store.cc | 71 ++++++++++++
src/io/store.cc | 57 ++++++++++
src/io/textfile_store.cc | 83 ++++++++++++++
src/test/test_store.cc | 92 +++++++++++++++
12 files changed, 937 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/d99b24cb/Makefile.am
----------------------------------------------------------------------
diff --git a/Makefile.am b/Makefile.am
index 77d6ded..f8e765d 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -46,7 +46,11 @@ SINGA_SRCS := src/driver.cc \
src/neuralnet/output_layer.cc \
src/neuralnet/neuralnet.cc \
src/comm/socket.cc \
- src/comm/msg.cc
+ src/comm/msg.cc \
+ src/io/kvfile.cc \
+ src/io/kvfile_store.cc \
+ src/io/textfile_store.cc \
+ src/io/store.cc
SINGA_HDRS := include/singa.h \
include/utils/cluster.h \
@@ -80,6 +84,10 @@ SINGA_HDRS := include/singa.h \
include/mshadow/tensor_random.h \
include/comm/msg.h \
include/comm/socket.h
+ src/io/store.h \
+ src/io/kvfile.h \
+ src/io/kvfile_store.h \
+ src/io/textfile_store.h
GTEST_SRCS := include/gtest/gtest-all.cc
GTEST_HRDS := include/gtest/gtest.h
@@ -89,7 +97,8 @@ TEST_SRCS := include/gtest/gtest_main.cc \
src/test/test_msg.cc \
src/test/test_neuralnet.cc \
src/test/test_paramslicer.cc \
- src/test/test_shard.cc
+ src/test/test_shard.cc \
+ src/test/test_store.cc
#EXTRA_PROGRAMS = $(PROGS)
EXTRA_PROGRAMS = singatest
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/d99b24cb/include/io/hdfs_store.h
----------------------------------------------------------------------
diff --git a/include/io/hdfs_store.h b/include/io/hdfs_store.h
new file mode 100644
index 0000000..f85615b
--- /dev/null
+++ b/include/io/hdfs_store.h
@@ -0,0 +1,22 @@
+/************************************************************
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+*************************************************************/
+
+// TODO(wangwei) use hdfs as data storage
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/d99b24cb/include/io/imagefolder_store.h
----------------------------------------------------------------------
diff --git a/include/io/imagefolder_store.h b/include/io/imagefolder_store.h
new file mode 100644
index 0000000..c05d92d
--- /dev/null
+++ b/include/io/imagefolder_store.h
@@ -0,0 +1,21 @@
+/************************************************************
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+*************************************************************/
+// TODO(wangwei) store images in a disk folder
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/d99b24cb/include/io/kvfile.h
----------------------------------------------------------------------
diff --git a/include/io/kvfile.h b/include/io/kvfile.h
new file mode 100644
index 0000000..d70f198
--- /dev/null
+++ b/include/io/kvfile.h
@@ -0,0 +1,182 @@
+/************************************************************
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+*************************************************************/
+
+#ifndef SINGA_IO_KVFILE_H_
+#define SINGA_IO_KVFILE_H_
+
+#include <fstream>
+#include <string>
+#include <unordered_set>
+
+#define USE_PROTOBUF 1
+
+#ifdef USE_PROTOBUF
+#include <google/protobuf/message.h>
+#endif
+
+namespace singa { namespace io {
+
+
+/**
+ * KVFile stores training/validation/test tuples.
+ * Every worker node should have a KVFile for training data (validation/test
+ * KVFile is optional).
+ * KVFile consists of a set of unordered tuples. Each tuple is
+ * encoded as [key_len key val_len val] (key_len and val_len are of type
+ * uint32, which indicate the bytes of key and value respectively.
+ *
+ * When KVFile is created, it will remove the last tuple if the value size
+ * and key size do not match because the last write crashed.
+ *
+ * TODO(wangwei) split one KVFile into multiple KVFile s.
+ *
+ */
+class KVFile {
+ public:
+ enum Mode {
+ // read only mode used in training
+ kRead = 0,
+ // write mode used in creating KVFile (will overwrite previous one)
+ kCreate = 1,
+ // append mode, e.g. used when previous creating crashes
+ kAppend = 2
+ };
+
+ /**
+ * KVFile constructor.
+ *
+ * @param path path to the disk KVFile, it can be
+ * - a path to local disk file.
+ * - a path to local directory. This is to be compatible with the older
+ * version (DataShard). The KVFile is shard.dat under that directory
+ * - a hdfs file starting with "hdfs://"
+ * @param mode KVFile open mode, KVFile::kRead, KVFile::kWrite or
+ * KVFile::kAppend
+ * @param bufsize Cache bufsize bytes data for every disk op (read or write),
+ * default is 10MB.
+ */
+ KVFile(const std::string& path, Mode mode, int bufsize = 10485760);
+ ~KVFile();
+
+#ifdef USE_PROTOBUF
+ /**
+ * read next tuple from the KVFile.
+ *
+ * @param key Tuple key
+ * @param val Record of type Message
+ * @return false if read unsuccess, e.g., the tuple was not inserted
+ * completely.
+ */
+ bool Next(std::string* key, google::protobuf::Message* val);
+ /**
+ * Append one tuple to the KVFile.
+ *
+ * @param key e.g., image path
+ * @param val
+ * @return false if unsucess, e.g., inserted before
+ */
+ bool Insert(const std::string& key, const google::protobuf::Message& tuple);
+#endif
+ /**
+ * read next tuple from the KVFile.
+ *
+ * @param key Tuple key
+ * @param val Record of type string
+ * @return false if unsuccess, e.g. the tuple was not inserted completely.
+ */
+ bool Next(std::string* key, std::string* val);
+ /**
+ * Append one tuple to the KVFile.
+ *
+ * @param key e.g., image path
+ * @param val
+ * @return false if unsucess, e.g., inserted before
+ */
+ bool Insert(const std::string& key, const std::string& tuple);
+ /**
+ * Move the read pointer to the head of the KVFile file.
+ * Used for repeated reading.
+ */
+ void SeekToFirst();
+ /**
+ * Flush buffered data to disk.
+ * Used only for kCreate or kAppend.
+ */
+ void Flush();
+ /**
+ * Iterate through all tuples to get the num of all tuples.
+ *
+ * @return num of tuples
+ */
+ int Count();
+ /**
+ * @return path to KVFile file
+ */
+ inline std::string path() { return path_; }
+
+ protected:
+ /**
+ * Read the next key and prepare buffer for reading value.
+ *
+ * @param key
+ * @return length (i.e., bytes) of value field.
+ */
+ int Next(std::string* key);
+ /**
+ * Setup the disk pointer to the right position for append in case that
+ * the pervious write crashes.
+ *
+ * @param path KVFile path.
+ * @return offset (end pos) of the last success written record.
+ */
+ int PrepareForAppend(const std::string& path);
+ /**
+ * Read data from disk if the current data in the buffer is not a full field.
+ *
+ * @param size size of the next field.
+ */
+ bool PrepareNextField(int size);
+
+ private:
+ std::string path_ = "";
+ Mode mode_;
+ //!< either ifstream or ofstream
+ std::fstream fdat_;
+ //!< to avoid replicated record
+ std::unordered_set<std::string> keys_;
+ //!< internal buffer
+ char* buf_ = nullptr;
+ //!< offset inside the buf_
+ int offset_ = 0;
+ //!< allocated bytes for the buf_
+ int capacity_ = 0;
+ //!< bytes in buf_, used in reading
+ int bufsize_ = 0;
+};
+} /* io */
+
+/**
+ * @deprecated {ShardData is deprecated! Use KVFile}.
+using ShardData = KVFile;
+*/
+} // namespace singa
+
+#endif // SINGA_IO_KVFILE_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/d99b24cb/include/io/kvfile_store.h
----------------------------------------------------------------------
diff --git a/include/io/kvfile_store.h b/include/io/kvfile_store.h
new file mode 100644
index 0000000..bda7409
--- /dev/null
+++ b/include/io/kvfile_store.h
@@ -0,0 +1,53 @@
+/************************************************************
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+*************************************************************/
+
+
+#ifndef SINGA_IO_KVFILE_STORE_H_
+#define SINGA_IO_KVFILE_STORE_H_
+
+#include <string>
+#include "io/store.h"
+#include "io/kvfile.h"
+
+namespace singa { namespace io {
+
+/**
+ * Use the KVFile as the data storage.
+ *
+ * KVFile is a binary file. Each tuple is stored as byte string.
+ */
+class KVFileStore : public Store {
+ public:
+ bool Open(const std::string& source, Mode mode) override;
+ void Close() override;
+ bool Read(std::string* key, std::string* value) override;
+ void SeekToFirst() override;
+ bool Write(const std::string& key, const std::string& value) override;
+ void Flush() override;
+
+ private:
+ KVFile* file_ = nullptr;
+ Mode mode_;
+};
+
+} /* io */
+} /* singa */
+#endif // SINGA_IO_KVFILE_STORE_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/d99b24cb/include/io/store.h
----------------------------------------------------------------------
diff --git a/include/io/store.h b/include/io/store.h
new file mode 100644
index 0000000..8665af0
--- /dev/null
+++ b/include/io/store.h
@@ -0,0 +1,79 @@
+/************************************************************
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+*************************************************************/
+
+#ifndef SINGA_IO_STORE_H_
+#define SINGA_IO_STORE_H_
+
+#include <string>
+
+namespace singa { namespace io {
+using std::string;
+enum Mode { kCreate, kRead, kAppend };
+
+/**
+ * General key-value store that provides functions for reading and writing
+ * tuples.
+ *
+ * Subclasses implement the functions for a specific data storage, e.g., CSV
+ * file, HDFS, image folder, singa::io::SFile, leveldb, lmdb, etc.
+ */
+class Store {
+ public:
+ Store() { }
+ virtual ~Store() { }
+ /**
+ * @param[in] source path to the storage, could be a file path, folder path
+ * or hdfs path, or even a http url.
+ * @param[in] mode
+ * @return true if open successfully, otherwise false.
+ */
+ virtual bool Open(const std::string& source, Mode mode) = 0;
+ virtual void Close() = 0;
+ /**
+ * Read a tuple.
+ *
+ * @param[out] key
+ * @param[out] value
+ * @return true if read successfully, otherwise false.
+ */
+ virtual bool Read(std::string* key, std::string* value) = 0;
+ /**
+ * Seek the read header to the first tuple.
+ */
+ virtual void SeekToFirst() = 0;
+ /**
+ * Write a tuple.
+ *
+ * @param[in] key
+ * @param[in] value
+ * @return true if success, otherwise false.
+ */
+ virtual bool Write(const std::string& key, const std::string& value) = 0;
+ /**
+ * Flush writing buffer if it has.
+ */
+ virtual void Flush() {}
+};
+
+Store* CreateStore(const std::string& store);
+} // namespace io
+} /* singa */
+#endif // SINGA_IO_STORE_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/d99b24cb/include/io/textfile_store.h
----------------------------------------------------------------------
diff --git a/include/io/textfile_store.h b/include/io/textfile_store.h
new file mode 100644
index 0000000..4c020e9
--- /dev/null
+++ b/include/io/textfile_store.h
@@ -0,0 +1,49 @@
+/************************************************************
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+*************************************************************/
+
+
+#include <fstream>
+#include "io/store.h"
+
+namespace singa { namespace io {
+/**
+ * Use text file as the data storage, one line per tuple.
+ *
+ * It is used for storeing CSV format data where the key is the line No. and
+ * the value is the line.
+ */
+class TextFileStore : public Store {
+ public:
+ bool Open(const std::string& source, Mode mode) override;
+ void Close() override;
+ bool Read(std::string* key, std::string* value) override;
+ void SeekToFirst() override;
+ bool Write(const std::string& key, const std::string& value) override;
+ void Flush() override;
+
+ private:
+ int lineNo_ = 0;
+ std::fstream* fs_ = nullptr;
+ Mode mode_;
+};
+} /* io */
+
+} /* singa */
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/d99b24cb/src/io/kvfile.cc
----------------------------------------------------------------------
diff --git a/src/io/kvfile.cc b/src/io/kvfile.cc
new file mode 100644
index 0000000..aa52150
--- /dev/null
+++ b/src/io/kvfile.cc
@@ -0,0 +1,217 @@
+/************************************************************
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+*************************************************************/
+
+#include "io/kvfile.h"
+
+#include <glog/logging.h>
+#include <string>
+
+namespace singa { namespace io {
+KVFile::KVFile(const std::string& path, Mode mode, int capacity) :
+path_(path), mode_(mode), capacity_(capacity) {
+ buf_ = new char[capacity];
+ switch (mode) {
+ case KVFile::kRead:
+ fdat_.open(path_, std::ios::in | std::ios::binary);
+ if (!fdat_.is_open()) {
+ // path may be a directory
+ path_ = path + "/shard.dat";
+ fdat_.open(path_, std::ios::in | std::ios::binary);
+ }
+ CHECK(fdat_.is_open()) << "Cannot create file " << path_;
+ break;
+ case KVFile::kCreate:
+ fdat_.open(path_, std::ios::binary | std::ios::out | std::ios::trunc);
+ CHECK(fdat_.is_open()) << "Cannot create file " << path_;
+ break;
+ case KVFile::kAppend:
+ fdat_.open(path_, std::ios::in | std::ios::binary);
+ if (!fdat_.is_open()) {
+ // path may be a directory
+ path_ = path + "/shard.dat";
+ fdat_.open(path_, std::ios::in | std::ios::binary);
+ }
+ CHECK(fdat_.is_open()) << "Cannot open file " << path_;
+ {
+ int last_tuple = PrepareForAppend(path_);
+ fdat_.open(path_, std::ios::binary | std::ios::out
+ | std::ios::in | std::ios::ate);
+ fdat_.seekp(last_tuple);
+ }
+ break;
+ default:
+ LOG(FATAL) << "unknown model to open KVFile " << mode;
+ break;
+ }
+}
+
+KVFile::~KVFile() {
+ if (mode_ != kRead)
+ Flush();
+ delete buf_;
+ fdat_.close();
+}
+#ifdef USE_PROTOBUF
+bool KVFile::Next(std::string* key, google::protobuf::Message* val) {
+ int vallen = Next(key);
+ if (vallen == 0) return false;
+ val->ParseFromArray(buf_ + offset_, vallen);
+ offset_ += vallen;
+ return true;
+}
+
+bool KVFile::Insert(const std::string& key,
+ const google::protobuf::Message& val) {
+ std::string str;
+ val.SerializeToString(&str);
+ return Insert(key, str);
+}
+#endif
+
+bool KVFile::Next(std::string *key, std::string* val) {
+ int vallen = Next(key);
+ if (vallen == 0) return false;
+ val->clear();
+ for (int i = 0; i < vallen; ++i)
+ val->push_back(buf_[offset_ + i]);
+ offset_ += vallen;
+ return true;
+}
+
+// insert one complete tuple
+bool KVFile::Insert(const std::string& key, const std::string& val) {
+ if (keys_.find(key) != keys_.end() || val.size() == 0)
+ return false;
+ int size = key.size() + val.size() + 2*sizeof(size_t);
+ if (bufsize_ + size > capacity_) {
+ fdat_.write(buf_, bufsize_);
+ bufsize_ = 0;
+ CHECK_LE(size, capacity_) << "Tuple size is larger than capacity "
+ << "Try a larger capacity size";
+ }
+ *reinterpret_cast<size_t*>(buf_ + bufsize_) = key.size();
+ bufsize_ += sizeof(size_t);
+ memcpy(buf_ + bufsize_, key.data(), key.size());
+ bufsize_ += key.size();
+ *reinterpret_cast<size_t*>(buf_ + bufsize_) = val.size();
+ bufsize_ += sizeof(size_t);
+ memcpy(buf_ + bufsize_, val.data(), val.size());
+ bufsize_ += val.size();
+ return true;
+}
+
+void KVFile::SeekToFirst() {
+ CHECK_EQ(mode_, kRead);
+ bufsize_ = 0;
+ offset_ = 0;
+ fdat_.clear();
+ fdat_.seekg(0);
+ CHECK(fdat_.is_open()) << "Cannot create file " << path_;
+}
+
+void KVFile::Flush() {
+ fdat_.write(buf_, bufsize_);
+ fdat_.flush();
+ bufsize_ = 0;
+}
+
+int KVFile::Count() {
+ std::ifstream fin(path_, std::ios::in | std::ios::binary);
+ CHECK(fdat_.is_open()) << "Cannot create file " << path_;
+ int count = 0;
+ while (true) {
+ size_t len;
+ fin.read(reinterpret_cast<char*>(&len), sizeof(len));
+ if (!fin.good()) break;
+ fin.seekg(len, std::ios_base::cur);
+ if (!fin.good()) break;
+ fin.read(reinterpret_cast<char*>(&len), sizeof(len));
+ if (!fin.good()) break;
+ fin.seekg(len, std::ios_base::cur);
+ if (!fin.good()) break;
+ count++;
+ }
+ fin.close();
+ return count;
+}
+
+int KVFile::Next(std::string *key) {
+ key->clear();
+ int ssize = sizeof(size_t);
+ if (!PrepareNextField(ssize)) return 0;
+ int keylen = *reinterpret_cast<size_t*>(buf_ + offset_);
+ offset_ += ssize;
+ if (!PrepareNextField(keylen)) return 0;
+ for (int i = 0; i < keylen; ++i)
+ key->push_back(buf_[offset_ + i]);
+ offset_ += keylen;
+ if (!PrepareNextField(ssize)) return 0;
+ int vallen = *reinterpret_cast<size_t*>(buf_ + offset_);
+ offset_ += ssize;
+ if (!PrepareNextField(vallen)) return 0;
+ return vallen;
+}
+
+int KVFile::PrepareForAppend(const std::string& path) {
+ std::ifstream fin(path, std::ios::in | std::ios::binary);
+ if (!fin.is_open()) return 0;
+ int last_tuple_offset = 0;
+ char buf[256];
+ size_t len;
+ while (true) {
+ fin.read(reinterpret_cast<char*>(&len), sizeof(len));
+ if (!fin.good()) break;
+ fin.read(buf, len);
+ buf[len] = '\0';
+ if (!fin.good()) break;
+ fin.read(reinterpret_cast<char*>(&len), sizeof(len));
+ if (!fin.good()) break;
+ fin.seekg(len, std::ios_base::cur);
+ if (!fin.good()) break;
+ keys_.insert(std::string(buf));
+ last_tuple_offset = fin.tellg();
+ }
+ fin.close();
+ return last_tuple_offset;
+}
+
+// if the buf does not have the next complete field, read data from disk
+bool KVFile::PrepareNextField(int size) {
+ if (offset_ + size > bufsize_) {
+ bufsize_ -= offset_;
+ // wangsh: commented, not sure what this check does
+ // CHECK_LE(bufsize_, offset_);
+ for (int i = 0; i < bufsize_; ++i)
+ buf_[i] = buf_[i + offset_];
+ offset_ = 0;
+ if (fdat_.eof()) {
+ return false;
+ } else {
+ fdat_.read(buf_ + bufsize_, capacity_ - bufsize_);
+ bufsize_ += fdat_.gcount();
+ if (size > bufsize_) return false;
+ }
+ }
+ return true;
+}
+
+} /* io */
+} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/d99b24cb/src/io/kvfile_store.cc
----------------------------------------------------------------------
diff --git a/src/io/kvfile_store.cc b/src/io/kvfile_store.cc
new file mode 100644
index 0000000..11609bf
--- /dev/null
+++ b/src/io/kvfile_store.cc
@@ -0,0 +1,71 @@
+/************************************************************
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+*************************************************************/
+
+
+#include "io/kvfile_store.h"
+#include <glog/logging.h>
+
+namespace singa { namespace io {
+
+bool KVFileStore::Open(const std::string& source, Mode mode) {
+ CHECK(file_ == nullptr);
+ if (mode == kRead)
+ file_ = new KVFile(source, KVFile::kRead);
+ else if (mode == kCreate)
+ file_ = new KVFile(source, KVFile::kCreate);
+ else if (mode == kAppend)
+ file_ = new KVFile(source, KVFile::kAppend);
+ mode_ = mode;
+ return file_ != nullptr;
+}
+
+void KVFileStore::Close() {
+ if (file_ != nullptr)
+ delete file_;
+ file_ = nullptr;
+}
+
+bool KVFileStore::Read(std::string* key, std::string* value) {
+ CHECK_EQ(mode_, kRead);
+ CHECK(file_ != nullptr);
+ return file_->Next(key, value);
+}
+
+void KVFileStore::SeekToFirst() {
+ CHECK_EQ(mode_, kRead);
+ CHECK(file_ != nullptr);
+ file_->SeekToFirst();
+}
+bool KVFileStore::Write(const std::string& key, const std::string& value) {
+ CHECK_NE(mode_, kRead);
+ CHECK(file_ != nullptr);
+ return file_->Insert(key, value);
+}
+
+void KVFileStore::Flush() {
+ CHECK_NE(mode_, kRead);
+ CHECK(file_!= nullptr);
+ file_->Flush();
+}
+
+} /* io */
+
+} /* singa */
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/d99b24cb/src/io/store.cc
----------------------------------------------------------------------
diff --git a/src/io/store.cc b/src/io/store.cc
new file mode 100644
index 0000000..6412628
--- /dev/null
+++ b/src/io/store.cc
@@ -0,0 +1,57 @@
+/************************************************************
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+*************************************************************/
+
+
+#include "io/store.h"
+#include "io/kvfile_store.h"
+#include "io/textfile_store.h"
+
+namespace singa { namespace io {
+Store* CreateStore(const std::string& backend) {
+ Store *store = nullptr;
+ if (backend.compare("textfile") == 0) {
+ store = new TextFileStore();
+ } else if (backend.compare("kvfile") == 0) {
+ store = new KVFileStore();
+ }
+
+#ifdef USE_LMDB
+ if (backend == "lmdb") {
+ return new LMDBStore();
+ }
+#endif
+
+#ifdef USE_OPENCV
+ if (backend == "imagefolder") {
+ return new ImageFolderStore();
+ }
+#endif
+
+#ifdef USE_HDFS
+ if (backend == "hdfs") {
+ return new HDFSStore();
+ }
+#endif
+ return store;
+}
+} /* io */
+
+} /* singa */
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/d99b24cb/src/io/textfile_store.cc
----------------------------------------------------------------------
diff --git a/src/io/textfile_store.cc b/src/io/textfile_store.cc
new file mode 100644
index 0000000..74ec9a4
--- /dev/null
+++ b/src/io/textfile_store.cc
@@ -0,0 +1,83 @@
+/************************************************************
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+*************************************************************/
+
+
+#include "io/textfile_store.h"
+#include <glog/logging.h>
+
+namespace singa { namespace io {
+bool TextFileStore::Open(const std::string& source, Mode mode) {
+ if (mode == kRead)
+ fs_ = new std::fstream(source, std::fstream::in);
+ else if (mode == kCreate) {
+ fs_ = new std::fstream(source, std::fstream::out);
+ }
+ mode_ = mode;
+ return fs_->is_open();
+}
+
+void TextFileStore::Close() {
+ if (fs_ != nullptr) {
+ if (fs_->is_open()) {
+ fs_->close();
+ }
+ delete fs_;
+ }
+}
+
+bool TextFileStore::Read(std::string* key, std::string* value) {
+ CHECK_EQ(mode_, kRead);
+ CHECK(fs_ != nullptr);
+ CHECK(value != nullptr);
+ CHECK(key != nullptr);
+ if (!std::getline(*fs_, *value)) {
+ if (fs_->eof())
+ return false;
+ else
+ LOG(FATAL) << "error in reading csv file";
+ }
+ *key = std::to_string(lineNo_++);
+ return true;
+}
+
+void TextFileStore::SeekToFirst() {
+ CHECK_EQ(mode_, kRead);
+ CHECK(fs_ != nullptr);
+ lineNo_ = 0;
+ fs_->clear();
+ fs_->seekg(0);
+}
+
+bool TextFileStore::Write(const std::string& key, const std::string& value) {
+ CHECK_NE(mode_, kRead);
+ CHECK(fs_ != nullptr);
+ // csv store does not write key
+ *fs_ << value << '\n';
+ return true;
+}
+
+void TextFileStore::Flush() {
+ fs_->flush();
+}
+
+} /* io */
+
+} /* singa */
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/d99b24cb/src/test/test_store.cc
----------------------------------------------------------------------
diff --git a/src/test/test_store.cc b/src/test/test_store.cc
new file mode 100644
index 0000000..f69aebb
--- /dev/null
+++ b/src/test/test_store.cc
@@ -0,0 +1,92 @@
+/************************************************************
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+*************************************************************/
+#include <string>
+#include "gtest/gtest.h"
+#include "io/store.h"
+
+TEST(TextFileStore, Open) {
+ auto store = singa::io::CreateStore("textfile");
+ EXPECT_EQ(store->Open("src/test/store.txt", singa::io::kCreate), true);
+ store->Close();
+ EXPECT_EQ(store->Open("src/test/store.txt", singa::io::kRead), true);
+ store->Close();
+}
+
+TEST(TextFileStore, Write) {
+ auto store = singa::io::CreateStore("textfile");
+ store->Open("src/test/store.txt", singa::io::kCreate);
+ store->Write("001", "first tuple");
+ store->Write("002", "second tuple");
+ store->Flush();
+ store->Write("003", "third tuple");
+ store->Close();
+}
+
+TEST(TextFileStore, Read) {
+ auto store = singa::io::CreateStore("textfile");
+ EXPECT_EQ(store->Open("src/test/store.txt", singa::io::kRead), true);
+ std::string key, value;
+ EXPECT_EQ(store->Read(&key, &value), true);
+ EXPECT_EQ(key, "0");
+ EXPECT_EQ(value, "first tuple");
+
+ EXPECT_EQ(store->Read(&key, &value), true);
+ EXPECT_EQ(store->Read(&key, &value), true);
+ EXPECT_EQ(store->Read(&key, &value), false);
+ store->SeekToFirst();
+
+ EXPECT_EQ(store->Read(&key, &value), true);
+ EXPECT_EQ(key, "0");
+ EXPECT_EQ(value, "first tuple");
+}
+TEST(KVFileStore, Open) {
+ auto store = singa::io::CreateStore("kvfile");
+ EXPECT_EQ(store->Open("src/test/store.bin", singa::io::kCreate), true);
+ store->Close();
+ EXPECT_EQ(store->Open("src/test/store.bin", singa::io::kRead), true);
+ store->Close();
+}
+TEST(KVFileStore, Write) {
+ auto store = singa::io::CreateStore("kvfile");
+ store->Open("src/test/store.bin", singa::io::kCreate);
+ store->Write("001", "first tuple");
+ store->Write("002", "second tuple");
+ store->Flush();
+ store->Write("003", "third tuple");
+ store->Close();
+}
+TEST(KVFileStore, Read) {
+ auto store = singa::io::CreateStore("kvfile");
+ store->Open("src/test/store.bin", singa::io::kRead);
+ std::string key, value;
+ EXPECT_EQ(store->Read(&key, &value), true);
+ EXPECT_EQ(key, "001");
+ EXPECT_EQ(value, "first tuple");
+
+ EXPECT_EQ(store->Read(&key, &value), true);
+ EXPECT_EQ(store->Read(&key, &value), true);
+ EXPECT_EQ(store->Read(&key, &value), false);
+ store->SeekToFirst();
+
+ EXPECT_EQ(store->Read(&key, &value), true);
+ EXPECT_EQ(key, "001");
+ EXPECT_EQ(value, "first tuple");
+}
[5/5] incubator-singa git commit: SINGA-82 Refactor input layers
using data store abstraction
Posted by wa...@apache.org.
SINGA-82 Refactor input layers using data store abstraction
* Enable StoreInputLayer to partition on dim = 0.
* Add random skip for StoreInputLayer, which randomly skip some records
at the beginning to make sure workers within the same group and running
in the same process train over different data records.
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/4afc7b89
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/4afc7b89
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/4afc7b89
Branch: refs/heads/master
Commit: 4afc7b893816f0fbb3141e72e3921fcd46feb8e4
Parents: 5f010ca
Author: Wei Wang <wa...@comp.nus.edu.sg>
Authored: Tue Oct 6 22:43:50 2015 +0800
Committer: wang sheng <wa...@gmail.com>
Committed: Wed Oct 7 15:19:59 2015 +0800
----------------------------------------------------------------------
include/neuralnet/input_layer.h | 3 ++-
include/utils/image_transform.h | 29 +++++++++++++++++++++++++++++
src/neuralnet/input_layer.cc | 12 ++++++++++++
3 files changed, 43 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/4afc7b89/include/neuralnet/input_layer.h
----------------------------------------------------------------------
diff --git a/include/neuralnet/input_layer.h b/include/neuralnet/input_layer.h
index 4dfded0..65e42cb 100644
--- a/include/neuralnet/input_layer.h
+++ b/include/neuralnet/input_layer.h
@@ -69,7 +69,8 @@ class StoreInputLayer : virtual public InputLayer {
virtual bool Parse(int k, int flag, const string& key, const string& val) = 0;
protected:
- int batchsize_;
+ int batchsize_ = 1;
+ int random_skip_ = 0;
io::Store* store_ = nullptr;
};
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/4afc7b89/include/utils/image_transform.h
----------------------------------------------------------------------
diff --git a/include/utils/image_transform.h b/include/utils/image_transform.h
new file mode 100644
index 0000000..24fe75d
--- /dev/null
+++ b/include/utils/image_transform.h
@@ -0,0 +1,29 @@
+/************************************************************
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+*************************************************************/
+#include <glog/logging.h>
+// TODO(wangwei) provide image transformation API, the implementation can be
+// done by opencv, manual transform, or mshadow.
+namespace singa {
+
+void ImageTransform(const float* in, const float* mean, bool mirror, int h_crop,
+ int w_crop, int h_offset, int w_offset, int channel, int height, int width,
+ float scale, float* out);
+} /* singa */
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/4afc7b89/src/neuralnet/input_layer.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/input_layer.cc b/src/neuralnet/input_layer.cc
index b4743f4..1599def 100644
--- a/src/neuralnet/input_layer.cc
+++ b/src/neuralnet/input_layer.cc
@@ -85,6 +85,11 @@ void StoreInputLayer::Setup(const LayerProto& conf,
const vector<Layer*>& srclayers) {
InputLayer::Setup(conf, srclayers);
batchsize_ = conf.store_conf().batchsize();
+ if (conf.partition_dim() == 0) {
+ batchsize_ /= conf.num_partitions();
+ }
+ if (conf.store_conf().random_skip() > 0)
+ random_skip_ = rand() % conf.store_conf().random_skip();
}
void StoreInputLayer::ComputeFeature(int flag,
@@ -94,6 +99,13 @@ void StoreInputLayer::ComputeFeature(int flag,
store_ = io::OpenStore(layer_conf_.store_conf().backend(),
layer_conf_.store_conf().path(),
io::kRead);
+ while (random_skip_ > 0) {
+ if (!store_->Read(&key, &val)) {
+ store_->SeekToFirst();
+ CHECK(store_->Read(&key, &val));
+ }
+ random_skip_ --;
+ }
}
for (int k = 0; k < batchsize_; k++){
if (!store_->Read(&key, &val)) {
[3/5] incubator-singa git commit: SINGA-82 Refactor input layers
using data store abstraction
Posted by wa...@apache.org.
SINGA-82 Refactor input layers using data store abstraction
* Add StoreLayer to read data from Store, e.g., KVFile, TextFile (will add support for HDFS later).
* Implemente subclasses of StoreLayer to parse different format tuples, e.g., SingleLabelImageRecord or CSV line.
* Update examples to use the new input layers.
* Add unit tests.
* Add a function for Layer class, which returns a vector<AuxType> for auxiliary data (e.g., label).
TODO
1. make AuxType a template argument of Layer class, and extend data() to return a vector of Blob for multiple dense features.
2. separate layer classeses into different files to make the structure of the source folder clear.
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/5f010caa
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/5f010caa
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/5f010caa
Branch: refs/heads/master
Commit: 5f010caabd7c09cd9fabee666d93a36377639270
Parents: d99b24c
Author: Wei Wang <wa...@comp.nus.edu.sg>
Authored: Tue Oct 6 01:10:40 2015 +0800
Committer: wang sheng <wa...@gmail.com>
Committed: Wed Oct 7 15:19:59 2015 +0800
----------------------------------------------------------------------
Makefile.am | 15 +-
examples/cifar10/Makefile.example | 11 +-
examples/cifar10/create_data.cc | 135 +++++++++
examples/cifar10/create_shard.cc | 131 ---------
examples/cifar10/job.conf | 42 ++-
examples/mnist/Makefile.example | 10 +-
examples/mnist/conv.conf | 42 ++-
examples/mnist/create_data.cc | 119 ++++++++
examples/mnist/create_shard.cc | 120 --------
examples/mnist/job.conf | 46 ++-
examples/rbm/autoencoder.conf | 41 ++-
examples/rbm/rbm1.conf | 52 ++--
examples/rbm/rbm2.conf | 52 ++--
examples/rbm/rbm3.conf | 51 ++--
examples/rbm/rbm4.conf | 53 ++--
examples/rnnlm/Makefile.example | 6 +-
examples/rnnlm/create_data.cc | 472 +++++++++++++++++++++++++++++++
examples/rnnlm/create_shard.cc | 471 ------------------------------
examples/rnnlm/job.conf | 13 +-
examples/rnnlm/main.cc | 1 -
examples/rnnlm/rnnlm.cc | 91 +++---
examples/rnnlm/rnnlm.h | 4 +-
examples/rnnlm/rnnlm.proto | 5 +-
include/io/kvfile_store.h | 1 +
include/io/store.h | 24 +-
include/io/textfile_store.h | 1 +
include/neuralnet/input_layer.h | 121 ++++++++
include/neuralnet/layer.h | 9 +
include/singa.h | 1 +
include/utils/tokenizer.h | 59 ++++
src/driver.cc | 5 +
src/io/store.cc | 7 +-
src/io/textfile_store.cc | 1 +
src/neuralnet/input_layer.cc | 218 +++++++++++++-
src/neuralnet/loss_layer.cc | 4 +-
src/proto/job.proto | 26 +-
src/test/test_csv_record_layer.cc | 92 ++++++
src/test/test_proto_record_layer.cc | 122 ++++++++
src/utils/image_transform.cc | 56 ++++
39 files changed, 1711 insertions(+), 1019 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5f010caa/Makefile.am
----------------------------------------------------------------------
diff --git a/Makefile.am b/Makefile.am
index f8e765d..a1496bd 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -35,6 +35,7 @@ SINGA_SRCS := src/driver.cc \
src/utils/updater.cc \
src/utils/data_shard.cc \
src/utils/blob.cc \
+ src/utils/image_transform.cc \
src/server.cc \
src/worker.cc \
src/stub.cc \
@@ -64,6 +65,8 @@ SINGA_HDRS := include/singa.h \
include/utils/blob.h \
include/utils/updater.h \
include/utils/tinydir.h \
+ include/utils/tokenizer.h \
+ include/utils/image_transform.h \
include/server.h \
include/worker.h \
include/stub.h \
@@ -84,10 +87,10 @@ SINGA_HDRS := include/singa.h \
include/mshadow/tensor_random.h \
include/comm/msg.h \
include/comm/socket.h
- src/io/store.h \
- src/io/kvfile.h \
- src/io/kvfile_store.h \
- src/io/textfile_store.h
+ include/io/store.h \
+ include/io/kvfile.h \
+ include/io/kvfile_store.h \
+ include/io/textfile_store.h
GTEST_SRCS := include/gtest/gtest-all.cc
GTEST_HRDS := include/gtest/gtest.h
@@ -98,7 +101,9 @@ TEST_SRCS := include/gtest/gtest_main.cc \
src/test/test_neuralnet.cc \
src/test/test_paramslicer.cc \
src/test/test_shard.cc \
- src/test/test_store.cc
+ src/test/test_store.cc \
+ src/test/test_proto_record_layer.cc \
+ src/test/test_csv_record_layer.cc
#EXTRA_PROGRAMS = $(PROGS)
EXTRA_PROGRAMS = singatest
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5f010caa/examples/cifar10/Makefile.example
----------------------------------------------------------------------
diff --git a/examples/cifar10/Makefile.example b/examples/cifar10/Makefile.example
index 9e65a58..dd65d7d 100644
--- a/examples/cifar10/Makefile.example
+++ b/examples/cifar10/Makefile.example
@@ -29,12 +29,7 @@ cifar-10-binary-bin:
tar xf cifar-10-binary.tar.gz
create:
- $(CXX) create_shard.cc -std=c++11 -lsinga -lprotobuf -lglog \
+ $(CXX) create_data.cc -std=c++11 -lsinga -lprotobuf -lglog \
-I../../include -L../../.libs/ -Wl,-unresolved-symbols=ignore-in-shared-libs \
- -Wl,-rpath=../../.libs/ -o create_shard.bin
- mkdir cifar10_train_shard
- mkdir cifar10_test_shard
- ./create_shard.bin cifar-10-batches-bin .
-
-
-
+ -Wl,-rpath=../../.libs/ -o create_data.bin
+ ./create_data.bin cifar-10-batches-bin .
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5f010caa/examples/cifar10/create_data.cc
----------------------------------------------------------------------
diff --git a/examples/cifar10/create_data.cc b/examples/cifar10/create_data.cc
new file mode 100644
index 0000000..5fddd1d
--- /dev/null
+++ b/examples/cifar10/create_data.cc
@@ -0,0 +1,135 @@
+/************************************************************
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+*************************************************************/
+
+//
+// This code creates training and test DataShard for CIFAR dataset.
+// It is adapted from the convert_cifar_data from Caffe
+//
+// Usage:
+// create_shard.bin input_folder output_folder
+//
+// The CIFAR dataset could be downloaded at
+// http://www.cs.toronto.edu/~kriz/cifar.html
+//
+
+#include <glog/logging.h>
+#include <fstream>
+#include <string>
+#include <cstdint>
+#include <iostream>
+
+#include "./singa.h"
+
+using std::string;
+
+const int kCIFARSize = 32;
+const int kCIFARImageNBytes = 3072;
+const int kCIFARBatchSize = 10000;
+const int kCIFARTrainBatches = 5;
+
+void read_image(std::ifstream* file, int* label, char* buffer) {
+ char label_char;
+ file->read(&label_char, 1);
+ *label = label_char;
+ file->read(buffer, kCIFARImageNBytes);
+ return;
+}
+
+void create_data(const string& input_folder, const string& output_folder) {
+ int label;
+ char str_buffer[kCIFARImageNBytes];
+ string rec_buf;
+
+ singa::SingleLabelImageRecord image;;
+ image.add_shape(3);
+ image.add_shape(kCIFARSize);
+ image.add_shape(kCIFARSize);
+
+ singa::SingleLabelImageRecord mean;
+ mean.CopyFrom(image);
+ for (int i = 0; i < kCIFARImageNBytes; i++)
+ mean.add_data(0.f);
+
+ auto store = singa::io::CreateStore("kvfile");
+ CHECK(store->Open(output_folder + "/train_data.bin", singa::io::kCreate));
+ LOG(INFO) << "Preparing training data";
+ int count = 0;
+ for (int fileid = 0; fileid < kCIFARTrainBatches; ++fileid) {
+ LOG(INFO) << "Training Batch " << fileid + 1;
+ snprintf(str_buffer, kCIFARImageNBytes, "/data_batch_%d.bin", fileid + 1);
+ std::ifstream data_file((input_folder + str_buffer).c_str(),
+ std::ios::in | std::ios::binary);
+ CHECK(data_file.is_open()) << "Unable to open train file #" << fileid + 1;
+ for (int itemid = 0; itemid < kCIFARBatchSize; ++itemid) {
+ read_image(&data_file, &label, str_buffer);
+ image.set_label(label);
+ image.set_pixel(str_buffer, kCIFARImageNBytes);
+ image.SerializeToString(&rec_buf);
+ int length = snprintf(str_buffer, kCIFARImageNBytes, "%05d", count);
+ CHECK(store->Write(string(str_buffer, length), rec_buf));
+
+ const string& pixels = image.pixel();
+ for (int i = 0; i < kCIFARImageNBytes; i++)
+ mean.set_data(i, mean.data(i) + static_cast<uint8_t>(pixels[i]));
+ count += 1;
+ }
+ }
+ store->Flush();
+ store->Close();
+
+ LOG(INFO) << "Create image mean";
+ store->Open(output_folder + "/image_mean.bin", singa::io::kCreate);
+ for (int i = 0; i < kCIFARImageNBytes; i++)
+ mean.set_data(i, mean.data(i) / count);
+ mean.SerializeToString(&rec_buf);
+ store->Write("mean", rec_buf);
+ store->Flush();
+ store->Close();
+
+ LOG(INFO) << "Create test data";
+ store->Open(output_folder + "/test_data.bin", singa::io::kCreate);
+ std::ifstream data_file((input_folder + "/test_batch.bin").c_str(),
+ std::ios::in | std::ios::binary);
+ CHECK(data_file.is_open()) << "Unable to open test file.";
+ for (int itemid = 0; itemid < kCIFARBatchSize; ++itemid) {
+ read_image(&data_file, &label, str_buffer);
+ image.set_label(label);
+ image.set_pixel(str_buffer, kCIFARImageNBytes);
+ image.SerializeToString(&rec_buf);
+ int length = snprintf(str_buffer, kCIFARImageNBytes, "%05d", itemid);
+ CHECK(store->Write(string(str_buffer, length), rec_buf));
+ }
+ store->Flush();
+ store->Close();
+}
+
+int main(int argc, char** argv) {
+ if (argc != 3) {
+ std::cout <<"Create train and test DataShard for Cifar dataset.\n"
+ << "Usage:\n"
+ << " create_data.bin input_folder output_folder\n"
+ << "Where the input folder should contain the binary batch files.\n";
+ } else {
+ google::InitGoogleLogging(argv[0]);
+ create_data(string(argv[1]), string(argv[2]));
+ }
+ return 0;
+}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5f010caa/examples/cifar10/create_shard.cc
----------------------------------------------------------------------
diff --git a/examples/cifar10/create_shard.cc b/examples/cifar10/create_shard.cc
deleted file mode 100644
index 0a00639..0000000
--- a/examples/cifar10/create_shard.cc
+++ /dev/null
@@ -1,131 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-//
-// This code creates training and test DataShard for CIFAR dataset.
-// It is adapted from the convert_cifar_data from Caffe
-//
-// Usage:
-// create_shard.bin input_folder output_folder
-//
-// The CIFAR dataset could be downloaded at
-// http://www.cs.toronto.edu/~kriz/cifar.html
-//
-
-#include <fstream>
-#include <string>
-
-#include <glog/logging.h>
-#include <cstdint>
-#include <iostream>
-
-#include "singa.h"
-
-using std::string;
-
-using singa::DataShard;
-using singa::WriteProtoToBinaryFile;
-
-const int kCIFARSize = 32;
-const int kCIFARImageNBytes = 3072;
-const int kCIFARBatchSize = 10000;
-const int kCIFARTrainBatches = 5;
-
-void read_image(std::ifstream* file, int* label, char* buffer) {
- char label_char;
- file->read(&label_char, 1);
- *label = label_char;
- file->read(buffer, kCIFARImageNBytes);
- return;
-}
-
-void create_shard(const string& input_folder, const string& output_folder) {
- int label;
- // Data buffer
- char str_buffer[kCIFARImageNBytes];
- singa::Record record;
- singa::SingleLabelImageRecord* image=record.mutable_image();;
- image->add_shape(3);
- image->add_shape(kCIFARSize);
- image->add_shape(kCIFARSize);
-
- singa::SingleLabelImageRecord mean;
- mean.CopyFrom(*image);
- for(int i=0;i<kCIFARImageNBytes;i++)
- mean.add_data(0.);
-
- DataShard train_shard(output_folder+"/cifar10_train_shard",DataShard::kCreate);
- LOG(INFO) << "Writing Training data";
- int count=0;
- for (int fileid = 0; fileid < kCIFARTrainBatches; ++fileid) {
- // Open files
- LOG(INFO) << "Training Batch " << fileid + 1;
- snprintf(str_buffer, kCIFARImageNBytes, "/data_batch_%d.bin", fileid + 1);
- std::ifstream data_file((input_folder + str_buffer).c_str(),
- std::ios::in | std::ios::binary);
- CHECK(data_file) << "Unable to open train file #" << fileid + 1;
- for (int itemid = 0; itemid < kCIFARBatchSize; ++itemid) {
- read_image(&data_file, &label, str_buffer);
- image->set_label(label);
- image->set_pixel(str_buffer, kCIFARImageNBytes);
- int length = snprintf(str_buffer, kCIFARImageNBytes, "%05d",
- fileid * kCIFARBatchSize + itemid);
- CHECK(train_shard.Insert(string(str_buffer, length), record));
-
- const string& pixels=image->pixel();
- for(int i=0;i<kCIFARImageNBytes;i++)
- mean.set_data(i, mean.data(i)+static_cast<uint8_t>(pixels[i]));
- count+=1;
- }
- }
- train_shard.Flush();
- for(int i=0;i<kCIFARImageNBytes;i++)
- mean.set_data(i, mean.data(i)/count);
- WriteProtoToBinaryFile(mean, (output_folder+"/image_mean.bin").c_str());
-
- LOG(INFO) << "Writing Testing data";
- DataShard test_shard(output_folder+"/cifar10_test_shard",DataShard::kCreate);
- // Open files
- std::ifstream data_file((input_folder + "/test_batch.bin").c_str(),
- std::ios::in | std::ios::binary);
- CHECK(data_file) << "Unable to open test file.";
- for (int itemid = 0; itemid < kCIFARBatchSize; ++itemid) {
- read_image(&data_file, &label, str_buffer);
- image->set_label(label);
- image->set_pixel(str_buffer, kCIFARImageNBytes);
- int length = snprintf(str_buffer, kCIFARImageNBytes, "%05d", itemid);
- CHECK(test_shard.Insert(string(str_buffer, length), record));
- }
- test_shard.Flush();
-}
-
-int main(int argc, char** argv) {
- if (argc != 3) {
- std::cout<<"Create train and test DataShard for Cifar dataset.\n"
- <<"Usage:\n"
- <<" create_shard.bin input_folder output_folder\n"
- <<"Where the input folder should contain the binary batch files.\n";
- } else {
- google::InitGoogleLogging(argv[0]);
- create_shard(string(argv[1]), string(argv[2]));
- }
- return 0;
-}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5f010caa/examples/cifar10/job.conf
----------------------------------------------------------------------
diff --git a/examples/cifar10/job.conf b/examples/cifar10/job.conf
index 343d969..57f4b36 100644
--- a/examples/cifar10/job.conf
+++ b/examples/cifar10/job.conf
@@ -1,7 +1,7 @@
name: "cifar10-convnet"
train_steps: 1000
test_steps: 100
-test_freq:300
+test_freq: 300
disp_freq: 30
train_one_batch {
alg: kBP
@@ -24,41 +24,38 @@ updater{
neuralnet {
layer{
name: "data"
- type: kShardData
- sharddata_conf {
- path: "examples/cifar10/cifar10_train_shard"
+ type: kProtoRecord
+ store_conf {
+ backend: "kvfile"
+ path: "examples/cifar10/train_data.bin"
+ mean_file: "examples/cifar10/image_mean.bin"
batchsize: 64
random_skip: 5000
+ shape: 3
+ shape: 32
+ shape: 32
}
exclude: kTest
}
layer{
name: "data"
- type: kShardData
- sharddata_conf {
- path: "examples/cifar10/cifar10_test_shard"
+ type: kProtoRecord
+ store_conf {
+ backend: "kvfile"
+ path: "examples/cifar10/test_data.bin"
+ mean_file: "examples/cifar10/image_mean.bin"
batchsize: 100
+ shape: 3
+ shape: 32
+ shape: 32
}
exclude: kTrain
}
- layer{
- name:"rgb"
- type: kRGBImage
- srclayers: "data"
- rgbimage_conf {
- meanfile: "examples/cifar10/image_mean.bin"
- }
- }
- layer{
- name: "label"
- type: kLabel
- srclayers: "data"
- }
layer {
name: "conv1"
type: kCConvolution
- srclayers: "rgb"
+ srclayers: "data"
convolution_conf {
num_filters: 32
kernel: 5
@@ -223,7 +220,6 @@ neuralnet {
}
}
}
-
layer{
name: "loss"
type: kSoftmaxLoss
@@ -231,7 +227,7 @@ neuralnet {
topk:1
}
srclayers:"ip1"
- srclayers: "label"
+ srclayers: "data"
}
}
cluster {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5f010caa/examples/mnist/Makefile.example
----------------------------------------------------------------------
diff --git a/examples/mnist/Makefile.example b/examples/mnist/Makefile.example
index 4df4edd..733633d 100644
--- a/examples/mnist/Makefile.example
+++ b/examples/mnist/Makefile.example
@@ -34,10 +34,8 @@ mnist:
gunzip t10k-images-idx3-ubyte.gz && gunzip t10k-labels-idx1-ubyte.gz
create:
- $(CXX) create_shard.cc -std=c++11 -lsinga -lprotobuf -lglog -I../../include \
+ $(CXX) create_data.cc -std=c++11 -lsinga -lprotobuf -lglog -I../../include \
-L../../.libs/ -Wl,-unresolved-symbols=ignore-in-shared-libs -Wl,-rpath=../../.libs/ \
- -o create_shard.bin
- mkdir mnist_train_shard
- mkdir mnist_test_shard
- ./create_shard.bin train-images-idx3-ubyte train-labels-idx1-ubyte mnist_train_shard
- ./create_shard.bin t10k-images-idx3-ubyte t10k-labels-idx1-ubyte mnist_test_shard
+ -o create_data.bin
+ ./create_data.bin train-images-idx3-ubyte train-labels-idx1-ubyte train_data.bin
+ ./create_data.bin t10k-images-idx3-ubyte t10k-labels-idx1-ubyte test_data.bin
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5f010caa/examples/mnist/conv.conf
----------------------------------------------------------------------
diff --git a/examples/mnist/conv.conf b/examples/mnist/conv.conf
index 7f7a158..ba631c1 100644
--- a/examples/mnist/conv.conf
+++ b/examples/mnist/conv.conf
@@ -22,43 +22,39 @@ updater {
neuralnet {
layer {
name: "data"
- type: kShardData
- sharddata_conf {
- path: "examples/mnist/mnist_train_shard"
+ type: kProtoRecord
+ store_conf {
+ backend: "kvfile"
+ path: "examples/mnist/train_data.bin"
batchsize: 64
+ std_value: 255
+ random_skip: 5000
+ shape: 1
+ shape: 28
+ shape: 28
}
exclude: kTest
}
layer {
name: "data"
- type: kShardData
- sharddata_conf {
- path: "examples/mnist/mnist_test_shard"
+ type: kProtoRecord
+ store_conf {
+ backend: "kvfile"
+ path: "examples/mnist/test_data.bin"
+ std_value: 255
batchsize: 100
+ shape: 1
+ shape: 28
+ shape: 28
}
exclude: kTrain
}
- layer{
- name:"mnist"
- type: kMnist
- srclayers: "data"
- mnist_conf {
- norm_a:255
- norm_b:0
- }
- }
-
- layer{
- name: "label"
- type: kLabel
- srclayers: "data"
- }
layer {
name: "conv1"
type: kCConvolution
- srclayers: "mnist"
+ srclayers: "data"
convolution_conf {
num_filters: 20
kernel: 5
@@ -181,7 +177,7 @@ neuralnet {
topk:1
}
srclayers:"ip2"
- srclayers:"label"
+ srclayers:"data"
}
}
cluster {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5f010caa/examples/mnist/create_data.cc
----------------------------------------------------------------------
diff --git a/examples/mnist/create_data.cc b/examples/mnist/create_data.cc
new file mode 100644
index 0000000..aad1f56
--- /dev/null
+++ b/examples/mnist/create_data.cc
@@ -0,0 +1,119 @@
+/************************************************************
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+*************************************************************/
+
+//
+// This code creates DataShard for MNIST dataset.
+// It is adapted from the convert_mnist_data from Caffe
+//
+// Usage:
+// create_shard.bin input_image_file input_label_file output_folder
+// The MNIST dataset could be downloaded at
+// http://yann.lecun.com/exdb/mnist/
+
+#include <glog/logging.h>
+#include <cstdint>
+#include <iostream>
+
+#include <fstream>
+#include <string>
+
+#include "io/store.h"
+#include "utils/common.h"
+#include "proto/common.pb.h"
+
+using std::string;
+
+uint32_t swap_endian(uint32_t val) {
+ val = ((val << 8) & 0xFF00FF00) | ((val >> 8) & 0xFF00FF);
+ return (val << 16) | (val >> 16);
+}
+
+void create_data(const char* image_filename, const char* label_filename,
+ const char* output) {
+ // Open files
+ std::ifstream image_file(image_filename, std::ios::in | std::ios::binary);
+ std::ifstream label_file(label_filename, std::ios::in | std::ios::binary);
+ CHECK(image_file) << "Unable to open file " << image_filename;
+ CHECK(label_file) << "Unable to open file " << label_filename;
+ // Read the magic and the meta data
+ uint32_t magic;
+ uint32_t num_items;
+ uint32_t num_labels;
+ uint32_t rows;
+ uint32_t cols;
+
+ image_file.read(reinterpret_cast<char*>(&magic), 4);
+ magic = swap_endian(magic);
+ CHECK_EQ(magic, 2051) << "Incorrect image file magic.";
+ label_file.read(reinterpret_cast<char*>(&magic), 4);
+ magic = swap_endian(magic);
+ CHECK_EQ(magic, 2049) << "Incorrect label file magic.";
+ image_file.read(reinterpret_cast<char*>(&num_items), 4);
+ num_items = swap_endian(num_items);
+ label_file.read(reinterpret_cast<char*>(&num_labels), 4);
+ num_labels = swap_endian(num_labels);
+ CHECK_EQ(num_items, num_labels);
+ image_file.read(reinterpret_cast<char*>(&rows), 4);
+ rows = swap_endian(rows);
+ image_file.read(reinterpret_cast<char*>(&cols), 4);
+ cols = swap_endian(cols);
+
+ auto store = singa::io::OpenStore("kvfile", output, singa::io::kCreate);
+ char label;
+ char* pixels = new char[rows * cols];
+ int count = 0;
+ const int kMaxKeyLength = 10;
+ char key[kMaxKeyLength];
+ string value;
+
+ singa::SingleLabelImageRecord image;
+ image.add_shape(rows);
+ image.add_shape(cols);
+ LOG(INFO) << "A total of " << num_items << " items.";
+ LOG(INFO) << "Rows: " << rows << " Cols: " << cols;
+ for (int item_id = 0; item_id < num_items; ++item_id) {
+ image_file.read(pixels, rows * cols);
+ label_file.read(&label, 1);
+ image.set_pixel(pixels, rows*cols);
+ image.set_label(label);
+ snprintf(key, kMaxKeyLength, "%08d", item_id);
+ image.SerializeToString(&value);
+ store->Write(string(key), value);
+ }
+ delete pixels;
+ store->Flush();
+ delete store;
+}
+
+int main(int argc, char** argv) {
+ if (argc != 4) {
+ std::cout<<"This program create a DataShard for a MNIST dataset\n"
+ "Usage:\n"
+ " create_shard.bin input_image_file input_label_file output_db_file\n"
+ "The MNIST dataset could be downloaded at\n"
+ " http://yann.lecun.com/exdb/mnist/\n"
+ "You should gunzip them after downloading.";
+ } else {
+ google::InitGoogleLogging(argv[0]);
+ create_data(argv[1], argv[2], argv[3]);
+ }
+ return 0;
+}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5f010caa/examples/mnist/create_shard.cc
----------------------------------------------------------------------
diff --git a/examples/mnist/create_shard.cc b/examples/mnist/create_shard.cc
deleted file mode 100644
index 3d7bd97..0000000
--- a/examples/mnist/create_shard.cc
+++ /dev/null
@@ -1,120 +0,0 @@
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-//
-// This code creates DataShard for MNIST dataset.
-// It is adapted from the convert_mnist_data from Caffe
-//
-// Usage:
-// create_shard.bin input_image_file input_label_file output_folder
-// The MNIST dataset could be downloaded at
-// http://yann.lecun.com/exdb/mnist/
-
-#include <glog/logging.h>
-#include <cstdint>
-#include <iostream>
-
-#include <fstream>
-#include <string>
-
-#include "utils/data_shard.h"
-#include "utils/common.h"
-#include "proto/common.pb.h"
-
-using singa::DataShard;
-using singa::WriteProtoToBinaryFile;
-using std::string;
-
-uint32_t swap_endian(uint32_t val) {
- val = ((val << 8) & 0xFF00FF00) | ((val >> 8) & 0xFF00FF);
- return (val << 16) | (val >> 16);
-}
-
-void create_shard(const char* image_filename, const char* label_filename,
- const char* output) {
- // Open files
- std::ifstream image_file(image_filename, std::ios::in | std::ios::binary);
- std::ifstream label_file(label_filename, std::ios::in | std::ios::binary);
- CHECK(image_file) << "Unable to open file " << image_filename;
- CHECK(label_file) << "Unable to open file " << label_filename;
- // Read the magic and the meta data
- uint32_t magic;
- uint32_t num_items;
- uint32_t num_labels;
- uint32_t rows;
- uint32_t cols;
-
- image_file.read(reinterpret_cast<char*>(&magic), 4);
- magic = swap_endian(magic);
- CHECK_EQ(magic, 2051) << "Incorrect image file magic.";
- label_file.read(reinterpret_cast<char*>(&magic), 4);
- magic = swap_endian(magic);
- CHECK_EQ(magic, 2049) << "Incorrect label file magic.";
- image_file.read(reinterpret_cast<char*>(&num_items), 4);
- num_items = swap_endian(num_items);
- label_file.read(reinterpret_cast<char*>(&num_labels), 4);
- num_labels = swap_endian(num_labels);
- CHECK_EQ(num_items, num_labels);
- image_file.read(reinterpret_cast<char*>(&rows), 4);
- rows = swap_endian(rows);
- image_file.read(reinterpret_cast<char*>(&cols), 4);
- cols = swap_endian(cols);
-
- DataShard shard(output, DataShard::kCreate);
- char label;
- char* pixels = new char[rows * cols];
- int count = 0;
- const int kMaxKeyLength = 10;
- char key[kMaxKeyLength];
- string value;
-
- singa::Record record;
- singa::SingleLabelImageRecord* image=record.mutable_image();
- image->add_shape(rows);
- image->add_shape(cols);
- LOG(INFO) << "A total of " << num_items << " items.";
- LOG(INFO) << "Rows: " << rows << " Cols: " << cols;
- for (int item_id = 0; item_id < num_items; ++item_id) {
- image_file.read(pixels, rows * cols);
- label_file.read(&label, 1);
- image->set_pixel(pixels, rows*cols);
- image->set_label(label);
- snprintf(key, kMaxKeyLength, "%08d", item_id);
- shard.Insert(string(key), record);
- }
- delete pixels;
- shard.Flush();
-}
-
-int main(int argc, char** argv) {
- if (argc != 4) {
- std::cout<<"This program create a DataShard for a MNIST dataset\n"
- "Usage:\n"
- " create_shard.bin input_image_file input_label_file output_db_file\n"
- "The MNIST dataset could be downloaded at\n"
- " http://yann.lecun.com/exdb/mnist/\n"
- "You should gunzip them after downloading.";
- } else {
- google::InitGoogleLogging(argv[0]);
- create_shard(argv[1], argv[2], argv[3]);
- }
- return 0;
-}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5f010caa/examples/mnist/job.conf
----------------------------------------------------------------------
diff --git a/examples/mnist/job.conf b/examples/mnist/job.conf
index 6d02561..bfbf17d 100644
--- a/examples/mnist/job.conf
+++ b/examples/mnist/job.conf
@@ -21,45 +21,37 @@ updater{
neuralnet {
layer {
name: "data"
- type: kShardData
- sharddata_conf {
- path: "examples/mnist/mnist_train_shard"
- batchsize: 1000
+ type: kProtoRecord
+ store_conf {
+ backend: "kvfile"
+ path: "examples/mnist/train_data.bin"
+ random_skip: 5000
+ batchsize: 64
+ shape: 784
+ std_value: 127.5
+ mean_value: 127.5
}
exclude: kTest
}
layer {
name: "data"
- type: kShardData
- sharddata_conf {
- path: "examples/mnist/mnist_test_shard"
- batchsize: 1000
+ type: kProtoRecord
+ store_conf {
+ backend: "kvfile"
+ path: "examples/mnist/test_data.bin"
+ batchsize: 100
+ shape: 784
+ std_value: 127.5
+ mean_value: 127.5
}
exclude: kTrain
}
layer{
- name:"mnist"
- type: kMnist
- srclayers: "data"
- mnist_conf {
- norm_a: 127.5
- norm_b: 1
- }
- }
-
-
- layer{
- name: "label"
- type: kLabel
- srclayers: "data"
- }
-
- layer{
name: "fc1"
type: kInnerProduct
- srclayers:"mnist"
+ srclayers:"data"
innerproduct_conf{
num_output: 2500
}
@@ -239,7 +231,7 @@ neuralnet {
topk:1
}
srclayers:"fc6"
- srclayers:"label"
+ srclayers:"data"
}
}
cluster {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5f010caa/examples/rbm/autoencoder.conf
----------------------------------------------------------------------
diff --git a/examples/rbm/autoencoder.conf b/examples/rbm/autoencoder.conf
index c818c6e..5799adf 100644
--- a/examples/rbm/autoencoder.conf
+++ b/examples/rbm/autoencoder.conf
@@ -21,44 +21,35 @@ updater{
neuralnet {
layer {
name: "data"
- type: kShardData
- sharddata_conf {
- path: "examples/mnist/mnist_train_shard"
- batchsize: 1000
+ type: kProtoRecord
+ store_conf {
+ backend: "kvfile"
+ path: "examples/mnist/train_data.bin"
+ batchsize: 100
+ std_value: 255
+ shape: 784
}
exclude: kTest
}
layer {
name: "data"
- type: kShardData
- sharddata_conf {
- path: "examples/mnist/mnist_test_shard"
- batchsize: 1000
+ type: kProtoRecord
+ store_conf {
+ backend: "kvfile"
+ path: "examples/mnist/test_data.bin"
+ std_value: 255
+ batchsize: 100
+ shape: 784
}
exclude: kTrain
}
- layer{
- name:"mnist"
- type: kMnist
- srclayers: "data"
- mnist_conf {
- norm_a: 255
- norm_b: 0
- }
- }
-
- layer{
- name: "label"
- type: kLabel
- srclayers: "data"
- }
layer{
name: "Inner1"
type: kInnerProduct
- srclayers:"mnist"
+ srclayers:"data"
innerproduct_conf{
num_output: 1000
}
@@ -228,7 +219,7 @@ neuralnet {
name: "loss"
type:kEuclideanLoss
srclayers:"Sigmoid8"
- srclayers:"mnist"
+ srclayers:"data"
}
}
cluster {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5f010caa/examples/rbm/rbm1.conf
----------------------------------------------------------------------
diff --git a/examples/rbm/rbm1.conf b/examples/rbm/rbm1.conf
index d185766..1c23d47 100644
--- a/examples/rbm/rbm1.conf
+++ b/examples/rbm/rbm1.conf
@@ -17,42 +17,36 @@ updater{
}
neuralnet {
-layer {
- name: "data"
- type: kShardData
- sharddata_conf {
- path: "examples/mnist/mnist_train_shard"
- batchsize: 100
- }
- exclude: kTest
-}
-
-
-layer {
- name: "data"
- type: kShardData
- sharddata_conf {
- path: "examples/mnist/mnist_test_shard"
- batchsize: 100
+ layer {
+ name: "data"
+ type: kProtoRecord
+ store_conf {
+ backend: "kvfile"
+ path: "examples/mnist/train_data.bin"
+ batchsize: 100
+ std_value: 255
+ shape: 784
+ }
+ exclude: kTest
}
- exclude: kTrain
-}
-
-layer{
- name:"mnist"
- type: kMnist
- srclayers: "data"
- mnist_conf {
- norm_a: 255
- norm_b: 0
+ layer {
+ name: "data"
+ type: kProtoRecord
+ store_conf {
+ backend: "kvfile"
+ path: "examples/mnist/test_data.bin"
+ std_value: 255
+ batchsize: 100
+ shape: 784
+ }
+ exclude: kTrain
}
-}
layer{
name: "RBMVis"
type: kRBMVis
- srclayers:"mnist"
+ srclayers:"data"
srclayers:"RBMHid"
rbm_conf{
hdim: 1000
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5f010caa/examples/rbm/rbm2.conf
----------------------------------------------------------------------
diff --git a/examples/rbm/rbm2.conf b/examples/rbm/rbm2.conf
index 52dc698..2f51208 100644
--- a/examples/rbm/rbm2.conf
+++ b/examples/rbm/rbm2.conf
@@ -18,42 +18,36 @@ updater{
}
neuralnet {
-layer {
- name: "data"
- type: kShardData
- sharddata_conf {
- path: "examples/mnist/mnist_train_shard"
- batchsize: 100
- }
- exclude: kTest
-}
-
-
-layer {
- name: "data"
- type: kShardData
- sharddata_conf {
- path: "examples/mnist/mnist_test_shard"
- batchsize: 100
+ layer {
+ name: "data"
+ type: kProtoRecord
+ store_conf {
+ backend: "kvfile"
+ path: "examples/mnist/train_data.bin"
+ batchsize: 100
+ std_value: 255
+ shape: 784
+ }
+ exclude: kTest
}
- exclude: kTrain
-}
-
-layer{
- name:"mnist"
- type: kMnist
- srclayers: "data"
- mnist_conf {
- norm_a: 255
- norm_b: 0
+ layer {
+ name: "data"
+ type: kProtoRecord
+ store_conf {
+ backend: "kvfile"
+ path: "examples/mnist/test_data.bin"
+ std_value: 255
+ batchsize: 100
+ shape: 784
+ }
+ exclude: kTrain
}
-}
layer{
name: "Inner1"
type: kInnerProduct
- srclayers:"mnist"
+ srclayers:"data"
innerproduct_conf{
num_output: 1000
}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5f010caa/examples/rbm/rbm3.conf
----------------------------------------------------------------------
diff --git a/examples/rbm/rbm3.conf b/examples/rbm/rbm3.conf
index 354fb3b..5df9ae3 100644
--- a/examples/rbm/rbm3.conf
+++ b/examples/rbm/rbm3.conf
@@ -20,42 +20,37 @@ updater{
neuralnet {
-layer {
- name: "data"
- type: kShardData
- sharddata_conf {
- path: "examples/mnist/mnist_train_shard"
- batchsize: 100
+ layer {
+ name: "data"
+ type: kProtoRecord
+ store_conf {
+ backend: "kvfile"
+ path: "examples/mnist/train_data.bin"
+ batchsize: 100
+ std_value: 255
+ shape: 784
+ }
+ exclude: kTest
}
- exclude: kTest
-}
-
-layer {
- name: "data"
- type: kShardData
- sharddata_conf {
- path: "examples/mnist/mnist_test_shard"
- batchsize: 100
+ layer {
+ name: "data"
+ type: kProtoRecord
+ store_conf {
+ backend: "kvfile"
+ path: "examples/mnist/test_data.bin"
+ std_value: 255
+ batchsize: 100
+ shape: 784
+ }
+ exclude: kTrain
}
- exclude: kTrain
-}
layer{
- name:"mnist"
- type: kMnist
- srclayers: "data"
- mnist_conf {
- norm_a: 255
- norm_b: 0
- }
-}
-
-layer{
name: "Inner1"
type: kInnerProduct
- srclayers:"mnist"
+ srclayers:"data"
innerproduct_conf{
num_output: 1000
}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5f010caa/examples/rbm/rbm4.conf
----------------------------------------------------------------------
diff --git a/examples/rbm/rbm4.conf b/examples/rbm/rbm4.conf
index ebf39fa..a34a75c 100644
--- a/examples/rbm/rbm4.conf
+++ b/examples/rbm/rbm4.conf
@@ -18,42 +18,37 @@ updater{
}
neuralnet {
-layer {
- name: "data"
- type: kShardData
- sharddata_conf {
- path: "examples/mnist/mnist_train_shard"
- batchsize: 100
+ layer {
+ name: "data"
+ type: kProtoRecord
+ store_conf {
+ backend: "kvfile"
+ path: "examples/mnist/train_data.bin"
+ batchsize: 100
+ std_value: 255
+ shape: 784
+ }
+ exclude: kTest
}
- exclude: kTest
-}
-
-layer {
- name: "data"
- type: kShardData
- sharddata_conf {
- path: "examples/mnist/mnist_test_shard"
- batchsize: 100
+ layer {
+ name: "data"
+ type: kProtoRecord
+ store_conf {
+ backend: "kvfile"
+ path: "examples/mnist/test_data.bin"
+ std_value: 255
+ batchsize: 100
+ shape: 784
+ }
+ exclude: kTrain
}
- exclude: kTrain
-}
-layer{
- name:"mnist"
- type: kMnist
- srclayers: "data"
- mnist_conf {
- norm_a: 255
- norm_b: 0
- }
-}
-
-layer{
+ layer{
name: "Inner1"
type: kInnerProduct
- srclayers:"mnist"
+ srclayers:"data"
innerproduct_conf{
num_output: 1000
}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5f010caa/examples/rnnlm/Makefile.example
----------------------------------------------------------------------
diff --git a/examples/rnnlm/Makefile.example b/examples/rnnlm/Makefile.example
index 48efd17..5c5ef97 100644
--- a/examples/rnnlm/Makefile.example
+++ b/examples/rnnlm/Makefile.example
@@ -38,11 +38,11 @@ download:
create:
protoc --proto_path=../../src/proto --proto_path=. --cpp_out=. rnnlm.proto
- $(CXX) create_shard.cc rnnlm.pb.cc -std=c++11 -lsinga -lprotobuf -lzookeeper_mt -lglog -I../../include -I../../include/proto \
+ $(CXX) create_data.cc rnnlm.pb.cc -std=c++11 -lsinga -lprotobuf -lzookeeper_mt -lglog -I../../include -I../../include/proto \
-L../../.libs/ -L/usr/local/lib -Wl,-unresolved-symbols=ignore-in-shared-libs -Wl,-rpath=../../.libs/ \
- -o create_shard.bin
+ -o create_data.bin
for d in $(dirshards); do mkdir -p $${d}; done
- ./create_shard.bin -train $(dirname)/train -test $(dirname)/test -valid $(dirname)/valid -class_size $(numclass)
+ ./create_data.bin -train $(dirname)/train -test $(dirname)/test -valid $(dirname)/valid -class_size $(numclass)
rnnlm:
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5f010caa/examples/rnnlm/create_data.cc
----------------------------------------------------------------------
diff --git a/examples/rnnlm/create_data.cc b/examples/rnnlm/create_data.cc
new file mode 100644
index 0000000..d63a8df
--- /dev/null
+++ b/examples/rnnlm/create_data.cc
@@ -0,0 +1,472 @@
+/*
+ * This file include code from rnnlmlib-0.4 whose licence is as follows:
+Copyright (c) 2010-2012 Tomas Mikolov
+Copyright (c) 2013 Cantab Research Ltd
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither name of copyright holders nor the names of its contributors
+may be used to endorse or promote products derived from this software
+without specific prior written permission.
+
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR
+CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+/************************************************************
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+*************************************************************/
+
+//
+// This code creates DataShard for RNNLM dataset.
+// The RNNLM dataset could be downloaded at
+// http://www.rnnlm.org/
+//
+// Usage:
+// create_shard.bin -train [train_file] -valid [valid_file]
+// -test [test_file] -class_size [# of classes]
+
+#include <cstring>
+#include <cstdlib>
+#include <cstdio>
+#include <cmath>
+#include <algorithm>
+#include <fstream>
+
+#include "io/store.h"
+#include "utils/common.h"
+#include "proto/common.pb.h"
+#include "./rnnlm.pb.h"
+
+#define MAX_STRING 100
+#define BUFFER_LEN 32
+#define NL_STRING "</s>"
+
+using std::string;
+using std::max;
+using std::min;
+
+struct vocab_word {
+ int cn;
+ char word[MAX_STRING];
+ int class_index;
+};
+
+struct vocab_word *vocab;
+int vocab_max_size;
+int vocab_size;
+int *vocab_hash;
+int vocab_hash_size;
+int debug_mode;
+int old_classes;
+int *class_start;
+int *class_end;
+int class_size;
+
+char train_file[MAX_STRING];
+char valid_file[MAX_STRING];
+char test_file[MAX_STRING];
+
+int valid_mode;
+int test_mode;
+
+unsigned int getWordHash(char *word) {
+ unsigned int hash, a;
+
+ hash = 0;
+ for (a = 0; a < strlen(word); a++) hash = hash * 237 + word[a];
+ hash = hash % vocab_hash_size;
+
+ return hash;
+}
+
+int searchVocab(char *word) {
+ int a;
+ unsigned int hash;
+
+ hash = getWordHash(word);
+
+ if (vocab_hash[hash] == -1) return -1;
+ if (!strcmp(word, vocab[vocab_hash[hash]].word)) return vocab_hash[hash];
+
+ for (a = 0; a < vocab_size; a++) { // search in vocabulary
+ if (!strcmp(word, vocab[a].word)) {
+ vocab_hash[hash] = a;
+ return a;
+ }
+ }
+
+ return -1; // return OOV if not found
+}
+
+int addWordToVocab(char *word) {
+ unsigned int hash;
+
+ snprintf(vocab[vocab_size].word, strlen(word)+1, "%s", word);
+ vocab[vocab_size].cn = 0;
+ vocab_size++;
+
+ if (vocab_size + 2 >= vocab_max_size) { // reallocate memory if needed
+ vocab_max_size += 100;
+ vocab = (struct vocab_word *) realloc(
+ vocab,
+ vocab_max_size * sizeof(struct vocab_word));
+ }
+
+ hash = getWordHash(word);
+ vocab_hash[hash] = vocab_size - 1;
+
+ return vocab_size - 1;
+}
+
+void readWord(char *word, FILE *fin) {
+ int a = 0, ch;
+
+ while (!feof(fin)) {
+ ch = fgetc(fin);
+
+ if (ch == 13) continue;
+
+ if ((ch == ' ') || (ch == '\t') || (ch == '\n')) {
+ if (a > 0) {
+ if (ch == '\n') ungetc(ch, fin);
+ break;
+ }
+
+ if (ch == '\n') {
+ snprintf(word, strlen(NL_STRING) + 1,
+ "%s", const_cast<char *>(NL_STRING));
+ return;
+ } else {
+ continue;
+ }
+ }
+
+ word[a] = static_cast<char>(ch);
+ a++;
+
+ if (a >= MAX_STRING) {
+ // printf("Too long word found!\n"); //truncate too long words
+ a--;
+ }
+ }
+ word[a] = 0;
+}
+
+void sortVocab() {
+ int a, b, max;
+ vocab_word swap;
+
+ for (a = 1; a < vocab_size; a++) {
+ max = a;
+ for (b = a + 1; b < vocab_size; b++)
+ if (vocab[max].cn < vocab[b].cn) max = b;
+
+ swap = vocab[max];
+ vocab[max] = vocab[a];
+ vocab[a] = swap;
+ }
+}
+
+int learnVocabFromTrainFile() {
+ char word[MAX_STRING];
+ FILE *fin;
+ int a, i, train_wcn;
+
+ for (a = 0; a < vocab_hash_size; a++) vocab_hash[a] = -1;
+
+ fin = fopen(train_file, "rb");
+
+ vocab_size = 0;
+
+ addWordToVocab(const_cast<char *>(NL_STRING));
+
+ train_wcn = 0;
+ while (1) {
+ readWord(word, fin);
+ if (feof(fin)) break;
+
+ train_wcn++;
+
+ i = searchVocab(word);
+ if (i == -1) {
+ a = addWordToVocab(word);
+ vocab[a].cn = 1;
+ } else {
+ vocab[i].cn++;
+ }
+ }
+
+ sortVocab();
+
+ if (debug_mode > 0) {
+ printf("Vocab size: %d\n", vocab_size);
+ printf("Words in train file: %d\n", train_wcn);
+ }
+
+ fclose(fin);
+ return 0;
+}
+
+int splitClasses() {
+ double df, dd;
+ int i, a, b;
+
+ df = 0;
+ dd = 0;
+ a = 0;
+ b = 0;
+
+ class_start = reinterpret_cast<int *>(calloc(class_size, sizeof(int)));
+ memset(class_start, 0x7f, sizeof(int) * class_size);
+ class_end = reinterpret_cast<int *>(calloc(class_size, sizeof(int)));
+ memset(class_end, 0, sizeof(int) * class_size);
+
+ if (old_classes) { // old classes
+ for (i = 0; i < vocab_size; i++)
+ b += vocab[i].cn;
+ for (i = 0; i < vocab_size; i++) {
+ df += vocab[i].cn / static_cast<double>(b);
+ if (df > 1) df = 1;
+ if (df > (a + 1) / static_cast<double>(class_size)) {
+ vocab[i].class_index = a;
+ if (a < class_size - 1) a++;
+ } else {
+ vocab[i].class_index = a;
+ }
+ }
+ } else { // new classes
+ for (i = 0; i < vocab_size; i++)
+ b += vocab[i].cn;
+ for (i = 0; i < vocab_size; i++)
+ dd += sqrt(vocab[i].cn / static_cast<double>(b));
+ for (i = 0; i < vocab_size; i++) {
+ df += sqrt(vocab[i].cn / static_cast<double>(b)) / dd;
+ if (df > 1) df = 1;
+ if (df > (a + 1) / static_cast<double>(class_size)) {
+ vocab[i].class_index = a;
+ if (a < class_size - 1) a++;
+ } else {
+ vocab[i].class_index = a;
+ }
+ }
+ }
+
+ // after dividing classes, update class start and class end information
+ for (i = 0; i < vocab_size; i++) {
+ a = vocab[i].class_index;
+ class_start[a] = min(i, class_start[a]);
+ class_end[a] = max(i + 1, class_end[a]);
+ }
+ return 0;
+}
+
+int init_class() {
+ // debug_mode = 1;
+ debug_mode = 0;
+ vocab_max_size = 100; // largest length value for each word
+ vocab_size = 0;
+ vocab = (struct vocab_word *) calloc(vocab_max_size,
+ sizeof(struct vocab_word));
+ vocab_hash_size = 100000000;
+ vocab_hash = reinterpret_cast<int *>(calloc(vocab_hash_size, sizeof(int)));
+ old_classes = 1;
+
+ // read vocab
+ learnVocabFromTrainFile();
+
+ // split classes
+ splitClasses();
+
+ return 0;
+}
+
+int create_data(const char *input_file, const char *output) {
+ auto* store = singa::io::OpenStore("kvfile", output, singa::io::kCreate);
+ WordRecord wordRecord;
+
+ FILE *fin;
+ int a, i;
+ fin = fopen(input_file, "rb");
+
+ int wcnt = 0;
+ char key[BUFFER_LEN];
+ char wordstr[MAX_STRING];
+ string value;
+ while (1) {
+ readWord(wordstr, fin);
+ if (feof(fin)) break;
+ i = searchVocab(wordstr);
+ if (i == -1) {
+ if (debug_mode) printf("unknown word [%s] detected!", wordstr);
+ } else {
+ wordRecord.set_word(string(wordstr));
+ wordRecord.set_word_index(i);
+ int class_idx = vocab[i].class_index;
+ wordRecord.set_class_index(class_idx);
+ wordRecord.set_class_start(class_start[class_idx]);
+ wordRecord.set_class_end(class_end[class_idx]);
+ int length = snprintf(key, BUFFER_LEN, "%05d", wcnt++);
+ wordRecord.SerializeToString(&value);
+ store->Write(string(key, length), value);
+ }
+ }
+
+ fclose(fin);
+ store->Flush();
+ delete store;
+ return 0;
+}
+
+int argPos(char *str, int argc, char **argv) {
+ int a;
+
+ for (a = 1; a < argc; a++)
+ if (!strcmp(str, argv[a]))
+ return a;
+
+ return -1;
+}
+
+int main(int argc, char **argv) {
+ int i;
+ FILE *f;
+
+ // set debug mode
+ i = argPos(const_cast<char *>("-debug"), argc, argv);
+ if (i > 0) {
+ debug_mode = 1;
+ if (debug_mode > 0)
+ printf("debug mode: %d\n", debug_mode);
+ }
+
+ // search for train file
+ i = argPos(const_cast<char *>("-train"), argc, argv);
+ if (i > 0) {
+ if (i + 1 == argc) {
+ printf("ERROR: training data file not specified!\n");
+ return 0;
+ }
+
+ snprintf(train_file, strlen(argv[i + 1])+1, "%s", argv[i + 1]);
+
+ if (debug_mode > 0)
+ printf("train file: %s\n", train_file);
+
+ f = fopen(train_file, "rb");
+ if (f == NULL) {
+ printf("ERROR: training data file not found!\n");
+ return 0;
+ }
+ fclose(f);
+ } else {
+ printf("ERROR: training data must be set.\n");
+ }
+
+ // search for valid file
+ i = argPos(const_cast<char *>("-valid"), argc, argv);
+ if (i > 0) {
+ if (i + 1 == argc) {
+ printf("ERROR: validating data file not specified!\n");
+ return 0;
+ }
+
+ snprintf(valid_file, strlen(argv[i + 1])+1, "%s", argv[i + 1]);
+
+ if (debug_mode > 0)
+ printf("valid file: %s\n", valid_file);
+
+ f = fopen(valid_file, "rb");
+ if (f == NULL) {
+ printf("ERROR: validating data file not found!\n");
+ return 0;
+ }
+ fclose(f);
+ valid_mode = 1;
+ }
+
+ // search for test file
+ i = argPos(const_cast<char *>("-test"), argc, argv);
+ if (i > 0) {
+ if (i + 1 == argc) {
+ printf("ERROR: testing data file not specified!\n");
+ return 0;
+ }
+
+ snprintf(test_file, strlen(argv[i + 1])+1, "%s", argv[i + 1]);
+
+ if (debug_mode > 0)
+ printf("test file: %s\n", test_file);
+
+ f = fopen(test_file, "rb");
+ if (f == NULL) {
+ printf("ERROR: testing data file not found!\n");
+ return 0;
+ }
+ fclose(f);
+ test_mode = 1;
+ }
+
+ // search for class size
+ i = argPos(const_cast<char *>("-class_size"), argc, argv);
+ if (i > 0) {
+ if (i + 1 == argc) {
+ printf("ERROR: class size not specified!\n");
+ return 0;
+ }
+
+ class_size = atoi(argv[i + 1]);
+
+ if (debug_mode > 0)
+ printf("class size: %d\n", class_size);
+ }
+ if (class_size <= 0) {
+ printf("ERROR: no or invalid class size received!\n");
+ return 0;
+ }
+
+ init_class();
+
+ create_data(train_file, "train_data.bin");
+ if (valid_mode) create_data(valid_file, "valid_data.bin");
+ if (test_mode) create_data(test_file, "test_data.bin");
+
+ return 0;
+}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5f010caa/examples/rnnlm/create_shard.cc
----------------------------------------------------------------------
diff --git a/examples/rnnlm/create_shard.cc b/examples/rnnlm/create_shard.cc
deleted file mode 100644
index 536ce1f..0000000
--- a/examples/rnnlm/create_shard.cc
+++ /dev/null
@@ -1,471 +0,0 @@
-/*
- * This file include code from rnnlmlib-0.4 whose licence is as follows:
-Copyright (c) 2010-2012 Tomas Mikolov
-Copyright (c) 2013 Cantab Research Ltd
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-
-1. Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright
-notice, this list of conditions and the following disclaimer in the
-documentation and/or other materials provided with the distribution.
-
-3. Neither name of copyright holders nor the names of its contributors
-may be used to endorse or promote products derived from this software
-without specific prior written permission.
-
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR
-CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
-EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
-PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
-PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
-LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
-NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*/
-
-/************************************************************
-*
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing,
-* software distributed under the License is distributed on an
-* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-* KIND, either express or implied. See the License for the
-* specific language governing permissions and limitations
-* under the License.
-*
-*************************************************************/
-
-//
-// This code creates DataShard for RNNLM dataset.
-// The RNNLM dataset could be downloaded at
-// http://www.rnnlm.org/
-//
-// Usage:
-// create_shard.bin -train [train_file] -valid [valid_file]
-// -test [test_file] -class_size [# of classes]
-
-#include <cstring>
-#include <cstdlib>
-#include <cstdio>
-#include <cmath>
-#include <algorithm>
-#include <fstream>
-
-#include "utils/data_shard.h"
-#include "utils/common.h"
-#include "proto/common.pb.h"
-#include "./rnnlm.pb.h"
-
-#define MAX_STRING 100
-#define BUFFER_LEN 32
-#define NL_STRING "</s>"
-
-using std::string;
-using std::max;
-using std::min;
-using singa::DataShard;
-
-struct vocab_word {
- int cn;
- char word[MAX_STRING];
- int class_index;
-};
-
-struct vocab_word *vocab;
-int vocab_max_size;
-int vocab_size;
-int *vocab_hash;
-int vocab_hash_size;
-int debug_mode;
-int old_classes;
-int *class_start;
-int *class_end;
-int class_size;
-
-char train_file[MAX_STRING];
-char valid_file[MAX_STRING];
-char test_file[MAX_STRING];
-
-int valid_mode;
-int test_mode;
-
-unsigned int getWordHash(char *word) {
- unsigned int hash, a;
-
- hash = 0;
- for (a = 0; a < strlen(word); a++) hash = hash * 237 + word[a];
- hash = hash % vocab_hash_size;
-
- return hash;
-}
-
-int searchVocab(char *word) {
- int a;
- unsigned int hash;
-
- hash = getWordHash(word);
-
- if (vocab_hash[hash] == -1) return -1;
- if (!strcmp(word, vocab[vocab_hash[hash]].word)) return vocab_hash[hash];
-
- for (a = 0; a < vocab_size; a++) { // search in vocabulary
- if (!strcmp(word, vocab[a].word)) {
- vocab_hash[hash] = a;
- return a;
- }
- }
-
- return -1; // return OOV if not found
-}
-
-int addWordToVocab(char *word) {
- unsigned int hash;
-
- snprintf(vocab[vocab_size].word, strlen(word)+1, "%s", word);
- vocab[vocab_size].cn = 0;
- vocab_size++;
-
- if (vocab_size + 2 >= vocab_max_size) { // reallocate memory if needed
- vocab_max_size += 100;
- vocab = (struct vocab_word *) realloc(
- vocab,
- vocab_max_size * sizeof(struct vocab_word));
- }
-
- hash = getWordHash(word);
- vocab_hash[hash] = vocab_size - 1;
-
- return vocab_size - 1;
-}
-
-void readWord(char *word, FILE *fin) {
- int a = 0, ch;
-
- while (!feof(fin)) {
- ch = fgetc(fin);
-
- if (ch == 13) continue;
-
- if ((ch == ' ') || (ch == '\t') || (ch == '\n')) {
- if (a > 0) {
- if (ch == '\n') ungetc(ch, fin);
- break;
- }
-
- if (ch == '\n') {
- snprintf(word, strlen(NL_STRING) + 1,
- "%s", const_cast<char *>(NL_STRING));
- return;
- } else {
- continue;
- }
- }
-
- word[a] = static_cast<char>(ch);
- a++;
-
- if (a >= MAX_STRING) {
- // printf("Too long word found!\n"); //truncate too long words
- a--;
- }
- }
- word[a] = 0;
-}
-
-void sortVocab() {
- int a, b, max;
- vocab_word swap;
-
- for (a = 1; a < vocab_size; a++) {
- max = a;
- for (b = a + 1; b < vocab_size; b++)
- if (vocab[max].cn < vocab[b].cn) max = b;
-
- swap = vocab[max];
- vocab[max] = vocab[a];
- vocab[a] = swap;
- }
-}
-
-int learnVocabFromTrainFile() {
- char word[MAX_STRING];
- FILE *fin;
- int a, i, train_wcn;
-
- for (a = 0; a < vocab_hash_size; a++) vocab_hash[a] = -1;
-
- fin = fopen(train_file, "rb");
-
- vocab_size = 0;
-
- addWordToVocab(const_cast<char *>(NL_STRING));
-
- train_wcn = 0;
- while (1) {
- readWord(word, fin);
- if (feof(fin)) break;
-
- train_wcn++;
-
- i = searchVocab(word);
- if (i == -1) {
- a = addWordToVocab(word);
- vocab[a].cn = 1;
- } else {
- vocab[i].cn++;
- }
- }
-
- sortVocab();
-
- if (debug_mode > 0) {
- printf("Vocab size: %d\n", vocab_size);
- printf("Words in train file: %d\n", train_wcn);
- }
-
- fclose(fin);
- return 0;
-}
-
-int splitClasses() {
- double df, dd;
- int i, a, b;
-
- df = 0;
- dd = 0;
- a = 0;
- b = 0;
-
- class_start = reinterpret_cast<int *>(calloc(class_size, sizeof(int)));
- memset(class_start, 0x7f, sizeof(int) * class_size);
- class_end = reinterpret_cast<int *>(calloc(class_size, sizeof(int)));
- memset(class_end, 0, sizeof(int) * class_size);
-
- if (old_classes) { // old classes
- for (i = 0; i < vocab_size; i++)
- b += vocab[i].cn;
- for (i = 0; i < vocab_size; i++) {
- df += vocab[i].cn / static_cast<double>(b);
- if (df > 1) df = 1;
- if (df > (a + 1) / static_cast<double>(class_size)) {
- vocab[i].class_index = a;
- if (a < class_size - 1) a++;
- } else {
- vocab[i].class_index = a;
- }
- }
- } else { // new classes
- for (i = 0; i < vocab_size; i++)
- b += vocab[i].cn;
- for (i = 0; i < vocab_size; i++)
- dd += sqrt(vocab[i].cn / static_cast<double>(b));
- for (i = 0; i < vocab_size; i++) {
- df += sqrt(vocab[i].cn / static_cast<double>(b)) / dd;
- if (df > 1) df = 1;
- if (df > (a + 1) / static_cast<double>(class_size)) {
- vocab[i].class_index = a;
- if (a < class_size - 1) a++;
- } else {
- vocab[i].class_index = a;
- }
- }
- }
-
- // after dividing classes, update class start and class end information
- for (i = 0; i < vocab_size; i++) {
- a = vocab[i].class_index;
- class_start[a] = min(i, class_start[a]);
- class_end[a] = max(i + 1, class_end[a]);
- }
- return 0;
-}
-
-int init_class() {
- // debug_mode = 1;
- debug_mode = 0;
- vocab_max_size = 100; // largest length value for each word
- vocab_size = 0;
- vocab = (struct vocab_word *) calloc(vocab_max_size,
- sizeof(struct vocab_word));
- vocab_hash_size = 100000000;
- vocab_hash = reinterpret_cast<int *>(calloc(vocab_hash_size, sizeof(int)));
- old_classes = 1;
-
- // read vocab
- learnVocabFromTrainFile();
-
- // split classes
- splitClasses();
-
- return 0;
-}
-
-int create_shard(const char *input_file, const char *output) {
- DataShard dataShard(output, DataShard::kCreate);
- singa::Record record;
- auto* wordRecord = record.MutableExtension(word);
-
- FILE *fin;
- int a, i;
- fin = fopen(input_file, "rb");
-
- int wcnt = 0;
- char key[BUFFER_LEN];
- char wordstr[MAX_STRING];
- while (1) {
- readWord(wordstr, fin);
- if (feof(fin)) break;
- i = searchVocab(wordstr);
- if (i == -1) {
- if (debug_mode) printf("unknown word [%s] detected!", wordstr);
- } else {
- wordRecord->set_word(string(wordstr));
- wordRecord->set_word_index(i);
- int class_idx = vocab[i].class_index;
- wordRecord->set_class_index(class_idx);
- wordRecord->set_class_start(class_start[class_idx]);
- wordRecord->set_class_end(class_end[class_idx]);
- int length = snprintf(key, BUFFER_LEN, "%05d", wcnt++);
- dataShard.Insert(string(key, length), record);
- }
- }
-
- dataShard.Flush();
- fclose(fin);
- return 0;
-}
-
-int argPos(char *str, int argc, char **argv) {
- int a;
-
- for (a = 1; a < argc; a++)
- if (!strcmp(str, argv[a]))
- return a;
-
- return -1;
-}
-
-int main(int argc, char **argv) {
- int i;
- FILE *f;
-
- // set debug mode
- i = argPos(const_cast<char *>("-debug"), argc, argv);
- if (i > 0) {
- debug_mode = 1;
- if (debug_mode > 0)
- printf("debug mode: %d\n", debug_mode);
- }
-
- // search for train file
- i = argPos(const_cast<char *>("-train"), argc, argv);
- if (i > 0) {
- if (i + 1 == argc) {
- printf("ERROR: training data file not specified!\n");
- return 0;
- }
-
- snprintf(train_file, strlen(argv[i + 1])+1, "%s", argv[i + 1]);
-
- if (debug_mode > 0)
- printf("train file: %s\n", train_file);
-
- f = fopen(train_file, "rb");
- if (f == NULL) {
- printf("ERROR: training data file not found!\n");
- return 0;
- }
- fclose(f);
- } else {
- printf("ERROR: training data must be set.\n");
- }
-
- // search for valid file
- i = argPos(const_cast<char *>("-valid"), argc, argv);
- if (i > 0) {
- if (i + 1 == argc) {
- printf("ERROR: validating data file not specified!\n");
- return 0;
- }
-
- snprintf(valid_file, strlen(argv[i + 1])+1, "%s", argv[i + 1]);
-
- if (debug_mode > 0)
- printf("valid file: %s\n", valid_file);
-
- f = fopen(valid_file, "rb");
- if (f == NULL) {
- printf("ERROR: validating data file not found!\n");
- return 0;
- }
- fclose(f);
- valid_mode = 1;
- }
-
- // search for test file
- i = argPos(const_cast<char *>("-test"), argc, argv);
- if (i > 0) {
- if (i + 1 == argc) {
- printf("ERROR: testing data file not specified!\n");
- return 0;
- }
-
- snprintf(test_file, strlen(argv[i + 1])+1, "%s", argv[i + 1]);
-
- if (debug_mode > 0)
- printf("test file: %s\n", test_file);
-
- f = fopen(test_file, "rb");
- if (f == NULL) {
- printf("ERROR: testing data file not found!\n");
- return 0;
- }
- fclose(f);
- test_mode = 1;
- }
-
- // search for class size
- i = argPos(const_cast<char *>("-class_size"), argc, argv);
- if (i > 0) {
- if (i + 1 == argc) {
- printf("ERROR: class size not specified!\n");
- return 0;
- }
-
- class_size = atoi(argv[i + 1]);
-
- if (debug_mode > 0)
- printf("class size: %d\n", class_size);
- }
- if (class_size <= 0) {
- printf("ERROR: no or invalid class size received!\n");
- return 0;
- }
-
- init_class();
-
- create_shard(train_file, "train_shard");
- if (valid_mode) create_shard(valid_file, "valid_shard");
- if (test_mode) create_shard(test_file, "test_shard");
-
- return 0;
-}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5f010caa/examples/rnnlm/job.conf
----------------------------------------------------------------------
diff --git a/examples/rnnlm/job.conf b/examples/rnnlm/job.conf
index 021692f..a1f803d 100644
--- a/examples/rnnlm/job.conf
+++ b/examples/rnnlm/job.conf
@@ -33,7 +33,8 @@ layer {
name: "data"
user_type: "kData"
[data_conf] {
- path: "examples/rnnlm/train_shard"
+ backend: "kvfile"
+ path: "examples/rnnlm/train_data.bin"
max_window: 10
}
exclude: kVal
@@ -43,19 +44,13 @@ layer {
name: "data"
user_type: "kData"
[data_conf] {
- path: "examples/rnnlm/valid_shard"
+ path: "examples/rnnlm/valid_data.bin"
max_window: 10
}
exclude: kTrain
}
layer{
- name:"label"
- user_type: "kLabel"
- srclayers: "data"
-}
-
-layer{
name: "embedding"
user_type: "kEmbedding"
srclayers: "data"
@@ -90,7 +85,7 @@ layer{
name: "loss"
user_type: "kLoss"
srclayers:"hidden"
- srclayers:"label"
+ srclayers:"data"
[loss_conf] {
nclass:100
vocab_size: 3720
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5f010caa/examples/rnnlm/main.cc
----------------------------------------------------------------------
diff --git a/examples/rnnlm/main.cc b/examples/rnnlm/main.cc
index ea1dcdd..ceb8eb7 100644
--- a/examples/rnnlm/main.cc
+++ b/examples/rnnlm/main.cc
@@ -36,7 +36,6 @@ int main(int argc, char **argv) {
driver.RegisterLayer<rnnlm::HiddenLayer, std::string>("kHidden");
driver.RegisterLayer<rnnlm::LossLayer, std::string>("kLoss");
driver.RegisterLayer<rnnlm::DataLayer, std::string>("kData");
- driver.RegisterLayer<rnnlm::LabelLayer, std::string>("kLabel");
singa::JobProto jobConf = driver.job_conf();
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5f010caa/examples/rnnlm/rnnlm.cc
----------------------------------------------------------------------
diff --git a/examples/rnnlm/rnnlm.cc b/examples/rnnlm/rnnlm.cc
index c086972..a6b65f7 100644
--- a/examples/rnnlm/rnnlm.cc
+++ b/examples/rnnlm/rnnlm.cc
@@ -52,42 +52,62 @@ inline Tensor<cpu, 1> RTensor1(Blob<float>* blob) {
/*******DataLayer**************/
DataLayer::~DataLayer() {
- if (shard_ != nullptr)
- delete shard_;
- shard_ = nullptr;
+ if (store_ != nullptr)
+ delete store_;
}
void DataLayer::Setup(const LayerProto& conf, const vector<Layer*>& srclayers) {
RNNLayer::Setup(conf, srclayers);
- shard_ = new singa::DataShard(
- conf.GetExtension(data_conf).path(),
- singa::DataShard::kRead);
string key;
max_window_ = conf.GetExtension(data_conf).max_window();
- records_.resize(max_window_ + 1); // resize to # of records in data layer
+ data_.Reshape(vector<int>{max_window_ + 1, 4});
window_ = 0;
- shard_->Next(&key, &records_[window_]);
+}
+
+void SetInst(int k, WordRecord& word, Blob<float>* to) {
+ float* dptr = to->mutable_cpu_data() + k * 4;
+ dptr[0] = static_cast<float>(word.word_index());
+ dptr[1] = static_cast<float>(word.class_index());
+ dptr[2] = static_cast<float>(word.class_start());
+ dptr[3] = static_cast<float>(word.class_end());
+}
+
+void ShiftInst(int from, int to, Blob<float>* data) {
+ const float* f = data->cpu_data() + from * 4;
+ float* t = data->mutable_cpu_data() + to * 4;
+ // hard code the feature dim to be 4;
+ t[0] = f[0]; t[1] = f[1]; t[2] = f[2]; t[3] = f[3];
}
void DataLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
- CHECK(records_.size() <= shard_->Count());
- records_[0] = records_[window_];
+ string key, value;
+ WordRecord word;
+ if (store_ == nullptr) {
+ store_ = singa::io::OpenStore(
+ layer_conf_.GetExtension(data_conf).backend(),
+ layer_conf_.GetExtension(data_conf).path(),
+ singa::io::kRead);
+ store_->Read(&key, &value);
+ word.ParseFromString(value);
+ SetInst(0, word, &data_);
+ }
+ ShiftInst(window_, 0, &data_);
window_ = max_window_;
for (int i = 1; i <= max_window_; i++) {
- string key;
- if (shard_->Next(&key, &records_[i])) {
- if (records_[i].GetExtension(word).word_index() == 0) {
- window_ = i;
- break;
- }
- } else {
- shard_->SeekToFirst();
- CHECK(shard_->Next(&key, &records_[i]));
+ if (!store_->Read(&key, &value)) {
+ store_->SeekToFirst();
+ CHECK(store_->Read(&key, &value));
+ }
+ word.ParseFromString(value);
+ SetInst(i, word, &data_);
+ if (word.word_index() == 0) {
+ window_ = i;
+ break;
}
}
}
-/*******LabelLayer**************/
+/*******LabelLayer**************
void LabelLayer::Setup(const LayerProto& conf,
const vector<Layer*>& srclayers) {
RNNLayer::Setup(conf, srclayers);
@@ -108,6 +128,7 @@ void LabelLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
label[4 * i + 3] = wordrecord.class_index();
}
}
+*/
/*******EmbeddingLayer**************/
EmbeddingLayer::~EmbeddingLayer() {
@@ -118,7 +139,7 @@ void EmbeddingLayer::Setup(const LayerProto& conf,
const vector<Layer*>& srclayers) {
RNNLayer::Setup(conf, srclayers);
CHECK_EQ(srclayers.size(), 1);
- int max_window = dynamic_cast<DataLayer*>(srclayers[0])->max_window();
+ int max_window = srclayers[0]->data(this).shape()[0];
word_dim_ = conf.GetExtension(embedding_conf).word_dim();
data_.Reshape(vector<int>{max_window, word_dim_});
grad_.ReshapeLike(data_);
@@ -130,12 +151,12 @@ void EmbeddingLayer::Setup(const LayerProto& conf,
void EmbeddingLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
auto datalayer = dynamic_cast<DataLayer*>(srclayers[0]);
window_ = datalayer->window();
- auto records = datalayer->records();
auto words = RTensor2(&data_);
auto embed = RTensor2(embed_->mutable_data());
+ const float* idxptr = datalayer->data(this).cpu_data();
for (int t = 0; t < window_; t++) {
- int idx = static_cast<int>(records[t].GetExtension(word).word_index());
+ int idx = static_cast<int>(idxptr[t * 4]);
CHECK_GE(idx, 0);
CHECK_LT(idx, vocab_size_);
Copy(words[t], embed[idx]);
@@ -147,10 +168,10 @@ void EmbeddingLayer::ComputeGradient(int flag,
auto grad = RTensor2(&grad_);
auto gembed = RTensor2(embed_->mutable_grad());
auto datalayer = dynamic_cast<DataLayer*>(srclayers[0]);
- auto records = datalayer->records();
gembed = 0;
+ const float* idxptr = datalayer->data(this).cpu_data();
for (int t = 0; t < window_; t++) {
- int idx = static_cast<int>(records[t].GetExtension(word).word_index());
+ int idx = static_cast<int>(idxptr[t * 4]);
Copy(gembed[idx], grad[t]);
}
}
@@ -241,8 +262,9 @@ void LossLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
float loss = 0.f, ppl = 0.f;
for (int t = 0; t < window_; t++) {
- int start = static_cast<int>(label[t * 4 + 0]);
- int end = static_cast<int>(label[t * 4 + 1]);
+ // label is the next word
+ int start = static_cast<int>(label[(t + 1) * 4 + 2]);
+ int end = static_cast<int>(label[(t + 1) * 4 + 3]);
auto wordWeight = word_weight.Slice(start, end);
CHECK_GT(end, start);
@@ -254,8 +276,8 @@ void LossLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
pclass[t] = dot(src[t], class_weight.T());
Softmax(pclass[t], pclass[t]);
- int wid = static_cast<int>(label[t * 4 + 2]);
- int cid = static_cast<int>(label[t * 4 + 3]);
+ int wid = static_cast<int>(label[(t + 1) * 4 + 0]);
+ int cid = static_cast<int>(label[(t + 1) * 4 + 1]);
CHECK_GT(end, wid);
CHECK_GE(wid, start);
loss_ += -log(std::max(pword[wid - start] * pclass[t][cid], FLT_MIN));
@@ -276,10 +298,10 @@ void LossLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers) {
gclass_weight = 0;
gword_weight = 0;
for (int t = 0; t < window_; t++) {
- int start = static_cast<int>(label[t * 4 + 0]);
- int end = static_cast<int>(label[t * 4 + 1]);
- int wid = static_cast<int>(label[t * 4 + 2]);
- int cid = static_cast<int>(label[t * 4 + 3]);
+ int start = static_cast<int>(label[(t + 1) * 4 + 2]);
+ int end = static_cast<int>(label[(t + 1) * 4 + 3]);
+ int wid = static_cast<int>(label[(t + 1) * 4 + 0]);
+ int cid = static_cast<int>(label[(t + 1) * 4 + 1]);
auto pword = RTensor1(&pword_[t]);
CHECK_GT(end, wid);
CHECK_GE(wid, start);
@@ -304,6 +326,9 @@ void LossLayer::ComputeGradient(int flag, const vector<Layer*>& srclayers) {
const std::string LossLayer::ToString(bool debug, int flag) {
float loss = loss_ / num_;
float ppl = exp10(- ppl_ / num_);
+ loss_ = 0;
+ num_ = 0;
+ ppl_ = 0;
return "loss = " + std::to_string(loss) + ", ppl = " + std::to_string(ppl);
}
} // end of namespace rnnlm
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5f010caa/examples/rnnlm/rnnlm.h
----------------------------------------------------------------------
diff --git a/examples/rnnlm/rnnlm.h b/examples/rnnlm/rnnlm.h
index 8ad7a68..cb3198e 100644
--- a/examples/rnnlm/rnnlm.h
+++ b/examples/rnnlm/rnnlm.h
@@ -69,20 +69,20 @@ class DataLayer : public RNNLayer, public singa::DataLayer {
private:
int max_window_;
- singa::DataShard* shard_;
+ singa::io::Store* store_ = nullptr;
};
/**
* LabelLayer that read records_[1] to records_[window_] from DataLayer to
* offer label information
- */
class LabelLayer : public RNNLayer {
public:
void Setup(const LayerProto& conf, const vector<Layer*>& srclayers) override;
void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
void ComputeGradient(int flag, const vector<Layer*>& srclayers) override {}
};
+ */
/**
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5f010caa/examples/rnnlm/rnnlm.proto
----------------------------------------------------------------------
diff --git a/examples/rnnlm/rnnlm.proto b/examples/rnnlm/rnnlm.proto
index 8feb3f9..8cfec86 100644
--- a/examples/rnnlm/rnnlm.proto
+++ b/examples/rnnlm/rnnlm.proto
@@ -35,6 +35,7 @@ message LossProto {
message DataProto {
required string path = 1;
optional int32 max_window = 2;
+ optional string backend = 3 [default = "kvfile"];
}
extend singa.LayerProto {
@@ -50,7 +51,3 @@ message WordRecord {
optional int32 class_start = 4;
optional int32 class_end = 5;
}
-
-extend singa.Record {
- optional WordRecord word = 101;
-}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5f010caa/include/io/kvfile_store.h
----------------------------------------------------------------------
diff --git a/include/io/kvfile_store.h b/include/io/kvfile_store.h
index bda7409..c3fd868 100644
--- a/include/io/kvfile_store.h
+++ b/include/io/kvfile_store.h
@@ -36,6 +36,7 @@ namespace singa { namespace io {
*/
class KVFileStore : public Store {
public:
+ ~KVFileStore() { Close();}
bool Open(const std::string& source, Mode mode) override;
void Close() override;
bool Read(std::string* key, std::string* value) override;
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5f010caa/include/io/store.h
----------------------------------------------------------------------
diff --git a/include/io/store.h b/include/io/store.h
index 8665af0..f3cc282 100644
--- a/include/io/store.h
+++ b/include/io/store.h
@@ -38,6 +38,10 @@ enum Mode { kCreate, kRead, kAppend };
class Store {
public:
Store() { }
+ /**
+ * In case that users forget to call Close() to release resources, e.g.,
+ * memory, you can release them here.
+ */
virtual ~Store() { }
/**
* @param[in] source path to the storage, could be a file path, folder path
@@ -46,6 +50,9 @@ class Store {
* @return true if open successfully, otherwise false.
*/
virtual bool Open(const std::string& source, Mode mode) = 0;
+ /**
+ * Release resources.
+ */
virtual void Close() = 0;
/**
* Read a tuple.
@@ -73,7 +80,22 @@ class Store {
virtual void Flush() {}
};
-Store* CreateStore(const std::string& store);
+/**
+ * Create a Store object.
+ *
+ * @param[in] backend identifier for a specific backend. Two backends are
+ * inluced currently, i.e., "kvfile", "textfile"
+ * @return a pointer to the newly created Store.
+ */
+Store* CreateStore(const string& backend);
+/**
+ * Create and open a Store object.
+ *
+ * @param[in] backend, @see CreateStore().
+ * @param[in] path
+ * @param[in] mode kRead or kCreate or kAppend
+ */
+Store* OpenStore(const string& backend, const string& path, Mode mode);
} // namespace io
} /* singa */
#endif // SINGA_IO_STORE_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5f010caa/include/io/textfile_store.h
----------------------------------------------------------------------
diff --git a/include/io/textfile_store.h b/include/io/textfile_store.h
index 4c020e9..788dc20 100644
--- a/include/io/textfile_store.h
+++ b/include/io/textfile_store.h
@@ -32,6 +32,7 @@ namespace singa { namespace io {
*/
class TextFileStore : public Store {
public:
+ ~TextFileStore() { Close(); }
bool Open(const std::string& source, Mode mode) override;
void Close() override;
bool Read(std::string* key, std::string* value) override;
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5f010caa/include/neuralnet/input_layer.h
----------------------------------------------------------------------
diff --git a/include/neuralnet/input_layer.h b/include/neuralnet/input_layer.h
index b5f2dd4..4dfded0 100644
--- a/include/neuralnet/input_layer.h
+++ b/include/neuralnet/input_layer.h
@@ -26,6 +26,7 @@
#include <vector>
#include "neuralnet/layer.h"
#include "utils/data_shard.h"
+#include "io/store.h"
/**
* \file this file includes the declarations of input layers that inherit the
* base InputLayer to load input features.
@@ -40,6 +41,126 @@
* ParserLayer.
*/
namespace singa {
+using std::string;
+using std::vector;
+
+/************************Start of new input layers***************************/
+/**
+ * Base class for loading data from Store.
+ */
+class StoreInputLayer : virtual public InputLayer {
+ public:
+ ~StoreInputLayer();
+ void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
+ void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
+
+ ConnectionType dst_layer_connection() const override { return kOneToMany; }
+
+ protected:
+ /**
+ * Parsing the (key, val) tuple to get feature (and label).
+ * Subclasses must implment this function.
+ * @param[in] k parse this tuple as the k-th instance of one mini-batch.
+ * @param[in] flag used to guide the parsing, e.g., kDeploy phase should not
+ * parse labels from the tuple.
+ * @param[in] key
+ * @param[in] val
+ */
+ virtual bool Parse(int k, int flag, const string& key, const string& val) = 0;
+
+ protected:
+ int batchsize_;
+ io::Store* store_ = nullptr;
+};
+
+/**
+ * Base layer for parsing a key-value tuple as a feature vector with fixed
+ * length. The feature shape is indicated by users in the configuration.
+ * Each tuple may has a label.
+ */
+class SingleLabelRecordLayer : public StoreInputLayer {
+ public:
+ void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
+ void ComputeFeature(int flag, const vector<Layer*>& srclayers) override;
+
+ protected:
+ /**
+ * Load a single record (tuple), e.g., the mean or standard variance vector.
+ */
+ virtual void LoadRecord(const string& backend, const string& path,
+ Blob<float>* to) = 0;
+
+ protected:
+ /**
+ * Feature standardization by processing each feature dimension via
+ * @f$ y = (x - mu)/ std @f$
+ * <a href= "http://ufldl.stanford.edu/wiki/index.php/Data_Preprocessing">
+ * UFLDL</a>
+ */
+ Blob<float> mean_, std_;
+};
+
+/**
+ * Specific layer that parses the value string loaded by Store into a
+ * SingleLabelImageRecord.
+ */
+class ProtoRecordLayer : public SingleLabelRecordLayer {
+ public:
+ void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
+
+ protected:
+ /**
+ * Parse key as instance ID and val into SingleLabelImageRecord.
+ * @copydetails StoreInputLayer::Parse()
+ */
+ bool Parse(int k, int flag, const string& key, const string& val) override;
+ void LoadRecord(const string& backend,
+ const string& path,
+ Blob<float>* to) override;
+
+ private:
+ // TODO(wangwei) decode the image
+ bool encoded_;
+};
+
+/**
+ * Specific layer that parses the value string loaded by Store as a line from
+ * a CSV file.
+ *
+ * It assumes the first column is the label except that has_label_ is configured
+ * to false. Or the data is used in deploy mode.
+ */
+class CSVRecordLayer : public SingleLabelRecordLayer {
+ public:
+ void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
+
+ protected:
+ bool Parse(int k, int flag, const string& key, const string& val) override;
+ void LoadRecord(const string& backend,
+ const string& path,
+ Blob<float>* to) override;
+
+ private:
+ std::string sep_;
+ bool has_label_;
+};
+
+/**
+ * Do preprocessing for images, including cropping, mirroring, resizing.
+ */
+class ImagePreprocessLayer : public InputLayer {
+ public:
+ void Setup(const LayerProto& proto, const vector<Layer*>& srclayers) override;
+ void ComputeFeature(int flag, const vector<Layer*>& srclayers);
+
+ private:
+ bool mirror_ = false;
+ int cropsize_ = 0;
+ int resize_ = 0;
+ float scale_ = 1;
+};
+
+/************************End of new input layers***************************/
/**
* Base layer for reading ::Record from local Shard, HDFS, lmdb, etc.
*/
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5f010caa/include/neuralnet/layer.h
----------------------------------------------------------------------
diff --git a/include/neuralnet/layer.h b/include/neuralnet/layer.h
index bf83163..5ed0c7e 100644
--- a/include/neuralnet/layer.h
+++ b/include/neuralnet/layer.h
@@ -34,6 +34,8 @@
namespace singa {
using std::vector;
+// TODO(wangwei) make AuxType a template argument for Layer.
+using AuxType = int;
/**
* Base layer class.
*
@@ -186,6 +188,12 @@ class Layer {
return &data_;
}
/**
+ * @return auxiliary data, e.g., image label.
+ */
+ virtual const vector<AuxType>& aux_data(const Layer* from = nullptr) const {
+ return aux_data_;
+ }
+ /**
* @see data().
* @return the const ref of the Blob for the gradient of this layer, mainly
* used in BP algorithm.
@@ -205,6 +213,7 @@ class Layer {
protected:
LayerProto layer_conf_;
Blob<float> data_, grad_;
+ vector<AuxType> aux_data_;
};
/**
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5f010caa/include/singa.h
----------------------------------------------------------------------
diff --git a/include/singa.h b/include/singa.h
index 6c801ab..63acb0a 100644
--- a/include/singa.h
+++ b/include/singa.h
@@ -31,6 +31,7 @@
#include "utils/param.h"
#include "utils/singleton.h"
#include "utils/factory.h"
+#include "io/store.h"
#include "./driver.h"
#endif // SINGA_SINGA_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5f010caa/include/utils/tokenizer.h
----------------------------------------------------------------------
diff --git a/include/utils/tokenizer.h b/include/utils/tokenizer.h
new file mode 100644
index 0000000..fc6ba8a
--- /dev/null
+++ b/include/utils/tokenizer.h
@@ -0,0 +1,59 @@
+/************************************************************
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+*************************************************************/
+#ifndef SINGA_UTILS_TOKENIER_H_
+#define SINGA_UTILS_TOKENIER_H_
+#include <string>
+#include <glog/logging.h>
+namespace singa {
+/**
+ * Tokenize a string.
+ *
+ * example:
+ * Tokenizer t("assa,asf;wes", ",;");
+ * string x;
+ * t >> x; // x is assa
+ * t >> x; // x is asf
+ * t >> x; // x is wes
+ * cout << (t >> x); // print 0.
+ */
+class Tokenizer {
+ public:
+ Tokenizer(const std::string& str, const std::string& sep): start_(0),
+ sep_(sep), buf_(str) {}
+ Tokenizer & operator>>(std::string& out) {
+ CHECK_LT(start_, buf_.length());
+ int start = start_;
+ auto pos = buf_.find_first_of(sep_, start);
+ if (pos == std::string::npos)
+ pos = buf_.length();
+ start_ = pos + 1;
+ out = buf_.substr(start, pos);
+ return *this;
+ }
+
+ bool Valid() { return start_ < buf_.length(); }
+ private:
+ unsigned start_;
+ std::string sep_;
+ const std::string& buf_;
+};
+} /* singa */
+#endif // SINGA_UTILS_TOKENIER_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5f010caa/src/driver.cc
----------------------------------------------------------------------
diff --git a/src/driver.cc b/src/driver.cc
index 28669fa..0d3bbfc 100644
--- a/src/driver.cc
+++ b/src/driver.cc
@@ -54,6 +54,11 @@ void Driver::Init(int argc, char **argv) {
ReadProtoFromTextFile(argv[arg_pos+1], &job_conf_);
// register layers
+
+ RegisterLayer<ProtoRecordLayer, int>(kProtoRecord);
+ RegisterLayer<CSVRecordLayer, int>(kCSVRecord);
+ RegisterLayer<ImagePreprocessLayer, int>(kImagePreprocess);
+
RegisterLayer<BridgeDstLayer, int>(kBridgeDst);
RegisterLayer<BridgeSrcLayer, int>(kBridgeSrc);
RegisterLayer<ConvolutionLayer, int>(kConvolution);
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5f010caa/src/io/store.cc
----------------------------------------------------------------------
diff --git a/src/io/store.cc b/src/io/store.cc
index 6412628..8d3bf13 100644
--- a/src/io/store.cc
+++ b/src/io/store.cc
@@ -19,7 +19,6 @@
*
*************************************************************/
-
#include "io/store.h"
#include "io/kvfile_store.h"
#include "io/textfile_store.h"
@@ -52,6 +51,12 @@ Store* CreateStore(const std::string& backend) {
#endif
return store;
}
+
+Store* OpenStore(const string& backend, const string& path, Mode mode) {
+ auto store = CreateStore(backend);
+ store->Open(path, mode);
+ return store;
+}
} /* io */
} /* singa */
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5f010caa/src/io/textfile_store.cc
----------------------------------------------------------------------
diff --git a/src/io/textfile_store.cc b/src/io/textfile_store.cc
index 74ec9a4..77694a0 100644
--- a/src/io/textfile_store.cc
+++ b/src/io/textfile_store.cc
@@ -40,6 +40,7 @@ void TextFileStore::Close() {
fs_->close();
}
delete fs_;
+ fs_ = nullptr;
}
}
[4/5] incubator-singa git commit: SINGA-82 Refactor input layers
using data store abstraction
Posted by wa...@apache.org.
SINGA-82 Refactor input layers using data store abstraction
add header guard for textfile_store.h, image_transform.h
format code
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/dc7f1996
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/dc7f1996
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/dc7f1996
Branch: refs/heads/master
Commit: dc7f1996df26687612f61945bbb58ccbe0db65f4
Parents: 4afc7b8
Author: wang sheng <wa...@gmail.com>
Authored: Wed Oct 7 15:09:21 2015 +0800
Committer: wang sheng <wa...@gmail.com>
Committed: Wed Oct 7 15:19:59 2015 +0800
----------------------------------------------------------------------
examples/mnist/create_data.cc | 5 +++--
examples/rnnlm/rnnlm.cc | 5 +++--
include/io/kvfile.h | 6 +++---
include/io/kvfile_store.h | 9 +++++----
include/io/store.h | 10 +++++++---
include/io/textfile_store.h | 12 +++++++++---
include/neuralnet/input_layer.h | 2 +-
include/singa.h | 2 +-
include/utils/image_transform.h | 8 +++++++-
include/utils/tokenizer.h | 25 +++++++++++++++----------
src/io/kvfile.cc | 7 ++++---
src/io/kvfile_store.cc | 10 +++++-----
src/io/textfile_store.cc | 12 ++++++------
src/neuralnet/input_layer.cc | 8 ++++----
src/test/test_csv_record_layer.cc | 2 +-
src/test/test_proto_record_layer.cc | 2 +-
src/utils/image_transform.cc | 3 ++-
17 files changed, 77 insertions(+), 51 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dc7f1996/examples/mnist/create_data.cc
----------------------------------------------------------------------
diff --git a/examples/mnist/create_data.cc b/examples/mnist/create_data.cc
index aad1f56..14b279f 100644
--- a/examples/mnist/create_data.cc
+++ b/examples/mnist/create_data.cc
@@ -105,9 +105,10 @@ void create_data(const char* image_filename, const char* label_filename,
int main(int argc, char** argv) {
if (argc != 4) {
- std::cout<<"This program create a DataShard for a MNIST dataset\n"
+ std::cout << "This program create a DataShard for a MNIST dataset\n"
"Usage:\n"
- " create_shard.bin input_image_file input_label_file output_db_file\n"
+ " create_shard.bin input_image_file input_label_file"
+ " output_db_file\n"
"The MNIST dataset could be downloaded at\n"
" http://yann.lecun.com/exdb/mnist/\n"
"You should gunzip them after downloading.";
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dc7f1996/examples/rnnlm/rnnlm.cc
----------------------------------------------------------------------
diff --git a/examples/rnnlm/rnnlm.cc b/examples/rnnlm/rnnlm.cc
index a6b65f7..641b465 100644
--- a/examples/rnnlm/rnnlm.cc
+++ b/examples/rnnlm/rnnlm.cc
@@ -18,12 +18,13 @@
* under the License.
*
*************************************************************/
+#include "./rnnlm.h"
+
#include <string>
#include <algorithm>
#include "mshadow/tensor.h"
#include "mshadow/tensor_expr.h"
#include "mshadow/cxxnet_op.h"
-#include "./rnnlm.h"
#include "./rnnlm.pb.h"
namespace rnnlm {
@@ -64,7 +65,7 @@ void DataLayer::Setup(const LayerProto& conf, const vector<Layer*>& srclayers) {
window_ = 0;
}
-void SetInst(int k, WordRecord& word, Blob<float>* to) {
+void SetInst(int k, const WordRecord& word, Blob<float>* to) {
float* dptr = to->mutable_cpu_data() + k * 4;
dptr[0] = static_cast<float>(word.word_index());
dptr[1] = static_cast<float>(word.class_index());
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dc7f1996/include/io/kvfile.h
----------------------------------------------------------------------
diff --git a/include/io/kvfile.h b/include/io/kvfile.h
index d70f198..27dd35e 100644
--- a/include/io/kvfile.h
+++ b/include/io/kvfile.h
@@ -32,8 +32,8 @@
#include <google/protobuf/message.h>
#endif
-namespace singa { namespace io {
-
+namespace singa {
+namespace io {
/**
* KVFile stores training/validation/test tuples.
@@ -171,7 +171,7 @@ class KVFile {
//!< bytes in buf_, used in reading
int bufsize_ = 0;
};
-} /* io */
+} // namespace io
/**
* @deprecated {ShardData is deprecated! Use KVFile}.
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dc7f1996/include/io/kvfile_store.h
----------------------------------------------------------------------
diff --git a/include/io/kvfile_store.h b/include/io/kvfile_store.h
index c3fd868..bcd70eb 100644
--- a/include/io/kvfile_store.h
+++ b/include/io/kvfile_store.h
@@ -19,7 +19,6 @@
*
*************************************************************/
-
#ifndef SINGA_IO_KVFILE_STORE_H_
#define SINGA_IO_KVFILE_STORE_H_
@@ -27,7 +26,8 @@
#include "io/store.h"
#include "io/kvfile.h"
-namespace singa { namespace io {
+namespace singa {
+namespace io {
/**
* Use the KVFile as the data storage.
@@ -49,6 +49,7 @@ class KVFileStore : public Store {
Mode mode_;
};
-} /* io */
-} /* singa */
+} // namespace io
+} // namespace singa
+
#endif // SINGA_IO_KVFILE_STORE_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dc7f1996/include/io/store.h
----------------------------------------------------------------------
diff --git a/include/io/store.h b/include/io/store.h
index f3cc282..15afb6a 100644
--- a/include/io/store.h
+++ b/include/io/store.h
@@ -24,7 +24,9 @@
#include <string>
-namespace singa { namespace io {
+namespace singa {
+namespace io {
+
using std::string;
enum Mode { kCreate, kRead, kAppend };
@@ -96,6 +98,8 @@ Store* CreateStore(const string& backend);
* @param[in] mode kRead or kCreate or kAppend
*/
Store* OpenStore(const string& backend, const string& path, Mode mode);
-} // namespace io
-} /* singa */
+
+} // namespace io
+} // namespace singa
+
#endif // SINGA_IO_STORE_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dc7f1996/include/io/textfile_store.h
----------------------------------------------------------------------
diff --git a/include/io/textfile_store.h b/include/io/textfile_store.h
index 788dc20..5450f00 100644
--- a/include/io/textfile_store.h
+++ b/include/io/textfile_store.h
@@ -19,11 +19,15 @@
*
*************************************************************/
+#ifndef SINGA_IO_TEXTFILE_STORE_H_
+#define SINGA_IO_TEXTFILE_STORE_H_
#include <fstream>
+#include <string>
#include "io/store.h"
-namespace singa { namespace io {
+namespace singa {
+namespace io {
/**
* Use text file as the data storage, one line per tuple.
*
@@ -45,6 +49,8 @@ class TextFileStore : public Store {
std::fstream* fs_ = nullptr;
Mode mode_;
};
-} /* io */
-} /* singa */
+} // namespace io
+} // namespace singa
+
+#endif // SINGA_IO_TEXTFILE_STORE_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dc7f1996/include/neuralnet/input_layer.h
----------------------------------------------------------------------
diff --git a/include/neuralnet/input_layer.h b/include/neuralnet/input_layer.h
index 65e42cb..2e8725e 100644
--- a/include/neuralnet/input_layer.h
+++ b/include/neuralnet/input_layer.h
@@ -24,9 +24,9 @@
#include <string>
#include <vector>
+#include "io/store.h"
#include "neuralnet/layer.h"
#include "utils/data_shard.h"
-#include "io/store.h"
/**
* \file this file includes the declarations of input layers that inherit the
* base InputLayer to load input features.
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dc7f1996/include/singa.h
----------------------------------------------------------------------
diff --git a/include/singa.h b/include/singa.h
index 63acb0a..ddfec0e 100644
--- a/include/singa.h
+++ b/include/singa.h
@@ -23,6 +23,7 @@
#define SINGA_SINGA_H_
#include "comm/socket.h"
+#include "io/store.h"
#include "neuralnet/neuralnet.h"
#include "neuralnet/layer.h"
#include "proto/job.pb.h"
@@ -31,7 +32,6 @@
#include "utils/param.h"
#include "utils/singleton.h"
#include "utils/factory.h"
-#include "io/store.h"
#include "./driver.h"
#endif // SINGA_SINGA_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dc7f1996/include/utils/image_transform.h
----------------------------------------------------------------------
diff --git a/include/utils/image_transform.h b/include/utils/image_transform.h
index 24fe75d..2867ad2 100644
--- a/include/utils/image_transform.h
+++ b/include/utils/image_transform.h
@@ -18,6 +18,10 @@
* under the License.
*
*************************************************************/
+
+#ifndef SINGA_UTILS_IMAGE_TRANSFORM_H_
+#define SINGA_UTILS_IMAGE_TRANSFORM_H_
+
#include <glog/logging.h>
// TODO(wangwei) provide image transformation API, the implementation can be
// done by opencv, manual transform, or mshadow.
@@ -26,4 +30,6 @@ namespace singa {
void ImageTransform(const float* in, const float* mean, bool mirror, int h_crop,
int w_crop, int h_offset, int w_offset, int channel, int height, int width,
float scale, float* out);
-} /* singa */
+} // namespace singa
+
+#endif // SINGA_UTILS_IMAGE_TRANSFORM_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dc7f1996/include/utils/tokenizer.h
----------------------------------------------------------------------
diff --git a/include/utils/tokenizer.h b/include/utils/tokenizer.h
index fc6ba8a..c66e0af 100644
--- a/include/utils/tokenizer.h
+++ b/include/utils/tokenizer.h
@@ -18,10 +18,13 @@
* under the License.
*
*************************************************************/
-#ifndef SINGA_UTILS_TOKENIER_H_
-#define SINGA_UTILS_TOKENIER_H_
-#include <string>
+
+#ifndef SINGA_UTILS_TOKENIZER_H_
+#define SINGA_UTILS_TOKENIZER_H_
+
#include <glog/logging.h>
+#include <string>
+
namespace singa {
/**
* Tokenize a string.
@@ -48,12 +51,14 @@ class Tokenizer {
out = buf_.substr(start, pos);
return *this;
}
-
bool Valid() { return start_ < buf_.length(); }
- private:
- unsigned start_;
- std::string sep_;
- const std::string& buf_;
+
+ private:
+ unsigned start_;
+ std::string sep_;
+ const std::string& buf_;
};
-} /* singa */
-#endif // SINGA_UTILS_TOKENIER_H_
+
+} // namespace singa
+
+#endif // SINGA_UTILS_TOKENIZER_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dc7f1996/src/io/kvfile.cc
----------------------------------------------------------------------
diff --git a/src/io/kvfile.cc b/src/io/kvfile.cc
index aa52150..465d3ee 100644
--- a/src/io/kvfile.cc
+++ b/src/io/kvfile.cc
@@ -22,9 +22,10 @@
#include "io/kvfile.h"
#include <glog/logging.h>
-#include <string>
-namespace singa { namespace io {
+namespace singa {
+namespace io {
+
KVFile::KVFile(const std::string& path, Mode mode, int capacity) :
path_(path), mode_(mode), capacity_(capacity) {
buf_ = new char[capacity];
@@ -213,5 +214,5 @@ bool KVFile::PrepareNextField(int size) {
return true;
}
-} /* io */
+} // namespace io
} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dc7f1996/src/io/kvfile_store.cc
----------------------------------------------------------------------
diff --git a/src/io/kvfile_store.cc b/src/io/kvfile_store.cc
index 11609bf..4d251f9 100644
--- a/src/io/kvfile_store.cc
+++ b/src/io/kvfile_store.cc
@@ -19,11 +19,12 @@
*
*************************************************************/
-
#include "io/kvfile_store.h"
+
#include <glog/logging.h>
-namespace singa { namespace io {
+namespace singa {
+namespace io {
bool KVFileStore::Open(const std::string& source, Mode mode) {
CHECK(file_ == nullptr);
@@ -66,6 +67,5 @@ void KVFileStore::Flush() {
file_->Flush();
}
-} /* io */
-
-} /* singa */
+} // namespace io
+} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dc7f1996/src/io/textfile_store.cc
----------------------------------------------------------------------
diff --git a/src/io/textfile_store.cc b/src/io/textfile_store.cc
index 77694a0..9371de7 100644
--- a/src/io/textfile_store.cc
+++ b/src/io/textfile_store.cc
@@ -23,13 +23,14 @@
#include "io/textfile_store.h"
#include <glog/logging.h>
-namespace singa { namespace io {
+namespace singa {
+namespace io {
+
bool TextFileStore::Open(const std::string& source, Mode mode) {
if (mode == kRead)
fs_ = new std::fstream(source, std::fstream::in);
- else if (mode == kCreate) {
+ else if (mode == kCreate)
fs_ = new std::fstream(source, std::fstream::out);
- }
mode_ = mode;
return fs_->is_open();
}
@@ -79,6 +80,5 @@ void TextFileStore::Flush() {
fs_->flush();
}
-} /* io */
-
-} /* singa */
+} // namespace io
+} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dc7f1996/src/neuralnet/input_layer.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/input_layer.cc b/src/neuralnet/input_layer.cc
index 1599def..d4bef70 100644
--- a/src/neuralnet/input_layer.cc
+++ b/src/neuralnet/input_layer.cc
@@ -104,10 +104,10 @@ void StoreInputLayer::ComputeFeature(int flag,
store_->SeekToFirst();
CHECK(store_->Read(&key, &val));
}
- random_skip_ --;
+ random_skip_--;
}
}
- for (int k = 0; k < batchsize_; k++){
+ for (int k = 0; k < batchsize_; k++) {
if (!store_->Read(&key, &val)) {
store_->SeekToFirst();
CHECK(store_->Read(&key, &val));
@@ -153,7 +153,7 @@ void SingleLabelRecordLayer::ComputeFeature(int flag,
if (mean_.count()) {
const float* mean = mean_.cpu_data();
- for (int k = 0; k < batchsize_; k++){
+ for (int k = 0; k < batchsize_; k++) {
float* dptr = data_.mutable_cpu_data() + k * mean_.count();
for (int i = 0; i < mean_.count(); i++) {
dptr[i] -= mean[i];
@@ -162,7 +162,7 @@ void SingleLabelRecordLayer::ComputeFeature(int flag,
}
if (std_.count()) {
const float* std = std_.cpu_data();
- for (int k = 0; k < batchsize_; k++){
+ for (int k = 0; k < batchsize_; k++) {
float* dptr = data_.mutable_cpu_data() + k * std_.count();
for (int i = 0; i < std_.count(); i++) {
dptr[i] /= std[i];
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dc7f1996/src/test/test_csv_record_layer.cc
----------------------------------------------------------------------
diff --git a/src/test/test_csv_record_layer.cc b/src/test/test_csv_record_layer.cc
index 4517698..2839db9 100644
--- a/src/test/test_csv_record_layer.cc
+++ b/src/test/test_csv_record_layer.cc
@@ -48,7 +48,7 @@ class CSVRecordLayerTest : public ::testing::Test {
TEST_F(CSVRecordLayerTest, Setup) {
singa::CSVRecordLayer layer;
layer.Setup(csv_conf, std::vector<singa::Layer*>{});
- EXPECT_EQ(2, layer.aux_data().size());
+ EXPECT_EQ(2, static_cast<int>(layer.aux_data().size()));
EXPECT_EQ(6, layer.data(nullptr).count());
}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dc7f1996/src/test/test_proto_record_layer.cc
----------------------------------------------------------------------
diff --git a/src/test/test_proto_record_layer.cc b/src/test/test_proto_record_layer.cc
index 0a39c13..03c7230 100644
--- a/src/test/test_proto_record_layer.cc
+++ b/src/test/test_proto_record_layer.cc
@@ -79,7 +79,7 @@ class ProtoRecordLayerTest : public ::testing::Test {
TEST_F(ProtoRecordLayerTest, Setup) {
singa::ProtoRecordLayer layer;
layer.Setup(image_conf, std::vector<singa::Layer*>{});
- EXPECT_EQ(2, layer.aux_data().size());
+ EXPECT_EQ(2, static_cast<int>(layer.aux_data().size()));
EXPECT_EQ(6, layer.data(nullptr).count());
}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/dc7f1996/src/utils/image_transform.cc
----------------------------------------------------------------------
diff --git a/src/utils/image_transform.cc b/src/utils/image_transform.cc
index 4851334..1e18e6b 100644
--- a/src/utils/image_transform.cc
+++ b/src/utils/image_transform.cc
@@ -53,4 +53,5 @@ void ImageTransform(const float* in, const float* mean, bool mirror, int h_crop,
}
}
}
-} /* singa */
+
+} // namespace singa
[2/5] incubator-singa git commit: SINGA-82 Refactor input layers
using data store abstraction
Posted by wa...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5f010caa/src/neuralnet/input_layer.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/input_layer.cc b/src/neuralnet/input_layer.cc
index f89369c..b4743f4 100644
--- a/src/neuralnet/input_layer.cc
+++ b/src/neuralnet/input_layer.cc
@@ -20,9 +20,9 @@
*************************************************************/
#include "neuralnet/input_layer.h"
-
#include "mshadow/tensor.h"
-
+#include "utils/image_transform.h"
+#include "utils/tokenizer.h"
namespace singa {
using namespace mshadow;
@@ -33,6 +33,219 @@ using mshadow::Tensor;
using std::string;
using std::vector;
+/*****************ImagePreprocess**************************************/
+void ImagePreprocessLayer::Setup(const LayerProto& conf,
+ const vector<Layer*>& srclayers) {
+ CHECK_EQ(srclayers.size(), 1);
+ InputLayer::Setup(conf, srclayers);
+ scale_ = conf.rgbimage_conf().scale();
+ cropsize_ = conf.rgbimage_conf().cropsize();
+ mirror_ = conf.rgbimage_conf().mirror();
+ const auto& src = srclayers.at(0)->data(this);
+ const auto& shape = src.shape();
+ CHECK_EQ(shape.size(), 4);
+ CHECK_EQ(shape.at(2), shape.at(3));
+ if (cropsize_ != 0 && cropsize_ != shape.at(2)) {
+ data_.Reshape(vector<int>{shape.at(0), shape.at(1), cropsize_, cropsize_});
+ } else {
+ data_ = src;
+ }
+}
+
+void ImagePreprocessLayer::ComputeFeature(int flag,
+ const vector<Layer*>& srclayers) {
+ const auto& srcdata = srclayers.at(0)->data(this);
+ int batchsize = srcdata.shape()[0], channel = srcdata.shape()[1];
+ int height = srcdata.shape()[2], width = srcdata.shape()[3];
+ const float* srcdptr = srcdata.cpu_data();
+ float* dptr = data_.mutable_cpu_data();
+ int srcimage_size = channel * height * width;
+ int image_size = channel * data_.shape()[2] * data_.shape()[3];
+ for (int k = 0; k < batchsize; k++) {
+ int h_offset = 0, w_offset = 0;
+ if (cropsize_> 0 && ((flag & kTrain) == kTrain)) {
+ h_offset = rand() % (srcdata.shape()[1] - cropsize_);
+ w_offset = rand() % (srcdata.shape()[2] - cropsize_);
+ }
+ bool do_mirror = mirror_ && rand() % 2 && ((flag & kTrain) == kTrain);
+ ImageTransform(srcdptr + k * srcimage_size, nullptr, do_mirror, cropsize_,
+ cropsize_, h_offset, w_offset, srcdata.shape()[1], height, width,
+ scale_, dptr + image_size);
+ }
+}
+
+/*************StoreInputLayer******************/
+StoreInputLayer::~StoreInputLayer() {
+ if (store_ != nullptr) {
+ delete store_;
+ }
+}
+
+void StoreInputLayer::Setup(const LayerProto& conf,
+ const vector<Layer*>& srclayers) {
+ InputLayer::Setup(conf, srclayers);
+ batchsize_ = conf.store_conf().batchsize();
+}
+
+void StoreInputLayer::ComputeFeature(int flag,
+ const vector<Layer*>& srclayers) {
+ string key, val;
+ if (store_ == nullptr) {
+ store_ = io::OpenStore(layer_conf_.store_conf().backend(),
+ layer_conf_.store_conf().path(),
+ io::kRead);
+ }
+ for (int k = 0; k < batchsize_; k++){
+ if (!store_->Read(&key, &val)) {
+ store_->SeekToFirst();
+ CHECK(store_->Read(&key, &val));
+ }
+ // TODO(wangwei) random skip and shuffle among this mini-batch
+ Parse(k, flag, key, val);
+ }
+}
+/*********SingleLabelRecordLayer******************/
+void SingleLabelRecordLayer::Setup(const LayerProto& conf,
+ const vector<Layer*>& srclayers) {
+ StoreInputLayer::Setup(conf, srclayers);
+
+ vector<int> shape {batchsize_};
+ for (int s : conf.store_conf().shape())
+ shape.push_back(s);
+ data_.Reshape(shape);
+ aux_data_.resize(batchsize_);
+}
+void SingleLabelRecordLayer::ComputeFeature(int flag,
+ const vector<Layer*>& srclayers) {
+ StoreInputLayer::ComputeFeature(flag, srclayers);
+
+ auto& store_conf = layer_conf_.store_conf();
+ if (store_conf.has_mean_file() && mean_.count() == 0) {
+ mean_.Reshape(vector<int>{data_.count() / batchsize_});
+ LoadRecord(store_conf.backend(), store_conf.mean_file(), &mean_);
+ } else if (store_conf.has_mean_value() && mean_.count() == 0) {
+ mean_.Reshape(vector<int>{data_.count() / batchsize_});
+ for (int i = 0; i < data_.count() / batchsize_; i++)
+ mean_.mutable_cpu_data()[i] = store_conf.mean_value();
+ }
+ if (store_conf.has_std_file() && std_.count() == 0) {
+ std_.Reshape(vector<int>{data_.count() / batchsize_});
+ LoadRecord(store_conf.backend(), store_conf.std_file(), &std_);
+ // TODO(wangwei) check std[i] != 0
+ } else if (store_conf.has_std_value() && std_.count() == 0) {
+ std_.Reshape(vector<int>{data_.count() / batchsize_});
+ CHECK_NE(store_conf.std_value(), 0);
+ for (int i = 0; i < data_.count() / batchsize_; i++)
+ std_.mutable_cpu_data()[i] = store_conf.std_value();
+ }
+
+ if (mean_.count()) {
+ const float* mean = mean_.cpu_data();
+ for (int k = 0; k < batchsize_; k++){
+ float* dptr = data_.mutable_cpu_data() + k * mean_.count();
+ for (int i = 0; i < mean_.count(); i++) {
+ dptr[i] -= mean[i];
+ }
+ }
+ }
+ if (std_.count()) {
+ const float* std = std_.cpu_data();
+ for (int k = 0; k < batchsize_; k++){
+ float* dptr = data_.mutable_cpu_data() + k * std_.count();
+ for (int i = 0; i < std_.count(); i++) {
+ dptr[i] /= std[i];
+ }
+ }
+ }
+}
+/*****************CSVRecordLayer*******************/
+void CSVRecordLayer::Setup(const LayerProto& conf,
+ const vector<Layer*>& srclayers) {
+ SingleLabelRecordLayer::Setup(conf, srclayers);
+ sep_ = conf.store_conf().separator();
+}
+
+void CSVRecordLayer::LoadRecord(const string& backend,
+ const string&path, Blob<float>* to) {
+ io::Store* store = io::OpenStore(backend, path, io::kRead);
+ string key, val;
+ CHECK(store->Read(&key, &val));
+ float* ptr = to->mutable_cpu_data();
+ Tokenizer t(val, sep_);
+ string x;
+ for (int i = 0; i< to->count(); i++) {
+ t >> x;
+ ptr[i] = stof(x);
+ }
+ CHECK(!t.Valid());
+ delete store;
+}
+
+bool CSVRecordLayer::Parse(int k, int flag, const string& key,
+ const string& value) {
+ float* ptr = data_.mutable_cpu_data() + k * data_.count() / batchsize_;
+ Tokenizer t(value, sep_);
+ string x;
+ // parse label if not deploy phase and has_label is set.
+ if ((flag & kDeploy) == 0 && layer_conf_.store_conf().has_label()) {
+ t >> x;
+ aux_data_[k] = stoi(x);
+ }
+ for (int i = 0; i< data_.count() / batchsize_; i++) {
+ t >> x;
+ ptr[i] = stof(x);
+ }
+ CHECK(!t.Valid());
+ return true;
+}
+
+
+/*********ProtoRecordLayer******************/
+void ProtoRecordLayer::Setup(const LayerProto& conf,
+ const vector<Layer*>& srclayers) {
+ SingleLabelRecordLayer::Setup(conf, srclayers);
+ encoded_ = conf.store_conf().encoded();
+}
+
+void ProtoRecordLayer::LoadRecord(const string& backend,
+ const string&path, Blob<float>* to) {
+ io::Store* store = io::OpenStore(backend, path, io::kRead);
+ string key, val;
+ CHECK(store->Read(&key, &val));
+ SingleLabelImageRecord image;
+ image.ParseFromString(val);
+ CHECK_EQ(to->count(), image.data_size());
+ float* ptr = to->mutable_cpu_data();
+ for (int i = 0; i< to->count(); i++)
+ ptr[i] = image.data(i);
+ delete store;
+}
+
+bool ProtoRecordLayer::Parse(int k, int flag, const string& key,
+ const string& value) {
+ SingleLabelImageRecord image;
+ image.ParseFromString(value);
+ int size = data_.count() / batchsize_;
+ if (image.data_size()) {
+ CHECK_EQ(size, image.data_size());
+ float* ptr = data_.mutable_cpu_data() + k * size;
+ for (int i = 0; i< size; i++)
+ ptr[i] = image.data(i);
+ } else if (image.pixel().size()) {
+ CHECK_EQ(size, image.pixel().size());
+ float* ptr = data_.mutable_cpu_data() + k * size;
+ string pixel = image.pixel();
+ for (int i = 0; i < size; i++)
+ ptr[i] = static_cast<float>(static_cast<uint8_t>(pixel[i]));
+ } else {
+ LOG(ERROR) << "not pixel nor pixel";
+ }
+ if ((flag & kDeploy) == 0) { // deploy mode does not have label
+ aux_data_.at(k) = image.label();
+ }
+ return true;
+}
+
/************* Implementation for ParserLayer ***********/
void ParserLayer::ComputeFeature(int flag, const vector<Layer*>& srclayers) {
CHECK_EQ(srclayers.size(), 1);
@@ -213,7 +426,6 @@ void LabelLayer::ParseRecords(int flag, const vector<Record>& records,
}
CHECK_EQ(rid, blob->shape()[0]);
}
-
/**************** Implementation for MnistLayer ******************/
void MnistLayer::ParseRecords(int flag, const vector<Record>& records,
Blob<float>* blob) {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5f010caa/src/neuralnet/loss_layer.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/loss_layer.cc b/src/neuralnet/loss_layer.cc
index b5447f6..3a5f5fa 100644
--- a/src/neuralnet/loss_layer.cc
+++ b/src/neuralnet/loss_layer.cc
@@ -92,7 +92,7 @@ void SoftmaxLossLayer::ComputeFeature(int flag,
Tensor<cpu, 2> prob(data_.mutable_cpu_data(), s);
Tensor<cpu, 2> src(srclayers[0]->mutable_data(this)->mutable_cpu_data(), s);
Softmax(prob, src);
- const float* label = srclayers[1]->data(this).cpu_data();
+ const auto& label = srclayers[1]->aux_data(this);
const float* probptr = prob.dptr;
float loss = 0, precision = 0;
for (int n = 0; n < batchsize_; n++) {
@@ -123,7 +123,7 @@ void SoftmaxLossLayer::ComputeFeature(int flag,
void SoftmaxLossLayer::ComputeGradient(int flag,
const vector<Layer*>& srclayers) {
- const float* label = srclayers[1]->data(this).cpu_data();
+ const auto& label = srclayers[1]->aux_data();
Blob<float>* gsrcblob = srclayers[0]->mutable_grad(this);
gsrcblob->CopyFrom(data_);
float* gsrcptr = gsrcblob->mutable_cpu_data();
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5f010caa/src/proto/job.proto
----------------------------------------------------------------------
diff --git a/src/proto/job.proto b/src/proto/job.proto
index 950f785..2d6dfbe 100644
--- a/src/proto/job.proto
+++ b/src/proto/job.proto
@@ -190,8 +190,6 @@ message LayerProto {
optional ConcateProto concate_conf = 31;
// configuration for dropout layer
optional DropoutProto dropout_conf = 33;
- // configuration for euclideanloss layer
- optional EuclideanLossProto euclideanloss_conf = 50;
// configuration for inner product layer
optional InnerProductProto innerproduct_conf = 34;
// configuration for local response normalization layer
@@ -218,6 +216,9 @@ message LayerProto {
optional SoftmaxLossProto softmaxloss_conf = 40;
// configuration for split layer
optional SplitProto split_conf = 42;
+ // configuration for store input layers
+ optional StoreProto store_conf = 51;
+
// overrides the partition dimension for neural net
@@ -316,9 +317,20 @@ message SplitProto {
optional int32 num_splits = 1 [default = 1];
}
-message EuclideanLossProto {
+message StoreProto {
+ required string backend = 1;
+ optional string path = 2;
+ optional string separator = 3 [default = ","];
+ optional string mean_file = 4;
+ optional string std_file = 5;
+ optional float mean_value = 6;
+ optional float std_value = 7;
+ optional int32 batchsize = 8 [default = 1];
+ repeated int32 shape = 9;
+ optional bool encoded = 10 [default = false];
+ optional int32 random_skip = 11 [default = 0];
+ optional bool has_label = 12 [default = true];
}
-
message SoftmaxLossProto {
// computing accuracy against topk results
optional int32 topk = 1 [default = 1];
@@ -525,11 +537,12 @@ enum InitMethod {
enum LayerType {
// Data layers
// - Load records from file, database
+ kProtoRecord = 29;
+ kCSVRecord = 30;
+ kImagePreprocess = 31;
kLMDBData = 17;
kPrefetch = 19;
kShardData = 3;
- // Parser layers
- // - Parse features from records, e.g., pixels
kLabel = 18;
kMnist = 7;
kRGBImage = 10;
@@ -582,6 +595,7 @@ enum Phase {
kForward = 32;
kBackward = 64;
kLoss = 128;
+ kDeploy = 256;
}
enum ParamType {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5f010caa/src/test/test_csv_record_layer.cc
----------------------------------------------------------------------
diff --git a/src/test/test_csv_record_layer.cc b/src/test/test_csv_record_layer.cc
new file mode 100644
index 0000000..4517698
--- /dev/null
+++ b/src/test/test_csv_record_layer.cc
@@ -0,0 +1,92 @@
+/************************************************************
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+*************************************************************/
+#include <string>
+#include <vector>
+#include <fstream>
+
+#include "gtest/gtest.h"
+#include "neuralnet/input_layer.h"
+#include "proto/job.pb.h"
+
+class CSVRecordLayerTest : public ::testing::Test {
+ protected:
+ virtual void SetUp() {
+ std::string path ="src/test/test.csv";
+ std::ofstream ofs(path, std::ofstream::out);
+ ASSERT_TRUE(ofs.is_open());
+ ofs << "12,3.2,1,14.1\n";
+ ofs << "2,0.2,0,1.1\n";
+ ofs << "1,2.2,1,4.1\n";
+ ofs.close();
+ auto conf = csv_conf.mutable_store_conf();
+ conf->set_path(path);
+ conf->set_batchsize(2);
+ conf->add_shape(3);
+ conf->set_backend("textfile");
+ }
+ singa::LayerProto csv_conf;
+};
+
+TEST_F(CSVRecordLayerTest, Setup) {
+ singa::CSVRecordLayer layer;
+ layer.Setup(csv_conf, std::vector<singa::Layer*>{});
+ EXPECT_EQ(2, layer.aux_data().size());
+ EXPECT_EQ(6, layer.data(nullptr).count());
+}
+
+TEST_F(CSVRecordLayerTest, ComputeFeature) {
+ singa::CSVRecordLayer csv;
+ csv.Setup(csv_conf, std::vector<singa::Layer*>{});
+ csv.ComputeFeature(singa::kTrain, std::vector<singa::Layer*>{});
+
+ EXPECT_EQ(12, csv.aux_data()[0]);
+ EXPECT_EQ(2, csv.aux_data()[1]);
+ auto data = csv.data(nullptr);
+ EXPECT_EQ(3.2f, data.cpu_data()[0]);
+ EXPECT_EQ(14.1f, data.cpu_data()[2]);
+ EXPECT_EQ(0.2f, data.cpu_data()[3]);
+ EXPECT_EQ(1.1f, data.cpu_data()[5]);
+}
+TEST_F(CSVRecordLayerTest, ComputeFeatureDeploy) {
+ singa::CSVRecordLayer csv;
+ csv_conf.mutable_store_conf()->set_shape(0, 4);
+ csv.Setup(csv_conf, std::vector<singa::Layer*>{});
+ csv.ComputeFeature(singa::kDeploy, std::vector<singa::Layer*>{});
+
+ auto data = csv.data(nullptr);
+ EXPECT_EQ(12.f, data.cpu_data()[0]);
+ EXPECT_EQ(1.f, data.cpu_data()[2]);
+ EXPECT_EQ(14.1f, data.cpu_data()[3]);
+ EXPECT_EQ(0.2f, data.cpu_data()[5]);
+}
+
+TEST_F(CSVRecordLayerTest, SeekToFirst) {
+ singa::CSVRecordLayer csv;
+ csv.Setup(csv_conf, std::vector<singa::Layer*>{});
+ csv.ComputeFeature(singa::kTrain, std::vector<singa::Layer*>{});
+ csv.ComputeFeature(singa::kTrain, std::vector<singa::Layer*>{});
+
+ auto data = csv.data(nullptr);
+ EXPECT_EQ(2.2f, data.cpu_data()[0]);
+ EXPECT_EQ(4.1f, data.cpu_data()[2]);
+ EXPECT_EQ(3.2f, data.cpu_data()[3]);
+ EXPECT_EQ(14.1f, data.cpu_data()[5]);
+}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5f010caa/src/test/test_proto_record_layer.cc
----------------------------------------------------------------------
diff --git a/src/test/test_proto_record_layer.cc b/src/test/test_proto_record_layer.cc
new file mode 100644
index 0000000..0a39c13
--- /dev/null
+++ b/src/test/test_proto_record_layer.cc
@@ -0,0 +1,122 @@
+/************************************************************
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+*************************************************************/
+#include <string>
+#include <vector>
+
+#include "gtest/gtest.h"
+#include "neuralnet/input_layer.h"
+#include "proto/job.pb.h"
+#include "proto/common.pb.h"
+
+class ProtoRecordLayerTest : public ::testing::Test {
+ protected:
+ virtual void SetUp() {
+ std::string path ="src/test/test.bin";
+ auto* store = singa::io::CreateStore("kvfile");
+ store->Open(path, singa::io::kCreate);
+ {
+ singa::SingleLabelImageRecord image;
+ image.add_data(3.2);
+ image.add_data(1);
+ image.add_data(14.1);
+ image.set_label(12);
+ std::string val;
+ image.SerializeToString(&val);
+ store->Write("0", val);
+ }
+
+ {
+ singa::SingleLabelImageRecord image;
+ image.add_data(0.2);
+ image.add_data(0);
+ image.add_data(1.1);
+ image.set_label(2);
+ std::string val;
+ image.SerializeToString(&val);
+ store->Write("1", val);
+ }
+
+ {
+ singa::SingleLabelImageRecord image;
+ image.add_data(2.2);
+ image.add_data(1);
+ image.add_data(4.1);
+ image.set_label(1);
+ std::string val;
+ image.SerializeToString(&val);
+ store->Write("2", val);
+ }
+ store->Flush();
+ store->Close();
+
+ auto conf = image_conf.mutable_store_conf();
+ conf->set_path(path);
+ conf->set_batchsize(2);
+ conf->add_shape(3);
+ conf->set_backend("kvfile");
+ }
+ singa::LayerProto image_conf;
+};
+
+TEST_F(ProtoRecordLayerTest, Setup) {
+ singa::ProtoRecordLayer layer;
+ layer.Setup(image_conf, std::vector<singa::Layer*>{});
+ EXPECT_EQ(2, layer.aux_data().size());
+ EXPECT_EQ(6, layer.data(nullptr).count());
+}
+
+TEST_F(ProtoRecordLayerTest, ComputeFeature) {
+ singa::ProtoRecordLayer image;
+ image.Setup(image_conf, std::vector<singa::Layer*>{});
+ image.ComputeFeature(singa::kTrain, std::vector<singa::Layer*>{});
+
+ EXPECT_EQ(12, image.aux_data()[0]);
+ EXPECT_EQ(2, image.aux_data()[1]);
+ auto data = image.data(nullptr);
+ EXPECT_EQ(3.2f, data.cpu_data()[0]);
+ EXPECT_EQ(14.1f, data.cpu_data()[2]);
+ EXPECT_EQ(0.2f, data.cpu_data()[3]);
+ EXPECT_EQ(1.1f, data.cpu_data()[5]);
+}
+TEST_F(ProtoRecordLayerTest, ComputeFeatureDeploy) {
+ singa::ProtoRecordLayer image;
+ image.Setup(image_conf, std::vector<singa::Layer*>{});
+ image.ComputeFeature(singa::kDeploy, std::vector<singa::Layer*>{});
+
+ auto data = image.data(nullptr);
+ EXPECT_EQ(3.2f, data.cpu_data()[0]);
+ EXPECT_EQ(14.1f, data.cpu_data()[2]);
+ EXPECT_EQ(0.2f, data.cpu_data()[3]);
+ EXPECT_EQ(1.1f, data.cpu_data()[5]);
+}
+
+TEST_F(ProtoRecordLayerTest, SeekToFirst) {
+ singa::ProtoRecordLayer image;
+ image.Setup(image_conf, std::vector<singa::Layer*>{});
+ image.ComputeFeature(singa::kTrain, std::vector<singa::Layer*>{});
+ image.ComputeFeature(singa::kTrain, std::vector<singa::Layer*>{});
+
+ auto data = image.data(nullptr);
+ EXPECT_EQ(2.2f, data.cpu_data()[0]);
+ EXPECT_EQ(4.1f, data.cpu_data()[2]);
+ EXPECT_EQ(3.2f, data.cpu_data()[3]);
+ EXPECT_EQ(14.1f, data.cpu_data()[5]);
+}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/5f010caa/src/utils/image_transform.cc
----------------------------------------------------------------------
diff --git a/src/utils/image_transform.cc b/src/utils/image_transform.cc
new file mode 100644
index 0000000..4851334
--- /dev/null
+++ b/src/utils/image_transform.cc
@@ -0,0 +1,56 @@
+/************************************************************
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+*************************************************************/
+#include "utils/image_transform.h"
+
+namespace singa {
+
+void ImageTransform(const float* in, const float* mean, bool mirror, int h_crop,
+ int w_crop, int h_offset, int w_offset, int channel, int height, int width,
+ float scale, float* out) {
+ if (h_crop == 0) {
+ CHECK_NE(h_offset, 0);
+ h_crop = height;
+ }
+ if (w_crop ==0) {
+ CHECK_NE(w_offset, 0);
+ w_crop = width;
+ }
+ CHECK_NE(scale, 0);
+
+ int out_idx = 0, in_idx = 0;
+ for (int c = 0; c < channel; c++) {
+ for (int h = 0; h < h_crop; h++) {
+ for (int w = 0; w < w_crop; w++) {
+ in_idx = (c * height + h_offset + h) * width + w_offset + w;
+ if (mirror) {
+ out_idx = (c * h_crop + h) * w_crop + (w_crop - 1 - w);
+ } else {
+ out_idx = (c * h_crop + h) * w_crop + w;
+ }
+ out[out_idx] = in[in_idx];
+ if (mean != nullptr)
+ out[out_idx] -= mean[in_idx];
+ out[out_idx] *= scale;
+ }
+ }
+ }
+}
+} /* singa */