You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@singa.apache.org by wa...@apache.org on 2016/01/02 16:20:04 UTC
[3/7] incubator-singa git commit: SINGA-97 Add HDFS Store
SINGA-97 Add HDFS Store
Revert changes to the examples directory, assuming that users upload data
to HDFS manually.
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/374f11d6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/374f11d6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/374f11d6
Branch: refs/heads/master
Commit: 374f11d62e83df6b681c36f9557b6825ab8c236a
Parents: 8a07a29
Author: Anh Dinh <ug...@gmail.com>
Authored: Mon Dec 28 15:51:40 2015 +0800
Committer: WANG Sheng <wa...@gmail.com>
Committed: Sat Jan 2 19:58:14 2016 +0800
----------------------------------------------------------------------
Makefile.am | 2 +-
examples/cifar10/Makefile.example | 5 -----
examples/cifar10/create_data.cc | 30 ++++++++++++------------------
examples/mnist/Makefile.example | 5 -----
examples/mnist/create_data.cc | 13 +------------
include/singa/io/hdfsfile.h | 2 +-
src/io/kvfile_store.cc | 4 +++-
7 files changed, 18 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/374f11d6/Makefile.am
----------------------------------------------------------------------
diff --git a/Makefile.am b/Makefile.am
index d78a150..cc308f7 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -33,7 +33,7 @@ CFLAGS += $(DEBUG)
CXXFLAGS += $(DEBUG)
AC_CXXFLAGS = $(DEBUG)
-INCLUDES = -I$(top_srcdir)/include -I/usr/local/include/hdfs
+INCLUDES = -I$(top_srcdir)/include -I/usr/local/include
PROTOS := $(top_srcdir)/src/proto/singa.proto \
$(top_srcdir)/src/proto/job.proto \
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/374f11d6/examples/cifar10/Makefile.example
----------------------------------------------------------------------
diff --git a/examples/cifar10/Makefile.example b/examples/cifar10/Makefile.example
index 775e165..dd65d7d 100644
--- a/examples/cifar10/Makefile.example
+++ b/examples/cifar10/Makefile.example
@@ -28,11 +28,6 @@ cifar-10-binary-bin:
wget http://www.cs.toronto.edu/~kriz/cifar-10-binary.tar.gz
tar xf cifar-10-binary.tar.gz
-compile:
- $(CXX) create_data.cc -std=c++11 -lsinga -lprotobuf -lglog -lhdfs3 \
- -I../../include -L../../.libs/ -Wl,-unresolved-symbols=ignore-in-shared-libs \
- -Wl,-rpath=../../.libs/ -o create_data.bin
-
create:
$(CXX) create_data.cc -std=c++11 -lsinga -lprotobuf -lglog \
-I../../include -L../../.libs/ -Wl,-unresolved-symbols=ignore-in-shared-libs \
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/374f11d6/examples/cifar10/create_data.cc
----------------------------------------------------------------------
diff --git a/examples/cifar10/create_data.cc b/examples/cifar10/create_data.cc
index 05169d8..5873c0e 100644
--- a/examples/cifar10/create_data.cc
+++ b/examples/cifar10/create_data.cc
@@ -19,16 +19,16 @@
*
*************************************************************/
-
-/**
- * Create training and test DataShard for CIFAR dataset.
- * It is adapted from convert_cifar_data from Caffe.
- * create_shard.bin <input> <output_folder>
- *
- * Read from JobConf object the option to use KVfile, HDFS or other (1st layer
- * store_conf object).
- * To load to HDFS, specify "hdfs://namenode/examples" as the output folder
- */
+//
+// This code creates training and test DataShard for CIFAR dataset.
+// It is adapted from the convert_cifar_data from Caffe
+//
+// Usage:
+// create_shard.bin input_folder output_folder
+//
+// The CIFAR dataset could be downloaded at
+// http://www.cs.toronto.edu/~kriz/cifar.html
+//
#include <glog/logging.h>
#include <fstream>
@@ -38,8 +38,6 @@
#include "singa/io/store.h"
#include "singa/proto/common.pb.h"
-#include "singa/utils/common.h"
-#include "singa/proto/job.pb.h"
using std::string;
@@ -47,7 +45,6 @@ const int kCIFARSize = 32;
const int kCIFARImageNBytes = 3072;
const int kCIFARBatchSize = 10000;
const int kCIFARTrainBatches = 5;
-const char JOB_CONFIG[] = "job.conf";
void read_image(std::ifstream* file, int* label, char* buffer) {
char label_char;
@@ -61,6 +58,7 @@ void create_data(const string& input_folder, const string& output_folder) {
int label;
char str_buffer[kCIFARImageNBytes];
string rec_buf;
+
singa::RecordProto image;
image.add_shape(3);
image.add_shape(kCIFARSize);
@@ -71,11 +69,7 @@ void create_data(const string& input_folder, const string& output_folder) {
for (int i = 0; i < kCIFARImageNBytes; i++)
mean.add_data(0.f);
- singa::JobProto job_proto;
- singa::ReadProtoFromTextFile(JOB_CONFIG, &job_proto);
- string store_backend =
- job_proto.neuralnet().layer(0).store_conf().backend();
- auto store = singa::io::CreateStore(store_backend);
+ auto store = singa::io::CreateStore("kvfile");
CHECK(store->Open(output_folder + "/train_data.bin", singa::io::kCreate));
LOG(INFO) << "Preparing training data";
int count = 0;
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/374f11d6/examples/mnist/Makefile.example
----------------------------------------------------------------------
diff --git a/examples/mnist/Makefile.example b/examples/mnist/Makefile.example
index ba2308b..733633d 100644
--- a/examples/mnist/Makefile.example
+++ b/examples/mnist/Makefile.example
@@ -33,11 +33,6 @@ mnist:
gunzip train-images-idx3-ubyte.gz && gunzip train-labels-idx1-ubyte.gz
gunzip t10k-images-idx3-ubyte.gz && gunzip t10k-labels-idx1-ubyte.gz
-compile:
- $(CXX) create_data.cc -std=c++11 -lsinga -lprotobuf -lglog -lhdfs3 -I../../include \
- -L../../.libs/ -Wl,-unresolved-symbols=ignore-in-shared-libs -Wl,-rpath=../../.libs/ \
- -o create_data.bin
-
create:
$(CXX) create_data.cc -std=c++11 -lsinga -lprotobuf -lglog -I../../include \
-L../../.libs/ -Wl,-unresolved-symbols=ignore-in-shared-libs -Wl,-rpath=../../.libs/ \
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/374f11d6/examples/mnist/create_data.cc
----------------------------------------------------------------------
diff --git a/examples/mnist/create_data.cc b/examples/mnist/create_data.cc
index 66a4905..5e51e97 100644
--- a/examples/mnist/create_data.cc
+++ b/examples/mnist/create_data.cc
@@ -38,19 +38,14 @@
#include "singa/io/store.h"
#include "singa/utils/common.h"
#include "singa/proto/common.pb.h"
-#include "singa/proto/job.pb.h"
using std::string;
-const char JOB_CONFIG[] = "job.conf";
-
uint32_t swap_endian(uint32_t val) {
val = ((val << 8) & 0xFF00FF00) | ((val >> 8) & 0xFF00FF);
return (val << 16) | (val >> 16);
}
-// output is the full path, unlike create_data in CIFAR with only
-// specifies the directory
void create_data(const char* image_filename, const char* label_filename,
const char* output) {
// Open files
@@ -81,13 +76,7 @@ void create_data(const char* image_filename, const char* label_filename,
image_file.read(reinterpret_cast<char*>(&cols), 4);
cols = swap_endian(cols);
- // read backend from the job.conf
- singa::JobProto job_proto;
- singa::ReadProtoFromTextFile(JOB_CONFIG.c_str(), &job_proto);
- string store_backend =
- job_proto.neuralnet().layer(0).store_conf().backend();
-
- auto store = singa::io::OpenStore(store_backend, output, singa::io::kCreate);
+ auto store = singa::io::OpenStore("kvfile", output, singa::io::kCreate);
char label;
char* pixels = new char[rows * cols];
int count = 0;
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/374f11d6/include/singa/io/hdfsfile.h
----------------------------------------------------------------------
diff --git a/include/singa/io/hdfsfile.h b/include/singa/io/hdfsfile.h
index f92910e..cd3ded3 100644
--- a/include/singa/io/hdfsfile.h
+++ b/include/singa/io/hdfsfile.h
@@ -33,7 +33,7 @@
#include <google/protobuf/message.h>
#endif
-#include <hdfs.h>
+#include <hdfs/hdfs.h>
namespace singa {
namespace io {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/374f11d6/src/io/kvfile_store.cc
----------------------------------------------------------------------
diff --git a/src/io/kvfile_store.cc b/src/io/kvfile_store.cc
index 79e2a40..e77f49b 100644
--- a/src/io/kvfile_store.cc
+++ b/src/io/kvfile_store.cc
@@ -56,7 +56,9 @@ void KVFileStore::SeekToFirst() {
file_->SeekToFirst();
}
-void KVFileStore::Seek(int offset){}
+void KVFileStore::Seek(int offset){
+ LOG(FATAL) << "Operation not supported.";
+}
bool KVFileStore::Write(const std::string& key, const std::string& value) {
CHECK_NE(mode_, kRead);