You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@singa.apache.org by wa...@apache.org on 2016/01/02 16:20:07 UTC
[6/7] incubator-singa git commit: SINGA-97 Add HDFS Store
SINGA-97 Add HDFS Store
Minor change to Makefile.example to get MNIST HDFS uploader to work.
Use HDFS_MNIST_TRAIN and HDFS_MNIST_TEST to specify the destination directories.
Fixed minor bug in mnist/create_data.cc
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/befe5ca5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/befe5ca5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/befe5ca5
Branch: refs/heads/master
Commit: befe5ca51c5959fec0f04b4ce06c02a3b9e409e7
Parents: aada365
Author: Anh Dinh <ug...@gmail.com>
Authored: Sat Jan 2 16:16:55 2016 +0800
Committer: WANG Sheng <wa...@gmail.com>
Committed: Sat Jan 2 19:58:14 2016 +0800
----------------------------------------------------------------------
examples/mnist/Makefile.example | 14 ++++++++------
examples/mnist/create_data.cc | 4 ++--
2 files changed, 10 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/befe5ca5/examples/mnist/Makefile.example
----------------------------------------------------------------------
diff --git a/examples/mnist/Makefile.example b/examples/mnist/Makefile.example
index 48d2fd8..8c1c838 100644
--- a/examples/mnist/Makefile.example
+++ b/examples/mnist/Makefile.example
@@ -23,7 +23,9 @@ libs :=singa glog protobuf
.PHONY: all download create
-HDFS_DIR := hdfs://node0:9000/examples/cifar10
+HDFS_MNIST_TRAIN := hdfs://node0:9000/examples/mnist/train_data.bin
+HDFS_MNIST_TEST := hdfs://node0:9000/examples/mnist/test_data.bin
+
download: mnist
@@ -48,8 +50,8 @@ create:
./create_data.bin t10k-images-idx3-ubyte t10k-labels-idx1-ubyte test_data.bin
create_hdfs:
- $(CXX) create_data.cc -std=c++11 -lsinga -lprotobuf -lglog \
- -I../../include -L../../.libs/ -Wl,-unresolved-symbols=ignore-in-shared-libs \
- -Wl,-rpath=../../.libs/ -o create_data.bin
- ./create_data.bin cifar-10-batches-bin $(HDFS_DIR)
-
+ $(CXX) create_data.cc -std=c++11 -lsinga -lprotobuf -lglog -I../../include \
+ -L../../.libs/ -Wl,-unresolved-symbols=ignore-in-shared-libs -Wl,-rpath=../../.libs/ \
+ -o create_data.bin
+ ./create_data.bin train-images-idx3-ubyte train-labels-idx1-ubyte $(HDFS_MNIST_TRAIN)
+ ./create_data.bin t10k-images-idx3-ubyte t10k-labels-idx1-ubyte $(HDFS_MNIST_TEST)
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/befe5ca5/examples/mnist/create_data.cc
----------------------------------------------------------------------
diff --git a/examples/mnist/create_data.cc b/examples/mnist/create_data.cc
index 59da860..ff166b4 100644
--- a/examples/mnist/create_data.cc
+++ b/examples/mnist/create_data.cc
@@ -79,8 +79,8 @@ void create_data(const char* image_filename, const char* label_filename,
cols = swap_endian(cols);
// read backend from the job.conf
- string store_backend = (output_folder.find("hdfs")!=-1) ? "hdfsfile" : "kvfile";
- auto store = singa::io::CreateStore(store_backend);
+ string store_backend = (string(output).find("hdfs")!=-1) ? "hdfsfile" : "kvfile";
+ auto store = singa::io::OpenStore(store_backend, output, singa::io::kCreate);
char label;
char* pixels = new char[rows * cols];
int count = 0;