You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@singa.apache.org by wa...@apache.org on 2015/07/23 08:49:30 UTC
[2/3] incubator-singa git commit: SINGA-36 Clean ModelProto,
ClusterProto, JobProto and driver program
SINGA-36 Clean ModelProto, ClusterProto, JobProto and driver program
The ClusterProto and ModelProto are merged into JobProto.
A SingaProto is added for global singa configuration including zookeeper info, log dir setting.
The driver program (i.e., main.cc) cleaned.
1. Users include singa.h and their own headers the the drvier program.
2. They prepare the job configuration
and singa configuration (may make this transparent to users, e.g., through environment variable, SINGA_HOME).
3. They register the layers them implemented.
4. Call the trainer to start the training
After linking the driver program with libsinga, users pass the executable to bin/singa-run.sh
together with the workspace where job.conf is stored to submit the job
Some layers are moved into optional_layer.cc(.h) if they have dependencies on external libs (e.g., lmdb and opencv).
If users do not need these layers, they do not need to install the dependencies. TODO update configure script to add
these options. USE_OPTIONAL_LAYER controls the compliation of optional_layer.cc(.h).
Move command line argument singa_conf into trainer.cc.
Usually, users do not need to pass this argument explicitly.
The sing-run.sh script will find it automatically.
modify scripts to support new singa command line
minor change on example job config
add SubmitJob func in singa.h to simplify the job submission API.
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/0478e8cf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/0478e8cf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/0478e8cf
Branch: refs/heads/master
Commit: 0478e8cfdb54688c7c627cd328af04aa6aa30a63
Parents: 29de863
Author: Wei Wang <wa...@comp.nus.edu.sg>
Authored: Wed Jul 22 15:51:08 2015 +0800
Committer: Wei Wang <wa...@comp.nus.edu.sg>
Committed: Thu Jul 23 11:52:12 2015 +0800
----------------------------------------------------------------------
.gitignore | 1 +
Makefile.am | 48 +---
Makefile.example | 5 +-
Makefile.in | 107 ++++----
bin/singa-console.sh | 6 +-
bin/singa-run.sh | 57 ++--
bin/singa-stop.sh | 14 +-
conf/singa.conf | 2 +-
examples/cifar10/Makefile.example | 6 +-
examples/cifar10/cluster.conf | 7 -
examples/cifar10/create_shard.cc | 5 +-
examples/cifar10/job.conf | 228 ++++++++++++++++
examples/cifar10/model.conf | 222 ---------------
examples/mnist/Makefile.example | 2 +-
examples/mnist/batch.sh | 41 ---
examples/mnist/cluster.conf | 7 -
examples/mnist/conv.conf | 11 +
examples/mnist/job.conf | 230 ++++++++++++++++
examples/mnist/model.conf | 224 ----------------
examples/mnist/run.sh | 40 ---
include/neuralnet/base_layer.h | 2 +-
include/neuralnet/layer.h | 18 +-
include/neuralnet/neuralnet.h | 2 +-
include/neuralnet/optional_layer.h | 28 ++
include/singa.h | 32 +++
include/trainer/server.h | 2 +-
include/trainer/trainer.h | 35 ++-
include/trainer/worker.h | 2 +-
include/utils/cluster.h | 27 +-
include/utils/common.h | 6 +
include/utils/param.h | 2 +-
include/utils/updater.h | 2 +-
src/main.cc | 50 ++--
src/neuralnet/layer.cc | 163 ++---------
src/neuralnet/neuralnet.cc | 5 +-
src/neuralnet/optional_layer.cc | 104 +++++++
src/proto/cluster.proto | 61 -----
src/proto/job.proto | 461 ++++++++++++++++++++++++++++++++
src/proto/model.proto | 427 -----------------------------
src/proto/singa.proto | 8 +
src/test/test_cluster.cc | 1 -
src/trainer/trainer.cc | 27 +-
src/trainer/worker.cc | 4 +-
src/utils/cluster.cc | 25 +-
src/utils/param.cc | 2 +-
src/utils/tool.cc | 4 +-
src/utils/updater.cc | 2 +-
tool/gen_hosts.py | 13 +-
48 files changed, 1348 insertions(+), 1430 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/0478e8cf/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 9abefe5..a88e846 100644
--- a/.gitignore
+++ b/.gitignore
@@ -18,6 +18,7 @@
*.pb.cc
*.hosts
*.id
+*.pids
*.tmp
*.out
tool/pb2/*
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/0478e8cf/Makefile.am
----------------------------------------------------------------------
diff --git a/Makefile.am b/Makefile.am
index cecc5ec..56facbb 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -10,20 +10,17 @@ AC_CXXFLAGS = -O3 -Wall -pthread -fPIC -std=c++11 -Wno-unknown-pragmas \
INCLUDES = -I$(top_srcdir)/include
-PROTOS := $(top_srcdir)/src/proto/cluster.proto \
- $(top_srcdir)/src/proto/model.proto \
+PROTOS := $(top_srcdir)/src/proto/singa.proto \
+ $(top_srcdir)/src/proto/job.proto \
$(top_srcdir)/src/proto/common.proto
-PROTO_SRCS := src/proto/cluster.pb.cc \
- src/proto/model.pb.cc \
+PROTO_SRCS := src/proto/singa.pb.cc \
+ src/proto/job.pb.cc \
src/proto/common.pb.cc
-PROTO_HDRS := src/proto/cluster.pb.h \
- src/proto/model.pb.h \
- src/proto/common.pb.h
-PROTO_OBJS := src/proto/cluster.pb.o \
- src/proto/model.pb.o \
- src/proto/common.pb.o
-PROTO_PYS := tool/pb2/cluster_pb2.py \
- tool/pb2/model_pb2.py \
+PROTO_HDRS := include/proto/singa.pb.h \
+ include/proto/job.pb.h \
+ include/proto/common.pb.h
+PROTO_PYS := tool/pb2/singa_pb2.py \
+ tool/pb2/job_pb2.py \
tool/pb2/common_pb2.py
SINGA_SRCS := src/utils/cluster.cc \
@@ -39,11 +36,13 @@ SINGA_SRCS := src/utils/cluster.cc \
src/trainer/trainer.cc \
src/neuralnet/base_layer.cc \
src/neuralnet/neuralnet.cc \
+ src/neuralnet/optional_layer.cc \
src/neuralnet/layer.cc \
src/communication/socket.cc \
src/communication/msg.cc
-SINGA_HDRS := include/utils/cluster.h \
+SINGA_HDRS := include/singa.h \
+ include/utils/cluster.h \
include/utils/cluster_rt.h \
include/utils/param.h \
include/utils/common.h \
@@ -59,6 +58,7 @@ SINGA_HDRS := include/utils/cluster.h \
include/trainer/trainer.h \
include/neuralnet/base_layer.h \
include/neuralnet/layer.h \
+ include/neuralnet/optional_layer.h \
include/neuralnet/neuralnet.h \
include/mshadow/tensor_expr.h \
include/mshadow/tensor_container.h \
@@ -71,29 +71,9 @@ SINGA_HDRS := include/utils/cluster.h \
include/communication/msg.h \
include/communication/socket.h
-SINGA_OBJS := src/proto/cluster.pb.o \
- src/proto/model.pb.o \
- src/utils/cluster.o \
- src/utils/cluster_rt.o \
- src/utils/graph.o \
- src/utils/common.o \
- src/utils/param.o \
- src/utils/updater.o \
- src/utils/data_shard.o \
- src/utils/blob.o \
- src/trainer/server.o \
- src/trainer/worker.o \
- src/trainer/pm_server.o \
- src/trainer/trainer.o \
- src/trainer/pm_worker.o \
- src/neuralnet/base_layer.o \
- src/neuralnet/neuralnet.o \
- src/neuralnet/layer.o \
- src/communication/socket.o \
- src/communication/msg.o
lib_LTLIBRARIES = libsinga.la
-libsinga_la_SOURCES = $(PROTO_HDRS) $(PROTO_SRCS) $(SINGA_SRCS) $(SINGA_HDRS)
+libsinga_la_SOURCES = $(PROTO_HDRS) $(PROTO_SRCS) $(SINGA_HDRS) $(SINGA_SRCS)
libsinga_la_CXXFLAGS = -O3 -Wall -pthread -fPIC -std=c++11 -Wno-unknown-pragmas \
$(MSHADOW_FLAGS) -DCPU_ONLY=1 -funroll-loops -gdwarf-2 \
-gstrict-dwarf -Woverloaded-virtual -DTHREADED -fpermissive
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/0478e8cf/Makefile.example
----------------------------------------------------------------------
diff --git a/Makefile.example b/Makefile.example
index 2195b91..79201cb 100644
--- a/Makefile.example
+++ b/Makefile.example
@@ -4,7 +4,7 @@ HOME_DIR := /usr
# Lib folder for system and external libs. You may need to change it.
LIBRARY_DIRS := $(HOME_DIR)/lib64 $(HOME_DIR)/lib $(HOME_DIR)/local/lib
# Header folder for system and external libs. You may need to change it.
-INCLUDE_DIRS := $(HOME_DIR)/include ./include $(HOME_DIR)/local/include/zookeeper
+INCLUDE_DIRS := $(HOME_DIR)/include ./include $(HOME_DIR)/local/include/zookeeper
# g++ location, should support c++11, tested with 4.8.1
CXX := g++
@@ -15,7 +15,7 @@ LIBRARIES := glog gflags protobuf rt opencv_highgui opencv_imgproc opencv_core\
LDFLAGS := $(foreach librarydir, $(LIBRARY_DIRS), -L$(librarydir))\
$(foreach library, $(LIBRARIES), -l$(library))
# Folder to store compiled files
-BUILD_DIR := build
+BUILD_DIR := .libs
MSHADOW_FLAGS :=-DMSHADOW_USE_CUDA=0 -DMSHADOW_USE_CBLAS=1 -DMSHADOW_USE_MKL=0
ZK_FLAGS :=-DTHREADED -fpermissive
CXXFLAGS := -O3 -Wall -pthread -fPIC -std=c++11 -Wno-unknown-pragmas \
@@ -51,6 +51,7 @@ OBJS := $(sort $(SINGA_OBJS) $(TEST_OBJS) )
.PHONY: singa test
singa: $(PROTO_OBJS) $(SINGA_OBJS)
+ $(CXX) -shared -o $(BUILD_DIR)/libsinga.so $(SINGA_OBJS)
$(CXX) $(SINGA_OBJS) src/main.cc -o singa $(CXXFLAGS) $(LDFLAGS)
@echo
$(CXX) $(SINGA_OBJS) src/utils/tool.cc -o singatool $(CXXFLAGS) $(LDFLAGS)
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/0478e8cf/Makefile.in
----------------------------------------------------------------------
diff --git a/Makefile.in b/Makefile.in
index 34ddd0c..c7fd578 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -82,8 +82,8 @@ LTLIBRARIES = $(lib_LTLIBRARIES)
libsinga_la_LIBADD =
am__objects_1 =
am__dirstamp = $(am__leading_dot)dirstamp
-am__objects_2 = src/proto/libsinga_la-cluster.pb.lo \
- src/proto/libsinga_la-model.pb.lo \
+am__objects_2 = src/proto/libsinga_la-singa.pb.lo \
+ src/proto/libsinga_la-job.pb.lo \
src/proto/libsinga_la-common.pb.lo
am__objects_3 = src/utils/libsinga_la-cluster.lo \
src/utils/libsinga_la-cluster_rt.lo \
@@ -97,11 +97,12 @@ am__objects_3 = src/utils/libsinga_la-cluster.lo \
src/trainer/libsinga_la-trainer.lo \
src/neuralnet/libsinga_la-base_layer.lo \
src/neuralnet/libsinga_la-neuralnet.lo \
+ src/neuralnet/libsinga_la-optional_layer.lo \
src/neuralnet/libsinga_la-layer.lo \
src/communication/libsinga_la-socket.lo \
src/communication/libsinga_la-msg.lo
am_libsinga_la_OBJECTS = $(am__objects_1) $(am__objects_2) \
- $(am__objects_3) $(am__objects_1)
+ $(am__objects_1) $(am__objects_3)
libsinga_la_OBJECTS = $(am_libsinga_la_OBJECTS)
AM_V_lt = $(am__v_lt_$(V))
am__v_lt_ = $(am__v_lt_$(AM_DEFAULT_VERBOSITY))
@@ -304,24 +305,20 @@ AC_CXXFLAGS = -O3 -Wall -pthread -fPIC -std=c++11 -Wno-unknown-pragmas \
$(MSHADOW_FLAGS) -DCPU_ONLY=1 -funroll-loops
INCLUDES = -I$(top_srcdir)/include
-PROTOS := $(top_srcdir)/src/proto/cluster.proto \
- $(top_srcdir)/src/proto/model.proto \
+PROTOS := $(top_srcdir)/src/proto/singa.proto \
+ $(top_srcdir)/src/proto/job.proto \
$(top_srcdir)/src/proto/common.proto
-PROTO_SRCS := src/proto/cluster.pb.cc \
- src/proto/model.pb.cc \
+PROTO_SRCS := src/proto/singa.pb.cc \
+ src/proto/job.pb.cc \
src/proto/common.pb.cc
-PROTO_HDRS := src/proto/cluster.pb.h \
- src/proto/model.pb.h \
- src/proto/common.pb.h
+PROTO_HDRS := include/proto/singa.pb.h \
+ include/proto/job.pb.h \
+ include/proto/common.pb.h
-PROTO_OBJS := src/proto/cluster.pb.o \
- src/proto/model.pb.o \
- src/proto/common.pb.o
-
-PROTO_PYS := tool/pb2/cluster_pb2.py \
- tool/pb2/model_pb2.py \
+PROTO_PYS := tool/pb2/singa_pb2.py \
+ tool/pb2/job_pb2.py \
tool/pb2/common_pb2.py
SINGA_SRCS := src/utils/cluster.cc \
@@ -337,11 +334,13 @@ SINGA_SRCS := src/utils/cluster.cc \
src/trainer/trainer.cc \
src/neuralnet/base_layer.cc \
src/neuralnet/neuralnet.cc \
+ src/neuralnet/optional_layer.cc \
src/neuralnet/layer.cc \
src/communication/socket.cc \
src/communication/msg.cc
-SINGA_HDRS := include/utils/cluster.h \
+SINGA_HDRS := include/singa.h \
+ include/utils/cluster.h \
include/utils/cluster_rt.h \
include/utils/param.h \
include/utils/common.h \
@@ -357,6 +356,7 @@ SINGA_HDRS := include/utils/cluster.h \
include/trainer/trainer.h \
include/neuralnet/base_layer.h \
include/neuralnet/layer.h \
+ include/neuralnet/optional_layer.h \
include/neuralnet/neuralnet.h \
include/mshadow/tensor_expr.h \
include/mshadow/tensor_container.h \
@@ -369,29 +369,8 @@ SINGA_HDRS := include/utils/cluster.h \
include/communication/msg.h \
include/communication/socket.h
-SINGA_OBJS := src/proto/cluster.pb.o \
- src/proto/model.pb.o \
- src/utils/cluster.o \
- src/utils/cluster_rt.o \
- src/utils/graph.o \
- src/utils/common.o \
- src/utils/param.o \
- src/utils/updater.o \
- src/utils/data_shard.o \
- src/utils/blob.o \
- src/trainer/server.o \
- src/trainer/worker.o \
- src/trainer/pm_server.o \
- src/trainer/trainer.o \
- src/trainer/pm_worker.o \
- src/neuralnet/base_layer.o \
- src/neuralnet/neuralnet.o \
- src/neuralnet/layer.o \
- src/communication/socket.o \
- src/communication/msg.o
-
lib_LTLIBRARIES = libsinga.la
-libsinga_la_SOURCES = $(PROTO_HDRS) $(PROTO_SRCS) $(SINGA_SRCS) $(SINGA_HDRS)
+libsinga_la_SOURCES = $(PROTO_HDRS) $(PROTO_SRCS) $(SINGA_HDRS) $(SINGA_SRCS)
libsinga_la_CXXFLAGS = -O3 -Wall -pthread -fPIC -std=c++11 -Wno-unknown-pragmas \
$(MSHADOW_FLAGS) -DCPU_ONLY=1 -funroll-loops -gdwarf-2 \
-gstrict-dwarf -Woverloaded-virtual -DTHREADED -fpermissive
@@ -520,9 +499,9 @@ src/proto/$(am__dirstamp):
src/proto/$(DEPDIR)/$(am__dirstamp):
@$(MKDIR_P) src/proto/$(DEPDIR)
@: > src/proto/$(DEPDIR)/$(am__dirstamp)
-src/proto/libsinga_la-cluster.pb.lo: src/proto/$(am__dirstamp) \
+src/proto/libsinga_la-singa.pb.lo: src/proto/$(am__dirstamp) \
src/proto/$(DEPDIR)/$(am__dirstamp)
-src/proto/libsinga_la-model.pb.lo: src/proto/$(am__dirstamp) \
+src/proto/libsinga_la-job.pb.lo: src/proto/$(am__dirstamp) \
src/proto/$(DEPDIR)/$(am__dirstamp)
src/proto/libsinga_la-common.pb.lo: src/proto/$(am__dirstamp) \
src/proto/$(DEPDIR)/$(am__dirstamp)
@@ -571,6 +550,9 @@ src/neuralnet/libsinga_la-base_layer.lo: \
src/neuralnet/$(DEPDIR)/$(am__dirstamp)
src/neuralnet/libsinga_la-neuralnet.lo: src/neuralnet/$(am__dirstamp) \
src/neuralnet/$(DEPDIR)/$(am__dirstamp)
+src/neuralnet/libsinga_la-optional_layer.lo: \
+ src/neuralnet/$(am__dirstamp) \
+ src/neuralnet/$(DEPDIR)/$(am__dirstamp)
src/neuralnet/libsinga_la-layer.lo: src/neuralnet/$(am__dirstamp) \
src/neuralnet/$(DEPDIR)/$(am__dirstamp)
src/communication/$(am__dirstamp):
@@ -659,12 +641,14 @@ mostlyclean-compile:
-rm -f src/neuralnet/libsinga_la-layer.lo
-rm -f src/neuralnet/libsinga_la-neuralnet.$(OBJEXT)
-rm -f src/neuralnet/libsinga_la-neuralnet.lo
- -rm -f src/proto/libsinga_la-cluster.pb.$(OBJEXT)
- -rm -f src/proto/libsinga_la-cluster.pb.lo
+ -rm -f src/neuralnet/libsinga_la-optional_layer.$(OBJEXT)
+ -rm -f src/neuralnet/libsinga_la-optional_layer.lo
-rm -f src/proto/libsinga_la-common.pb.$(OBJEXT)
-rm -f src/proto/libsinga_la-common.pb.lo
- -rm -f src/proto/libsinga_la-model.pb.$(OBJEXT)
- -rm -f src/proto/libsinga_la-model.pb.lo
+ -rm -f src/proto/libsinga_la-job.pb.$(OBJEXT)
+ -rm -f src/proto/libsinga_la-job.pb.lo
+ -rm -f src/proto/libsinga_la-singa.pb.$(OBJEXT)
+ -rm -f src/proto/libsinga_la-singa.pb.lo
-rm -f src/singa-main.$(OBJEXT)
-rm -f src/trainer/libsinga_la-server.$(OBJEXT)
-rm -f src/trainer/libsinga_la-server.lo
@@ -699,9 +683,10 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote@src/neuralnet/$(DEPDIR)/libsinga_la-base_layer.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@src/neuralnet/$(DEPDIR)/libsinga_la-layer.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@src/neuralnet/$(DEPDIR)/libsinga_la-neuralnet.Plo@am__quote@
-@AMDEP_TRUE@@am__include@ @am__quote@src/proto/$(DEPDIR)/libsinga_la-cluster.pb.Plo@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@src/neuralnet/$(DEPDIR)/libsinga_la-optional_layer.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@src/proto/$(DEPDIR)/libsinga_la-common.pb.Plo@am__quote@
-@AMDEP_TRUE@@am__include@ @am__quote@src/proto/$(DEPDIR)/libsinga_la-model.pb.Plo@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@src/proto/$(DEPDIR)/libsinga_la-job.pb.Plo@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@src/proto/$(DEPDIR)/libsinga_la-singa.pb.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@src/trainer/$(DEPDIR)/libsinga_la-server.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@src/trainer/$(DEPDIR)/libsinga_la-trainer.Plo@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@src/trainer/$(DEPDIR)/libsinga_la-worker.Plo@am__quote@
@@ -742,21 +727,21 @@ distclean-compile:
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(LTCXXCOMPILE) -c -o $@ $<
-src/proto/libsinga_la-cluster.pb.lo: src/proto/cluster.pb.cc
-@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libsinga_la_CXXFLAGS) $(CXXFLAGS) -MT src/proto/libsinga_la-cluster.pb.lo -MD -MP -MF src/proto/$(DEPDIR)/libsinga_la-cluster.pb.Tpo -c -o src/proto/libsinga_la-cluster.pb.lo `test -f 'src/proto/cluster.pb.cc' || echo '$(srcdir)/'`src/proto/cluster.pb.cc
-@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) src/proto/$(DEPDIR)/libsinga_la-cluster.pb.Tpo src/proto/$(DEPDIR)/libsinga_la-cluster.pb.Plo
+src/proto/libsinga_la-singa.pb.lo: src/proto/singa.pb.cc
+@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libsinga_la_CXXFLAGS) $(CXXFLAGS) -MT src/proto/libsinga_la-singa.pb.lo -MD -MP -MF src/proto/$(DEPDIR)/libsinga_la-singa.pb.Tpo -c -o src/proto/libsinga_la-singa.pb.lo `test -f 'src/proto/singa.pb.cc' || echo '$(srcdir)/'`src/proto/singa.pb.cc
+@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) src/proto/$(DEPDIR)/libsinga_la-singa.pb.Tpo src/proto/$(DEPDIR)/libsinga_la-singa.pb.Plo
@am__fastdepCXX_FALSE@ $(AM_V_CXX) @AM_BACKSLASH@
-@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='src/proto/cluster.pb.cc' object='src/proto/libsinga_la-cluster.pb.lo' libtool=yes @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='src/proto/singa.pb.cc' object='src/proto/libsinga_la-singa.pb.lo' libtool=yes @AMDEPBACKSLASH@
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
-@am__fastdepCXX_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libsinga_la_CXXFLAGS) $(CXXFLAGS) -c -o src/proto/libsinga_la-cluster.pb.lo `test -f 'src/proto/cluster.pb.cc' || echo '$(srcdir)/'`src/proto/cluster.pb.cc
+@am__fastdepCXX_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libsinga_la_CXXFLAGS) $(CXXFLAGS) -c -o src/proto/libsinga_la-singa.pb.lo `test -f 'src/proto/singa.pb.cc' || echo '$(srcdir)/'`src/proto/singa.pb.cc
-src/proto/libsinga_la-model.pb.lo: src/proto/model.pb.cc
-@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libsinga_la_CXXFLAGS) $(CXXFLAGS) -MT src/proto/libsinga_la-model.pb.lo -MD -MP -MF src/proto/$(DEPDIR)/libsinga_la-model.pb.Tpo -c -o src/proto/libsinga_la-model.pb.lo `test -f 'src/proto/model.pb.cc' || echo '$(srcdir)/'`src/proto/model.pb.cc
-@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) src/proto/$(DEPDIR)/libsinga_la-model.pb.Tpo src/proto/$(DEPDIR)/libsinga_la-model.pb.Plo
+src/proto/libsinga_la-job.pb.lo: src/proto/job.pb.cc
+@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libsinga_la_CXXFLAGS) $(CXXFLAGS) -MT src/proto/libsinga_la-job.pb.lo -MD -MP -MF src/proto/$(DEPDIR)/libsinga_la-job.pb.Tpo -c -o src/proto/libsinga_la-job.pb.lo `test -f 'src/proto/job.pb.cc' || echo '$(srcdir)/'`src/proto/job.pb.cc
+@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) src/proto/$(DEPDIR)/libsinga_la-job.pb.Tpo src/proto/$(DEPDIR)/libsinga_la-job.pb.Plo
@am__fastdepCXX_FALSE@ $(AM_V_CXX) @AM_BACKSLASH@
-@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='src/proto/model.pb.cc' object='src/proto/libsinga_la-model.pb.lo' libtool=yes @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='src/proto/job.pb.cc' object='src/proto/libsinga_la-job.pb.lo' libtool=yes @AMDEPBACKSLASH@
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
-@am__fastdepCXX_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libsinga_la_CXXFLAGS) $(CXXFLAGS) -c -o src/proto/libsinga_la-model.pb.lo `test -f 'src/proto/model.pb.cc' || echo '$(srcdir)/'`src/proto/model.pb.cc
+@am__fastdepCXX_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libsinga_la_CXXFLAGS) $(CXXFLAGS) -c -o src/proto/libsinga_la-job.pb.lo `test -f 'src/proto/job.pb.cc' || echo '$(srcdir)/'`src/proto/job.pb.cc
src/proto/libsinga_la-common.pb.lo: src/proto/common.pb.cc
@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libsinga_la_CXXFLAGS) $(CXXFLAGS) -MT src/proto/libsinga_la-common.pb.lo -MD -MP -MF src/proto/$(DEPDIR)/libsinga_la-common.pb.Tpo -c -o src/proto/libsinga_la-common.pb.lo `test -f 'src/proto/common.pb.cc' || echo '$(srcdir)/'`src/proto/common.pb.cc
@@ -870,6 +855,14 @@ src/neuralnet/libsinga_la-neuralnet.lo: src/neuralnet/neuralnet.cc
@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCXX_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libsinga_la_CXXFLAGS) $(CXXFLAGS) -c -o src/neuralnet/libsinga_la-neuralnet.lo `test -f 'src/neuralnet/neuralnet.cc' || echo '$(srcdir)/'`src/neuralnet/neuralnet.cc
+src/neuralnet/libsinga_la-optional_layer.lo: src/neuralnet/optional_layer.cc
+@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libsinga_la_CXXFLAGS) $(CXXFLAGS) -MT src/neuralnet/libsinga_la-optional_layer.lo -MD -MP -MF src/neuralnet/$(DEPDIR)/libsinga_la-optional_layer.Tpo -c -o src/neuralnet/libsinga_la-optional_layer.lo `test -f 'src/neuralnet/optional_layer.cc' || echo '$(srcdir)/'`src/neuralnet/optional_layer.cc
+@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) src/neuralnet/$(DEPDIR)/libsinga_la-optional_layer.Tpo src/neuralnet/$(DEPDIR)/libsinga_la-optional_layer.Plo
+@am__fastdepCXX_FALSE@ $(AM_V_CXX) @AM_BACKSLASH@
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@ source='src/neuralnet/optional_layer.cc' object='src/neuralnet/libsinga_la-optional_layer.lo' libtool=yes @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCXX_FALSE@ $(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libsinga_la_CXXFLAGS) $(CXXFLAGS) -c -o src/neuralnet/libsinga_la-optional_layer.lo `test -f 'src/neuralnet/optional_layer.cc' || echo '$(srcdir)/'`src/neuralnet/optional_layer.cc
+
src/neuralnet/libsinga_la-layer.lo: src/neuralnet/layer.cc
@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libsinga_la_CXXFLAGS) $(CXXFLAGS) -MT src/neuralnet/libsinga_la-layer.lo -MD -MP -MF src/neuralnet/$(DEPDIR)/libsinga_la-layer.Tpo -c -o src/neuralnet/libsinga_la-layer.lo `test -f 'src/neuralnet/layer.cc' || echo '$(srcdir)/'`src/neuralnet/layer.cc
@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) src/neuralnet/$(DEPDIR)/libsinga_la-layer.Tpo src/neuralnet/$(DEPDIR)/libsinga_la-layer.Plo
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/0478e8cf/bin/singa-console.sh
----------------------------------------------------------------------
diff --git a/bin/singa-console.sh b/bin/singa-console.sh
index 8844bca..9ed3075 100755
--- a/bin/singa-console.sh
+++ b/bin/singa-console.sh
@@ -24,9 +24,9 @@
#
usage="Usage:\n
- singa-console.sh list : list running singa jobs\n
- singa-console.sh view JOB_ID : view procs of a singa job\n
- singa-console.sh kill JOB_ID : kill a singa job"
+ # singa-console.sh list : list running singa jobs\n
+ # singa-console.sh view JOB_ID : view procs of a singa job\n
+ # singa-console.sh kill JOB_ID : kill a singa job"
if [ $# == 0 ]; then
echo -e $usage
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/0478e8cf/bin/singa-run.sh
----------------------------------------------------------------------
diff --git a/bin/singa-run.sh b/bin/singa-run.sh
index a478221..d434331 100755
--- a/bin/singa-run.sh
+++ b/bin/singa-run.sh
@@ -23,23 +23,38 @@
# run a Singa job
#
-usage="Usage: singa-run.sh -conf=CONF_DIR
- (CONF_DIR should contain cluster.conf && model.conf)"
+usage="Usage: singa-run.sh -workspace=YOUR_WORKSPACE [ --resume ]\n
+ # workspace should contain job.conf\n
+ # set --resume if want to recover a job\n
+ ### NOTICE ###\n
+ # if you are using model.conf + cluster.conf,\n
+ # please see how to combine them to a job.conf:\n
+ # http://singa.incubator.apache.org/quick-start.html"
# check arguments
-if [ $# != 1 ] || [[ $1 != "-conf="* ]]; then
- echo $usage
+while [ $# != 0 ]; do
+ if [[ $1 == "-workspace="* ]]; then
+ workspace=$1
+ elif [ $1 == "--resume" ]; then
+ resume=1
+ else
+ echo -e $usage
+ exit 1
+ fi
+ shift
+done
+if [ -z $workspace ]; then
+ echo -e $usage
exit 1
fi
# get environment variables
. `dirname "${BASH_SOURCE-$0}"`/singa-env.sh
# get workspace path
-workspace=`cd "${1:6}">/dev/null; pwd`
-cluster_conf=$workspace/cluster.conf
-model_conf=$workspace/model.conf
-if [ ! -f $cluster_conf ] || [ ! -f $model_conf ]; then
- echo cluster.conf or model.conf not exists in $workspace
+workspace=`cd "${workspace:11}">/dev/null; pwd`
+job_conf=$workspace/job.conf
+if [ ! -f $job_conf ]; then
+ echo job.conf not exists in $workspace
exit 1
fi
cd $SINGA_HOME
@@ -51,7 +66,7 @@ fi
# generate host file
host_file=$workspace/job.hosts
-python $SINGA_HOME/tool/gen_hosts.py -conf=$cluster_conf \
+python $SINGA_HOME/tool/gen_hosts.py -conf=$job_conf \
-hosts=$SINGA_CONF/hostfile \
-output=$host_file \
|| exit 1
@@ -59,24 +74,32 @@ python $SINGA_HOME/tool/gen_hosts.py -conf=$cluster_conf \
# generate unique job id
./singatool create 1>$workspace/job.id || exit 1
job_id=`cat $workspace/job.id`
-echo generate job id at $workspace/job.id [job_id = $job_id]
+echo Generate job id to $workspace/job.id [job_id = $job_id]
+
+# set command to run singa
+singa_run="./singa -workspace=$workspace -job=$job_id"
+if [ ! -z $resume ]; then
+ singa_run="$singa_run --resume"
+fi
+singa_sshrun="cd $SINGA_HOME; $singa_run"
# ssh and start singa processes
ssh_options="-oStrictHostKeyChecking=no \
-oUserKnownHostsFile=/dev/null \
-oLogLevel=quiet"
hosts=`cat $host_file | cut -d ' ' -f 1`
-singa_run="./singa -cluster=$cluster_conf -model=$model_conf \
- -job=$job_id"
-singa_sshrun="cd $SINGA_HOME; $singa_run"
-
for i in ${hosts[@]} ; do
if [ $i = localhost ] ; then
- echo executing : $singa_run
+ echo Executing : $singa_run
$singa_run &
else
- echo executing @ $i : $singa_sshrun
+ echo Executing @ $i : $singa_sshrun
ssh $ssh_options $i $singa_sshrun &
fi
done
+
+# generate pid list for this job
+sleep 2
+./singatool view $job_id 1>$workspace/job.pids || exit
+echo Generate pid list to $workspace/job.pids
wait
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/0478e8cf/bin/singa-stop.sh
----------------------------------------------------------------------
diff --git a/bin/singa-stop.sh b/bin/singa-stop.sh
index dd3e7bf..ff67f32 100755
--- a/bin/singa-stop.sh
+++ b/bin/singa-stop.sh
@@ -23,15 +23,6 @@
# clean up singa processes and zookeeper metadata
#
-# usage="Usage: \n \
-# (local process): singa-stop.sh \n \
-# (distributed): singa-stop.sh HOST_FILE"
-#
-# if [ $# -gt 1 ]; then
-# echo -e $usage
-# exit 1
-# fi
-
# get environment variables
. `dirname "${BASH_SOURCE-$0}"`/singa-env.sh
cd $SINGA_HOME
@@ -44,7 +35,7 @@ ssh_options="-oStrictHostKeyChecking=no \
hosts=`cat $host_file | cut -d ' ' -f 1`
singa_kill="killall -s SIGKILL -r singa"
for i in ${hosts[@]}; do
- echo kill singa @ $i ...
+ echo Kill singa @ $i ...
if [ $i == localhost ]; then
$singa_kill
else
@@ -55,6 +46,5 @@ done
sleep 2
# remove zk data
-# singatool need global conf under SINGA_HOME
-echo cleanning metadata in zookeeper ...
+echo Cleanning metadata in zookeeper ...
./singatool cleanup || exit 1
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/0478e8cf/conf/singa.conf
----------------------------------------------------------------------
diff --git a/conf/singa.conf b/conf/singa.conf
index f6c351b..fad37d5 100644
--- a/conf/singa.conf
+++ b/conf/singa.conf
@@ -2,4 +2,4 @@
zookeeper_host: "localhost:2181"
# set if you want to change log directory
-# log_dir: "/tmp/singa-log/"
+log_dir: "/tmp/singa-log/"
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/0478e8cf/examples/cifar10/Makefile.example
----------------------------------------------------------------------
diff --git a/examples/cifar10/Makefile.example b/examples/cifar10/Makefile.example
index aa9bf6f..5d3fa21 100644
--- a/examples/cifar10/Makefile.example
+++ b/examples/cifar10/Makefile.example
@@ -9,9 +9,9 @@ cifar-10-binary-bin:
tar xf cifar-10-binary.tar.gz
create:
- $(CXX) create_shard.cc -std=c++11 -lsinga -lprotobuf -lglog -I../../include \
- -L../../.libs/ -Wl,-unresolved-symbols=ignore-in-shared-libs -Wl,-rpath=../../.libs/ \
- -o create_shard.bin
+ $(CXX) create_shard.cc -std=c++11 -lsinga -lprotobuf -lglog -lzookeeper_mt \
+ -I../../include -L../../.libs/ -Wl,-unresolved-symbols=ignore-in-shared-libs \
+ -Wl,-rpath=../../.libs/ -o create_shard.bin
mkdir cifar10_train_shard
mkdir cifar10_test_shard
./create_shard.bin cifar-10-batches-bin .
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/0478e8cf/examples/cifar10/cluster.conf
----------------------------------------------------------------------
diff --git a/examples/cifar10/cluster.conf b/examples/cifar10/cluster.conf
deleted file mode 100644
index e7e3400..0000000
--- a/examples/cifar10/cluster.conf
+++ /dev/null
@@ -1,7 +0,0 @@
-nworker_groups: 1
-nserver_groups: 1
-nservers_per_group: 1
-nworkers_per_group: 1
-nworkers_per_procs: 1
-nservers_per_procs: 1
-workspace: "examples/cifar10/"
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/0478e8cf/examples/cifar10/create_shard.cc
----------------------------------------------------------------------
diff --git a/examples/cifar10/create_shard.cc b/examples/cifar10/create_shard.cc
index 9df92d6..5b8c245 100644
--- a/examples/cifar10/create_shard.cc
+++ b/examples/cifar10/create_shard.cc
@@ -16,10 +16,7 @@
#include <cstdint>
#include <iostream>
-#include "utils/data_shard.h"
-#include "utils/common.h"
-#include "proto/model.pb.h"
-#include "proto/common.pb.h"
+#include "singa.h"
using std::string;
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/0478e8cf/examples/cifar10/job.conf
----------------------------------------------------------------------
diff --git a/examples/cifar10/job.conf b/examples/cifar10/job.conf
new file mode 100644
index 0000000..f3b21b8
--- /dev/null
+++ b/examples/cifar10/job.conf
@@ -0,0 +1,228 @@
+cluster {
+ nworker_groups: 1
+ nserver_groups: 1
+}
+
+model {
+ name: "cifar10-convnet"
+ train_steps: 1000
+ test_steps: 100
+ test_frequency:300
+ display_frequency:30
+ alg: kBackPropagation
+ updater{
+ weight_decay:0.004
+ lr_change: kFixedStep
+ type: kSGD
+ fixedstep_conf:{
+ step:0
+ step:60000
+ step:65000
+ step_lr:0.001
+ step_lr:0.0001
+ step_lr:0.00001
+ }
+ }
+ neuralnet {
+ layer{
+ name: "data"
+ type: kShardData
+ sharddata_conf {
+ path: "examples/cifar10/cifar10_train_shard"
+ batchsize: 16
+ random_skip: 5000
+ }
+ exclude: kTest
+ }
+ layer{
+ name: "data"
+ type: kShardData
+ sharddata_conf {
+ path: "examples/cifar10/cifar10_test_shard"
+ batchsize: 100
+ }
+ exclude: kTrain
+ }
+ layer{
+ name:"rgb"
+ type: kRGBImage
+ srclayers: "data"
+ rgbimage_conf {
+ meanfile: "examples/cifar10/image_mean.bin"
+ }
+ }
+ layer{
+ name: "label"
+ type: kLabel
+ srclayers: "data"
+ }
+
+ layer {
+ name: "conv1"
+ type: kConvolution
+ srclayers: "rgb"
+ convolution_conf {
+ num_filters: 32
+ kernel: 5
+ stride: 1
+ pad:2
+ }
+ param {
+ name: "w1"
+ init_method:kGaussian
+ std:0.0001
+ learning_rate_multiplier:1.0
+ }
+ param {
+ name: "b1"
+ init_method: kConstant
+ learning_rate_multiplier:2.0
+ value:0
+ }
+ }
+
+ layer {
+ name: "pool1"
+ type: kPooling
+ srclayers: "conv1"
+ pooling_conf {
+ pool: MAX
+ kernel: 3
+ stride: 2
+ }
+ }
+ layer {
+ name: "relu1"
+ type: kReLU
+ srclayers:"pool1"
+ }
+ layer {
+ name: "norm1"
+ type: kLRN
+ lrn_conf {
+ norm_region: WITHIN_CHANNEL
+ local_size: 3
+ alpha: 5e-05
+ beta: 0.75
+ }
+ srclayers:"relu1"
+ }
+ layer {
+ name: "conv2"
+ type: kConvolution
+ srclayers: "norm1"
+ convolution_conf {
+ num_filters: 32
+ kernel: 5
+ stride: 1
+ pad:2
+ }
+ param {
+ name: "w2"
+ init_method:kGaussian
+ std:0.01
+ learning_rate_multiplier:1.0
+ }
+ param {
+ name: "b2"
+ init_method: kConstant
+ learning_rate_multiplier:2.0
+ value:0
+ }
+ }
+ layer {
+ name: "relu2"
+ type: kReLU
+ srclayers:"conv2"
+ }
+ layer {
+ name: "pool2"
+ type: kPooling
+ srclayers: "relu2"
+ pooling_conf {
+ pool: MAX
+ kernel: 3
+ stride: 2
+ }
+ }
+ layer {
+ name: "norm2"
+ type: kLRN
+ lrn_conf {
+ norm_region: WITHIN_CHANNEL
+ local_size: 3
+ alpha: 5e-05
+ beta: 0.75
+ }
+ srclayers:"pool2"
+ }
+ layer {
+ name: "conv3"
+ type: kConvolution
+ srclayers: "norm2"
+ convolution_conf {
+ num_filters: 64
+ kernel: 5
+ stride: 1
+ pad:2
+ }
+ param {
+ name: "w3"
+ init_method:kGaussian
+ std:0.01
+ }
+ param {
+ name: "b3"
+ init_method: kConstant
+ value:0
+ }
+ }
+ layer {
+ name: "relu3"
+ type: kReLU
+ srclayers:"conv3"
+ }
+ layer {
+ name: "pool3"
+ type: kPooling
+ srclayers: "relu3"
+ pooling_conf {
+ pool: AVE
+ kernel: 3
+ stride: 2
+ }
+ }
+ layer {
+ name: "ip1"
+ type: kInnerProduct
+ srclayers:"pool3"
+ innerproduct_conf {
+ num_output: 10
+ }
+ param {
+ name: "w4"
+ init_method:kGaussian
+ std:0.01
+ learning_rate_multiplier:1.0
+ weight_decay_multiplier:250
+ }
+ param {
+ name: "b4"
+ init_method: kConstant
+ learning_rate_multiplier:2.0
+ weight_decay_multiplier:0
+ value:0
+ }
+ }
+
+ layer{
+ name: "loss"
+ type: kSoftmaxLoss
+ softmaxloss_conf{
+ topk:1
+ }
+ srclayers:"ip1"
+ srclayers: "label"
+ }
+}
+}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/0478e8cf/examples/cifar10/model.conf
----------------------------------------------------------------------
diff --git a/examples/cifar10/model.conf b/examples/cifar10/model.conf
deleted file mode 100644
index f0e677c..0000000
--- a/examples/cifar10/model.conf
+++ /dev/null
@@ -1,222 +0,0 @@
-name: "cifar10-convnet"
-train_steps: 1000
-test_steps: 100
-test_frequency:300
-display_frequency:30
-#checkpoint: "examples/cifar10/checkpoint/step2200-worker0.bin"
-alg: kBackPropagation
-updater{
- weight_decay:0.004
- lr_change: kFixedStep
- type: kSGD
- fixedstep_conf:{
- step:0
- step:60000
- step:65000
- step_lr:0.001
- step_lr:0.0001
- step_lr:0.00001
- }
-}
-neuralnet {
-layer{
- name: "data"
- type: kShardData
- sharddata_conf {
- path: "examples/cifar10/cifar10_train_shard"
- batchsize: 16
- random_skip: 5000
- }
- exclude: kTest
-}
-layer{
- name: "data"
- type: kShardData
- sharddata_conf {
- path: "examples/cifar10/cifar10_test_shard"
- batchsize: 100
- }
- exclude: kTrain
-}
-layer{
- name:"rgb"
- type: kRGBImage
- srclayers: "data"
- rgbimage_conf {
- meanfile: "examples/cifar10/image_mean.bin"
- }
-}
-layer{
- name: "label"
- type: kLabel
- srclayers: "data"
-}
-
-layer {
- name: "conv1"
- type: kConvolution
- srclayers: "rgb"
- convolution_conf {
- num_filters: 32
- kernel: 5
- stride: 1
- pad:2
- }
- param {
- name: "w1"
- init_method:kGaussian
- std:0.0001
- learning_rate_multiplier:1.0
- }
- param {
- name: "b1"
- init_method: kConstant
- learning_rate_multiplier:2.0
- value:0
- }
-}
-
-layer {
- name: "pool1"
- type: kPooling
- srclayers: "conv1"
- pooling_conf {
- pool: MAX
- kernel: 3
- stride: 2
- }
-}
-layer {
- name: "relu1"
- type: kReLU
- srclayers:"pool1"
-}
-layer {
- name: "norm1"
- type: kLRN
- lrn_conf {
- norm_region: WITHIN_CHANNEL
- local_size: 3
- alpha: 5e-05
- beta: 0.75
- }
- srclayers:"relu1"
-}
-layer {
- name: "conv2"
- type: kConvolution
- srclayers: "norm1"
- convolution_conf {
- num_filters: 32
- kernel: 5
- stride: 1
- pad:2
- }
- param {
- name: "w2"
- init_method:kGaussian
- std:0.01
- learning_rate_multiplier:1.0
- }
- param {
- name: "b2"
- init_method: kConstant
- learning_rate_multiplier:2.0
- value:0
- }
-}
-layer {
- name: "relu2"
- type: kReLU
- srclayers:"conv2"
-}
-layer {
- name: "pool2"
- type: kPooling
- srclayers: "relu2"
- pooling_conf {
- pool: MAX
- kernel: 3
- stride: 2
- }
-}
-layer {
- name: "norm2"
- type: kLRN
- lrn_conf {
- norm_region: WITHIN_CHANNEL
- local_size: 3
- alpha: 5e-05
- beta: 0.75
- }
- srclayers:"pool2"
-}
-layer {
- name: "conv3"
- type: kConvolution
- srclayers: "norm2"
- convolution_conf {
- num_filters: 64
- kernel: 5
- stride: 1
- pad:2
- }
- param {
- name: "w3"
- init_method:kGaussian
- std:0.01
- }
- param {
- name: "b3"
- init_method: kConstant
- value:0
- }
-}
-layer {
- name: "relu3"
- type: kReLU
- srclayers:"conv3"
-}
-layer {
- name: "pool3"
- type: kPooling
- srclayers: "relu3"
- pooling_conf {
- pool: AVE
- kernel: 3
- stride: 2
- }
-}
-layer {
- name: "ip1"
- type: kInnerProduct
- srclayers:"pool3"
- innerproduct_conf {
- num_output: 10
- }
- param {
- name: "w4"
- init_method:kGaussian
- std:0.01
- learning_rate_multiplier:1.0
- weight_decay_multiplier:250
- }
- param {
- name: "b4"
- init_method: kConstant
- learning_rate_multiplier:2.0
- weight_decay_multiplier:0
- value:0
- }
-}
-
-layer{
- name: "loss"
- type: kSoftmaxLoss
- softmaxloss_conf{
- topk:1
- }
- srclayers:"ip1"
- srclayers: "label"
-}
-}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/0478e8cf/examples/mnist/Makefile.example
----------------------------------------------------------------------
diff --git a/examples/mnist/Makefile.example b/examples/mnist/Makefile.example
index 9016887..87399fe 100644
--- a/examples/mnist/Makefile.example
+++ b/examples/mnist/Makefile.example
@@ -13,7 +13,7 @@ mnist:
gunzip t10k-images-idx3-ubyte.gz && gunzip t10k-labels-idx1-ubyte.gz
create:
- $(CXX) create_shard.cc -std=c++11 -lsinga -lprotobuf -lglog -I../../include \
+ $(CXX) create_shard.cc -std=c++11 -lsinga -lprotobuf -lglog -lzookeeper_mt -I../../include \
-L../../.libs/ -Wl,-unresolved-symbols=ignore-in-shared-libs -Wl,-rpath=../../.libs/ \
-o create_shard.bin
mkdir mnist_train_shard
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/0478e8cf/examples/mnist/batch.sh
----------------------------------------------------------------------
diff --git a/examples/mnist/batch.sh b/examples/mnist/batch.sh
deleted file mode 100644
index ff6b8b4..0000000
--- a/examples/mnist/batch.sh
+++ /dev/null
@@ -1,41 +0,0 @@
-#!/bin/bash
-
-for nservers in 1
-do
- for nthreads in 2 4
- do
- for nworkers in 1 2 4 8 16
- do
- echo "nworkers: $nworkers" >examples/mnist/cluster.conf
- echo "nservers: $nservers" >>examples/mnist/cluster.conf
- echo "nthreads_per_server: $nthreads" >>examples/mnist/cluster.conf
- echo "workspace:\" /data1/wangwei/singa\"">>examples/mnist/cluster.conf
- cat examples/mnist/cluster.conf
- nprocs=$(($nworkers+$nservers))
- log=log1k/${nworkers}w${nservers}s${nthreads}t
- echo $log $nprocs
- ./examples/mnist/run.sh start $nprocs >$log 2>&1
- sleep 4
-
- while true
- do
- nstopped=0
- to=$(($nprocs-1))
- for worker in $(eval echo "{0..$to}")
- do
- if [ ! -e /home/wangwei/program/singa/$worker.lock ]
- then
- echo "$worker.lock is free"
- nstopped=$(($nstopped+1))
- fi
- done
- if [ $nstopped -eq $(($nprocs)) ]
- then
- break
- else
- sleep 5
- fi
- done
- done
- done
-done
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/0478e8cf/examples/mnist/cluster.conf
----------------------------------------------------------------------
diff --git a/examples/mnist/cluster.conf b/examples/mnist/cluster.conf
deleted file mode 100644
index ff25b8c..0000000
--- a/examples/mnist/cluster.conf
+++ /dev/null
@@ -1,7 +0,0 @@
-nworker_groups: 1
-nserver_groups: 1
-nservers_per_group: 1
-nworkers_per_group: 1
-nservers_per_procs: 1
-nworkers_per_procs: 1
-workspace: "examples/mnist/"
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/0478e8cf/examples/mnist/conv.conf
----------------------------------------------------------------------
diff --git a/examples/mnist/conv.conf b/examples/mnist/conv.conf
index 5f3bf58..0bd014d 100644
--- a/examples/mnist/conv.conf
+++ b/examples/mnist/conv.conf
@@ -1,3 +1,13 @@
+cluster {
+ nworker_groups: 1
+ nserver_groups: 1
+ nservers_per_group: 1
+ nworkers_per_group: 1
+ nservers_per_procs: 1
+ nworkers_per_procs: 1
+ workspace: "examples/mnist/"
+}
+model {
name: "mnist-conv"
train_steps: 10000
test_steps:100
@@ -173,3 +183,4 @@ layer{
srclayers:"label"
}
}
+}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/0478e8cf/examples/mnist/job.conf
----------------------------------------------------------------------
diff --git a/examples/mnist/job.conf b/examples/mnist/job.conf
new file mode 100644
index 0000000..ca54f92
--- /dev/null
+++ b/examples/mnist/job.conf
@@ -0,0 +1,230 @@
+cluster {
+ nworker_groups: 1
+ nserver_groups: 1
+}
+model {
+ name: "deep-big-simple-mlp"
+ train_steps: 1000
+ test_steps:10
+ test_frequency:60
+ display_frequency:30
+ alg: kBackPropagation
+ updater{
+ base_lr: 0.001
+ lr_change: kStep
+ type: kSGD
+ step_conf{
+ change_freq: 60
+ gamma: 0.997
+ }
+ }
+
+ neuralnet {
+ layer {
+ name: "data"
+ type: kShardData
+ sharddata_conf {
+ path: "examples/mnist/mnist_train_shard"
+ batchsize: 1000
+ }
+ exclude: kTest
+ }
+
+ layer {
+ name: "data"
+ type: kShardData
+ sharddata_conf {
+ path: "examples/mnist/mnist_test_shard"
+ batchsize: 1000
+ }
+ exclude: kTrain
+ }
+
+ layer{
+ name:"mnist"
+ type: kMnist
+ srclayers: "data"
+ mnist_conf {
+# sigma: 6
+# alpha: 38
+# gamma: 15
+# kernel: 21
+# elastic_freq:100
+# beta:15
+# resize: 29
+ norm_a: 127.5
+ norm_b: 1
+ }
+ }
+
+
+ layer{
+ name: "label"
+ type: kLabel
+ srclayers: "data"
+ }
+
+ layer{
+ name: "fc1"
+ type: kInnerProduct
+ srclayers:"mnist"
+ innerproduct_conf{
+ num_output: 2500
+ }
+ param{
+ name: "w1"
+ init_method: kUniform
+ low:-0.05
+ high:0.05
+ }
+ param{
+ name: "b1"
+ init_method: kUniform
+ low: -0.05
+ high:0.05
+ }
+ }
+
+ layer{
+ name: "tanh1"
+ type: kTanh
+ srclayers:"fc1"
+ }
+ layer{
+ name: "fc2"
+ type: kInnerProduct
+ srclayers:"tanh1"
+ innerproduct_conf{
+ num_output: 2000
+ }
+ param{
+ name: "w2"
+ init_method: kUniform
+ low:-0.05
+ high:0.05
+ }
+ param{
+ name: "b2"
+ init_method: kUniform
+ low: -0.05
+ high:0.05
+ }
+ }
+
+ layer{
+ name: "tanh2"
+ type: kTanh
+ srclayers:"fc2"
+ }
+ layer{
+ name: "fc3"
+ type: kInnerProduct
+ srclayers:"tanh2"
+ innerproduct_conf{
+ num_output: 1500
+ }
+ param{
+ name: "w3"
+ init_method: kUniform
+ low:-0.05
+ high:0.05
+ }
+ param{
+ name: "b3"
+ init_method: kUniform
+ low: -0.05
+ high:0.05
+ }
+
+ }
+
+ layer{
+ name: "tanh3"
+ type: kTanh
+ srclayers:"fc3"
+ }
+ layer{
+ name: "fc4"
+ type: kInnerProduct
+ srclayers:"tanh3"
+ innerproduct_conf{
+ num_output: 1000
+ }
+ param{
+ name: "w4"
+ init_method: kUniform
+ low:-0.05
+ high:0.05
+ }
+ param{
+ name: "b4"
+ init_method: kUniform
+ low: -0.05
+ high:0.05
+ }
+
+ }
+
+ layer{
+ name: "tanh4"
+ type: kTanh
+ srclayers:"fc4"
+ }
+ layer{
+ name: "fc5"
+ type: kInnerProduct
+ srclayers:"tanh4"
+ innerproduct_conf{
+ num_output: 500
+ }
+ param{
+ name: "w5"
+ init_method: kUniform
+ low:-0.05
+ high:0.05
+ }
+ param{
+ name: "b5"
+ init_method: kUniform
+ low: -0.05
+ high:0.05
+ }
+
+ }
+
+ layer{
+ name: "tanh5"
+ type: kTanh
+ srclayers:"fc5"
+ }
+ layer{
+ name: "fc6"
+ type: kInnerProduct
+ srclayers:"tanh5"
+ innerproduct_conf{
+ num_output: 10
+ }
+ param{
+ name: "w6"
+ init_method: kUniform
+ low:-0.05
+ high:0.05
+ }
+ param{
+ name: "b6"
+ init_method: kUniform
+ low: -0.05
+ high:0.05
+ }
+ }
+ layer{
+ name: "loss"
+ type:kSoftmaxLoss
+ softmaxloss_conf{
+ topk:1
+ }
+ srclayers:"fc6"
+ srclayers:"label"
+ }
+}
+}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/0478e8cf/examples/mnist/model.conf
----------------------------------------------------------------------
diff --git a/examples/mnist/model.conf b/examples/mnist/model.conf
deleted file mode 100644
index 4b704bc..0000000
--- a/examples/mnist/model.conf
+++ /dev/null
@@ -1,224 +0,0 @@
-name: "deep-big-simple-mlp"
-train_steps: 1000
-test_steps:10
-test_frequency:60
-display_frequency:30
-alg: kBackPropagation
-updater{
- base_lr: 0.001
- lr_change: kStep
- type: kSGD
- step_conf{
- change_freq: 60
- gamma: 0.997
- }
-}
-
-neuralnet {
-layer {
- name: "data"
- type: kShardData
- sharddata_conf {
- path: "examples/mnist/mnist_train_shard"
- batchsize: 1000
- }
- exclude: kTest
-}
-
-layer {
- name: "data"
- type: kShardData
- sharddata_conf {
- path: "examples/mnist/mnist_test_shard"
- batchsize: 1000
- }
- exclude: kTrain
-}
-
-layer{
- name:"mnist"
- type: kMnist
- srclayers: "data"
- mnist_conf {
-# sigma: 6
-# alpha: 38
-# gamma: 15
-# kernel: 21
-# elastic_freq:100
-# beta:15
-# resize: 29
- norm_a: 127.5
- norm_b: 1
- }
-}
-
-
-layer{
- name: "label"
- type: kLabel
- srclayers: "data"
-}
-
-layer{
- name: "fc1"
- type: kInnerProduct
- srclayers:"mnist"
- innerproduct_conf{
- num_output: 2500
- }
- param{
- name: "w1"
- init_method: kUniform
- low:-0.05
- high:0.05
- }
- param{
- name: "b1"
- init_method: kUniform
- low: -0.05
- high:0.05
- }
-}
-
-layer{
- name: "tanh1"
- type: kTanh
- srclayers:"fc1"
-}
-layer{
- name: "fc2"
- type: kInnerProduct
- srclayers:"tanh1"
- innerproduct_conf{
- num_output: 2000
- }
- param{
- name: "w2"
- init_method: kUniform
- low:-0.05
- high:0.05
- }
- param{
- name: "b2"
- init_method: kUniform
- low: -0.05
- high:0.05
- }
-}
-
-layer{
- name: "tanh2"
- type: kTanh
- srclayers:"fc2"
-}
-layer{
- name: "fc3"
- type: kInnerProduct
- srclayers:"tanh2"
- innerproduct_conf{
- num_output: 1500
- }
- param{
- name: "w3"
- init_method: kUniform
- low:-0.05
- high:0.05
- }
- param{
- name: "b3"
- init_method: kUniform
- low: -0.05
- high:0.05
- }
-
-}
-
-layer{
- name: "tanh3"
- type: kTanh
- srclayers:"fc3"
-}
-layer{
- name: "fc4"
- type: kInnerProduct
- srclayers:"tanh3"
- innerproduct_conf{
- num_output: 1000
- }
- param{
- name: "w4"
- init_method: kUniform
- low:-0.05
- high:0.05
- }
- param{
- name: "b4"
- init_method: kUniform
- low: -0.05
- high:0.05
- }
-
-}
-
-layer{
- name: "tanh4"
- type: kTanh
- srclayers:"fc4"
-}
-layer{
- name: "fc5"
- type: kInnerProduct
- srclayers:"tanh4"
- innerproduct_conf{
- num_output: 500
- }
- param{
- name: "w5"
- init_method: kUniform
- low:-0.05
- high:0.05
- }
- param{
- name: "b5"
- init_method: kUniform
- low: -0.05
- high:0.05
- }
-
-}
-
-layer{
- name: "tanh5"
- type: kTanh
- srclayers:"fc5"
-}
-layer{
- name: "fc6"
- type: kInnerProduct
- srclayers:"tanh5"
- innerproduct_conf{
- num_output: 10
- }
- param{
- name: "w6"
- init_method: kUniform
- low:-0.05
- high:0.05
- }
- param{
- name: "b6"
- init_method: kUniform
- low: -0.05
- high:0.05
- }
-}
-layer{
- name: "loss"
- type:kSoftmaxLoss
- softmaxloss_conf{
- topk:1
- }
- srclayers:"fc6"
- srclayers:"label"
-}
-}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/0478e8cf/examples/mnist/run.sh
----------------------------------------------------------------------
diff --git a/examples/mnist/run.sh b/examples/mnist/run.sh
deleted file mode 100755
index 1b16ca9..0000000
--- a/examples/mnist/run.sh
+++ /dev/null
@@ -1,40 +0,0 @@
-#!/bin/bash
-
-if [ $# -ne 2 ];then
- echo "Usage: run.sh [start|stop] num_procs"
- exit
-fi
-
-script_path=`readlink -f $0`
-script_dir=`dirname $script_path`
-example_dir=`dirname $script_dir`
-singa_dir=`dirname $example_dir`
-exec_path=${singa_dir}/build/pm
-host_path=$script_dir/hostfile
-model_path=$script_dir/mlp.conf
-cluster_path=$script_dir/cluster.conf
-ssh_options="-oStrictHostKeyChecking=no \
--oUserKnownHostsFile=/dev/null"
-
-hosts=(`cat $host_path |cut -d ' ' -f 1`)
-params=(`cat $host_path | cut -d ' ' -f 2`)
-if [ $1 == "start" ]
-then
- rm -rf $singa_dir/log*
- for (( i=0; i<$2; i++ ))
- do
- cmd="source ~/.bash_profile; touch $singa_dir/$i.lock;\
- $exec_path --hostfile=$script_dir/hostfile --procs_id=$i\
- --model=${modelfile} --cluster=${clusterfile}"
- echo ${hosts[$i]} $ssh_options $cmd
- ssh $ssh_options ${hosts[$i]} $cmd &
- done
-elif [ $1 == "stop" ]
-then
- for (( idx=0 ; idx<$2 ; idx++ ))
- do
- echo "ssh ${hosts[$idx]} \"kill pm\""
- ssh $ssh_options ${hosts[$idx]} "killall -q pm"
- sleep 1
- done
-fi
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/0478e8cf/include/neuralnet/base_layer.h
----------------------------------------------------------------------
diff --git a/include/neuralnet/base_layer.h b/include/neuralnet/base_layer.h
index 6ae7c50..c00a1c9 100644
--- a/include/neuralnet/base_layer.h
+++ b/include/neuralnet/base_layer.h
@@ -8,7 +8,7 @@
#include <memory>
#include <thread>
-#include "proto/model.pb.h"
+#include "proto/job.pb.h"
#include "proto/common.pb.h"
#include "utils/param.h"
#include "utils/common.h"
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/0478e8cf/include/neuralnet/layer.h
----------------------------------------------------------------------
diff --git a/include/neuralnet/layer.h b/include/neuralnet/layer.h
index 9a79e3f..7f3b256 100644
--- a/include/neuralnet/layer.h
+++ b/include/neuralnet/layer.h
@@ -12,7 +12,7 @@
#include <chrono>
#include <random>
-#include "proto/model.pb.h"
+#include "proto/job.pb.h"
#include "utils/data_shard.h"
#include "neuralnet/base_layer.h"
@@ -237,22 +237,6 @@ class ShardDataLayer: public DataLayer{
private:
shared_ptr<DataShard> shard_;
};
-class LMDBDataLayer: public DataLayer{
- public:
- using Layer::ComputeFeature;
-
- void Setup(const LayerProto& proto, int npartitions) override;
- void ComputeFeature(Phase phase, Metric *perf) override;
- void ConvertCaffeDatumToRecord(const CaffeDatum& datum,
- SingleLabelImageRecord* record);
-
- private:
- MDB_env* mdb_env_;
- MDB_dbi mdb_dbi_;
- MDB_txn* mdb_txn_;
- MDB_cursor* mdb_cursor_;
- MDB_val mdb_key_, mdb_value_;
-};
/**
* This layer apply Tan function to neuron activations.
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/0478e8cf/include/neuralnet/neuralnet.h
----------------------------------------------------------------------
diff --git a/include/neuralnet/neuralnet.h b/include/neuralnet/neuralnet.h
index 6aec88e..1dbf44a 100644
--- a/include/neuralnet/neuralnet.h
+++ b/include/neuralnet/neuralnet.h
@@ -6,7 +6,7 @@
#include <memory>
#include <string>
-#include "proto/model.pb.h"
+#include "proto/job.pb.h"
#include "neuralnet/layer.h"
#include "utils/factory.h"
#include "utils/graph.h"
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/0478e8cf/include/neuralnet/optional_layer.h
----------------------------------------------------------------------
diff --git a/include/neuralnet/optional_layer.h b/include/neuralnet/optional_layer.h
new file mode 100644
index 0000000..2cbcdb8
--- /dev/null
+++ b/include/neuralnet/optional_layer.h
@@ -0,0 +1,28 @@
+#ifdef USE_OPTIONAL_LAYER
+#ifndef SINGA_NEURALNET_OPTIONAL_LAYER_
+#define SINGA_NEURALNET_OPTIONAL_LAYER_
+#include "neuralnet/layer.h"
+
+namespace singa {
+
+class LMDBDataLayer: public DataLayer{
+ public:
+ using Layer::ComputeFeature;
+
+ void Setup(const LayerProto& proto, int npartitions) override;
+ void ComputeFeature(Phase phase, Metric *perf) override;
+ void ConvertCaffeDatumToRecord(const CaffeDatum& datum,
+ SingleLabelImageRecord* record);
+
+ private:
+ MDB_env* mdb_env_;
+ MDB_dbi mdb_dbi_;
+ MDB_txn* mdb_txn_;
+ MDB_cursor* mdb_cursor_;
+ MDB_val mdb_key_, mdb_value_;
+};
+} /* singa */
+
+#endif // SINGA_NEURALNET_OPTIONAL_LAYER_
+#endif
+
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/0478e8cf/include/singa.h
----------------------------------------------------------------------
diff --git a/include/singa.h b/include/singa.h
new file mode 100644
index 0000000..6eca6c4
--- /dev/null
+++ b/include/singa.h
@@ -0,0 +1,32 @@
+#ifndef SINGA_SINGA_H_
+#define SINGA_SINGA_H_
+#include <gflags/gflags.h>
+#include <glog/logging.h>
+
+#include "utils/common.h"
+#include "proto/job.pb.h"
+#include "proto/singa.pb.h"
+
+#include "utils/param.h"
+#include "utils/singleton.h"
+#include "utils/factory.h"
+
+#include "neuralnet/neuralnet.h"
+#include "trainer/trainer.h"
+#include "communication/socket.h"
+
+DEFINE_string(singa_conf, "conf/singa.conf", "Global config file");
+
+namespace singa {
+void SubmitJob(int job, bool resume, const JobProto& jobConf) {
+ SingaProto singaConf;
+ ReadProtoFromTextFile(FLAGS_singa_conf.c_str(), &singaConf);
+ if (singaConf.has_log_dir())
+ SetupLog(singaConf.log_dir(),
+ std::to_string(job) + "-" + jobConf.model().name());
+ Trainer trainer;
+ trainer.Start(job, resume, jobConf, singaConf);
+}
+} /* singa */
+#endif // SINGA_SINGA_H_
+
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/0478e8cf/include/trainer/server.h
----------------------------------------------------------------------
diff --git a/include/trainer/server.h b/include/trainer/server.h
index 7fb60c4..ef6e599 100644
--- a/include/trainer/server.h
+++ b/include/trainer/server.h
@@ -4,7 +4,7 @@
#include <unordered_map>
#include <utils/param.h>
#include <utils/updater.h>
-#include "proto/model.pb.h"
+#include "proto/job.pb.h"
#include "communication/socket.h"
namespace singa {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/0478e8cf/include/trainer/trainer.h
----------------------------------------------------------------------
diff --git a/include/trainer/trainer.h b/include/trainer/trainer.h
index 9f47ccd..911a4c4 100644
--- a/include/trainer/trainer.h
+++ b/include/trainer/trainer.h
@@ -2,8 +2,8 @@
#define INCLUDE_TRAINER_TRAINER_H_
#include <unordered_map>
#include <queue>
-#include "proto/cluster.pb.h"
-#include "proto/model.pb.h"
+#include "proto/job.pb.h"
+#include "proto/singa.pb.h"
#include "utils/param.h"
#include "utils/singleton.h"
#include "utils/factory.h"
@@ -25,17 +25,16 @@ class Trainer{
~Trainer();
/**
* Entrance function which construct the workers and servers, and luanch
- * one thread per worker/server. TODO rename variables about cluster config,
- * job config, etc.
+ * one thread per worker/server.
*
- * @param resume if true resume the training from the latest checkpoint files
* @param job job ID
- * @param mconf model configuration
- * @param globalconf global singa configuration
- * @param cconf cluster configuration
+ * @param resume if true resume the training from the latest checkpoint files
+ * @param jobConf job configuration, including cluster and model configuration
+ * @param singaConf global singa configuration including zookeeper and
+ * log dir setting.
*/
- void Start(ModelProto& mconf, const GlobalProto& gconf,
- const ClusterProto& cconf, int job, bool resume);
+ void Start(int job, bool resume,
+ const JobProto& jobConf, const SingaProto& singaConf);
protected:
/**
@@ -45,27 +44,27 @@ class Trainer{
* checkpoint, which will be added into the checkpoint field. The workers
* would then load the values of params from the checkpoint files.
*
- * @param model_conf model configuration
+ * @param modelConf model configuration
*/
- void Resume(ModelProto& model_conf);
+ void Resume(ModelProto* modelConf);
/**
* Create server instances.
* @param nthread total num of threads in current procs which is used to
* assign each thread a local thread ID. The number of workers is extracted
* from Cluster
- * @param model_conf
+ * @param modelConf
* @return server instances
*/
- vector<Server*> CreateServers(int nthread, const ModelProto& mproto);
+ vector<Server*> CreateServers(int nthread, const ModelProto& modelConf);
/**
* Create workers instances.
* @param nthread total num of threads in current procs which is used to
* assign each thread a local thread ID. The number of workers is extracted
* from Cluster
- * @param model_conf
+ * @param modelConf
* @return worker instances
*/
- vector<Worker*> CreateWorkers(int nthread, const ModelProto& mproto);
+ vector<Worker*> CreateWorkers(int nthread, const ModelProto& modelConf);
/**
* Setup workers and servers.
@@ -73,12 +72,12 @@ class Trainer{
* For each worker, create and assign a neuralnet to it.
* For each server, create and assign the param shard to it.
* Create the partition map from slice ID to server
- * @param model_conf
+ * @param modelConf
* @param workers
* @param servers
*/
void SetupWorkerServer(
- const ModelProto& model_conf,
+ const ModelProto& modelConf,
const vector<Worker*>& workers,
const vector<Server*>& servers);
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/0478e8cf/include/trainer/worker.h
----------------------------------------------------------------------
diff --git a/include/trainer/worker.h b/include/trainer/worker.h
index 9bda99c..db8cac3 100644
--- a/include/trainer/worker.h
+++ b/include/trainer/worker.h
@@ -1,7 +1,7 @@
#ifndef SINGA_TRAINER_WORKER_H_
#define SINGA_TRAINER_WORKER_H_
#include "neuralnet/neuralnet.h"
-#include "proto/model.pb.h"
+#include "proto/job.pb.h"
#include "utils/updater.h"
#include "communication/socket.h"
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/0478e8cf/include/utils/cluster.h
----------------------------------------------------------------------
diff --git a/include/utils/cluster.h b/include/utils/cluster.h
index b48c1c5..e2c979d 100644
--- a/include/utils/cluster.h
+++ b/include/utils/cluster.h
@@ -7,7 +7,8 @@
#include <vector>
#include <unordered_map>
#include "utils/common.h"
-#include "proto/cluster.pb.h"
+#include "proto/job.pb.h"
+#include "proto/singa.pb.h"
#include "utils/cluster_rt.h"
using std::shared_ptr;
@@ -24,8 +25,8 @@ namespace singa {
class Cluster {
public:
static shared_ptr<Cluster> Get();
- static shared_ptr<Cluster> Get(const GlobalProto& global,
- const ClusterProto& cluster, int job_id);
+ static shared_ptr<Cluster> Get(int job_id,
+ const SingaProto& singaConf, const ClusterProto& clusterConf);
const int nserver_groups()const{ return cluster_.nserver_groups(); }
const int nworker_groups()const { return cluster_.nworker_groups(); }
@@ -59,7 +60,7 @@ class Cluster {
/**
* @return global procs id, which starts from 0.
*/
- int procs_id()const {return procs_id_;}
+ int procs_id() const {return procs_id_;}
void set_procs_id(int procs_id) {procs_id_ = procs_id;}
bool server_worker_separate() const {
return cluster_.server_worker_separate();
@@ -88,6 +89,7 @@ class Cluster {
const string checkpoint_folder() const {
return cluster_.workspace()+"/checkpoint";
}
+ /*
const int stub_timeout() const {
return cluster_.stub_timeout();
}
@@ -97,6 +99,7 @@ class Cluster {
const int server_timeout() const {
return cluster_.server_timeout();
}
+ */
const bool server_update() const {
return cluster_.server_update();
@@ -121,14 +124,24 @@ class Cluster {
return cluster_rt_;
}
+ /**
+ * @return logical procs ID
+ */
int ProcsIDOf(int group_id, int id, int flag);
const string hostip() const {
return hostip_;
}
- void Register(const string& endpoint, int pid);
+ /**
+ * Register this process.
+ *
+ * @param pid physical process id get from OS, all other procs ID refers to
+ * logical process ID.
+ * @param endpoint unique string for other procs to connect
+ */
+ void Register(int pid, const string& endpoint);
private:
- Cluster(const GlobalProto& global, const ClusterProto &cluster, int job_id);
+ Cluster(int job, const SingaProto& singaConf, const ClusterProto& clusterConf);
void SetupFolders(const ClusterProto &cluster);
int Hash(int gid, int id, int flag);
@@ -139,7 +152,7 @@ class Cluster {
std::vector<std::string> endpoints_;
// cluster config proto
ClusterProto cluster_;
- GlobalProto global_;
+ SingaProto singa_;
shared_ptr<ClusterRuntime> cluster_rt_;
// make this class a singlton
static shared_ptr<Cluster> instance_;
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/0478e8cf/include/utils/common.h
----------------------------------------------------------------------
diff --git a/include/utils/common.h b/include/utils/common.h
index ef83031..c4ff4eb 100644
--- a/include/utils/common.h
+++ b/include/utils/common.h
@@ -2,6 +2,7 @@
#define SINGA_UTILS_COMMON_H_
#include <google/protobuf/message.h>
+#include <gflags/gflags.h>
#include <stdlib.h>
#include <unordered_map>
#include <sstream>
@@ -9,6 +10,11 @@
#include <vector>
#include "proto/common.pb.h"
+#ifndef GFLAGS_GFLAGS_H_
+ namespace gflags = google;
+#endif // GFLAGS_GFLAGS_H_
+
+
namespace singa {
using std::vector;
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/0478e8cf/include/utils/param.h
----------------------------------------------------------------------
diff --git a/include/utils/param.h b/include/utils/param.h
index 0273519..18293b6 100644
--- a/include/utils/param.h
+++ b/include/utils/param.h
@@ -2,7 +2,7 @@
#define INCLUDE_UTILS_PARAM_H_
#include <vector>
#include <string>
-#include "proto/model.pb.h"
+#include "proto/job.pb.h"
#include "utils/blob.h"
#include "communication/msg.h"
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/0478e8cf/include/utils/updater.h
----------------------------------------------------------------------
diff --git a/include/utils/updater.h b/include/utils/updater.h
index ea6d74a..d2f4dc1 100644
--- a/include/utils/updater.h
+++ b/include/utils/updater.h
@@ -1,6 +1,6 @@
#ifndef INCLUDE_UTILS_UPDATER_H_
#define INCLUDE_UTILS_UPDATER_H_
-#include "proto/model.pb.h"
+#include "proto/job.pb.h"
#include "utils/param.h"
namespace singa{
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/0478e8cf/src/main.cc
----------------------------------------------------------------------
diff --git a/src/main.cc b/src/main.cc
index cda1294..87ab384 100644
--- a/src/main.cc
+++ b/src/main.cc
@@ -1,29 +1,19 @@
-#include <gflags/gflags.h>
-#include <glog/logging.h>
-#include "trainer/trainer.h"
-#include "utils/common.h"
-#ifndef GFLAGS_GFLAGS_H_
- namespace gflags = google;
-#endif // GFLAGS_GFLAGS_H_
+#include "singa.h"
/**
* \file main.cc is the main entry of SINGA, like the driver program for Hadoop.
*
* 1. Users register their own implemented classes, e.g., layer, updater, etc.
- * 2. Users prepare the google protobuf object for the model configuration and
- * the cluster configuration.
+ * 2. Users prepare the google protobuf object for the job configuration.
* 3. Users call trainer to start the training.
*
* TODO
- * 1. Add helper functions for users to configure their model and cluster
- * easily, e.g., AddLayer(layer_type, source_layers, meta_data).
+ * 1. Add helper functions for users to configure their model easily,
+ * e.g., AddLayer(layer_type, source_layers, meta_data).
*/
-// TODO: re-organize ClusterProto, GlobalProto and ModelProto.
-DEFINE_int32(job, -1, "Unique job ID");
-DEFINE_bool(resume, false, "Resume from checkpoint");
-DEFINE_string(cluster, "examples/mnist/cluster.conf", "Cluster config file");
-DEFINE_string(model, "examples/mnist/conv.conf", "Model config file");
-DEFINE_string(global, "conf/singa.conf", "Global config file");
+DEFINE_int32(job, -1, "Unique job ID generated from singa-run.sh");
+DEFINE_bool(resume, false, "Resume from checkpoint passed at cmd line");
+DEFINE_string(workspace, "./workspace", "workspace passed at cmd line");
/**
* Register layers, and other customizable classes.
@@ -31,7 +21,8 @@ DEFINE_string(global, "conf/singa.conf", "Global config file");
* If users want to use their own implemented classes, they should register
* them here. Refer to the Worker::RegisterDefaultClasses()
*/
-void RegisterClasses(const singa::ModelProto& proto) {
+void RegisterClasses() {
+
}
@@ -39,20 +30,15 @@ int main(int argc, char **argv) {
google::InitGoogleLogging(argv[0]);
gflags::ParseCommandLineFlags(&argc, &argv, true);
- singa::ClusterProto cluster;
- singa::ReadProtoFromTextFile(FLAGS_cluster.c_str(), &cluster);
- singa::ModelProto model;
- singa::ReadProtoFromTextFile(FLAGS_model.c_str(), &model);
- singa::GlobalProto global;
- singa::ReadProtoFromTextFile(FLAGS_global.c_str(), &global);
- singa::SetupLog(global.log_dir(), model.name());
-
- LOG(INFO) << "The cluster config is\n" << cluster.DebugString();
- LOG(INFO) << "The model config is\n" << model.DebugString();
- LOG(INFO) << "The global config is\n" << global.DebugString();
+ singa::JobProto jobConf;
+ std::string job_file = FLAGS_workspace + "/job.conf";
+ singa::ReadProtoFromTextFile(job_file.c_str(), &jobConf);
+ CHECK(jobConf.has_cluster());
+ CHECK(jobConf.has_model());
+ if (!jobConf.cluster().has_workspace())
+ jobConf.mutable_cluster()->set_workspace(FLAGS_workspace);
- RegisterClasses(model);
- singa::Trainer trainer;
- trainer.Start(model, global, cluster, FLAGS_job, FLAGS_resume);
+ RegisterClasses();
+ singa::SubmitJob(FLAGS_job, FLAGS_resume, jobConf);
return 0;
}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/0478e8cf/src/neuralnet/layer.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/layer.cc b/src/neuralnet/layer.cc
index e965888..3d9def4 100644
--- a/src/neuralnet/layer.cc
+++ b/src/neuralnet/layer.cc
@@ -1,8 +1,6 @@
#include <glog/logging.h>
#include <memory>
#include <algorithm>
-#include <opencv2/highgui/highgui.hpp>
-#include <opencv2/imgproc/imgproc.hpp>
#include "mshadow/tensor.h"
#include "mshadow/cxxnet_op.h"
#include "neuralnet/layer.h"
@@ -231,103 +229,6 @@ void LabelLayer::ParseRecords(Phase phase, const vector<Record>& records,
CHECK_EQ(rid, blob->shape()[0]);
}
-
-/*********************LMDBDataLayer**********************************/
-void LMDBDataLayer::ComputeFeature(Phase phase, Metric* perf){
- if(random_skip_){
- int nskip = rand() % random_skip_;
- int n=0;
- CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_,
- &mdb_value_, MDB_FIRST), MDB_SUCCESS);
- while (mdb_cursor_get(mdb_cursor_, &mdb_key_,
- &mdb_value_, MDB_NEXT) == MDB_SUCCESS)
- n++;
- LOG(INFO)<<"Random Skip "<<nskip<<" records of total "<<n<<"records";
- // We have reached the end. Restart from the first.
- CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_,
- &mdb_value_, MDB_FIRST), MDB_SUCCESS);
- for(int i=0;i<nskip;i++){
- if (mdb_cursor_get(mdb_cursor_, &mdb_key_,
- &mdb_value_, MDB_NEXT) != MDB_SUCCESS) {
- // We have reached the end. Restart from the first.
- DLOG(INFO) << "Restarting data prefetching from start.";
- CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_,
- &mdb_value_, MDB_FIRST), MDB_SUCCESS);
- }
- }
- random_skip_=0;
- }
- CaffeDatum datum;
- for(auto& record: records_){
- SingleLabelImageRecord* image=record.mutable_image();
- CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_,
- &mdb_value_, MDB_GET_CURRENT), MDB_SUCCESS);
- datum.ParseFromArray(mdb_value_.mv_data, mdb_value_.mv_size);
- ConvertCaffeDatumToRecord(datum, image);
- if (mdb_cursor_get(mdb_cursor_, &mdb_key_,
- &mdb_value_, MDB_NEXT) != MDB_SUCCESS) {
- // We have reached the end. Restart from the first.
- DLOG(INFO) << "Restarting data prefetching from start.";
- CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_,
- &mdb_value_, MDB_FIRST), MDB_SUCCESS);
- }
- }
-}
-
-void LMDBDataLayer::ConvertCaffeDatumToRecord(const CaffeDatum& datum,
- SingleLabelImageRecord* record){
- record->set_label(datum.label());
- record->clear_shape();
- if(datum.has_channels())
- record->add_shape(datum.channels());
- if(datum.has_height())
- record->add_shape(datum.height());
- if(datum.has_width())
- record->add_shape(datum.width());
- if(datum.has_data())
- record->set_pixel(datum.data());
- if(datum.float_data_size()){
- record->clear_data();
- for(float x: datum.float_data())
- record->add_data(x);
- }
-}
-
-void LMDBDataLayer::Setup(const LayerProto& proto, int npartitions) {
- Layer::Setup(proto, npartitions);
- CHECK_EQ(mdb_env_create(&mdb_env_), MDB_SUCCESS) << "mdb_env_create failed";
- CHECK_EQ(mdb_env_set_mapsize(mdb_env_, 1099511627776), MDB_SUCCESS); // 1TB
- CHECK_EQ(mdb_env_open(mdb_env_,
- proto.lmdbdata_conf().path().c_str(),
- MDB_RDONLY, 0664), MDB_SUCCESS) << "cannot open lmdb "
- << proto.lmdbdata_conf().path();
- CHECK_EQ(mdb_txn_begin(mdb_env_, NULL, MDB_RDONLY, &mdb_txn_), MDB_SUCCESS)
- << "mdb_txn_begin failed";
- CHECK_EQ(mdb_open(mdb_txn_, NULL, 0, &mdb_dbi_), MDB_SUCCESS)
- << "mdb_open failed";
- CHECK_EQ(mdb_cursor_open(mdb_txn_, mdb_dbi_, &mdb_cursor_), MDB_SUCCESS)
- << "mdb_cursor_open failed";
- LOG(INFO) << "Opening lmdb " << proto.lmdbdata_conf().path();
- CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_, &mdb_value_, MDB_FIRST),
- MDB_SUCCESS) << "mdb_cursor_get failed";
-
- if (mdb_cursor_get(mdb_cursor_, &mdb_key_, &mdb_value_, MDB_NEXT)
- != MDB_SUCCESS) {
- CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_, &mdb_value_,
- MDB_FIRST), MDB_SUCCESS);
- }
- CaffeDatum datum;
- datum.ParseFromArray(mdb_value_.mv_data, mdb_value_.mv_size);
- SingleLabelImageRecord* record=sample_.mutable_image();
- ConvertCaffeDatumToRecord(datum, record);
-
- batchsize_=batchsize();
- if(partition_dim() == 0)
- batchsize_ /= npartitions;
- records_.resize(batchsize_);
- random_skip_=proto.lmdbdata_conf().random_skip();
-}
-
/***************** Implementation for LRNLayer *************************/
void LRNLayer::Setup(const LayerProto& proto, int npartitions) {
Layer::Setup(proto, npartitions);
@@ -377,63 +278,29 @@ void MnistLayer::ParseRecords(Phase phase,
LOG_IF(ERROR, records.size()==0)<<"Empty records to parse";
int ndim=records.at(0).image().shape_size();
int inputsize =records.at(0).image().shape(ndim-1);
+ CHECK_EQ(inputsize, blob->shape()[1]);
float* dptr=blob->mutable_cpu_data();
for(const Record& record: records){
- // copy from record to cv::Mat
- cv::Mat input(inputsize, inputsize, CV_32FC1);
const SingleLabelImageRecord& imagerecord=record.image();
- if(imagerecord.pixel().size()){
+ if(imagerecord.pixel().size()) {
string pixel=imagerecord.pixel();
- for(int i=0,k=0;i<inputsize;i++)
- for(int j=0;j<inputsize;j++)
+ for(int i = 0, k = 0; i < inputsize; i++) {
+ for(int j = 0; j < inputsize; j++) {
// NOTE!!! must cast pixel to uint8_t then to float!!! waste a lot of
// time to debug this
- input.at<float>(i,j)=static_cast<float>(static_cast<uint8_t>(pixel[k++]));
- }else{
- for(int i=0,k=0;i<inputsize;i++)
- for(int j=0;j<inputsize;j++)
- input.at<float>(i,j)=imagerecord.data(k++);
- }
- int size=blob->shape()[1];
- /*
- cv::Mat resizeMat=input;
- // affine transform, scaling, rotation and shearing
- if(gamma_){
- float r1=rand_real()*2-1;
- float r2=rand_real()*2-1;
- int h=static_cast<int>(inputsize*(1.+r1*gamma_/100.0));
- int w=static_cast<int>(inputsize*(1.+r2*gamma_/100.0));
- cv::resize(input, resizeMat, cv::Size(h,w));
- }
- cv::Mat betaMat=resizeMat;
- cv::Mat warpmat(2,3, CV_32FC1);
- warpmat.at<float>(0,0)=1.0;
- warpmat.at<float>(0,1)=0.0;
- warpmat.at<float>(0,2)=0.0;
- warpmat.at<float>(1,0)=0.0;
- warpmat.at<float>(1,1)=1.0;
- warpmat.at<float>(1,2)=0.0;
-
- if(beta_){
- float r=rand_real()*2-1;
- if(rand() % 2){ // rotation
- cv::Point center(resizeMat.rows/2, resizeMat.cols/2);
- warpmat=cv::getRotationMatrix2D(center, r*beta_, 1.0);
- }else{
- //shearing
- warpmat.at<float>(0,1)=r*beta_/90;
- if(imagerecord.label()==1 ||imagerecord.label()==7)
- warpmat.at<float>(0,1)/=2.0;
+ float x = static_cast<float>(static_cast<uint8_t>(pixel[k++]));
+ x = x / norm_a_-norm_b_;
+ *dptr = x;
+ dptr++;
+ }
}
- }
- cv::warpAffine(resizeMat, betaMat, warpmat, cv::Size(size, size));
- */
-
- for(int i=0;i<size;i++){
- for(int j=0;j<size;j++){
- *dptr=input.at<float>(i,j)/norm_a_-norm_b_;
- dptr++;
+ } else {
+ for(int i = 0, k = 0; i < inputsize; i++) {
+ for(int j = 0; j < inputsize; j++) {
+ *dptr = imagerecord.data(k++) / norm_a_ - norm_b_;
+ dptr++;
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/0478e8cf/src/neuralnet/neuralnet.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/neuralnet.cc b/src/neuralnet/neuralnet.cc
index 173f08f..7769e45 100644
--- a/src/neuralnet/neuralnet.cc
+++ b/src/neuralnet/neuralnet.cc
@@ -22,7 +22,6 @@ void NeuralNet::RegisterLayers() {
RegisterLayer(factory, Dropout);
RegisterLayer(factory, InnerProduct);
RegisterLayer(factory, Label);
- RegisterLayer(factory, LMDBData);
RegisterLayer(factory, LRN);
RegisterLayer(factory, Mnist);
RegisterLayer(factory, Prefetch);
@@ -34,6 +33,10 @@ void NeuralNet::RegisterLayers() {
RegisterLayer(factory, SoftmaxLoss);
RegisterLayer(factory, Split);
RegisterLayer(factory, Tanh);
+
+#ifdef USE_OPTIONAL_LAYER
+ RegisterLayer(factory, LMDBData);
+#endif
}
shared_ptr<NeuralNet> NeuralNet::Create(
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/0478e8cf/src/neuralnet/optional_layer.cc
----------------------------------------------------------------------
diff --git a/src/neuralnet/optional_layer.cc b/src/neuralnet/optional_layer.cc
new file mode 100644
index 0000000..ba85807
--- /dev/null
+++ b/src/neuralnet/optional_layer.cc
@@ -0,0 +1,104 @@
+#ifdef USE_OPTIONAL_LAYER
+#include "neuralnet/optional_layer.h"
+namespace singa {
+
+/*********************LMDBDataLayer**********************************/
+void LMDBDataLayer::ComputeFeature(Phase phase, Metric* perf){
+ if(random_skip_){
+ int nskip = rand() % random_skip_;
+ int n=0;
+ CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_,
+ &mdb_value_, MDB_FIRST), MDB_SUCCESS);
+ while (mdb_cursor_get(mdb_cursor_, &mdb_key_,
+ &mdb_value_, MDB_NEXT) == MDB_SUCCESS)
+ n++;
+ LOG(INFO)<<"Random Skip "<<nskip<<" records of total "<<n<<"records";
+ // We have reached the end. Restart from the first.
+ CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_,
+ &mdb_value_, MDB_FIRST), MDB_SUCCESS);
+ for(int i=0;i<nskip;i++){
+ if (mdb_cursor_get(mdb_cursor_, &mdb_key_,
+ &mdb_value_, MDB_NEXT) != MDB_SUCCESS) {
+ // We have reached the end. Restart from the first.
+ DLOG(INFO) << "Restarting data prefetching from start.";
+ CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_,
+ &mdb_value_, MDB_FIRST), MDB_SUCCESS);
+ }
+ }
+ random_skip_=0;
+ }
+ CaffeDatum datum;
+ for(auto& record: records_){
+ SingleLabelImageRecord* image=record.mutable_image();
+ CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_,
+ &mdb_value_, MDB_GET_CURRENT), MDB_SUCCESS);
+ datum.ParseFromArray(mdb_value_.mv_data, mdb_value_.mv_size);
+ ConvertCaffeDatumToRecord(datum, image);
+ if (mdb_cursor_get(mdb_cursor_, &mdb_key_,
+ &mdb_value_, MDB_NEXT) != MDB_SUCCESS) {
+ // We have reached the end. Restart from the first.
+ DLOG(INFO) << "Restarting data prefetching from start.";
+ CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_,
+ &mdb_value_, MDB_FIRST), MDB_SUCCESS);
+ }
+ }
+}
+
+void LMDBDataLayer::ConvertCaffeDatumToRecord(const CaffeDatum& datum,
+ SingleLabelImageRecord* record){
+ record->set_label(datum.label());
+ record->clear_shape();
+ if(datum.has_channels())
+ record->add_shape(datum.channels());
+ if(datum.has_height())
+ record->add_shape(datum.height());
+ if(datum.has_width())
+ record->add_shape(datum.width());
+ if(datum.has_data())
+ record->set_pixel(datum.data());
+ if(datum.float_data_size()){
+ record->clear_data();
+ for(float x: datum.float_data())
+ record->add_data(x);
+ }
+}
+
+void LMDBDataLayer::Setup(const LayerProto& proto, int npartitions) {
+ Layer::Setup(proto, npartitions);
+ CHECK_EQ(mdb_env_create(&mdb_env_), MDB_SUCCESS) << "mdb_env_create failed";
+ CHECK_EQ(mdb_env_set_mapsize(mdb_env_, 1099511627776), MDB_SUCCESS); // 1TB
+ CHECK_EQ(mdb_env_open(mdb_env_,
+ proto.lmdbdata_conf().path().c_str(),
+ MDB_RDONLY, 0664), MDB_SUCCESS) << "cannot open lmdb "
+ << proto.lmdbdata_conf().path();
+ CHECK_EQ(mdb_txn_begin(mdb_env_, NULL, MDB_RDONLY, &mdb_txn_), MDB_SUCCESS)
+ << "mdb_txn_begin failed";
+ CHECK_EQ(mdb_open(mdb_txn_, NULL, 0, &mdb_dbi_), MDB_SUCCESS)
+ << "mdb_open failed";
+ CHECK_EQ(mdb_cursor_open(mdb_txn_, mdb_dbi_, &mdb_cursor_), MDB_SUCCESS)
+ << "mdb_cursor_open failed";
+ LOG(INFO) << "Opening lmdb " << proto.lmdbdata_conf().path();
+ CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_, &mdb_value_, MDB_FIRST),
+ MDB_SUCCESS) << "mdb_cursor_get failed";
+
+ if (mdb_cursor_get(mdb_cursor_, &mdb_key_, &mdb_value_, MDB_NEXT)
+ != MDB_SUCCESS) {
+ CHECK_EQ(mdb_cursor_get(mdb_cursor_, &mdb_key_, &mdb_value_,
+ MDB_FIRST), MDB_SUCCESS);
+ }
+ CaffeDatum datum;
+ datum.ParseFromArray(mdb_value_.mv_data, mdb_value_.mv_size);
+ SingleLabelImageRecord* record=sample_.mutable_image();
+ ConvertCaffeDatumToRecord(datum, record);
+
+ batchsize_=batchsize();
+ if(partition_dim() == 0)
+ batchsize_ /= npartitions;
+ records_.resize(batchsize_);
+ random_skip_=proto.lmdbdata_conf().random_skip();
+}
+
+} /* singa */
+
+#endif
+
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/0478e8cf/src/proto/cluster.proto
----------------------------------------------------------------------
diff --git a/src/proto/cluster.proto b/src/proto/cluster.proto
deleted file mode 100644
index 7afb866..0000000
--- a/src/proto/cluster.proto
+++ /dev/null
@@ -1,61 +0,0 @@
-package singa;
-
-message GlobalProto {
- // ip/hostname:port[,ip/hostname:port]
- required string zookeeper_host = 1;
- // if not set, use the default dir of glog
- optional string log_dir = 2 [default = "/tmp/singa-log/"];
-}
-
-message ClusterProto {
- optional int32 nworker_groups = 1;
- optional int32 nserver_groups = 2;
- optional int32 nworkers_per_group = 3 [default = 1];
- optional int32 nservers_per_group = 4 [default = 1];
- optional int32 nworkers_per_procs = 5 [default = 1];
- optional int32 nservers_per_procs = 6 [default = 1];
-
- // Used in standalone mode, one ip or hostname per line
- // For YARN or Mesos version, the processes are allocted dynamically,
- // hence no need to specify the hosts statically
- optional string hostfile = 10 [default=""];
-
- // servers and workers in different processes?
- optional bool server_worker_separate = 11 [default = false];
-
- // port number is used by ZeroMQ
- optional int32 start_port = 13 [default = 6723];
- // local workspace, train/val/test shards, checkpoint files
- required string workspace = 14;
- // relative path to workspace. if not set, use the default dir of glog
- //optional string log_dir = 15;
- // ip/hostname : port [, ip/hostname : port]
- //optional string zookeeper_host = 16 [default = "localhost:2181"];
- // message size limit, default 1MB
- // optional int32 largest_message = 20 [default = 1048576];
- // optional float bandwidth = 21 [default = 100]; // MB/s
-
- //repeated ServerTopology server_group = 20;
-
- optional int32 stub_timeout = 30 [default = 5000];
- optional int32 worker_timeout = 31 [default = 5000];
- optional int32 server_timeout = 32 [default = 5000];
-
- // conduct updates at server side; otherwise do it at worker side
- optional bool server_update = 40 [default = true];
- // share memory space between worker groups in one procs
- optional bool share_memory = 41 [default = true];
-
- // bandwidth of ethernet, Bytes per second, default is 1 Gbps
- optional int32 bandwidth=50 [default=134217728];
- // poll time in milliseconds
- optional int32 poll_time=51 [default =100];
-}
-
-message ServerTopology {
- // group id
- required int32 id = 1;
- optional int32 sync_interval = 2;
- // neighbor group id
- repeated int32 neighbor = 3;
-}