You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@singa.apache.org by wa...@apache.org on 2015/05/26 16:04:13 UTC
[1/2] incubator-singa git commit: add checking on test/validation
frequency for TestNow and ValidateNow functions.
Repository: incubator-singa
Updated Branches:
refs/heads/master 679573adf -> af235456b
add checking on test/validation frequency for TestNow and ValidateNow functions.
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/011823d1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/011823d1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/011823d1
Branch: refs/heads/master
Commit: 011823d144eeaa01a48b5c10bc3a4ab727f61312
Parents: 679573a
Author: wang wei <wa...@comp.nus.edu.sg>
Authored: Tue May 26 10:41:38 2015 +0800
Committer: wang wei <wa...@comp.nus.edu.sg>
Committed: Tue May 26 10:41:38 2015 +0800
----------------------------------------------------------------------
Makefile.example | 91 +++++++++++++++++++++++++++++++++++++++++++
include/trainer/worker.h | 2 +
2 files changed, 93 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/011823d1/Makefile.example
----------------------------------------------------------------------
diff --git a/Makefile.example b/Makefile.example
new file mode 100644
index 0000000..80dfc26
--- /dev/null
+++ b/Makefile.example
@@ -0,0 +1,91 @@
+###################User Config Varaibles #############################
+# third-party library installation folder
+HOME_DIR := /usr/
+# Lib folder for system and external libs. You may need to change it.
+LIBRARY_DIRS := $(HOME_DIR)/lib64 $(HOME_DIR)/lib $(HOME_DIR)/local/lib
+# Header folder for system and external libs. You may need to change it.
+INCLUDE_DIRS := $(HOME_DIR)/include ./include
+# g++ location, should support c++11, tested with 4.8.1
+CXX := g++
+
+######################Setting Varialbes#######################################
+LIBRARIES := glog gflags protobuf rt opencv_highgui opencv_imgproc opencv_core\
+ lmdb openblas zmq czmq
+
+LDFLAGS := $(foreach librarydir, $(LIBRARY_DIRS), -L$(librarydir))\
+ $(foreach library, $(LIBRARIES), -l$(library))
+# Folder to store compiled files
+BUILD_DIR := build
+MSHADOW_FLAGS :=-DMSHADOW_USE_CUDA=0 -DMSHADOW_USE_CBLAS=1 -DMSHADOW_USE_MKL=0
+CXXFLAGS := -O3 -Wall -pthread -fPIC -std=c++11 -Wno-unknown-pragmas \
+ $(MSHADOW_FLAGS) -DCPU_ONLY=1 \
+ -funroll-loops $(foreach includedir, $(INCLUDE_DIRS), -I$(includedir))
+
+# find user defined .proto file, and then compute the corresponding .h, .cc
+# files, which cannot be found by shell find, because they haven't been
+# generated currently
+PROTOS := $(shell find src/proto/ -name "*.proto")
+PROTO_SRCS :=$(PROTOS:.proto=.pb.cc)
+PROTO_HDRS :=$(patsubst src%, include%, $(PROTOS:.proto=.pb.h))
+PROTO_OBJS :=$(addprefix $(BUILD_DIR)/, $(PROTO_SRCS:.cc=.o))
+
+# each singa src file will generate a .o file
+SINGA_SRCS := $(shell find src/ \( -path "src/test" -o -path "src/main.cc" \) \
+ -prune -o \( -name "*.cc" -type f \) -print )
+SINGA_OBJS := $(sort $(addprefix $(BUILD_DIR)/, $(SINGA_SRCS:.cc=.o)) \
+ $(PROTO_OBJS) )
+-include $(SINGA_OBJS:%.o=%.P)
+
+TEST_SRCS :=$(shell find src/test/ -maxdepth 1 -name "*.cc")
+TEST_OBJS := $(sort $(addprefix $(BUILD_DIR)/, $(TEST_SRCS:.cc=.o)))
+-include $(TEST_OBJS:%.o=%.P)
+
+GTEST_SRC := include/gtest/gtest-all.cc
+GTEST_HDR := include/gtest/gtest.h
+GTEST_LIB := $(BUILD_DIR)/libgtest.a
+
+OBJS := $(sort $(SINGA_OBJS) $(TEST_OBJS) )
+
+########################Compilation Section###################################
+.PHONY: singa test
+
+singa: $(PROTO_OBJS) $(SINGA_OBJS)
+ $(CXX) $(SINGA_OBJS) src/main.cc -o $(BUILD_DIR)/singa $(CXXFLAGS) $(LDFLAGS)
+ @echo
+
+loader: proto $(LOADER_OBJS)
+ $(CXX) $(LOADER_OBJS) -o $(BUILD_DIR)/loader $(CXXFLAGS) $(LDFLAGS)
+ @echo
+
+test: proto $(GTEST_LIB) $(TEST_OBJS) $(SINGA_OBJS)
+ $(CXX) $(TEST_OBJS) include/gtest/gtest_main.cc $(GTEST_LIB) \
+ $(SINGA_OBJS) -o $(BUILD_DIR)/test $(CXXFLAGS) $(LDFLAGS)
+ @echo
+
+$(GTEST_LIB): $(GTEST_HDR) $(GTEST_SRC)
+ $(CXX) $(GTEST_SRC) -c -o $(BUILD_DIR)/gtest-all.o $(CXXFLAGS)
+ ar -rv $(GTEST_LIB) $(BUILD_DIR)/gtest-all.o
+
+# compile all files
+$(OBJS):$(BUILD_DIR)/%.o : %.cc
+ @mkdir -p $(dir $@)
+ $(CXX) $< $(CXXFLAGS) -MMD -c -o $@
+ cp $(BUILD_DIR)/$*.d $(BUILD_DIR)/$*.P; \
+ sed -e 's/#.*//' -e 's/^[^:]*: *//' -e 's/ *\\$$//' \
+ -e '/^$$/ d' -e 's/$$/ :/' < $(BUILD_DIR)/$*.d >> $(BUILD_DIR)/$*.P; \
+ rm -f $*.d
+
+proto: $(PROTO_OBJS)
+
+$(PROTO_SRCS): $(PROTOS)
+ protoc --proto_path=src/proto --cpp_out=src/proto $(PROTOS)
+ mkdir -p include/proto/
+ cp src/proto/*.pb.h include/proto/
+ @echo
+
+clean:
+ rm -rf *.a *.so
+ rm -rf include/proto/*
+ rm -rf src/proto/*.pb.h src/proto/*.pb.cc
+ rm -rf $(BUILD_DIR)
+ @echo
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/011823d1/include/trainer/worker.h
----------------------------------------------------------------------
diff --git a/include/trainer/worker.h b/include/trainer/worker.h
index a202e93..1a8a52d 100644
--- a/include/trainer/worker.h
+++ b/include/trainer/worker.h
@@ -109,6 +109,7 @@ class Worker {
const bool TestNow(const int step) const{
return (group_id_==0
&& modelproto_.test_frequency() > 0
+ && modelproto_.test_steps() > 0
&& step >= modelproto_.test_after_steps()
&& ((step - modelproto_.test_after_steps())
% modelproto_.test_frequency() == 0));
@@ -120,6 +121,7 @@ class Worker {
const bool ValidateNow(const int step) {
return (group_id_==0
&& modelproto_.validation_frequency() > 0
+ && modelproto_.validation_steps() > 0
&& step >= modelproto_.validation_after_steps()
&& ((step - modelproto_.validation_after_steps())
% modelproto_.validation_frequency() == 0));
[2/2] incubator-singa git commit: fix bug from zsock_connect by
binding router firstly and then connecting dealers to it.
Posted by wa...@apache.org.
fix bug from zsock_connect by binding router firstly and then connecting dealers to it.
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/af235456
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/af235456
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/af235456
Branch: refs/heads/master
Commit: af235456bc348d33502162f2b291557c69662d23
Parents: 011823d
Author: wang wei <wa...@comp.nus.edu.sg>
Authored: Tue May 26 17:05:42 2015 +0800
Committer: wang wei <wa...@comp.nus.edu.sg>
Committed: Tue May 26 17:05:42 2015 +0800
----------------------------------------------------------------------
Makefile.example | 2 +-
include/trainer/trainer.h | 2 ++
src/trainer/trainer.cc | 14 +++++++-------
3 files changed, 10 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/af235456/Makefile.example
----------------------------------------------------------------------
diff --git a/Makefile.example b/Makefile.example
index 80dfc26..9a1b514 100644
--- a/Makefile.example
+++ b/Makefile.example
@@ -50,7 +50,7 @@ OBJS := $(sort $(SINGA_OBJS) $(TEST_OBJS) )
.PHONY: singa test
singa: $(PROTO_OBJS) $(SINGA_OBJS)
- $(CXX) $(SINGA_OBJS) src/main.cc -o $(BUILD_DIR)/singa $(CXXFLAGS) $(LDFLAGS)
+ $(CXX) $(SINGA_OBJS) src/main.cc -o singa $(CXXFLAGS) $(LDFLAGS)
@echo
loader: proto $(LOADER_OBJS)
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/af235456/include/trainer/trainer.h
----------------------------------------------------------------------
diff --git a/include/trainer/trainer.h b/include/trainer/trainer.h
index f5d2591..fed011d 100644
--- a/include/trainer/trainer.h
+++ b/include/trainer/trainer.h
@@ -9,6 +9,7 @@
#include "neuralnet/neuralnet.h"
#include "trainer/worker.h"
#include "trainer/server.h"
+#include "communication/socket.h"
namespace singa {
/**
@@ -130,6 +131,7 @@ class Trainer{
protected:
int procs_id_;
+ shared_ptr<Router> router_;
};
} /* singa */
#endif // INCLUDE_TRAINER_TRAINER_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/af235456/src/trainer/trainer.cc
----------------------------------------------------------------------
diff --git a/src/trainer/trainer.cc b/src/trainer/trainer.cc
index bc6867d..00202ae 100644
--- a/src/trainer/trainer.cc
+++ b/src/trainer/trainer.cc
@@ -42,6 +42,11 @@ void Trainer::Start(const ModelProto& mproto, const ClusterProto& cproto,
RegisterDefaultClasses(mproto);
auto cluster=Cluster::Get(cproto, procs_id);
+ router_=make_shared<Router>();
+ router_->Bind(kInprocRouterEndpoint);
+ if(cluster->nprocs()>1)
+ router_->Bind(cluster->endpoint());
+
// create servers
vector<shared_ptr<Server>> servers;
int nthreads=1; // the first socket is the router
@@ -160,14 +165,9 @@ void Trainer::Start(const ModelProto& mproto, const ClusterProto& cproto,
void Trainer::Run(const std::map<int, shared_ptr<Trainer::ParamShard>>& shards){
auto cluster=Cluster::Get();
procs_id_=cluster->procs_id();
- auto router=make_shared<Router>();
- router->Bind(kInprocRouterEndpoint);
- if(cluster->nprocs()>1)
- router->Bind(cluster->endpoint());
-
map<int, shared_ptr<Dealer>> interprocs_dealers;
while(true){
- Msg* msg=router->Receive();
+ Msg* msg=router_->Receive();
if(msg==nullptr){
LOG(ERROR)<<"Connection broken!";
exit(0);
@@ -218,7 +218,7 @@ void Trainer::Run(const std::map<int, shared_ptr<Trainer::ParamShard>>& shards){
interprocs_dealers[procs_id]->Send(&msg);
*/
}else{
- router->Send(&msg);
+ router_->Send(&msg);
}
}
}