You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@singa.apache.org by wa...@apache.org on 2016/04/02 05:46:35 UTC
[2/2] incubator-singa git commit: SINGA-155 Remove zookeeper for
single-process training
SINGA-155 Remove zookeeper for single-process training
make zookeeper optional, configure with --enable-zookeeper to use zookeeper
* add USE_ZOOKEEPER flags
* update makefile
* check cpplint
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/0233049c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/0233049c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/0233049c
Branch: refs/heads/master
Commit: 0233049ce13d3f430affe4e32a88cbaf4a3e17e4
Parents: adeced6
Author: WANG Sheng <wa...@gmail.com>
Authored: Thu Mar 31 22:03:42 2016 +0800
Committer: WANG Sheng <wa...@gmail.com>
Committed: Fri Apr 1 12:59:59 2016 +0800
----------------------------------------------------------------------
Makefile.am | 94 ++++++++++++++++++++--------------
configure.ac | 95 +++++++++++++++++++----------------
include/singa/utils/cluster_rt.h | 3 ++
src/utils/cluster.cc | 3 +-
src/utils/cluster_rt.cc | 12 +++++
5 files changed, 124 insertions(+), 83 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/0233049c/Makefile.am
----------------------------------------------------------------------
diff --git a/Makefile.am b/Makefile.am
index f185a9d..95f7e81 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -32,19 +32,22 @@ CUDA_OBJS := src/utils/math_kernel.o
CUDA_HDRS := include/singa/utils/math_kernel.h
CUDNN_SRCS := src/neuralnet/loss_layer/cudnn_softmaxloss.cc \
- src/neuralnet/neuron_layer/cudnn_softmax.cc \
- src/neuralnet/neuron_layer/cudnn_pooling.cc \
- src/neuralnet/neuron_layer/cudnn_activation.cc \
- src/neuralnet/neuron_layer/cudnn_lrn.cc \
- src/neuralnet/neuron_layer/cudnn_convolution.cc
+ src/neuralnet/neuron_layer/cudnn_softmax.cc \
+ src/neuralnet/neuron_layer/cudnn_pooling.cc \
+ src/neuralnet/neuron_layer/cudnn_activation.cc \
+ src/neuralnet/neuron_layer/cudnn_lrn.cc \
+ src/neuralnet/neuron_layer/cudnn_convolution.cc
PY_SRCS := tool/python/singa/driver_wrap.cxx \
- src/driver.cc
+ src/driver.cc
+
+ZOOKEEPER_SRCS := src/utils/zk_service.cc
+ZOOKEEPER_HDRS := include/singa/utils/zk_service.h
HDFS_SRCS := src/io/hdfsfile.cc \
- src/io/hdfsfile_store.cc
+ src/io/hdfsfile_store.cc
HDFS_HDRS := include/singa/io/hdfsfile.h \
- include/singa/io/hdfsfile_store.h
+ include/singa/io/hdfsfile_store.h
SINGA_SRCS := src/driver.cc \
src/server.cc \
@@ -101,14 +104,13 @@ SINGA_SRCS := src/driver.cc \
src/utils/updater.cc \
src/utils/blob.cc \
src/utils/image_transform.cc \
- src/utils/job_manager.cc \
- src/utils/zk_service.cc
+ src/utils/job_manager.cc
SINGA_HDRS := include/singa.h \
- include/singa/utils/math_blob.h \
- include/singa/utils/math_addr.h \
- include/singa/utils/cluster.h \
+ include/singa/utils/math_blob.h \
+ include/singa/utils/math_addr.h \
+ include/singa/utils/cluster.h \
include/utils/cluster_rt.h \
include/utils/param.h \
include/utils/common.h \
@@ -121,8 +123,7 @@ SINGA_HDRS := include/singa.h \
include/utils/tinydir.h \
include/utils/tokenizer.h \
include/utils/image_transform.h \
- include/utils/job_manager.cc \
- include/utils/zk_service.cc \
+ include/utils/job_manager.h \
include/server.h \
include/worker.h \
include/stub.h \
@@ -151,19 +152,19 @@ SINGA_HDRS := include/singa.h \
GTEST_SRCS := include/gtest/gtest-all.cc
GTEST_HRDS := include/gtest/gtest.h
TEST_SRCS := include/gtest/gtest_main.cc \
- src/test/test_cluster.cc \
- src/test/test_common.cc \
- src/test/test_msg.cc \
- src/test/test_math.cc \
- src/test/test_neuralnet.cc \
- src/test/test_paramslicer.cc \
- src/test/test_kvfile.cc \
- src/test/test_store.cc \
- src/test/test_connection_layers.cc \
- src/test/test_record_input_layer.cc \
- src/test/test_csv_input_layer.cc \
- src/test/test_gru_layer.cc \
- src/test/test_unrolling.cc
+ src/test/test_cluster.cc \
+ src/test/test_common.cc \
+ src/test/test_msg.cc \
+ src/test/test_math.cc \
+ src/test/test_neuralnet.cc \
+ src/test/test_paramslicer.cc \
+ src/test/test_kvfile.cc \
+ src/test/test_store.cc \
+ src/test/test_connection_layers.cc \
+ src/test/test_record_input_layer.cc \
+ src/test/test_csv_input_layer.cc \
+ src/test/test_gru_layer.cc \
+ src/test/test_unrolling.cc
#EXTRA_PROGRAMS = $(PROGS)
EXTRA_PROGRAMS = singatest test
@@ -194,6 +195,11 @@ libsinga_la_SOURCES += $(CUDNN_SRCS)
libsinga_la_CXXFLAGS += $(CUDNN_CFLAGS)
libsinga_la_LDFLAGS += $(CUDNN_LDFLAGS) $(CUDNN_LIBS)
endif
+if DZOOKEEPER
+libsinga_la_SOURCES += $(ZOOKEEPER_SRCS)
+libsinga_la_CXXFLAGS += $(ZOOKEEPER_CFLAGS)
+libsinga_la_LDFLAGS += $(ZOOKEEPER_LDFLAGS) $(ZOOKEEPER_LIBS)
+endif
if DHDFS
libsinga_la_SOURCES += $(HDFS_SRCS)
@@ -209,8 +215,7 @@ singa_LDFLAGS = -lsinga \
-lprotobuf \
-lopenblas \
-lzmq \
- -lczmq \
- -lzookeeper_mt
+ -lczmq
if LMDB
singa_LDFLAGS += -llmdb
endif
@@ -221,6 +226,12 @@ singa_CXXFLAGS += $(CUDA_CFLAGS)
singa_LDFLAGS += $(CUDA_LDFLAGS) $(CUDA_LIBS)
endif
+if DZOOKEEPER
+singa_SOURCES += $(ZOOKEEPER_SRCS)
+singa_CXXFLAGS += $(ZOOKEEPER_CFLAGS)
+singa_LDFLAGS += $(ZOOKEEPER_LDFLAGS) $(ZOOKEEPER_LIBS)
+endif
+
if DCUDNN
singa_SOURCES += $(CUDNN_SRCS)
singa_CXXFLAGS += $(CUDNN_CFLAGS)
@@ -235,17 +246,17 @@ endif
#bin_PROGRAMS += singatool
singatool_SOURCES = src/utils/tool.cc #$(CUDA_SRCS) $(CUDA_HDRS) $(CUDNN_SRCS)
singatool_CXXFLAGS = -Wall -pthread -fPIC -std=c++11 -MMD -Wno-unknown-pragmas \
- -funroll-loops -DTHREADED -I$(top_srcdir)/include $(DEFAULT_FLAGS)
+ -funroll-loops -DTHREADED -I$(top_srcdir)/include $(DEFAULT_FLAGS)
singatool_LDFLAGS = -lsinga \
-lglog \
- -lprotobuf \
- -lzookeeper_mt
+ -lprotobuf
-if DHDFS
-singatool_SOURCES += $(HDFS_SRCS)
-singatool_CXXFLAGS += $(HDFS_CFLAGS)
-singatool_LDFLAGS += $(HDFS_LDFLAGS) $(HDFS_LIBS)
+if DZOOKEEPER
+singatool_SOURCES += $(ZOOKEEPER_SRCS)
+singatool_CXXFLAGS += $(ZOOKEEPER_CFLAGS)
+singatool_LDFLAGS += $(ZOOKEEPER_LDFLAGS) $(ZOOKEEPER_LIBS)
endif
+
#lib_LTLIBRARIES += libgtest.la
libgtest_la_SOURCES = $(GTEST_HDRS) $(GTEST_SRCS)
libgtest_la_CXXFLAGS = $(DEFAULT_FLAGS) -msse3 -fpermissive -I$(top_srcdir)/include
@@ -265,8 +276,7 @@ singatest_LDFLAGS = -lsinga \
-lopenblas \
-lzmq \
-lczmq \
- -lzookeeper_mt \
- -lgtest
+ -lgtest
if LMDB
singatest_LDFLAGS += -llmdb
endif
@@ -283,6 +293,12 @@ singatest_CXXFLAGS += $(CUDNN_CFLAGS)
singatest_LDFLAGS += $(CUDNN_LDFLAGS) $(CUDNN_LIBS)
endif
+if DZOOKEEPER
+singatest_SOURCES += $(ZOOKEEPER_SRCS)
+singatest_CXXFLAGS += $(ZOOKEEPER_CFLAGS)
+singatest_LDFLAGS += $(ZOOKEEPER_LDFLAGS) $(ZOOKEEPER_LIBS)
+endif
+
_driver_la_SOURCES = $(PY_SRCS)
_driver_la_CXXFLAGS = $(DEFAULT_FLAGS) $(MSHADOW_FLAGS) -I$(top_srcdir)/include $(PYFLAGS)
_driver_la_LDFLAGS = -lsinga -module -shared $(PYLIBS) -avoid-version -rpath $(pydir)
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/0233049c/configure.ac
----------------------------------------------------------------------
diff --git a/configure.ac b/configure.ac
index 41ff510..af80c1e 100644
--- a/configure.ac
+++ b/configure.ac
@@ -36,13 +36,32 @@ AC_PROG_CC
AC_DISABLE_STATIC
AC_PROG_LIBTOOL
+# Checks for libraries.
+AC_SEARCH_LIBS([cblas_sgemm], [openblas], [], [
+ AC_MSG_ERROR([unable to find cblas_sgemm() function])
+ ])
+AC_SEARCH_LIBS([zmq_ctx_new], [zmq], [], [
+ AC_MSG_ERROR([unable to find zmq_ctx_new() function])
+ ])
+AC_SEARCH_LIBS([zmsg_new], [czmq], [], [
+ AC_MSG_ERROR([unable to find zmsg_new() function])
+ ])
+AC_CHECK_LIB([glog], [main], [], [
+ AC_MSG_ERROR([unable to find glog library])
+ ])
+AC_CHECK_LIB([protobuf], [main], [], [
+ AC_MSG_ERROR([unable to find protobuf library])
+ ])
+
+PROGS=''
+LTLIBS=''
+
# Setup custom CUDA paths
AC_ARG_ENABLE(cuda,
[AS_HELP_STRING(--enable-cuda,enable CUDA support)],
cudaval="yes",
cudaval="no")
AM_CONDITIONAL(DCUDA, [test "$cudaval" = "yes"])
-
AC_ARG_WITH([cuda],
[AS_HELP_STRING(
[--with-cuda=PATH],
@@ -54,7 +73,6 @@ if test "$cuda_prefix" == "yes"; then
cuda_prefix="/usr/local/cuda"
fi
fi
-
if test x"$cudaval" = x"yes"; then
AC_MSG_CHECKING([nvcc in $cuda_prefix/bin])
if test -x "$cuda_prefix/bin/nvcc"; then
@@ -100,7 +118,6 @@ AC_ARG_ENABLE([cudnn],
[AS_HELP_STRING(--enable-cudnn,enable CUDNN support)],
[enable_cudnn=yes], [enable_cudnn=no])
AM_CONDITIONAL(DCUDNN, [test "$enable_cudnn" = "yes"])
-
AC_ARG_WITH([cudnn],
[AS_HELP_STRING([--with-cudnn=PATH], [prefix where CUDNN is installed])],
[cudnn_prefix=$withval], [cudnn_prefix="/usr/local/cuda"])
@@ -109,7 +126,6 @@ if test "$cudnn_prefix" == "yes"; then
cudnn_prefix="/usr/local/cuda"
fi
fi
-
if test x"$enable_cudnn" == x"yes"; then
CUDNN_CFLAGS="-I$cudnn_prefix/include"
CUDNN_LDFLAGS="-L$cudnn_prefix/lib64 -L$cudnn_prefix/lib"
@@ -126,27 +142,42 @@ else
CUDNN_LDFLAGS=""
CUDNN_LIBS=""
fi
-
AC_SUBST(CUDNN_CFLAGS)
AC_SUBST(CUDNN_LDFLAGS)
AC_SUBST(CUDNN_LIBS)
-# Checks for libraries.
-AC_SEARCH_LIBS([cblas_sgemm], [openblas], [], [
- AC_MSG_ERROR([unable to find cblas_sgemm() function])
- ])
-AC_SEARCH_LIBS([zmq_ctx_new], [zmq], [], [
- AC_MSG_ERROR([unable to find zmq_ctx_new() function])
- ])
-AC_SEARCH_LIBS([zmsg_new], [czmq], [], [
- AC_MSG_ERROR([unable to find zmsg_new() function])
- ])
-AC_CHECK_LIB([glog], [main], [], [
- AC_MSG_ERROR([unable to find glog library])
- ])
-AC_CHECK_LIB([protobuf], [main], [], [
- AC_MSG_ERROR([unable to find protobuf library])
- ])
+# Setup custom zookeeper paths
+AC_ARG_ENABLE(zookeeper,
+ AS_HELP_STRING([--enable-zookeeper],[enable zookeeper support]),
+ [enable_zookeeper=yes],[enable_zookeeper=no])
+AM_CONDITIONAL(DZOOKEEPER, test "$enable_zookeeper" = yes)
+AC_ARG_WITH([zookeeper],
+ [AS_HELP_STRING([--with-zookeeper=PATH], [prefix where zookeeper is installed])],
+ [zookeeper_prefix=$withval], [zookeeper_prefix="/usr/local"])
+if test "$zookeeper_prefix" == "yes"; then
+ if test "$withval" == "yes"; then
+ cudnn_prefix="/usr/local"
+ fi
+fi
+if test x"$enable_zookeeper" != x"no"; then
+ ZOOKEEPER_CFLAGS="-I$zookeeper_prefix/include"
+ ZOOKEEPER_LDFLAGS="-L$zookeeper_prefix/lib"
+ ZOOKEEPER_LIBS="-lzookeeper_mt"
+ LIBS="$LIBS $ZOOKEEPER_LIBS"
+ LDFLAGS="$LDFLAGS $ZOOKEEPER_LDFLAGS"
+ DEBUG+=" -DUSE_ZOOKEEPER"
+ AC_DEFINE(DZOOKEEPER,[1],[Defined if zookeeper should be used])
+ AC_CHECK_LIB([zookeeper_mt], [main], [], [
+ AC_MSG_ERROR([unable to find zookeeper library])
+ ])
+else
+ ZOOKEEPER_CFLAGS=""
+ ZOOKEEPER_LDFLAGS=""
+ ZOOKEEPER_LIBS=""
+fi
+AC_SUBST(ZOOKEEPER_CFLAGS)
+AC_SUBST(ZOOKEEPER_LDFLAGS)
+AC_SUBST(ZOOKEEPER_LIBS)
# Setup custom lmdb paths
AC_ARG_ENABLE(lmdb,
@@ -160,14 +191,11 @@ if test x"$enable_lmdb" = x"yes"; then
AC_DEFINE(LMDB, 1, [Enable Option layer])
fi
-PROGS=''
-LTLIBS=''
# Setup custom libhdfs paths
AC_ARG_ENABLE(hdfs,
AS_HELP_STRING([--enable-hdfs],[enable hdfs support]),
[enable_hdfs=yes],[enable_hdfs=no])
AM_CONDITIONAL(DHDFS, test "$enable_hdfs" = yes)
-
AC_ARG_WITH([libhdfs],
[AS_HELP_STRING([--with-libhdfs=PATH], [prefix where libhdfs is installed])],
[hdfs_prefix=$withval], [hdfs_prefix="/usr/local"])
@@ -176,7 +204,6 @@ if test "$hdfs_prefix" == "yes"; then
cudnn_prefix="/usr/local"
fi
fi
-
if test x"$enable_hdfs" != x"no"; then
HDFS_CFLAGS="-I$hdfs_prefix/include"
HDFS_LDFLAGS="-L$hdfs_prefix/lib"
@@ -193,7 +220,6 @@ else
HDFS_LDFLAGS=""
HDFS_LIBS=""
fi
-
AC_SUBST(HDFS_CFLAGS)
AC_SUBST(HDFS_LDFLAGS)
AC_SUBST(HDFS_LIBS)
@@ -246,8 +272,6 @@ if test "$python_prefix" == "yes"; then
python_prefix="/usr/include/python`python -V 2>&1 | awk '{print substr($2,1,3)}'`"
fi
fi
-
-
if test x"$enable_python" != x"no"; then
AC_MSG_CHECKING([Python.h in $python_prefix])
if test -f "$python_prefix/Python.h"; then
@@ -264,27 +288,12 @@ else
PY_PROGS=''
PYFLAGS=''
fi
-
AC_SUBST([PROGS])
AC_SUBST([LTLIBS])
AC_SUBST([DEBUG])
AC_SUBST([PYFLAGS])
AC_SUBST([PY_PROGS])
-# Setup for opencv libs
-#AC_CHECK_LIB([opencv_imgproc], [main], [], [
-# AC_MSG_ERROR([unable to find opencv_imgproc lib])
-# ])
-#AC_CHECK_LIB([opencv_highgui], [main], [], [
-# AC_MSG_ERROR([unable to find opencv_highgui lib])
-# ])
-#AC_CHECK_LIB([opencv_core], [main], [], [
-# AC_MSG_ERROR([unable to find opencv_core lib])
-# ])
-AC_CHECK_LIB([zookeeper_mt], [main], [], [
- AC_MSG_ERROR([unable to find zookeeper])
- ])
-
# Checks for header files.
AC_HEADER_STDC
AC_CHECK_HEADERS([fcntl.h malloc.h stdlib.h])
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/0233049c/include/singa/utils/cluster_rt.h
----------------------------------------------------------------------
diff --git a/include/singa/utils/cluster_rt.h b/include/singa/utils/cluster_rt.h
index e0125be..4ab48bd 100644
--- a/include/singa/utils/cluster_rt.h
+++ b/include/singa/utils/cluster_rt.h
@@ -44,6 +44,9 @@ struct RTCallback {
*/
class ClusterRuntime {
public:
+ // ClusterRuntime have different implementation determined when compiling
+ static ClusterRuntime* Create(const std::string&host, int job_id);
+
virtual ~ClusterRuntime() {}
/**
* Initialize the runtime instance
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/0233049c/src/utils/cluster.cc
----------------------------------------------------------------------
diff --git a/src/utils/cluster.cc b/src/utils/cluster.cc
index 53c0444..13729f2 100644
--- a/src/utils/cluster.cc
+++ b/src/utils/cluster.cc
@@ -82,7 +82,8 @@ void Cluster::Init(int job, const SingaProto& singaConf,
}
}
// cluster_rt_ = new ZKClusterRT(singa_.zookeeper_host(), job);
- cluster_rt_ = new SPClusterRT();
+ // cluster_rt_ = new SPClusterRT();
+ cluster_rt_ = ClusterRuntime::Create(singa_.zookeeper_host(), job);
cluster_rt_->Init();
hostip_ = GetHostIP();
}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/0233049c/src/utils/cluster_rt.cc
----------------------------------------------------------------------
diff --git a/src/utils/cluster_rt.cc b/src/utils/cluster_rt.cc
index 33e04dc..7a04ff7 100644
--- a/src/utils/cluster_rt.cc
+++ b/src/utils/cluster_rt.cc
@@ -29,12 +29,24 @@
#include <iostream>
#include "singa/proto/job.pb.h"
+#ifdef USE_ZOOKEEPER
+#include "singa/utils/zk_service.h"
+#endif
+
using std::string;
using std::to_string;
using std::vector;
namespace singa {
+ClusterRuntime* ClusterRuntime::Create(const std::string&host, int job_id) {
+#ifdef USE_ZOOKEEPER
+ return new ZKClusterRT(host, job_id);
+#else
+ return new SPClusterRT();
+#endif
+}
+
SPClusterRT::~SPClusterRT() {
// release callback vector
for (auto list : grp_callbacks_)