You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@singa.apache.org by wa...@apache.org on 2016/01/01 13:29:15 UTC
[06/10] incubator-singa git commit: SINGA-81 Add Python Helper
SINGA-81 Add Python Helper
update singa-run.sh to support running in a single process without job.conf
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/2a6645cf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/2a6645cf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/2a6645cf
Branch: refs/heads/master
Commit: 2a6645cf2b7b625795b58b619e6cc544c08b63ff
Parents: d47ed4b
Author: WANG Sheng <wa...@gmail.com>
Authored: Thu Dec 31 11:02:22 2015 +0800
Committer: chonho <le...@comp.nus.edu.sg>
Committed: Fri Jan 1 15:59:15 2016 +0800
----------------------------------------------------------------------
bin/singa-run.sh | 44 +++++++++++++++++++++++---------------------
src/utils/cluster_rt.cc | 26 ++++++++++++++------------
src/utils/tool.cc | 2 +-
3 files changed, 38 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/2a6645cf/bin/singa-run.sh
----------------------------------------------------------------------
diff --git a/bin/singa-run.sh b/bin/singa-run.sh
index 548f8f1..836342e 100755
--- a/bin/singa-run.sh
+++ b/bin/singa-run.sh
@@ -23,16 +23,14 @@
# run a singa job
#
-usage="Usage: singa-run.sh -conf <job config file> [ other arguments ]\n
- -resume : if want to recover a job\n
- -exec <path to mysinga> : if want to use own singa driver\n
- ### NOTICE ###\n
- if you are using model.conf + cluster.conf,\n
- please see how to combine them to a job.conf:\n
- http://singa.incubator.apache.org/quick-start.html"
+usage="Usage: singa-run.sh [ arguments ]\n
+ -exec <binary or python script> : if want to use own singa driver\n
+ -conf <job config file> : need cluster conf if train in a cluster
+ -resume : if want to recover a job"
# parse arguments
-# make sure we have '-conf' and remove '-exec'
+# extract and remove '-exec' and '-conf'
+# other arguments remain untouched
exe=./singa
while [ $# != 0 ]; do
if [ $1 == "-exec" ]; then
@@ -46,23 +44,23 @@ while [ $# != 0 ]; do
fi
shift
done
-if [ -z $conf ]; then
- echo -e $usage
- exit 1
-fi
# get environment variables
. `dirname "${BASH_SOURCE-$0}"`/singa-env.sh
# change conf to an absolute path
-conf_dir=`dirname "$conf"`
-conf_dir=`cd "$conf_dir">/dev/null; pwd`
-conf_base=`basename "$conf"`
-job_conf=$conf_dir/$conf_base
-if [ ! -f $job_conf ]; then
- echo $job_conf not exists
- exit 1
+if [ ! -z $conf ]; then
+ conf_dir=`dirname "$conf"`
+ conf_dir=`cd "$conf_dir">/dev/null; pwd`
+ conf_base=`basename "$conf"`
+ job_conf=$conf_dir/$conf_base
+ if [ ! -f $job_conf ]; then
+ echo $job_conf not exists
+ exit 1
+ fi
fi
+
+# go to singa home to execute binary
cd $SINGA_HOME
# generate unique job id
@@ -81,9 +79,13 @@ host_file=$log_dir/job.hosts
./singatool genhost $job_conf 1>$host_file || exit 1
# set command to run singa
-singa_run="$exe $args -conf $job_conf \
+singa_run="$exe $args \
-singa_conf $SINGA_HOME/conf/singa.conf \
- -singa_job $job_id"
+ -singa_job $job_id"
+# add -conf if exists
+if [ ! -z $job_conf ]; then
+ singa_run="$singa_run -conf $job_conf"
+fi
singa_sshrun="cd $SINGA_HOME; $singa_run"
# ssh and start singa processes
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/2a6645cf/src/utils/cluster_rt.cc
----------------------------------------------------------------------
diff --git a/src/utils/cluster_rt.cc b/src/utils/cluster_rt.cc
index 5ac3812..cdf8aab 100644
--- a/src/utils/cluster_rt.cc
+++ b/src/utils/cluster_rt.cc
@@ -367,19 +367,21 @@ bool JobManager::GenerateJobID(int* id) {
bool JobManager::GenerateHostList(const char* host_file, const char* job_file,
vector<string>* list) {
+ int nprocs = 1;
// compute required #process from job conf
- ClusterProto cluster;
- google::protobuf::TextFormat::ParseFromString(ExtractClusterConf(job_file),
- &cluster);
- int nworker_procs = cluster.nworker_groups() * cluster.nworkers_per_group()
- / cluster.nworkers_per_procs();
- int nserver_procs = cluster.nserver_groups() * cluster.nservers_per_group()
- / cluster.nservers_per_procs();
- int nprocs = 0;
- if (cluster.server_worker_separate())
- nprocs = nworker_procs + nserver_procs;
- else
- nprocs = std::max(nworker_procs, nserver_procs);
+ if (job_file != nullptr) {
+ ClusterProto cluster;
+ google::protobuf::TextFormat::ParseFromString(ExtractClusterConf(job_file),
+ &cluster);
+ int nworker_procs = cluster.nworker_groups() * cluster.nworkers_per_group()
+ / cluster.nworkers_per_procs();
+ int nserver_procs = cluster.nserver_groups() * cluster.nservers_per_group()
+ / cluster.nservers_per_procs();
+ if (cluster.server_worker_separate())
+ nprocs = nworker_procs + nserver_procs;
+ else
+ nprocs = std::max(nworker_procs, nserver_procs);
+ }
// get available host list from global conf
std::ifstream hostfile(host_file);
if (!hostfile.is_open()) {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/2a6645cf/src/utils/tool.cc
----------------------------------------------------------------------
diff --git a/src/utils/tool.cc b/src/utils/tool.cc
index 18df826..4b50214 100644
--- a/src/utils/tool.cc
+++ b/src/utils/tool.cc
@@ -147,7 +147,7 @@ int main(int argc, char **argv) {
else if (!strcmp(argv[1], "create"))
stat = create();
else if (!strcmp(argv[1], "genhost"))
- stat = (argc > 2) ? genhost(argv[2]) : ARG_ERR;
+ stat = (argc > 2) ? genhost(argv[2]) : genhost(nullptr);
else if (!strcmp(argv[1], "list"))
stat = list(false);
else if (!strcmp(argv[1], "listall"))