You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@singa.apache.org by wa...@apache.org on 2015/08/18 16:13:20 UTC
[3/4] incubator-singa git commit: SINGA-21 Code review 3
SINGA-21 Code review 3
format following files:
- msg.h, msg.cc
- socket.h, socket.cc
- common.h, common.cc
-- remove Debug(), CurrentDateTime() functions
- factory.h
- graph.h, graph.cc
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/04b24e7f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/04b24e7f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/04b24e7f
Branch: refs/heads/master
Commit: 04b24e7fb538e182ad9df6c940189ff0b78c05fe
Parents: b24f0a3
Author: wang sheng <wa...@gmail.com>
Authored: Tue Aug 18 13:15:55 2015 +0800
Committer: wang sheng <wa...@gmail.com>
Committed: Tue Aug 18 19:55:43 2015 +0800
----------------------------------------------------------------------
include/communication/msg.h | 25 ++--
include/communication/socket.h | 10 +-
include/utils/common.h | 22 ++-
include/utils/factory.h | 2 +-
include/utils/graph.h | 50 ++++---
src/communication/msg.cc | 9 +-
src/communication/socket.cc | 16 +--
src/trainer/server.cc | 2 +
src/utils/common.cc | 263 +++++++++++++++++-------------------
src/utils/graph.cc | 49 +++----
10 files changed, 210 insertions(+), 238 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/04b24e7f/include/communication/msg.h
----------------------------------------------------------------------
diff --git a/include/communication/msg.h b/include/communication/msg.h
index 1570936..75c91a7 100644
--- a/include/communication/msg.h
+++ b/include/communication/msg.h
@@ -5,7 +5,6 @@
#define USE_ZMQ
#include <utility>
-
#ifdef USE_ZMQ
#include <czmq.h>
#endif
@@ -29,6 +28,7 @@ inline int Addr(int grp, int id_or_proc, int type) {
inline int AddrGrp(int addr) {
return addr >> 16;
}
+
/**
* Parse worker/server id from addr.
*
@@ -47,6 +47,7 @@ inline int AddrID(int addr) {
inline int AddrProc(int addr) {
return AddrID(addr);
}
+
/**
* Parse msg type from addr
* @return msg type
@@ -159,44 +160,40 @@ class Msg {
* Set source addr.
* @param addr unique identify one worker/server/stub in the current job
*/
- void set_src(int addr) { src_ = addr; }
+ inline void set_src(int addr) { src_ = addr; }
/**
* @return source addr.
*/
- int src() const { return src_; }
+ inline int src() const { return src_; }
/**
* Set destination addr.
* @param addr unique identify one worker/server/stub in the current job
*/
- void set_dst(int addr) { dst_ = addr; }
+ inline void set_dst(int addr) { dst_ = addr; }
/**
* @return dst addr.
*/
- int dst() const { return dst_; }
+ inline int dst() const { return dst_; }
/**
* Set msg type, e.g., kPut, kGet, kUpdate, kRequest
*/
- void set_type(int type) { type_ = type; }
+ inline void set_type(int type) { type_ = type; }
/**
* @return msg type.
*/
- int type() const { return type_; }
+ inline int type() const { return type_; }
/**
* Set msg target.
*
* One msg has a target to identify some entity in worker/server/stub.
* The target is associated with a version, e.g., Param version.
*/
- void set_trgt(int val, int version) {
+ inline void set_trgt(int val, int version) {
trgt_val_ = val;
trgt_version_ = version;
}
- int trgt_val() const {
- return trgt_val_;
- }
- int trgt_version() const {
- return trgt_version_;
- }
+ inline int trgt_val() const { return trgt_val_; }
+ inline int trgt_version() const { return trgt_version_; }
protected:
int src_ = 0;
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/04b24e7f/include/communication/socket.h
----------------------------------------------------------------------
diff --git a/include/communication/socket.h b/include/communication/socket.h
index fe06aad..d76a163 100644
--- a/include/communication/socket.h
+++ b/include/communication/socket.h
@@ -1,16 +1,14 @@
#ifndef SINGA_COMMUNICATION_SOCKET_H_
#define SINGA_COMMUNICATION_SOCKET_H_
+#ifdef USE_ZMQ
+#include <czmq.h>
+#endif
#include <map>
#include <string>
#include <vector>
-
#include "communication/msg.h"
-#ifdef USE_ZMQ
-#include <czmq.h>
-#endif
-
namespace singa {
const std::string kInprocRouterEndpoint = "inproc://router";
@@ -43,7 +41,7 @@ class SocketInterface {
class Poller {
public:
Poller();
- Poller(SocketInterface* socket);
+ explicit Poller(SocketInterface* socket);
/**
* Add a socket for polling; Multiple sockets can be polled together by
* adding them into the same poller.
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/04b24e7f/include/utils/common.h
----------------------------------------------------------------------
diff --git a/include/utils/common.h b/include/utils/common.h
index acdb20e..868047a 100644
--- a/include/utils/common.h
+++ b/include/utils/common.h
@@ -2,15 +2,14 @@
#define SINGA_UTILS_COMMON_H_
#include <google/protobuf/message.h>
-#include <stdlib.h>
#include <unordered_map>
#include <sstream>
#include <string>
#include <vector>
+#include <utility>
#include "proto/common.pb.h"
namespace singa {
-using std::vector;
std::string IntVecToString(const std::vector<int>& vec);
std::string VStringPrintf(std::string fmt, va_list l);
@@ -23,7 +22,6 @@ void ReadProtoFromBinaryFile(const char* filename,
google::protobuf::Message* proto);
void WriteProtoToBinaryFile(const google::protobuf::Message& proto,
const char* filename);
-
/**
* Locate the position of the arg in arglist.
*
@@ -33,9 +31,7 @@ void WriteProtoToBinaryFile(const google::protobuf::Message& proto,
* @return the position of arg in the arglist; -1 if not found.
*/
int ArgPos(int argc, char** arglist, const char* arg);
-
-const std::string CurrentDateTime();
-void CreateFolder(const std::string name);
+void CreateFolder(const std::string name);
/**
* Slice a set of large Params into small pieces such that they can be roughtly
* equally partitioned into a fixed number of boxes.
@@ -44,7 +40,8 @@ void CreateFolder(const std::string name);
* @param sizes size of all Params
* @return all slices for each Param
*/
-const vector<vector<int>> Slice(int num, const vector<int>& sizes);
+const std::vector<std::vector<int>> Slice(int num,
+ const std::vector<int>& sizes);
/**
* Partition slices into boxes.
*
@@ -52,14 +49,12 @@ const vector<vector<int>> Slice(int num, const vector<int>& sizes);
* @param slices slice sizes
* @return box id for each slice
*/
-const vector<int> PartitionSlices(int num, const vector<int>& slices);
-
+const std::vector<int> PartitionSlices(int num, const std::vector<int>& slices);
/*
inline void Sleep(int millisec=1){
std::this_thread::sleep_for(std::chrono::milliseconds(millisec));
}
*/
-
int gcd(int a, int b);
int LeastCommonMultiple(int a, int b);
/*
@@ -67,7 +62,7 @@ inline float rand_real() {
return static_cast<float>(rand_r())/(RAND_MAX+1.0f);
}
*/
-const std::string GetHostIP();
+std::string GetHostIP();
void SetupLog(const std::string& workspace, const std::string& model);
/**
@@ -93,11 +88,11 @@ class Metric {
/**
* Generate a one-line string for logging
*/
- const std::string ToLogString() const;
+ std::string ToLogString() const;
/**
* Serialize the object into a string
*/
- const std::string ToString() const;
+ std::string ToString() const;
/**
* Parse the metric from a string
*/
@@ -106,6 +101,7 @@ class Metric {
private:
std::unordered_map<std::string, std::pair<int, float>> entry_;
};
+
} // namespace singa
#endif // SINGA_UTILS_COMMON_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/04b24e7f/include/utils/factory.h
----------------------------------------------------------------------
diff --git a/include/utils/factory.h b/include/utils/factory.h
index 1e3069c..e0cd6e3 100644
--- a/include/utils/factory.h
+++ b/include/utils/factory.h
@@ -49,7 +49,6 @@ class Factory {
<< "The id has been registered by another function";
id2func_[id] = func;
}
-
/**
* create an instance by providing its id
*
@@ -70,6 +69,7 @@ class Factory {
<< "The creation function for " << id << " has not been registered";
return id2func_[id]();
}
+
private:
// Map that stores the registered creation functions
std::map<std::string, std::function<T*(void)>> str2func_;
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/04b24e7f/include/utils/graph.h
----------------------------------------------------------------------
diff --git a/include/utils/graph.h b/include/utils/graph.h
index 04e122d..d2a5d38 100644
--- a/include/utils/graph.h
+++ b/include/utils/graph.h
@@ -1,15 +1,12 @@
#ifndef SINGA_UTILS_GRAPH_H_
#define SINGA_UTILS_GRAPH_H_
-#include <vector>
+
+#include <stack>
#include <string>
#include <map>
-#include <stack>
-#include <memory>
+#include <vector>
namespace singa {
-using std::vector;
-using std::string;
-using std::map;
class Node {
public:
@@ -18,7 +15,7 @@ class Node {
*
* @param name name of the corresponding layer
*/
- explicit Node(string name);
+ explicit Node(std::string name);
/**
* Node constructor.
*
@@ -28,24 +25,22 @@ class Node {
* @param id partition id of this node
* @param proto conf of the corresponding layer
*/
- Node(const string& name, const string& origin, int id, void* proto);
- ~Node();
+ Node(const std::string& name, const std::string& origin, int id, void* proto);
+ ~Node() {} // the proto field is deleted outside by other functions
void AddDstNode(Node* dstnode);
void AddSrcNode(Node* srcnode);
void RemoveDstNode(Node* dst);
void RemoveSrcNode(Node* src);
- public:
- string name;
+ std::string name = "";
//! name of the origin node/layer from which is node is derived
- string origin;
+ std::string origin = "";
//! partition id
- int partition_id;
+ int partition_id = 0;
//! proto of the corresponding layer
- void* proto;
-
- vector<Node*> srcnodes;
- vector<Node*> dstnodes;
+ void* proto = nullptr;
+ std::vector<Node*> srcnodes;
+ std::vector<Node*> dstnodes;
};
/**
@@ -60,42 +55,43 @@ class Graph {
/**
* @return all nodes of the graph
*/
- const vector<Node*>& nodes() const {
+ inline const std::vector<Node*>& nodes() const {
return nodes_;
}
/**
* @param name node name
* @return return the node of given name
*/
- Node* node(const string& name) const {
+ inline Node* node(const std::string& name) const {
return name2node_.at(name);
}
-
void AddNode(Node* node);
- Node* AddNode(const string& name);
+ Node* AddNode(const std::string& name);
void AddEdge(Node* srcnode, Node* dstnode);
- void AddEdge(const string& src, const string& dst);
+ void AddEdge(const std::string& src, const std::string& dst);
void RemoveEdge(Node* src, Node* dst);
- void RemoveEdge(const string &src, const string& dst);
+ void RemoveEdge(const std::string &src, const std::string& dst);
/**
* Dump the graph into json string which can be used to draw a picture by
* graphviz
*/
- const string ToJson() const;
+ std::string ToJson() const;
/**
* \copybreif ToJson()
*
* @param info info associated with each node
*/
- const string ToJson(const map<string, string>& info) const;
+ std::string ToJson(const std::map<std::string, std::string>& info) const;
/**
* Do topology sort for all nodes of the graph.
*/
void Sort();
private:
- vector<Node*> nodes_;
- map<string, Node*> name2node_;
+ std::vector<Node*> nodes_;
+ std::map<std::string, Node*> name2node_;
};
+
} // namespace singa
+
#endif // SINGA_UTILS_GRAPH_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/04b24e7f/src/communication/msg.cc
----------------------------------------------------------------------
diff --git a/src/communication/msg.cc b/src/communication/msg.cc
index ccf02c4..41a8ecc 100644
--- a/src/communication/msg.cc
+++ b/src/communication/msg.cc
@@ -1,6 +1,7 @@
-#include <glog/logging.h>
#include "communication/msg.h"
+#include <glog/logging.h>
+
namespace singa {
#ifdef USE_ZMQ
@@ -20,7 +21,7 @@ Msg::Msg(const Msg& msg) {
type_ = msg.type_;
trgt_val_ = msg.trgt_val_;
trgt_version_ = msg.trgt_version_;
- msg_=zmsg_dup(msg.msg_);
+ msg_ = zmsg_dup(msg.msg_);
}
Msg::Msg(int src, int dst) {
@@ -155,7 +156,7 @@ int Msg::ParseFormatFrame(const char *format, ...) {
CHECK_EQ(src[size++], 'f');
memcpy(x, src + size, sizeof(*x));
size += sizeof(*x);
- }else if (*format == '1') {
+ } else if (*format == '1') {
uint8_t *x = va_arg(argptr, uint8_t *);
memcpy(x, src + size, sizeof(*x));
size += sizeof(*x);
@@ -188,8 +189,6 @@ int Msg::ParseFormatFrame(const char *format, ...) {
delete src;
return size;
}
-
#endif
} // namespace singa
-
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/04b24e7f/src/communication/socket.cc
----------------------------------------------------------------------
diff --git a/src/communication/socket.cc b/src/communication/socket.cc
index 4e6ec2c..f21edc1 100644
--- a/src/communication/socket.cc
+++ b/src/communication/socket.cc
@@ -28,7 +28,7 @@ SocketInterface* Poller::Wait(int timeout) {
return nullptr;
}
-bool Poller::Terminated(){
+bool Poller::Terminated() {
return zpoller_terminated(poller_);
}
@@ -94,13 +94,13 @@ Router::~Router() {
zmsg_destroy(&msg);
}
}
-int Router::Bind(const std::string& endpoint){
- int port=-1;
- if(endpoint.length()){
- port=zsock_bind(router_, "%s", endpoint.c_str());
+int Router::Bind(const std::string& endpoint) {
+ int port = -1;
+ if (endpoint.length()) {
+ port = zsock_bind(router_, "%s", endpoint.c_str());
}
- CHECK_NE(port,-1)<<endpoint;
- LOG(INFO)<<"bind successfully to "<<endpoint+":"+std::to_string(port);
+ CHECK_NE(port, -1) << endpoint;
+ LOG(INFO) << "bind successfully to " << endpoint + ":" + std::to_string(port);
return port;
}
@@ -128,7 +128,7 @@ int Router::Send(Msg **msg) {
Msg* Router::Receive() {
zmsg_t* zmsg = zmsg_recv(router_);
if (zmsg == nullptr) {
- LOG(ERROR)<<"Connection broken!";
+ LOG(ERROR) << "Connection broken!";
exit(0);
}
zframe_t* dealer = zmsg_pop(zmsg);
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/04b24e7f/src/trainer/server.cc
----------------------------------------------------------------------
diff --git a/src/trainer/server.cc b/src/trainer/server.cc
index a3749d1..e0eb821 100644
--- a/src/trainer/server.cc
+++ b/src/trainer/server.cc
@@ -9,7 +9,9 @@
#include "proto/common.pb.h"
namespace singa {
+
using namespace mshadow;
+using std::vector;
Server::Server(int thread_id,int group_id, int server_id):
thread_id_(thread_id),grp_id_(group_id), id_(server_id){
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/04b24e7f/src/utils/common.cc
----------------------------------------------------------------------
diff --git a/src/utils/common.cc b/src/utils/common.cc
index 01ee317..1888380 100644
--- a/src/utils/common.cc
+++ b/src/utils/common.cc
@@ -1,22 +1,21 @@
#include "utils/common.h"
+#include <arpa/inet.h>
#include <fcntl.h>
#include <glog/logging.h>
#include <google/protobuf/io/coded_stream.h>
-#include <google/protobuf/text_format.h>
#include <google/protobuf/io/zero_copy_stream_impl.h>
+#include <google/protobuf/text_format.h>
#include <stdarg.h>
#include <stdio.h>
-#include <unistd.h>
+#include <sys/ioctl.h>
+#include <sys/socket.h>
#include <sys/stat.h>
#include <sys/types.h>
-#include <sys/socket.h>
-#include <sys/ioctl.h>
#include <netinet/in.h>
#include <net/if.h>
-#include <arpa/inet.h>
#include <time.h>
-#include <string>
+#include <unistd.h>
namespace singa {
@@ -29,12 +28,14 @@ using google::protobuf::io::ZeroCopyInputStream;
using google::protobuf::Message;
const int kBufLen = 1024;
+
string IntVecToString(const vector<int>& vec) {
string disp = "(";
for (int x : vec)
disp += std::to_string(x) + ", ";
return disp + ")";
}
+
/**
* * Formatted string.
* */
@@ -55,31 +56,6 @@ string StringPrintf(string fmt, ...) {
return result;
}
-void Debug() {
- int i = 0;
- char hostname[256];
- gethostname(hostname, sizeof(hostname));
- printf("PID %d on %s ready for attach\n", getpid(), hostname);
- fflush(stdout);
- while (0 == i)
- sleep(5);
-}
-int ArgPos(int argc, char** arglist, const char* arg) {
- for (int i = 0; i < argc; i++) {
- if (strcmp(arglist[i], arg) == 0) {
- return i;
- }
- }
- return -1;
-}
-void CreateFolder(const std::string name) {
- struct stat buffer;
- if (stat(name.c_str(), &buffer) != 0) {
- mkdir(name.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
- CHECK_EQ(stat(name.c_str(), &buffer), 0);
- }
-}
-
// the proto related functions are from Caffe.
void ReadProtoFromTextFile(const char* filename, Message* proto) {
int fd = open(filename, O_RDONLY);
@@ -117,136 +93,49 @@ void WriteProtoToBinaryFile(const Message& proto, const char* filename) {
CHECK(proto.SerializeToFileDescriptor(fd));
}
-int gcd(int a, int b) {
- for (;;) {
- if (a == 0) return b;
- b %= a;
- if (b == 0) return a;
- a %= b;
- }
-}
-int LeastCommonMultiple(int a, int b) {
- int temp = gcd(a, b);
-
- return temp ? (a / temp * b) : 0;
-}
-
-const string GetHostIP() {
- int fd;
- struct ifreq ifr;
-
- fd = socket(AF_INET, SOCK_DGRAM, 0);
-
- /* I want to get an IPv4 IP address */
- ifr.ifr_addr.sa_family = AF_INET;
-
- /* I want IP address attached to "eth0" */
- strncpy(ifr.ifr_name, "eth0", IFNAMSIZ-1);
-
- ioctl(fd, SIOCGIFADDR, &ifr);
-
- close(fd);
-
- string ip(inet_ntoa(((struct sockaddr_in *)&ifr.ifr_addr)->sin_addr));
- /* display result */
- LOG(INFO) << "Host IP=(" << ip;
- return ip;
-}
-
-void SetupLog(const std::string& log_dir, const std::string& model) {
- // TODO check if NFS, then create folder using script otherwise may have
- // problems due to multiple processes create the same folder.
- CreateFolder(log_dir);
- std::string warn = log_dir + "/" + model + "-warn-";
- std::string info = log_dir + "/" + model + "-info-";
- std::string error = log_dir + "/" + model + "-error-";
- std::string fatal = log_dir + "/" + model + "-fatal-";
- google::SetLogDestination(google::WARNING, warn.c_str());
- google::SetLogDestination(google::INFO, info.c_str());
- google::SetLogDestination(google::ERROR, error.c_str());
- google::SetLogDestination(google::FATAL, fatal.c_str());
-}
-
-Metric::Metric(const std::string& str) {
- ParseFrom(str);
-}
-
-void Metric::Add(const string& name, float value) {
- if(entry_.find(name) == entry_.end())
- entry_[name] = std::make_pair(1, value);
- else{
- auto& e = entry_.at(name);
- e.first += 1;
- e.second += value;
- }
-}
-
-void Metric::Reset() {
- for(auto& e : entry_) {
- e.second.first = 0;
- e.second.second = 0;
- }
-}
-const string Metric::ToLogString() const {
- string ret;
- size_t k = 0;
- for(auto e : entry_) {
- ret += e.first + " : " ;
- ret += std::to_string(e.second.second / e.second.first);
- if(++k < entry_.size())
- ret += ", ";
- }
- return ret;
-}
-
-const string Metric::ToString() const {
- MetricProto proto;
- for(auto e : entry_) {
- proto.add_name(e.first);
- proto.add_count(e.second.first);
- proto.add_val(e.second.second);
+int ArgPos(int argc, char** arglist, const char* arg) {
+ for (int i = 0; i < argc; i++) {
+ if (strcmp(arglist[i], arg) == 0) {
+ return i;
+ }
}
- string ret;
- proto.SerializeToString(&ret);
- return ret;
+ return -1;
}
-void Metric::ParseFrom(const string& msg) {
- MetricProto proto;
- proto.ParseFromString(msg);
- Reset();
- for(int i = 0; i < proto.name_size(); i++) {
- entry_[proto.name(i)] = std::make_pair(proto.count(i), proto.val(i));
+void CreateFolder(const string name) {
+ struct stat buffer;
+ if (stat(name.c_str(), &buffer) != 0) {
+ mkdir(name.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
+ CHECK_EQ(stat(name.c_str(), &buffer), 0);
}
}
-
const vector<vector<int>> Slice(int num, const vector<int>& sizes) {
vector<vector<int>> slices;
if (num == 0)
return slices;
int avg = 0;
- for(int x : sizes)
+ for (int x : sizes)
avg += x;
avg = avg / num + avg % num;
int diff = avg / 10;
- LOG(INFO) << "Slicer, param avg=" << avg << ", diff= " << diff;
+ LOG(INFO) << "Slicer, param avg = " << avg << ", diff = " << diff;
int capacity = avg, nbox = 0;
for (int x : sizes) {
vector<int> slice;
string slicestr = "";
while (x > 0) {
- int size=0;
+ int size = 0;
if (capacity >= x) {
capacity -= x;
size = x;
x = 0;
- }else if(capacity + diff >= x) {
+ } else if (capacity + diff >= x) {
size = x;
x = 0;
capacity = 0;
- }else if (capacity >= diff) {
+ } else if (capacity >= diff) {
x -= capacity;
size = capacity;
capacity = avg;
@@ -272,7 +161,7 @@ const vector<int> PartitionSlices(int num, const vector<int>& slices) {
if (num == 0)
return slice2box;
int avg = 0;
- for(int x : slices)
+ for (int x : slices)
avg += x;
avg = avg / num + avg % num;
int box = avg, boxid = 0, diff = avg / 10;
@@ -293,7 +182,7 @@ const vector<int> PartitionSlices(int num, const vector<int>& slices) {
}
CHECK_EQ(slice2box.size(), slices.size());
int previd = -1;
- std::string disp;
+ string disp;
for (size_t i = 0; i < slice2box.size(); i++) {
if (previd != slice2box[i]) {
previd = slice2box[i];
@@ -301,7 +190,107 @@ const vector<int> PartitionSlices(int num, const vector<int>& slices) {
}
disp += " " + std::to_string(slices[i]);
}
- LOG(INFO) << "partition slice (avg =" << avg << ", num="<<num<<"):" << disp;
+ LOG(INFO) << "partition slice (avg = " << avg
+ << ", num = " << num << "):" << disp;
return slice2box;
}
+
+int gcd(int a, int b) {
+ for (;;) {
+ if (a == 0) return b;
+ b %= a;
+ if (b == 0) return a;
+ a %= b;
+ }
+}
+
+int LeastCommonMultiple(int a, int b) {
+ int temp = gcd(a, b);
+ return temp ? (a / temp * b) : 0;
+}
+
+string GetHostIP() {
+ int fd;
+ struct ifreq ifr;
+ fd = socket(AF_INET, SOCK_DGRAM, 0);
+ /* I want to get an IPv4 IP address */
+ ifr.ifr_addr.sa_family = AF_INET;
+ /* I want IP address attached to "eth0" */
+ strncpy(ifr.ifr_name, "eth0", IFNAMSIZ-1);
+ ioctl(fd, SIOCGIFADDR, &ifr);
+ close(fd);
+ string ip(inet_ntoa(((struct sockaddr_in *)&ifr.ifr_addr)->sin_addr));
+ /* display result */
+ LOG(INFO) << "Host IP=(" << ip;
+ return ip;
+}
+
+void SetupLog(const string& log_dir, const string& model) {
+ // TODO(wangwei) check if NFS, then create folder using script, otherwise
+ // may have problems due to multiple processes create the same folder.
+ CreateFolder(log_dir);
+ string warn = log_dir + "/" + model + "-warn-";
+ string info = log_dir + "/" + model + "-info-";
+ string error = log_dir + "/" + model + "-error-";
+ string fatal = log_dir + "/" + model + "-fatal-";
+ google::SetLogDestination(google::WARNING, warn.c_str());
+ google::SetLogDestination(google::INFO, info.c_str());
+ google::SetLogDestination(google::ERROR, error.c_str());
+ google::SetLogDestination(google::FATAL, fatal.c_str());
+}
+
+Metric::Metric(const string& str) {
+ ParseFrom(str);
+}
+
+void Metric::Add(const string& name, float value) {
+ if (entry_.find(name) == entry_.end()) {
+ entry_[name] = std::make_pair(1, value);
+ } else {
+ auto& e = entry_.at(name);
+ e.first += 1;
+ e.second += value;
+ }
+}
+
+void Metric::Reset() {
+ for (auto& e : entry_) {
+ e.second.first = 0;
+ e.second.second = 0;
+ }
+}
+
+string Metric::ToLogString() const {
+ string ret;
+ size_t k = 0;
+ for (auto e : entry_) {
+ ret += e.first + " : ";
+ ret += std::to_string(e.second.second / e.second.first);
+ if (++k < entry_.size())
+ ret += ", ";
+ }
+ return ret;
+}
+
+string Metric::ToString() const {
+ MetricProto proto;
+ for (auto e : entry_) {
+ proto.add_name(e.first);
+ proto.add_count(e.second.first);
+ proto.add_val(e.second.second);
+ }
+ string ret;
+ proto.SerializeToString(&ret);
+ return ret;
+}
+
+void Metric::ParseFrom(const string& msg) {
+ MetricProto proto;
+ proto.ParseFromString(msg);
+ Reset();
+ for (int i = 0; i < proto.name_size(); i++) {
+ entry_[proto.name(i)] = std::make_pair(proto.count(i), proto.val(i));
+ }
+}
+
} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/04b24e7f/src/utils/graph.cc
----------------------------------------------------------------------
diff --git a/src/utils/graph.cc b/src/utils/graph.cc
index 1f7df06..192ac3c 100644
--- a/src/utils/graph.cc
+++ b/src/utils/graph.cc
@@ -1,16 +1,15 @@
-
#include "utils/graph.h"
+
#include <glog/logging.h>
#include <algorithm>
#include <queue>
#include <unordered_set>
namespace singa {
-/************************Node********************************/
-Node::~Node() {
- // the proto field is deleted outside by other functions
-}
+using std::map;
+using std::string;
+using std::vector;
Node::Node(string name) {
this->name = name;
@@ -43,11 +42,10 @@ void Node::RemoveSrcNode(Node* src) {
auto iter = srcnodes.begin();
while ((*iter)->name != src->name && iter != srcnodes.end())
iter++;
- CHECK((*iter)->name == src->name);
+ CHECK_STREQ((*iter)->name.c_str(), src->name.c_str());
srcnodes.erase(iter);
}
-/*************************Graph****************************/
Graph::~Graph() {
for (Node* node : nodes_)
delete node;
@@ -55,6 +53,8 @@ Graph::~Graph() {
void Graph::AddNode(Node* node) {
nodes_.push_back(node);
+ CHECK(name2node_.find(node->name) == name2node_.end())
+ << "node " << node->name << " already exists";
name2node_[node->name] = node;
}
@@ -70,13 +70,11 @@ void Graph::AddEdge(Node* srcnode, Node* dstnode) {
}
void Graph::AddEdge(const string& src, const string& dst) {
- CHECK(name2node_.find(src) != name2node_.end())
- <<"can't find src node " << src;
- CHECK(name2node_.find(dst) != name2node_.end())
- <<"can't find dst node " << dst;
-
- Node* srcnode = name2node_[src], *dstnode = name2node_[dst];
- AddEdge(srcnode, dstnode);
+ auto srcnode = name2node_.find(src);
+ CHECK(srcnode != name2node_.end()) << "can't find src node " << src;
+ auto dstnode = name2node_.find(dst);
+ CHECK(dstnode != name2node_.end()) << "can't find dst node " << dst;
+ AddEdge(srcnode->second, dstnode->second);
}
void Graph::RemoveEdge(Node* src, Node* dst) {
@@ -85,29 +83,27 @@ void Graph::RemoveEdge(Node* src, Node* dst) {
}
void Graph::RemoveEdge(const string &src, const string& dst) {
- CHECK(name2node_.find(src) != name2node_.end())
- <<"can't find src node " << src;
- CHECK(name2node_.find(dst) != name2node_.end())
- <<"can't find dst node " << dst;
-
- Node* srcnode = name2node_[src], *dstnode = name2node_[dst];
- RemoveEdge(srcnode, dstnode);
+ auto srcnode = name2node_.find(src);
+ CHECK(srcnode != name2node_.end()) << "can't find src node " << src;
+ auto dstnode = name2node_.find(dst);
+ CHECK(dstnode != name2node_.end()) << "can't find dst node " << dst;
+ RemoveEdge(srcnode->second, dstnode->second);
}
-const string Graph::ToJson() const {
+string Graph::ToJson() const {
map<string, string> info;
return ToJson(info);
}
-const string Graph::ToJson(const map<string, string>& info) const {
+string Graph::ToJson(const map<string, string>& info) const {
map<string, int> nodeid;
string disp = "{\"directed\":1,\n";
// add nodes
disp += "\"nodes\":[\n";
bool first = true;
-
vector<string> colors = {"red", "blue", "black", "green"};
+
// see for more shapes at http://www.graphviz.org/doc/info/shapes.html
vector<string> shapes = {"box", "ellipse"};
int id = 0;
@@ -185,12 +181,11 @@ void Graph::Sort() {
}
// check whether its src nodes number greater than 1
if (bi_direction && (node->srcnodes).size() > 1) {
- auto src = node->srcnodes.at(0);
+ auto src = node->srcnodes.at(0);
if (visited_set.find(src) == visited_set.end()) {
visit = false;
}
- }
- else {
+ } else {
for (auto src : node->srcnodes)
if (visited_set.find(src) == visited_set.end()) {
visit = false;