You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@singa.apache.org by wa...@apache.org on 2016/06/13 09:53:56 UTC
[1/3] incubator-singa git commit: SINGA-195 Channel for sending
training statistics
Repository: incubator-singa
Updated Branches:
refs/heads/dev 21e4b2d79 -> b167dfa5b
SINGA-195 Channel for sending training statistics
Add utils/channel.h, utils/channel.cc, test/test_channel.cc
Channel provides functionalities of unifily sending training statistics
to different terminals.
Upon setup, config it in global:
InitChannel();
SetChannelDirectory(path);
To get channel instance:
GetChannel(channel_name);
Then send messages:
channel->Send(message);
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/a2a8e34b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/a2a8e34b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/a2a8e34b
Branch: refs/heads/dev
Commit: a2a8e34b4c33be406af081c731d7ee990936701f
Parents: 26df5ac
Author: WANG Sheng <wa...@gmail.com>
Authored: Mon Jun 13 11:20:43 2016 +0800
Committer: WANG Sheng <wa...@gmail.com>
Committed: Mon Jun 13 11:28:16 2016 +0800
----------------------------------------------------------------------
include/singa/utils/channel.h | 76 +++++++++++++++++++++++++++++
src/utils/channel.cc | 99 ++++++++++++++++++++++++++++++++++++++
test/singa/test_channel.cc | 39 +++++++++++++++
3 files changed, 214 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/a2a8e34b/include/singa/utils/channel.h
----------------------------------------------------------------------
diff --git a/include/singa/utils/channel.h b/include/singa/utils/channel.h
new file mode 100644
index 0000000..7cd7aa3
--- /dev/null
+++ b/include/singa/utils/channel.h
@@ -0,0 +1,76 @@
+/************************************************************
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+*************************************************************/
+
+#ifndef SINGA_UTILS_CHANNEL_H_
+#define SINGA_UTILS_CHANNEL_H_
+
+#include <google/protobuf/message.h>
+
+#include <iostream>
+#include <fstream>
+#include <map>
+#include <string>
+
+namespace singa {
+
+class Channel {
+ public:
+ explicit Channel(const std::string& name);
+ ~Channel();
+
+ inline const std::string& GetName() { return name_; }
+ inline void EnableDestStderr(bool enable) { stderr_ = enable; }
+ inline void EnableDestFile(bool enable) { file_ = enable; }
+ void SetDestFilePath(const std::string& file);
+ void Send(const std::string& message);
+ void Send(const google::protobuf::Message& message);
+
+ private:
+ std::string name_ = "";
+ bool stderr_ = false;
+ bool file_ = false;
+ std::ofstream os_;
+};
+
+class ChannelManager {
+ public:
+ ChannelManager() {}
+ ~ChannelManager();
+
+ void Init();
+ void SetDefaultDir(const char* dir);
+ Channel* GetInstance(const std::string& channel);
+
+ private:
+ std::string dir_ = "";
+ std::map<std::string, Channel*> name2ptr_;
+};
+
+/// Initial function for global usage of channel
+void InitChannel(const char* argv);
+/// Set the directory name for persisting channel content
+void SetChannelDirectory(const char* path);
+/// Get the channel instance
+Channel* GetChannel(const std::string& channel_name);
+
+} // namespace singa
+
+#endif // SINGA_UTILS_CHANNEL_H__
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/a2a8e34b/src/utils/channel.cc
----------------------------------------------------------------------
diff --git a/src/utils/channel.cc b/src/utils/channel.cc
new file mode 100644
index 0000000..52909a3
--- /dev/null
+++ b/src/utils/channel.cc
@@ -0,0 +1,99 @@
+/************************************************************
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+*************************************************************/
+
+#include "singa/utils/channel.h"
+
+#include "singa/utils/logging.h"
+#include "singa/utils/singleton.h"
+
+namespace singa {
+
+ChannelManager::~ChannelManager() {
+ for (auto it : name2ptr_) {
+ if (it.second != nullptr) delete(it.second);
+ }
+}
+
+void ChannelManager::Init() {
+ // do nothing here
+}
+
+void ChannelManager::SetDefaultDir(const char* dir) {
+ if (dir != nullptr) {
+ dir_ = dir;
+ if (dir[dir_.length()-1] != '/') dir_ += '/';
+ }
+}
+
+Channel* ChannelManager::GetInstance(const std::string& channel) {
+ // find the channel
+ if (name2ptr_.find(channel) == name2ptr_.end()) {
+ // create new channel
+ Channel* chn = new Channel(channel);
+ chn->SetDestFilePath(dir_+channel);
+ chn->EnableDestFile(true);
+ name2ptr_[channel] = chn;
+ }
+ return name2ptr_[channel];
+}
+
+Channel::Channel(const std::string& name) {
+ name_ = name;
+}
+
+Channel::~Channel() {
+ if (os_.is_open()) os_.close();
+}
+
+void Channel::SetDestFilePath(const std::string& file) {
+ // file is append only
+ if (os_.is_open()) os_.close();
+ os_.open(file.c_str(), std::ios::app);
+ if (os_.is_open() == false)
+ LOG(WARNING) << "Cannot open channel file (" << file << ")";
+}
+
+void Channel::Send(const std::string& message) {
+ if (stderr_) fprintf(stderr, "%s\n", message.c_str());
+ if (file_ && os_.is_open()) os_ << message << "\n";
+}
+
+void Channel::Send(const google::protobuf::Message& message) {
+ if (stderr_) fprintf(stderr, "%s\n", message.DebugString().c_str());
+ if (file_ && os_.is_open()) message.SerializeToOstream(&os_);
+}
+
+void InitChannel(const char* argv) {
+ ChannelManager* mng = Singleton<ChannelManager>().Instance();
+ mng->Init();
+}
+
+void SetChannelDirectory(const char* path) {
+ ChannelManager * mng = Singleton<ChannelManager>().Instance();
+ mng->SetDefaultDir(path);
+}
+
+Channel* GetChannel(const std::string& channel_name) {
+ ChannelManager * mng = Singleton<ChannelManager>().Instance();
+ return mng->GetInstance(channel_name);
+}
+
+} // namespace singa
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/a2a8e34b/test/singa/test_channel.cc
----------------------------------------------------------------------
diff --git a/test/singa/test_channel.cc b/test/singa/test_channel.cc
new file mode 100644
index 0000000..77d7cbc
--- /dev/null
+++ b/test/singa/test_channel.cc
@@ -0,0 +1,39 @@
+/************************************************************
+*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied. See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*
+*************************************************************/
+
+#include "gtest/gtest.h"
+#include "singa/utils/channel.h"
+
+TEST(Channel, InitChannel) {
+ singa::InitChannel("");
+ singa::SetChannelDirectory("/tmp");
+}
+
+TEST(Channel, SendStringToFile) {
+ singa::Channel* chn = singa::GetChannel("test_channel");
+ chn->Send("test to file");
+}
+
+TEST(Channel, SendStringToFileAndStderr) {
+ singa::Channel* chn = singa::GetChannel("test_channel");
+ chn->EnableDestStderr(true);
+ chn->Send("test to both file and stderr");
+}
[3/3] incubator-singa git commit: SINGA-195 Channel for sending
training statistics
Posted by wa...@apache.org.
SINGA-195 Channel for sending training statistics
Merge branch PR#166
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/b167dfa5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/b167dfa5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/b167dfa5
Branch: refs/heads/dev
Commit: b167dfa5bbf6a84af87a4db0f3946659fb62c4ca
Parents: 21e4b2d a4fc4ea
Author: Wei Wang <wa...@comp.nus.edu.sg>
Authored: Mon Jun 13 17:50:20 2016 +0800
Committer: Wei Wang <wa...@comp.nus.edu.sg>
Committed: Mon Jun 13 17:53:29 2016 +0800
----------------------------------------------------------------------
include/singa/utils/channel.h | 85 ++++++++++++++++++++++++++++++
src/utils/channel.cc | 104 +++++++++++++++++++++++++++++++++++++
test/singa/test_channel.cc | 39 ++++++++++++++
3 files changed, 228 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/b167dfa5/src/utils/channel.cc
----------------------------------------------------------------------
diff --cc src/utils/channel.cc
index 0000000,95daed6..588a11a
mode 000000,100644..100644
--- a/src/utils/channel.cc
+++ b/src/utils/channel.cc
@@@ -1,0 -1,104 +1,104 @@@
+ /************************************************************
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ *************************************************************/
+
+ #include "singa/utils/channel.h"
+
+ #include "singa/utils/logging.h"
+ #include "singa/utils/singleton.h"
+
+ namespace singa {
+
+ ChannelManager::~ChannelManager() {
+ for (auto it : name2ptr_) {
+ if (it.second != nullptr) delete (it.second);
+ }
+ }
+
+ void ChannelManager::Init() {
+ // do nothing here
+ }
+
+ void ChannelManager::SetDefaultDir(const char* dir) {
+ if (dir != nullptr) {
+ dir_ = dir;
+ if (dir[dir_.length() - 1] != '/') dir_ += '/';
+ }
+ }
+
+ Channel* ChannelManager::GetInstance(const std::string& channel) {
+ // find the channel
+ if (name2ptr_.find(channel) == name2ptr_.end()) {
+ // create new channel
+ Channel* chn = new Channel(channel);
+ chn->SetDestFilePath(dir_ + channel);
+ chn->EnableDestFile(true);
+ name2ptr_[channel] = chn;
+ }
+ return name2ptr_[channel];
+ }
+
+ Channel::Channel(const std::string& name) { name_ = name; }
+
+ Channel::~Channel() {
+ if (os_.is_open()) os_.close();
+ }
+
+ void Channel::SetDestFilePath(const std::string& file) {
+ // file is append only
+ if (os_.is_open()) os_.close();
+ {
- ifstream fin(file.c_str());
++ std::ifstream fin(file.c_str());
+ if (fin.good())
+ LOG(WARNING) << "Messages will be appended to an existed file: " << file;
+ }
+ os_.open(file.c_str(), std::ios::app);
+ if (os_.is_open() == false)
+ LOG(WARNING) << "Cannot open channel file (" << file << ")";
+ }
+
+ void Channel::Send(const std::string& message) {
+ if (stderr_) fprintf(stderr, "%s\n", message.c_str());
+ if (file_ && os_.is_open()) os_ << message << "\n";
+ // TODO(wangwei) flush
+ }
+
+ void Channel::Send(const google::protobuf::Message& message) {
+ if (stderr_) fprintf(stderr, "%s\n", message.DebugString().c_str());
+ if (file_ && os_.is_open()) message.SerializeToOstream(&os_);
+ // TODO(wangwei) flush
+ }
+
+ void InitChannel(const char* argv) {
+ ChannelManager* mng = Singleton<ChannelManager>().Instance();
+ mng->Init();
+ }
+
+ void SetChannelDirectory(const char* path) {
+ ChannelManager* mng = Singleton<ChannelManager>().Instance();
+ mng->SetDefaultDir(path);
+ }
+
+ Channel* GetChannel(const std::string& channel_name) {
+ ChannelManager* mng = Singleton<ChannelManager>().Instance();
+ return mng->GetInstance(channel_name);
+ }
+
+ } // namespace singa
[2/3] incubator-singa git commit: SINGA-195 Channel for sending
training statistics
Posted by wa...@apache.org.
SINGA-195 Channel for sending training statistics
Add comments and TODOs.
Reformat some code.
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/a4fc4ea1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/a4fc4ea1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/a4fc4ea1
Branch: refs/heads/dev
Commit: a4fc4ea1d251242129be2e1a3cd388ca145223ca
Parents: a2a8e34
Author: Wei Wang <wa...@comp.nus.edu.sg>
Authored: Mon Jun 13 17:47:44 2016 +0800
Committer: Wei Wang <wa...@comp.nus.edu.sg>
Committed: Mon Jun 13 17:48:36 2016 +0800
----------------------------------------------------------------------
include/singa/utils/channel.h | 11 ++++++++++-
src/utils/channel.cc | 25 +++++++++++++++----------
test/singa/test_channel.cc | 4 ++--
3 files changed, 27 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/a4fc4ea1/include/singa/utils/channel.h
----------------------------------------------------------------------
diff --git a/include/singa/utils/channel.h b/include/singa/utils/channel.h
index 7cd7aa3..b640e90 100644
--- a/include/singa/utils/channel.h
+++ b/include/singa/utils/channel.h
@@ -31,16 +31,24 @@
namespace singa {
+/// Channel for appending metrics or other information into files or screen.
class Channel {
public:
explicit Channel(const std::string& name);
~Channel();
+ /// Return the channel name, which is also used for naming the output file.
inline const std::string& GetName() { return name_; }
+ /// Disabled by default.
inline void EnableDestStderr(bool enable) { stderr_ = enable; }
+ /// Enabled by default.
inline void EnableDestFile(bool enable) { file_ = enable; }
+ /// Reset the output file path.
+ /// The dest file is named as global dir + channel name by default.
void SetDestFilePath(const std::string& file);
+ /// Append a string message
void Send(const std::string& message);
+ /// Append a protobuf message
void Send(const google::protobuf::Message& message);
private:
@@ -64,7 +72,8 @@ class ChannelManager {
std::map<std::string, Channel*> name2ptr_;
};
-/// Initial function for global usage of channel
+/// Initial function for global usage of channel.
+/// 'argv' is for future use.
void InitChannel(const char* argv);
/// Set the directory name for persisting channel content
void SetChannelDirectory(const char* path);
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/a4fc4ea1/src/utils/channel.cc
----------------------------------------------------------------------
diff --git a/src/utils/channel.cc b/src/utils/channel.cc
index 52909a3..95daed6 100644
--- a/src/utils/channel.cc
+++ b/src/utils/channel.cc
@@ -7,9 +7,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
-*
+*
* http://www.apache.org/licenses/LICENSE-2.0
-*
+*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -28,7 +28,7 @@ namespace singa {
ChannelManager::~ChannelManager() {
for (auto it : name2ptr_) {
- if (it.second != nullptr) delete(it.second);
+ if (it.second != nullptr) delete (it.second);
}
}
@@ -39,7 +39,7 @@ void ChannelManager::Init() {
void ChannelManager::SetDefaultDir(const char* dir) {
if (dir != nullptr) {
dir_ = dir;
- if (dir[dir_.length()-1] != '/') dir_ += '/';
+ if (dir[dir_.length() - 1] != '/') dir_ += '/';
}
}
@@ -48,16 +48,14 @@ Channel* ChannelManager::GetInstance(const std::string& channel) {
if (name2ptr_.find(channel) == name2ptr_.end()) {
// create new channel
Channel* chn = new Channel(channel);
- chn->SetDestFilePath(dir_+channel);
+ chn->SetDestFilePath(dir_ + channel);
chn->EnableDestFile(true);
name2ptr_[channel] = chn;
}
return name2ptr_[channel];
}
-Channel::Channel(const std::string& name) {
- name_ = name;
-}
+Channel::Channel(const std::string& name) { name_ = name; }
Channel::~Channel() {
if (os_.is_open()) os_.close();
@@ -66,6 +64,11 @@ Channel::~Channel() {
void Channel::SetDestFilePath(const std::string& file) {
// file is append only
if (os_.is_open()) os_.close();
+ {
+ ifstream fin(file.c_str());
+ if (fin.good())
+ LOG(WARNING) << "Messages will be appended to an existed file: " << file;
+ }
os_.open(file.c_str(), std::ios::app);
if (os_.is_open() == false)
LOG(WARNING) << "Cannot open channel file (" << file << ")";
@@ -74,11 +77,13 @@ void Channel::SetDestFilePath(const std::string& file) {
void Channel::Send(const std::string& message) {
if (stderr_) fprintf(stderr, "%s\n", message.c_str());
if (file_ && os_.is_open()) os_ << message << "\n";
+ // TODO(wangwei) flush
}
void Channel::Send(const google::protobuf::Message& message) {
if (stderr_) fprintf(stderr, "%s\n", message.DebugString().c_str());
if (file_ && os_.is_open()) message.SerializeToOstream(&os_);
+ // TODO(wangwei) flush
}
void InitChannel(const char* argv) {
@@ -87,12 +92,12 @@ void InitChannel(const char* argv) {
}
void SetChannelDirectory(const char* path) {
- ChannelManager * mng = Singleton<ChannelManager>().Instance();
+ ChannelManager* mng = Singleton<ChannelManager>().Instance();
mng->SetDefaultDir(path);
}
Channel* GetChannel(const std::string& channel_name) {
- ChannelManager * mng = Singleton<ChannelManager>().Instance();
+ ChannelManager* mng = Singleton<ChannelManager>().Instance();
return mng->GetInstance(channel_name);
}
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/a4fc4ea1/test/singa/test_channel.cc
----------------------------------------------------------------------
diff --git a/test/singa/test_channel.cc b/test/singa/test_channel.cc
index 77d7cbc..68b0017 100644
--- a/test/singa/test_channel.cc
+++ b/test/singa/test_channel.cc
@@ -7,9 +7,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
-*
+*
* http://www.apache.org/licenses/LICENSE-2.0
-*
+*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY