You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@singa.apache.org by wa...@apache.org on 2016/04/16 11:36:19 UTC
incubator-singa git commit: SINGA-159 Rewrite safe_queue.h to avoid
the license issue
Repository: incubator-singa
Updated Branches:
refs/heads/master fb5b21841 -> fde87ed22
SINGA-159 Rewrite safe_queue.h to avoid the license issue
Rewrite the safe_queue.h.
Add a PriorityQueue which is a thread safe priority queue.
It is implemented using the base SafeQueue.
Fixed a bug from the protobuf extension field ID in rnnlm example.
Remove "Copyright 2015 The Apache Software Foundation" from Apache
headers in all files.
Project: http://git-wip-us.apache.org/repos/asf/incubator-singa/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-singa/commit/fde87ed2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-singa/tree/fde87ed2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-singa/diff/fde87ed2
Branch: refs/heads/master
Commit: fde87ed2283e3fcef188666bad503b94f343cca3
Parents: fb5b218
Author: Wei Wang <wa...@comp.nus.edu.sg>
Authored: Sat Apr 16 16:58:35 2016 +0800
Committer: Wei Wang <wa...@comp.nus.edu.sg>
Committed: Sat Apr 16 17:11:52 2016 +0800
----------------------------------------------------------------------
LICENSE | 6 -
Makefile.example | 2 -
Makefile.gpu | 2 -
autogen.sh | 2 -
bin/singa-cleanup.sh | 4 +-
bin/singa-console.sh | 6 +-
bin/singa-env.sh | 4 +-
bin/singa-run.sh | 2 -
bin/singa-stop.sh | 4 +-
bin/zk-service.sh | 6 +-
configure.ac | 14 +-
examples/alexnet/Makefile.example | 2 -
examples/cifar10/Makefile.example | 4 +-
examples/mnist/Makefile.example | 2 -
examples/rnnlm/rnnlm.proto | 6 +-
include/singa/utils/safe_queue.h | 305 ++++++++++++---------------------
src/comm/socket.cc | 12 +-
thirdparty/install.sh | 2 -
tool/docker/mesos/.bashrc | 2 -
tool/docker/mesos/Dockerfile | 6 +-
tool/docker/mesos/install.sh | 4 +-
tool/docker/singa/.bashrc | 2 -
tool/docker/singa/.vimrc | 27 ---
tool/docker/singa/Dockerfile | 2 -
tool/docker/singa/Dockerfile_gpu | 8 +-
tool/graph.py | 2 -
tool/node.sh | 2 -
tool/python/singa/generatepy.sh | 4 +-
28 files changed, 136 insertions(+), 308 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/fde87ed2/LICENSE
----------------------------------------------------------------------
diff --git a/LICENSE b/LICENSE
index b835024..75001c1 100644
--- a/LICENSE
+++ b/LICENSE
@@ -244,9 +244,3 @@ Copyright (c) 2010-2012 Tomas Mikolov
Copyright (c) 2013 Cantab Research Ltd
All rights reserved.
http://www.fit.vutbr.cz/~imikolov/rnnlm/
-
-====================================================================
-SINGA bundles the following under Creative Commons Attribution 4.0 International
-Public License: include/singa/utils/safe_queue.h
-
-http://gnodebian.blogspot.sg/2013/07/a-thread-safe-asynchronous-queue-in-c11.html
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/fde87ed2/Makefile.example
----------------------------------------------------------------------
diff --git a/Makefile.example b/Makefile.example
index 76b8c0d..4fb0c66 100644
--- a/Makefile.example
+++ b/Makefile.example
@@ -1,6 +1,4 @@
#/**
-# * Copyright 2015 The Apache Software Foundation
-# *
# * Licensed to the Apache Software Foundation (ASF) under one
# * or more contributor license agreements. See the NOTICE file
# * distributed with this work for additional information
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/fde87ed2/Makefile.gpu
----------------------------------------------------------------------
diff --git a/Makefile.gpu b/Makefile.gpu
index 11d6827..0615f6b 100644
--- a/Makefile.gpu
+++ b/Makefile.gpu
@@ -1,6 +1,4 @@
#/**
-# * Copyright 2015 The Apache Software Foundation
-# *
# * Licensed to the Apache Software Foundation (ASF) under one
# * or more contributor license agreements. See the NOTICE file
# * distributed with this work for additional information
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/fde87ed2/autogen.sh
----------------------------------------------------------------------
diff --git a/autogen.sh b/autogen.sh
index f41a815..ff8eacb 100755
--- a/autogen.sh
+++ b/autogen.sh
@@ -1,7 +1,5 @@
#!/bin/sh
#/**
-# * Copyright 2015 The Apache Software Foundation
-# *
# * Licensed to the Apache Software Foundation (ASF) under one
# * or more contributor license agreements. See the NOTICE file
# * distributed with this work for additional information
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/fde87ed2/bin/singa-cleanup.sh
----------------------------------------------------------------------
diff --git a/bin/singa-cleanup.sh b/bin/singa-cleanup.sh
index 2c78dfb..88b947d 100755
--- a/bin/singa-cleanup.sh
+++ b/bin/singa-cleanup.sh
@@ -1,8 +1,6 @@
#!/usr/bin/env bash
#
#/**
-# * Copyright 2015 The Apache Software Foundation
-# *
# * Licensed to the Apache Software Foundation (ASF) under one
# * or more contributor license agreements. See the NOTICE file
# * distributed with this work for additional information
@@ -19,7 +17,7 @@
# * See the License for the specific language governing permissions and
# * limitations under the License.
# */
-#
+#
# kill all singa jobs and clean up zookeeper
#
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/fde87ed2/bin/singa-console.sh
----------------------------------------------------------------------
diff --git a/bin/singa-console.sh b/bin/singa-console.sh
index 9600132..06ee945 100755
--- a/bin/singa-console.sh
+++ b/bin/singa-console.sh
@@ -1,8 +1,6 @@
#!/usr/bin/env bash
#
#/**
-# * Copyright 2015 The Apache Software Foundation
-# *
# * Licensed to the Apache Software Foundation (ASF) under one
# * or more contributor license agreements. See the NOTICE file
# * distributed with this work for additional information
@@ -19,7 +17,7 @@
# * See the License for the specific language governing permissions and
# * limitations under the License.
# */
-#
+#
# console to list/view/kill singa jobs
#
@@ -80,7 +78,7 @@ case $1 in
done
./singatool remove $2 || exit 1
;;
-
+
*)
echo -e $usage
exit 1
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/fde87ed2/bin/singa-env.sh
----------------------------------------------------------------------
diff --git a/bin/singa-env.sh b/bin/singa-env.sh
index 0a3db9e..8adeade 100755
--- a/bin/singa-env.sh
+++ b/bin/singa-env.sh
@@ -1,8 +1,6 @@
#!/usr/bin/env bash
#
#/**
-# * Copyright 2015 The Apache Software Foundation
-# *
# * Licensed to the Apache Software Foundation (ASF) under one
# * or more contributor license agreements. See the NOTICE file
# * distributed with this work for additional information
@@ -52,7 +50,7 @@ fi
if [ -z $SINGA_LOG ]; then
# add -confdir arg, so no need to run under SINGA_HOME
SINGA_LOG=`"$SINGA_HOME"/singatool getlogdir -confdir "$SINGA_CONF"`
- [ $? == 0 ] || exit 1
+ [ $? == 0 ] || exit 1
fi
# set ZK_HOME
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/fde87ed2/bin/singa-run.sh
----------------------------------------------------------------------
diff --git a/bin/singa-run.sh b/bin/singa-run.sh
index 7f40046..763130d 100755
--- a/bin/singa-run.sh
+++ b/bin/singa-run.sh
@@ -1,8 +1,6 @@
#!/usr/bin/env bash
#
#/**
-# * Copyright 2015 The Apache Software Foundation
-# *
# * Licensed to the Apache Software Foundation (ASF) under one
# * or more contributor license agreements. See the NOTICE file
# * distributed with this work for additional information
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/fde87ed2/bin/singa-stop.sh
----------------------------------------------------------------------
diff --git a/bin/singa-stop.sh b/bin/singa-stop.sh
index 5dce790..446eaf8 100755
--- a/bin/singa-stop.sh
+++ b/bin/singa-stop.sh
@@ -1,8 +1,6 @@
#!/usr/bin/env bash
#
#/**
-# * Copyright 2015 The Apache Software Foundation
-# *
# * Licensed to the Apache Software Foundation (ASF) under one
# * or more contributor license agreements. See the NOTICE file
# * distributed with this work for additional information
@@ -19,7 +17,7 @@
# * See the License for the specific language governing permissions and
# * limitations under the License.
# */
-#
+#
# kill all singa jobs
#
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/fde87ed2/bin/zk-service.sh
----------------------------------------------------------------------
diff --git a/bin/zk-service.sh b/bin/zk-service.sh
index 9999913..e7bb67b 100755
--- a/bin/zk-service.sh
+++ b/bin/zk-service.sh
@@ -1,8 +1,6 @@
#!/usr/bin/env bash
#
#/**
-# * Copyright 2015 The Apache Software Foundation
-# *
# * Licensed to the Apache Software Foundation (ASF) under one
# * or more contributor license agreements. See the NOTICE file
# * distributed with this work for additional information
@@ -19,7 +17,7 @@
# * See the License for the specific language governing permissions and
# * limitations under the License.
# */
-#
+#
# manage ZooKeeper service
#
@@ -66,7 +64,7 @@ case $1 in
# stop zk service
$ZK_HOME/bin/zkServer.sh stop
;;
-
+
*)
echo $usage
exit 1
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/fde87ed2/configure.ac
----------------------------------------------------------------------
diff --git a/configure.ac b/configure.ac
index c54d6a9..21f699c 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1,6 +1,4 @@
#/**
-# * Copyright 2015 The Apache Software Foundation
-# *
# * Licensed to the Apache Software Foundation (ASF) under one
# * or more contributor license agreements. See the NOTICE file
# * distributed with this work for additional information
@@ -143,7 +141,7 @@ AC_SUBST(CUDA_LDFLAGS)
AC_SUBST(CUDA_LIBS)
AC_SUBST(CUDA_CFLAGS)
-# Setup custom zookeeper and zmq paths
+# Setup custom zookeeper and zmq paths
AC_ARG_ENABLE(dist,
AS_HELP_STRING([--enable-dist],[enable dist support]),
[enable_dist="yes"],[enable_dist="no"])
@@ -158,10 +156,10 @@ if test "$dist_prefix" == "yes"; then
fi
fi
if test x"$enable_dist" == x"yes"; then
- AC_CHECK_LIB([zookeeper_mt], [main], [], [
+ AC_CHECK_LIB([zookeeper_mt], [main], [], [
AC_MSG_ERROR([unable to find zookeeper library])
])
- AC_SEARCH_LIBS([zmq_ctx_new], [zmq], [], [
+ AC_SEARCH_LIBS([zmq_ctx_new], [zmq], [], [
AC_MSG_ERROR([unable to find zmq_ctx_new() function])
])
DIST_CFLAGS="-I$dist_prefix/include"
@@ -180,7 +178,7 @@ AC_SUBST(DIST_CFLAGS)
AC_SUBST(DIST_LDFLAGS)
AC_SUBST(DIST_LIBS)
-# Setup custom lmdb paths
+# Setup custom lmdb paths
AC_ARG_ENABLE(lmdb,
AS_HELP_STRING([--enable-lmdb],[enable debug option]),
[enable_lmdb=yes],[enable_lmdb=no])
@@ -192,7 +190,7 @@ if test x"$enable_lmdb" = x"yes"; then
AC_DEFINE(LMDB, 1, [Enable Option layer])
fi
-# Setup custom libhdfs paths
+# Setup custom libhdfs paths
AC_ARG_ENABLE(hdfs,
AS_HELP_STRING([--enable-hdfs],[enable hdfs support]),
[enable_hdfs=yes],[enable_hdfs=no])
@@ -225,7 +223,7 @@ AC_SUBST(HDFS_CFLAGS)
AC_SUBST(HDFS_LDFLAGS)
AC_SUBST(HDFS_LIBS)
-# Setup for test args
+# Setup for test args
AC_ARG_ENABLE(test,
AS_HELP_STRING([--enable-test],[enable singa test]),
[enable_test=yes],[enable_test=no])
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/fde87ed2/examples/alexnet/Makefile.example
----------------------------------------------------------------------
diff --git a/examples/alexnet/Makefile.example b/examples/alexnet/Makefile.example
index 78ca7c3..f895146 100644
--- a/examples/alexnet/Makefile.example
+++ b/examples/alexnet/Makefile.example
@@ -1,6 +1,4 @@
#/**
-# * Copyright 2015 The Apache Software Foundation
-# *
# * Licensed to the Apache Software Foundation (ASF) under one
# * or more contributor license agreements. See the NOTICE file
# * distributed with this work for additional information
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/fde87ed2/examples/cifar10/Makefile.example
----------------------------------------------------------------------
diff --git a/examples/cifar10/Makefile.example b/examples/cifar10/Makefile.example
index 72c326b..40e85b1 100644
--- a/examples/cifar10/Makefile.example
+++ b/examples/cifar10/Makefile.example
@@ -1,6 +1,4 @@
#/**
-# * Copyright 2015 The Apache Software Foundation
-# *
# * Licensed to the Apache Software Foundation (ASF) under one
# * or more contributor license agreements. See the NOTICE file
# * distributed with this work for additional information
@@ -40,4 +38,4 @@ create_hdfs:
$(CXX) create_data.cc -std=c++11 -lsinga -lprotobuf -lglog -lhdfs3 \
-I../../include -L../../.libs/ -Wl,-unresolved-symbols=ignore-in-shared-libs \
-Wl,-rpath=../../.libs/ -o create_data.bin
- ./create_data.bin cifar-10-batches-bin $(HDFS_DIR)
+ ./create_data.bin cifar-10-batches-bin $(HDFS_DIR)
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/fde87ed2/examples/mnist/Makefile.example
----------------------------------------------------------------------
diff --git a/examples/mnist/Makefile.example b/examples/mnist/Makefile.example
index 7b516e7..a041359 100644
--- a/examples/mnist/Makefile.example
+++ b/examples/mnist/Makefile.example
@@ -1,7 +1,5 @@
#
#/**
-# * Copyright 2015 The Apache Software Foundation
-# *
# * Licensed to the Apache Software Foundation (ASF) under one
# * or more contributor license agreements. See the NOTICE file
# * distributed with this work for additional information
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/fde87ed2/examples/rnnlm/rnnlm.proto
----------------------------------------------------------------------
diff --git a/examples/rnnlm/rnnlm.proto b/examples/rnnlm/rnnlm.proto
index 8cfec86..4a4dcbc 100644
--- a/examples/rnnlm/rnnlm.proto
+++ b/examples/rnnlm/rnnlm.proto
@@ -39,9 +39,9 @@ message DataProto {
}
extend singa.LayerProto {
- optional EmbeddingProto embedding_conf = 101;
- optional LossProto loss_conf = 102;
- optional DataProto data_conf = 103;
+ optional EmbeddingProto embedding_conf = 1001;
+ optional LossProto loss_conf = 1002;
+ optional DataProto data_conf = 1003;
}
message WordRecord {
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/fde87ed2/include/singa/utils/safe_queue.h
----------------------------------------------------------------------
diff --git a/include/singa/utils/safe_queue.h b/include/singa/utils/safe_queue.h
index 31df1ef..1f791a4 100644
--- a/include/singa/utils/safe_queue.h
+++ b/include/singa/utils/safe_queue.h
@@ -19,268 +19,175 @@
*
*************************************************************/
-/**
- * The code is adapted from following source:
- * http://gnodebian.blogspot.sg/2013/07/a-thread-safe-asynchronous-queue-in-c11.html
- * under Creative Commons Attribution 4.0 International Public License
- */
-
#ifndef SINGA_UTILS_SAFE_QUEUE_H_
#define SINGA_UTILS_SAFE_QUEUE_H_
-// source: http://gnodebian.blogspot.sg/2013/07/a-thread-safe-asynchronous-queue-in-c11.html
#include <algorithm>
#include <queue>
#include <list>
#include <mutex>
-#include <thread>
-#include <cstdint>
#include <condition_variable>
+#include <thread>
-/** A thread-safe asynchronous queue */
-template <class T, class Container = std::list<T>>
+/**
+ * Thread-safe queue.
+ */
+template <typename T, class Container = std::queue<T>>
class SafeQueue {
- typedef typename Container::value_type value_type;
- typedef typename Container::size_type size_type;
- typedef Container container_type;
-
public:
- /*! Create safe queue. */
SafeQueue() = default;
- SafeQueue(SafeQueue&& sq) {
- m_queue = std::move(sq.m_queue);
- }
- SafeQueue(const SafeQueue& sq) {
- std::lock_guard<std::mutex> lock(sq.m_mutex);
- m_queue = sq.m_queue;
- }
-
- /*! Destroy safe queue. */
~SafeQueue() {
- std::lock_guard<std::mutex> lock(m_mutex);
- }
-
- /**
- * Sets the maximum number of items in the queue. Defaults is 0: No limit
- * \param[in] item An item.
- */
- void set_max_num_items(unsigned int max_num_items) {
- m_max_num_items = max_num_items;
+ std::lock_guard<std::mutex> lock(mutex_);
}
/**
- * Pushes the item into the queue.
- * \param[in] item An item.
- * \return true if an item was pushed into the queue
+ * Push an element into the queue. Blocking operation.
+ * @return true if success;
*/
- bool push(const value_type& item) {
- std::lock_guard<std::mutex> lock(m_mutex);
-
- if (m_max_num_items > 0 && m_queue.size() > m_max_num_items)
- return false;
-
- m_queue.push(item);
- m_condition.notify_one();
+ bool Push(const T& e) {
+ std::lock_guard<std::mutex> lock(mutex_);
+ queue_.push(e);
+ condition_.notify_one();
return true;
}
/**
- * Pushes the item into the queue.
- * \param[in] item An item.
- * \return true if an item was pushed into the queue
+ * Pop an element from the queue.
+ * It will be blocked until one element is poped.
*/
- bool push(const value_type&& item) {
- std::lock_guard<std::mutex> lock(m_mutex);
-
- if (m_max_num_items > 0 && m_queue.size() > m_max_num_items)
- return false;
-
- m_queue.push(item);
- m_condition.notify_one();
- return true;
- }
-
- /**
- * Pops item from the queue. If queue is empty, this function blocks until item becomes available.
- * \param[out] item The item.
- */
- void pop(value_type& item) {
- std::unique_lock<std::mutex> lock(m_mutex);
- m_condition.wait(lock, [this]() { // Lambda funct
- return !m_queue.empty();
- });
- item = m_queue.front();
- m_queue.pop();
+ void Pop(T& e) {
+ std::unique_lock<std::mutex> lock(mutex_);
+ condition_.wait(lock, [this]() { return !queue_.empty(); });
+ e = queue_.front();
+ queue_.pop();
}
-
/**
- * Pops item from the queue using the contained type's move assignment operator, if it has one..
- * This method is identical to the pop() method if that type has no move assignment operator.
- * If queue is empty, this function blocks until item becomes available.
- * \param[out] item The item.
+ * Pop an item from the queue until one element is poped or timout.
+ * @param[in] timeout, return false after waiting this number of microseconds
*/
- void move_pop(value_type& item) {
- std::unique_lock<std::mutex> lock(m_mutex);
- m_condition.wait(lock, [this]() { // Lambda funct
- return !m_queue.empty();
- });
- item = std::move(m_queue.front());
- m_queue.pop();
- }
+ bool Pop(T& item, std::uint64_t timeout) {
+ std::unique_lock<std::mutex> lock(mutex_);
- /**
- * Tries to pop item from the queue.
- * \param[out] item The item.
- * \return False is returned if no item is available.
- */
- bool try_pop(value_type& item) {
- std::unique_lock<std::mutex> lock(m_mutex);
+ if (queue_.empty()) {
+ if (timeout == 0)
+ return false;
- if (m_queue.empty())
- return false;
+ if (condition_.wait_for(lock, std::chrono::microseconds(timeout))
+ == std::cv_status::timeout)
+ return false;
+ }
- item = m_queue.front();
- m_queue.pop();
+ item = queue_.front();
+ queue_.pop();
return true;
}
/**
- * Tries to pop item from the queue using the contained type's move assignment operator, if it has one..
- * This method is identical to the try_pop() method if that type has no move assignment operator.
- * \param[out] item The item.
- * \return False is returned if no item is available.
+ * Try to pop an element from the queue.
+ * \return false the queue is empty now.
*/
- bool try_move_pop(value_type& item) {
- std::unique_lock<std::mutex> lock(m_mutex);
+ bool TryPop(T& e) {
+ std::unique_lock<std::mutex> lock(mutex_);
- if (m_queue.empty())
+ if (queue_.empty())
return false;
- item = std::move(m_queue.front());
- m_queue.pop();
+ e = queue_.front();
+ queue_.pop();
return true;
}
+
/**
- * Pops item from the queue. If the queue is empty, blocks for timeout microseconds, or until item becomes available.
- * \param[out] t An item.
- * \param[in] timeout The number of microseconds to wait.
- * \return true if get an item from the queue, false if no item is received before the timeout.
+ * @return Number of elements in the queue.
*/
- bool timeout_pop(value_type& item, std::uint64_t timeout) {
- std::unique_lock<std::mutex> lock(m_mutex);
-
- if (m_queue.empty()) {
- if (timeout == 0)
- return false;
-
- if (m_condition.wait_for(lock, std::chrono::microseconds(timeout))
- == std::cv_status::timeout)
- return false;
- }
-
- item = m_queue.front();
- m_queue.pop();
- return true;
+ unsigned int Size() const {
+ std::lock_guard<std::mutex> lock(mutex_);
+ return queue_.size();
}
+ private:
+ Container queue_;
+ mutable std::mutex mutex_;
+ std::condition_variable condition_;
+};
+
+/**
+ * Thread safe priority queue.
+ */
+template<typename T>
+class PriorityQueue {
+ public:
+ PriorityQueue() = default;
/**
- * Pops item from the queue using the contained type's move assignment operator, if it has one..
- * If the queue is empty, blocks for timeout microseconds, or until item becomes available.
- * This method is identical to the try_pop() method if that type has no move assignment operator.
- * \param[out] t An item.
- * \param[in] timeout The number of microseconds to wait.
- * \return true if get an item from the queue, false if no item is received before the timeout.
+ * Push an element into the queue with a given priority.
+ * The queue should not be a priority queue.
+ * @return true if success; otherwise false, e.g., due to capacity constraint.
*/
- bool timeout_move_pop(value_type& item, std::uint64_t timeout) {
- std::unique_lock<std::mutex> lock(m_mutex);
-
- if (m_queue.empty()) {
- if (timeout == 0)
- return false;
-
- if (m_condition.wait_for(lock, std::chrono::microseconds(timeout))
- == std::cv_status::timeout)
- return false;
- }
-
- item = std::move(m_queue.front());
- m_queue.pop();
+ bool Push(const T& e, int priority) {
+ Element ele;
+ ele.data = e;
+ ele.priority = priority;
+ queue_.push(ele);
return true;
}
/**
- * Gets the number of items in the queue.
- * \return Number of items in the queue.
+ * Pop an element from the queue with the highest priority.
+ * It blocks until one element is poped.
*/
- size_type size() const {
- std::lock_guard<std::mutex> lock(m_mutex);
- return m_queue.size();
+ void Pop(T& e) {
+ Element ele;
+ queue_.pop(ele);
+ e = ele.data;
}
-
/**
- * Check if the queue is empty.
- * \return true if queue is empty.
+ * Pop the item with the highest priority from the queue until one element is
+ * poped or timeout.
+ * @param[in] timeout, return false if no element is poped after this number
+ * of microseconds.
*/
- bool empty() const {
- std::lock_guard<std::mutex> lock(m_mutex);
- return m_queue.empty();
+ bool Pop(T& e, std::uint64_t timeout) {
+ Element ele;
+ if (queue_.pop(ele, timeout)) {
+ e = ele.data;
+ return true;
+ } else {
+ return false;
+ }
}
/**
- * Swaps the contents.
- * \param[out] sq The SafeQueue to swap with 'this'.
+ * Try to pop an element from the queue.
+ * @return false if the queue is empty now.
*/
- void swap(SafeQueue& sq) {
- if (this != &sq) {
- std::lock_guard<std::mutex> lock1(m_mutex);
- std::lock_guard<std::mutex> lock2(sq.m_mutex);
- m_queue.swap(sq.m_queue);
-
- if (!m_queue.empty())
- m_condition.notify_all();
-
- if (!sq.m_queue.empty())
- sq.m_condition.notify_all();
- }
- }
-
- /*! The copy assignment operator */
- SafeQueue& operator= (const SafeQueue& sq) {
- if (this != &sq) {
- std::lock_guard<std::mutex> lock1(m_mutex);
- std::lock_guard<std::mutex> lock2(sq.m_mutex);
- std::queue<T, Container> temp{sq.m_queue};
- m_queue.swap(temp);
-
- if (!m_queue.empty())
- m_condition.notify_all();
+ bool TryPop(T& e) {
+ Element ele;
+ if (queue_.TryPop(ele)) {
+ e = ele.data;
+ return true;
+ } else {
+ return false;
}
-
- return *this;
}
- /*! The move assignment operator */
- SafeQueue& operator= (SafeQueue && sq) {
- std::lock_guard<std::mutex> lock(m_mutex);
- m_queue = std::move(sq.m_queue);
-
- if (!m_queue.empty()) m_condition.notify_all();
-
- return *this;
+ /**
+ * @return Number of elements in the queue.
+ */
+ unsigned int Size() const {
+ return queue_.Size();
}
private:
- std::queue<T, Container> m_queue;
- mutable std::mutex m_mutex;
- std::condition_variable m_condition;
- unsigned int m_max_num_items = 0;
-};
+ struct Element {
+ T data;
+ int priority;
+ inline bool operator<(const Element &other) const {
+ return priority < other.priority;
+ }
+ };
-/*! Swaps the contents of two SafeQueue objects. */
-template <class T, class Container>
-void swap(SafeQueue<T, Container>& q1, SafeQueue<T, Container>& q2) {
- q1.swap(q2);
-}
+ SafeQueue<Element, std::priority_queue<Element>> queue_;
+};
#endif // SINGA_UTILS_SAFE_QUEUE_H_
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/fde87ed2/src/comm/socket.cc
----------------------------------------------------------------------
diff --git a/src/comm/socket.cc b/src/comm/socket.cc
index 9afc54c..eba6a0c 100644
--- a/src/comm/socket.cc
+++ b/src/comm/socket.cc
@@ -60,7 +60,7 @@ int Dealer::Send(Msg** msg) {
delete *msg;
*msg = nullptr;
} else {
- msgQueues.at(-1).push(*msg);
+ msgQueues.at(-1).Push(*msg);
}
return 1;
}
@@ -68,10 +68,10 @@ int Dealer::Send(Msg** msg) {
Msg* Dealer::Receive(int timeout) {
Msg* msg = nullptr;
if (timeout > 0) {
- if (!msgQueues.at(id_).timeout_pop(msg, timeout))
+ if (!msgQueues.at(id_).Pop(msg, timeout))
return nullptr;
} else {
- msgQueues.at(id_).pop(msg);
+ msgQueues.at(id_).Pop(msg);
}
msg->FirstFrame();
return msg;
@@ -108,7 +108,7 @@ int Router::Bind(const std::string& endpoint) {
int Router::Send(Msg **msg) {
int dstid = (*msg)->dst();
if (msgQueues.find(dstid) != msgQueues.end()) {
- msgQueues.at(dstid).push(*msg);
+ msgQueues.at(dstid).Push(*msg);
} else {
LOG(FATAL) << "The dst queue not exist for dstid = " << dstid;
}
@@ -133,11 +133,11 @@ Msg* Router::Receive(int timeout) {
zframe_destroy(&dealer);
Msg* remote_msg = new Msg();
remote_msg->ParseFromZmsg(zmsg);
- msgQueues.at(-1).push(remote_msg);
+ msgQueues.at(-1).Push(remote_msg);
}
}
#endif
- msgQueues.at(-1).timeout_pop(msg, timeout * 10);
+ msgQueues.at(-1).Pop(msg, timeout * 10);
}
msg->FirstFrame();
return msg;
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/fde87ed2/thirdparty/install.sh
----------------------------------------------------------------------
diff --git a/thirdparty/install.sh b/thirdparty/install.sh
index 99403b9..bc3c159 100755
--- a/thirdparty/install.sh
+++ b/thirdparty/install.sh
@@ -1,6 +1,4 @@
#/**
-# * Copyright 2015 The Apache Software Foundation
-# *
# * Licensed to the Apache Software Foundation (ASF) under one
# * or more contributor license agreements. See the NOTICE file
# * distributed with this work for additional information
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/fde87ed2/tool/docker/mesos/.bashrc
----------------------------------------------------------------------
diff --git a/tool/docker/mesos/.bashrc b/tool/docker/mesos/.bashrc
index 96dbc19..c796a27 100644
--- a/tool/docker/mesos/.bashrc
+++ b/tool/docker/mesos/.bashrc
@@ -1,6 +1,4 @@
#/**
-# * Copyright 2015 The Apache Software Foundation
-# *
# * Licensed to the Apache Software Foundation (ASF) under one
# * or more contributor license agreements. See the NOTICE file
# * distributed with this work for additional information
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/fde87ed2/tool/docker/mesos/Dockerfile
----------------------------------------------------------------------
diff --git a/tool/docker/mesos/Dockerfile b/tool/docker/mesos/Dockerfile
index d2da1f3..83142b3 100644
--- a/tool/docker/mesos/Dockerfile
+++ b/tool/docker/mesos/Dockerfile
@@ -1,6 +1,4 @@
#/**
-# * Copyright 2015 The Apache Software Foundation
-# *
# * Licensed to the Apache Software Foundation (ASF) under one
# * or more contributor license agreements. See the NOTICE file
# * distributed with this work for additional information
@@ -33,9 +31,9 @@ RUN apt-get -y install curl cmake libxml2 libxml2-dev uuid-dev protobuf-compiler
RUN cd /opt && source /root/.bashrc && wget -c http://www.eu.apache.org/dist/hadoop/common/hadoop-2.6.0/hadoop-2.6.0.tar.gz && tar -zxvf hadoop-2.6.0.tar.gz && git clone https://github.com/PivotalRD/libhdfs3.git && cd libhdfs3 && mkdir build && cd build && cmake ../ && make && make install
#to be removed after SINGA-11 is merged (pulling from Anh's branch instead of pulling from the master)
-#RUN source ~/.bashrc && mkdir /root/mesos && cd /root/mesos && git clone https://github.com/ug93tad/incubator-singa && cd incubator-singa && git checkout SINGA-11 && cp -r tool/mesos /root/incubator-singa/tool/ && cd /root/incubator-singa/tool/mesos && make
+#RUN source ~/.bashrc && mkdir /root/mesos && cd /root/mesos && git clone https://github.com/ug93tad/incubator-singa && cd incubator-singa && git checkout SINGA-11 && cp -r tool/mesos /root/incubator-singa/tool/ && cd /root/incubator-singa/tool/mesos && make
-# update SINGA, but keep the examples directory
+# update SINGA, but keep the examples directory
RUN source ~/.bashrc && cd /root/incubator-singa/tool/mesos && make
COPY incubator-singa/tool/docker/mesos/*.xml /opt/hadoop-2.6.0/etc/hadoop/
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/fde87ed2/tool/docker/mesos/install.sh
----------------------------------------------------------------------
diff --git a/tool/docker/mesos/install.sh b/tool/docker/mesos/install.sh
index ac7106c..4b6ae79 100644
--- a/tool/docker/mesos/install.sh
+++ b/tool/docker/mesos/install.sh
@@ -1,8 +1,6 @@
#!/bin/bash
#/**
-# * Copyright 2015 The Apache Software Foundation
-# *
# * Licensed to the Apache Software Foundation (ASF) under one
# * or more contributor license agreements. See the NOTICE file
# * distributed with this work for additional information
@@ -24,7 +22,7 @@ source /root/.bashrc
#download
cd /opt
wget -c http://archive.apache.org/dist/mesos/0.22.0/mesos-0.22.0.tar.gz
-wget https://www.comp.nus.edu.sg/~dinhtta/files/mesos_patch
+wget https://www.comp.nus.edu.sg/~dinhtta/files/mesos_patch
tar -zxvf mesos-0.22.0.tar.gz
#patch and install mesos
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/fde87ed2/tool/docker/singa/.bashrc
----------------------------------------------------------------------
diff --git a/tool/docker/singa/.bashrc b/tool/docker/singa/.bashrc
index 818592b..cf0712f 100644
--- a/tool/docker/singa/.bashrc
+++ b/tool/docker/singa/.bashrc
@@ -1,6 +1,4 @@
#/**
-# * Copyright 2015 The Apache Software Foundation
-# *
# * Licensed to the Apache Software Foundation (ASF) under one
# * or more contributor license agreements. See the NOTICE file
# * distributed with this work for additional information
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/fde87ed2/tool/docker/singa/.vimrc
----------------------------------------------------------------------
diff --git a/tool/docker/singa/.vimrc b/tool/docker/singa/.vimrc
deleted file mode 100644
index a537894..0000000
--- a/tool/docker/singa/.vimrc
+++ /dev/null
@@ -1,27 +0,0 @@
-"/**
-" * Copyright 2015 The Apache Software Foundation
-" *
-" * Licensed to the Apache Software Foundation (ASF) under one
-" * or more contributor license agreements. See the NOTICE file
-" * distributed with this work for additional information
-" * regarding copyright ownership. The ASF licenses this file
-" * to you under the Apache License, Version 2.0 (the
-" * "License"); you may not use this file except in compliance
-" * with the License. You may obtain a copy of the License at
-" *
-" * http://www.apache.org/licenses/LICENSE-2.0
-" *
-" * Unless required by applicable law or agreed to in writing, software
-" * distributed under the License is distributed on an "AS IS" BASIS,
-" * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-" * See the License for the specific language governing permissions and
-" * limitations under the License.
-" */
-
-set autoindent " always set autoindenting on
-set smartindent " use smart indent if there is no indent file
-set tabstop=2 " <tab> inserts 4 spaces
-set shiftwidth=2 " but an indent level is 2 spaces wide.
-set softtabstop=2 " <BS> over an autoindent deletes both spaces.
-"set expandtab " Use spaces, not tabs, for autoindent/tab key.
-set shiftround " rounds indent to a multiple of shiftwidth
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/fde87ed2/tool/docker/singa/Dockerfile
----------------------------------------------------------------------
diff --git a/tool/docker/singa/Dockerfile b/tool/docker/singa/Dockerfile
index 07ca2cc..0fc16bc 100644
--- a/tool/docker/singa/Dockerfile
+++ b/tool/docker/singa/Dockerfile
@@ -1,6 +1,4 @@
#/**
-# * Copyright 2015 The Apache Software Foundation
-# *
# * Licensed to the Apache Software Foundation (ASF) under one
# * or more contributor license agreements. See the NOTICE file
# * distributed with this work for additional information
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/fde87ed2/tool/docker/singa/Dockerfile_gpu
----------------------------------------------------------------------
diff --git a/tool/docker/singa/Dockerfile_gpu b/tool/docker/singa/Dockerfile_gpu
index 4afd931..da57479 100644
--- a/tool/docker/singa/Dockerfile_gpu
+++ b/tool/docker/singa/Dockerfile_gpu
@@ -1,6 +1,4 @@
#/**
-# * Copyright 2015 The Apache Software Foundation
-# *
# * Licensed to the Apache Software Foundation (ASF) under one
# * or more contributor license agreements. See the NOTICE file
# * distributed with this work for additional information
@@ -44,7 +42,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends --force-yes cur
cuda-cusparse-$CUDA_PKG_VERSION \
cuda-npp-$CUDA_PKG_VERSION \
cuda-cudart-$CUDA_PKG_VERSION && \
- ln -s cuda-$CUDA_VERSION /usr/local/cuda
+ ln -s cuda-$CUDA_VERSION /usr/local/cuda
RUN apt-get install -y --no-install-recommends --force-yes \
cuda-core-$CUDA_PKG_VERSION \
@@ -59,7 +57,7 @@ RUN apt-get install -y --no-install-recommends --force-yes \
cuda-cusparse-dev-$CUDA_PKG_VERSION \
cuda-npp-dev-$CUDA_PKG_VERSION \
cuda-cudart-dev-$CUDA_PKG_VERSION \
- cuda-driver-dev-$CUDA_PKG_VERSION
+ cuda-driver-dev-$CUDA_PKG_VERSION
RUN echo "/usr/local/cuda/lib" >> /etc/ld.so.conf.d/cuda.conf && \
echo "/usr/local/cuda/lib64" >> /etc/ld.so.conf.d/cuda.conf && \
@@ -79,4 +77,4 @@ RUN echo "/usr/local/nvidia/lib" >> /etc/ld.so.conf.d/nvidia.conf && \
ENV PATH /usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH}
ENV LD_LIBRARY_PATH /usr/local/nvidia/lib:/usr/local/nvidia/lib64:${LD_LIBRARY_PATH}
-RUN cd && source /root/.bashrc && cd incubator-singa && ./configure --enable-cuda --with-cuda=/usr/local/cuda --enable-cudnn --with-cudnn=/usr/local/cuda && make && make install
+RUN cd && source /root/.bashrc && cd incubator-singa && ./configure --enable-cuda --with-cuda=/usr/local/cuda --enable-cudnn --with-cudnn=/usr/local/cuda && make && make install
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/fde87ed2/tool/graph.py
----------------------------------------------------------------------
diff --git a/tool/graph.py b/tool/graph.py
index 74ad98f..e2870ed 100644
--- a/tool/graph.py
+++ b/tool/graph.py
@@ -1,6 +1,4 @@
#/**
-# * Copyright 2015 The Apache Software Foundation
-# *
# * Licensed to the Apache Software Foundation (ASF) under one
# * or more contributor license agreements. See the NOTICE file
# * distributed with this work for additional information
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/fde87ed2/tool/node.sh
----------------------------------------------------------------------
diff --git a/tool/node.sh b/tool/node.sh
index 808ea03..653806f 100755
--- a/tool/node.sh
+++ b/tool/node.sh
@@ -1,6 +1,4 @@
#/**
-# * Copyright 2015 The Apache Software Foundation
-# *
# * Licensed to the Apache Software Foundation (ASF) under one
# * or more contributor license agreements. See the NOTICE file
# * distributed with this work for additional information
http://git-wip-us.apache.org/repos/asf/incubator-singa/blob/fde87ed2/tool/python/singa/generatepy.sh
----------------------------------------------------------------------
diff --git a/tool/python/singa/generatepy.sh b/tool/python/singa/generatepy.sh
index 8b24635..488d96a 100755
--- a/tool/python/singa/generatepy.sh
+++ b/tool/python/singa/generatepy.sh
@@ -1,7 +1,5 @@
#!/usr/bin/env bash
#/**
-# * Copyright 2015 The Apache Software Foundation
-# *
# * Licensed to the Apache Software Foundation (ASF) under one
# * or more contributor license agreements. See the NOTICE file
# * distributed with this work for additional information
@@ -19,7 +17,7 @@
# * limitations under the License.
# */
-#The following commands are only for developers adding new py apis.
+#The following commands are only for developers adding new py apis.
swig -c++ -python driver.i
#g++ -fPIC ../../../src/driver.cc driver_wrap.cxx -shared -o _driver.so \
# -L../../../.libs/ -lsinga -DMSHADOW_USE_CUDA=0 \