You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by uw...@apache.org on 2017/03/20 09:58:02 UTC
arrow git commit: ARROW-502 [C++/Python]: Logging memory pool
Repository: arrow
Updated Branches:
refs/heads/master cd4544df8 -> 02bdbf48a
ARROW-502 [C++/Python]: Logging memory pool
This is a simple decorator on MemoryPool that logs it call to ``std::cout``. I can improve it later if you need to log to other supports. Are you ok with the current logging format ?
Also, I'm not a cython expert so I hope the implementation of ``CLoggingMemoryPool`` is correct.
Author: Johan Mabille <jo...@gmail.com>
Closes #395 from JohanMabille/memory_pool and squashes the following commits:
aa8ad5f [Johan Mabille] cython fix
f70e78a [Johan Mabille] python logging memory pool
9d1d144 [Johan Mabille] formatting
8f9164c [Johan Mabille] Logging memory pool
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/02bdbf48
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/02bdbf48
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/02bdbf48
Branch: refs/heads/master
Commit: 02bdbf48a483b224ebfd61cf9be69cb0807e6e50
Parents: cd4544d
Author: Johan Mabille <jo...@gmail.com>
Authored: Mon Mar 20 10:57:57 2017 +0100
Committer: Uwe L. Korn <uw...@xhochy.com>
Committed: Mon Mar 20 10:57:57 2017 +0100
----------------------------------------------------------------------
cpp/src/arrow/memory_pool-test.cc | 17 ++++++++++++++++
cpp/src/arrow/memory_pool.cc | 32 +++++++++++++++++++++++++++++++
cpp/src/arrow/memory_pool.h | 18 +++++++++++++++++
python/pyarrow/includes/libarrow.pxd | 3 +++
python/pyarrow/memory.pxd | 5 ++++-
python/pyarrow/memory.pyx | 5 ++++-
6 files changed, 78 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/02bdbf48/cpp/src/arrow/memory_pool-test.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/memory_pool-test.cc b/cpp/src/arrow/memory_pool-test.cc
index 6ab73fb..8a185ab 100644
--- a/cpp/src/arrow/memory_pool-test.cc
+++ b/cpp/src/arrow/memory_pool-test.cc
@@ -78,4 +78,21 @@ TEST(DefaultMemoryPoolDeathTest, MaxMemory) {
#endif // ARROW_VALGRIND
+TEST(LoggingMemoryPool, Logging) {
+ DefaultMemoryPool pool;
+ LoggingMemoryPool lp(&pool);
+
+ ASSERT_EQ(0, lp.max_memory());
+
+ uint8_t* data;
+ ASSERT_OK(pool.Allocate(100, &data));
+
+ uint8_t* data2;
+ ASSERT_OK(pool.Allocate(100, &data2));
+
+ pool.Free(data, 100);
+ pool.Free(data2, 100);
+
+ ASSERT_EQ(200, pool.max_memory());
+}
} // namespace arrow
http://git-wip-us.apache.org/repos/asf/arrow/blob/02bdbf48/cpp/src/arrow/memory_pool.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/memory_pool.cc b/cpp/src/arrow/memory_pool.cc
index 5a63027..cf01a02 100644
--- a/cpp/src/arrow/memory_pool.cc
+++ b/cpp/src/arrow/memory_pool.cc
@@ -22,6 +22,7 @@
#include <mutex>
#include <sstream>
#include <stdlib.h>
+#include <iostream>
#include "arrow/status.h"
#include "arrow/util/logging.h"
@@ -134,4 +135,35 @@ MemoryPool* default_memory_pool() {
return &default_memory_pool_;
}
+LoggingMemoryPool::LoggingMemoryPool(MemoryPool* pool) : pool_(pool) {}
+
+Status LoggingMemoryPool::Allocate(int64_t size, uint8_t** out) {
+ Status s = pool_->Allocate(size, out);
+ std::cout << "Allocate: size = " << size << " - out = " << *out << std::endl;
+ return s;
+}
+
+Status LoggingMemoryPool::Reallocate(int64_t old_size, int64_t new_size, uint8_t** ptr) {
+ Status s = pool_->Reallocate(old_size, new_size, ptr);
+ std::cout << "Reallocate: old_size = " << old_size << " - new_size = " << new_size
+ << " - ptr = " << *ptr << std::endl;
+ return s;
+}
+
+void LoggingMemoryPool::Free(uint8_t* buffer, int64_t size) {
+ pool_->Free(buffer, size);
+ std::cout << "Free: buffer = " << buffer << " - size = " << size << std::endl;
+}
+
+int64_t LoggingMemoryPool::bytes_allocated() const {
+ int64_t nb_bytes = pool_->bytes_allocated();
+ std::cout << "bytes_allocated: " << nb_bytes << std::endl;
+ return nb_bytes;
+}
+
+int64_t LoggingMemoryPool::max_memory() const {
+ int64_t mem = pool_->max_memory();
+ std::cout << "max_memory: " << mem << std::endl;
+ return mem;
+}
} // namespace arrow
http://git-wip-us.apache.org/repos/asf/arrow/blob/02bdbf48/cpp/src/arrow/memory_pool.h
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/memory_pool.h b/cpp/src/arrow/memory_pool.h
index 0edfda6..90bc593 100644
--- a/cpp/src/arrow/memory_pool.h
+++ b/cpp/src/arrow/memory_pool.h
@@ -89,6 +89,24 @@ class ARROW_EXPORT DefaultMemoryPool : public MemoryPool {
std::atomic<int64_t> max_memory_;
};
+class ARROW_EXPORT LoggingMemoryPool : public MemoryPool {
+ public:
+ explicit LoggingMemoryPool(MemoryPool* pool);
+ virtual ~LoggingMemoryPool() = default;
+
+ Status Allocate(int64_t size, uint8_t** out) override;
+ Status Reallocate(int64_t old_size, int64_t new_size, uint8_t** ptr) override;
+
+ void Free(uint8_t* buffer, int64_t size) override;
+
+ int64_t bytes_allocated() const override;
+
+ int64_t max_memory() const override;
+
+ private:
+ MemoryPool* pool_;
+};
+
ARROW_EXPORT MemoryPool* default_memory_pool();
} // namespace arrow
http://git-wip-us.apache.org/repos/asf/arrow/blob/02bdbf48/python/pyarrow/includes/libarrow.pxd
----------------------------------------------------------------------
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index dee7fd4..705fe6b 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -104,6 +104,9 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
cdef cppclass CMemoryPool" arrow::MemoryPool":
int64_t bytes_allocated()
+ cdef cppclass CLoggingMemoryPool" arrow::LoggingMemoryPool"(CMemoryPool):
+ CLoggingMemoryPool(CMemoryPool*)
+
cdef cppclass CBuffer" arrow::Buffer":
uint8_t* data()
int64_t size()
http://git-wip-us.apache.org/repos/asf/arrow/blob/02bdbf48/python/pyarrow/memory.pxd
----------------------------------------------------------------------
diff --git a/python/pyarrow/memory.pxd b/python/pyarrow/memory.pxd
index 3079ccb..bb1af85 100644
--- a/python/pyarrow/memory.pxd
+++ b/python/pyarrow/memory.pxd
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
-from pyarrow.includes.libarrow cimport CMemoryPool
+from pyarrow.includes.libarrow cimport CMemoryPool, CLoggingMemoryPool
cdef class MemoryPool:
@@ -24,4 +24,7 @@ cdef class MemoryPool:
cdef init(self, CMemoryPool* pool)
+cdef class LoggingMemoryPool(MemoryPool):
+ pass
+
cdef CMemoryPool* maybe_unbox_memory_pool(MemoryPool memory_pool)
http://git-wip-us.apache.org/repos/asf/arrow/blob/02bdbf48/python/pyarrow/memory.pyx
----------------------------------------------------------------------
diff --git a/python/pyarrow/memory.pyx b/python/pyarrow/memory.pyx
index 18a6de4..98dbf66 100644
--- a/python/pyarrow/memory.pyx
+++ b/python/pyarrow/memory.pyx
@@ -19,7 +19,7 @@
# distutils: language = c++
# cython: embedsignature = True
-from pyarrow.includes.libarrow cimport CMemoryPool
+from pyarrow.includes.libarrow cimport CMemoryPool, CLoggingMemoryPool
from pyarrow.includes.pyarrow cimport set_default_memory_pool, get_memory_pool
cdef class MemoryPool:
@@ -35,6 +35,9 @@ cdef CMemoryPool* maybe_unbox_memory_pool(MemoryPool memory_pool):
else:
return memory_pool.pool
+cdef class LoggingMemoryPool(MemoryPool):
+ pass
+
def default_pool():
cdef:
MemoryPool pool = MemoryPool()