You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by uw...@apache.org on 2018/04/15 08:00:24 UTC
[arrow] branch master updated: ARROW-2435: [Rust] Add memory pool
abstraction.
This is an automated email from the ASF dual-hosted git repository.
uwe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new b2167e4 ARROW-2435: [Rust] Add memory pool abstraction.
b2167e4 is described below
commit b2167e4103212e73ae51c36a70b2c44c259c271d
Author: liurenjie1024 <li...@gmail.com>
AuthorDate: Sun Apr 15 10:00:16 2018 +0200
ARROW-2435: [Rust] Add memory pool abstraction.
Current implementation use libc for memory allocation and deallocation. This can't be ported cross platforms. We can add a memory pool abstraction for many use cases, such as:
* Easy migration to other allocation api, such as the api use in rust alloc crate.
* Logging.
* Statistics updating.
Author: liurenjie1024 <li...@gmail.com>
Closes #1875 from liurenjie1024/arrow-2435 and squashes the following commits:
bdb50ce <liurenjie1024> Fix test
af9d08d <liurenjie1024> Add test for reallocate
083487e <liurenjie1024> Free old pointer
9f79e10 <liurenjie1024> Fix format
f57923b <liurenjie1024> Add header, reallocation method of memory pool
d74667b <liurenjie1024> Add memory pool abstraction.
---
.gitignore | 1 +
rust/src/error.rs | 2 +
rust/src/lib.rs | 1 +
rust/src/memory_pool.rs | 105 ++++++++++++++++++++++++++++++++++++++++++++++++
4 files changed, 109 insertions(+)
diff --git a/.gitignore b/.gitignore
index c902ba3..d9c69e9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -30,6 +30,7 @@ MANIFEST
*.vcxproj
*.vcxproj.*
*.sln
+*.iml
cpp/.idea/
python/.eggs/
diff --git a/rust/src/error.rs b/rust/src/error.rs
index 6a342e0..d82ee11 100644
--- a/rust/src/error.rs
+++ b/rust/src/error.rs
@@ -20,3 +20,5 @@ pub enum ArrowError {
MemoryError(String),
ParseError(String),
}
+
+pub type Result<T> = ::std::result::Result<T, ArrowError>;
diff --git a/rust/src/lib.rs b/rust/src/lib.rs
index e0c14db..80b53a1 100644
--- a/rust/src/lib.rs
+++ b/rust/src/lib.rs
@@ -30,3 +30,4 @@ pub mod error;
pub mod list;
pub mod list_builder;
pub mod memory;
+pub mod memory_pool;
diff --git a/rust/src/memory_pool.rs b/rust/src/memory_pool.rs
new file mode 100644
index 0000000..acfcc30
--- /dev/null
+++ b/rust/src/memory_pool.rs
@@ -0,0 +1,105 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use libc;
+use std::mem;
+use std::cmp;
+
+use super::error::ArrowError;
+use super::error::Result;
+
+const ALIGNMENT: usize = 64;
+
+/// Memory pool for allocating memory. It's also responsible for tracking memory usage.
+pub trait MemoryPool {
+ /// Allocate memory.
+ /// The implementation should ensures that allocated memory is aligned.
+ fn allocate(&self, size: usize) -> Result<*mut u8>;
+
+ /// Reallocate memory.
+ /// If the implementation doesn't support reallocating aligned memory, it allocates new memory
+ /// and copied old memory to it.
+ fn reallocate(&self, old_size: usize, new_size: usize, pointer: *mut u8) -> Result<*mut u8>;
+
+ /// Free memory.
+ fn free(&self, ptr: *mut u8);
+}
+
+/// Implementation of memory pool using libc api.
+#[allow(dead_code)]
+struct LibcMemoryPool;
+
+impl MemoryPool for LibcMemoryPool {
+ fn allocate(&self, size: usize) -> Result<*mut u8> {
+ unsafe {
+ let mut page: *mut libc::c_void = mem::uninitialized();
+ let result = libc::posix_memalign(&mut page, ALIGNMENT, size);
+ match result {
+ 0 => Ok(mem::transmute::<*mut libc::c_void, *mut u8>(page)),
+ _ => Err(ArrowError::MemoryError(
+ "Failed to allocate memory".to_string(),
+ )),
+ }
+ }
+ }
+
+ fn reallocate(&self, old_size: usize, new_size: usize, pointer: *mut u8) -> Result<*mut u8> {
+ unsafe {
+ let old_src = mem::transmute::<*mut u8, *mut libc::c_void>(pointer);
+ let result = self.allocate(new_size)?;
+ let dst = mem::transmute::<*mut u8, *mut libc::c_void>(result);
+ libc::memcpy(dst, old_src, cmp::min(old_size, new_size));
+ libc::free(old_src);
+ Ok(result)
+ }
+ }
+
+ fn free(&self, ptr: *mut u8) {
+ unsafe { libc::free(mem::transmute::<*mut u8, *mut libc::c_void>(ptr)) }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn test_allocate() {
+ let memory_pool = LibcMemoryPool {};
+
+ for _ in 0..10 {
+ let p = memory_pool.allocate(1024).unwrap();
+ // make sure this is 64-byte aligned
+ assert_eq!(0, (p as usize) % ALIGNMENT);
+ memory_pool.free(p);
+ }
+ }
+
+ #[test]
+ fn test_reallocate() {
+ let memory_pool = LibcMemoryPool {};
+
+ for _ in 0..10 {
+ let p1 = memory_pool.allocate(1024).unwrap();
+ let p2 = memory_pool.reallocate(1024, 2048, p1).unwrap();
+ // make sure this is 64-byte aligned
+ assert_eq!(0, (p1 as usize) % ALIGNMENT);
+ assert_eq!(0, (p2 as usize) % ALIGNMENT);
+ memory_pool.free(p2);
+ }
+ }
+}
--
To stop receiving notification emails like this one, please contact
uwe@apache.org.