You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2017/01/02 07:37:46 UTC
arrow git commit: ARROW-450: Fixes for PARQUET-818
Repository: arrow
Updated Branches:
refs/heads/master e15c6a0b3 -> e8b6231b2
ARROW-450: Fixes for PARQUET-818
Author: Uwe L. Korn <uw...@xhochy.com>
Closes #263 from xhochy/ARROW-450 and squashes the following commits:
287015a [Uwe L. Korn] ARROW-450: Fixes for PARQUET-818
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/e8b6231b
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/e8b6231b
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/e8b6231b
Branch: refs/heads/master
Commit: e8b6231b29f59b2978b78a33eff73697d537c5dd
Parents: e15c6a0
Author: Uwe L. Korn <uw...@xhochy.com>
Authored: Mon Jan 2 02:37:37 2017 -0500
Committer: Wes McKinney <we...@twosigma.com>
Committed: Mon Jan 2 02:37:37 2017 -0500
----------------------------------------------------------------------
python/pyarrow/includes/parquet.pxd | 19 ++-----------------
python/pyarrow/parquet.pyx | 15 ++++++---------
2 files changed, 8 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/e8b6231b/python/pyarrow/includes/parquet.pxd
----------------------------------------------------------------------
diff --git a/python/pyarrow/includes/parquet.pxd b/python/pyarrow/includes/parquet.pxd
index cb791e1..b4d127c 100644
--- a/python/pyarrow/includes/parquet.pxd
+++ b/python/pyarrow/includes/parquet.pxd
@@ -120,24 +120,9 @@ cdef extern from "parquet/api/writer.h" namespace "parquet" nogil:
shared_ptr[WriterProperties] build()
-cdef extern from "parquet/arrow/io.h" namespace "parquet::arrow" nogil:
- cdef cppclass ParquetAllocator:
- ParquetAllocator()
- ParquetAllocator(MemoryPool* pool)
- MemoryPool* pool()
- void set_pool(MemoryPool* pool)
-
- cdef cppclass ParquetReadSource:
- ParquetReadSource(ParquetAllocator* allocator)
- Open(const shared_ptr[ReadableFileInterface]& file)
-
- cdef cppclass ParquetWriteSink:
- ParquetWriteSink(const shared_ptr[OutputStream]& file)
-
-
cdef extern from "parquet/arrow/reader.h" namespace "parquet::arrow" nogil:
CStatus OpenFile(const shared_ptr[ReadableFileInterface]& file,
- ParquetAllocator* allocator,
+ MemoryPool* allocator,
unique_ptr[FileReader]* reader)
cdef cppclass FileReader:
@@ -157,6 +142,6 @@ cdef extern from "parquet/arrow/schema.h" namespace "parquet::arrow" nogil:
cdef extern from "parquet/arrow/writer.h" namespace "parquet::arrow" nogil:
cdef CStatus WriteFlatTable(
const CTable* table, MemoryPool* pool,
- const shared_ptr[ParquetWriteSink]& sink,
+ const shared_ptr[OutputStream]& sink,
int64_t chunk_size,
const shared_ptr[WriterProperties]& properties)
http://git-wip-us.apache.org/repos/asf/arrow/blob/e8b6231b/python/pyarrow/parquet.pyx
----------------------------------------------------------------------
diff --git a/python/pyarrow/parquet.pyx b/python/pyarrow/parquet.pyx
index 043ccf1..7379456 100644
--- a/python/pyarrow/parquet.pyx
+++ b/python/pyarrow/parquet.pyx
@@ -42,12 +42,12 @@ __all__ = [
cdef class ParquetReader:
cdef:
- ParquetAllocator allocator
+ MemoryPool* allocator
unique_ptr[FileReader] reader
column_idx_map
def __cinit__(self):
- self.allocator.set_pool(default_memory_pool())
+ self.allocator = default_memory_pool()
def open(self, source):
self._open(source)
@@ -69,7 +69,7 @@ cdef class ParquetReader:
ParquetFileReader.OpenFile(path)))
else:
get_reader(source, &rd_handle)
- check_status(OpenFile(rd_handle, &self.allocator, &self.reader))
+ check_status(OpenFile(rd_handle, self.allocator, &self.reader))
def read_all(self):
cdef:
@@ -174,10 +174,8 @@ def write_table(table, sink, chunk_size=None, version=None,
"""
cdef Table table_ = table
cdef CTable* ctable_ = table_.table
- cdef shared_ptr[ParquetWriteSink] sink_
-
cdef shared_ptr[FileOutputStream] filesink_
- cdef shared_ptr[OutputStream] general_sink
+ cdef shared_ptr[OutputStream] sink_
cdef WriterProperties.Builder properties_builder
cdef int64_t chunk_size_ = 0
@@ -237,10 +235,9 @@ def write_table(table, sink, chunk_size=None, version=None,
if isinstance(sink, six.string_types):
check_status(FileOutputStream.Open(tobytes(sink), &filesink_))
- sink_.reset(new ParquetWriteSink(<shared_ptr[OutputStream]>filesink_))
+ sink_ = <shared_ptr[OutputStream]>filesink_
else:
- get_writer(sink, &general_sink)
- sink_.reset(new ParquetWriteSink(general_sink))
+ get_writer(sink, &sink_)
with nogil:
check_status(WriteFlatTable(ctable_, default_memory_pool(), sink_,