You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2017/09/19 04:23:09 UTC
arrow git commit: ARROW-1550: [Python] Explicitly close owned file
handles in ParquetWriter.close to avoid Windows flakiness
Repository: arrow
Updated Branches:
refs/heads/master 4a65fea01 -> e1d9c7fc7
ARROW-1550: [Python] Explicitly close owned file handles in ParquetWriter.close to avoid Windows flakiness
I can reproduce this failure locally, but I'm unsure why this just now started happening. The 0.7.0 release build passed (https://ci.appveyor.com/project/ApacheSoftwareFoundation/arrow/build/1.0.3357/job/477b1iicmwuy51l8) and there haven't been related code changes since then. Either way it's better to close the sink explicitly
Author: Wes McKinney <we...@twosigma.com>
Closes #1114 from wesm/ARROW-1550 and squashes the following commits:
863827c4 [Wes McKinney] Check status
7248c793 [Wes McKinney] Explicitly close owned file handles in ParquetWriter.close to avoid flakiness on Windows
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/e1d9c7fc
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/e1d9c7fc
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/e1d9c7fc
Branch: refs/heads/master
Commit: e1d9c7fc7b3bbbb390f336551b95d3ca881c9285
Parents: 4a65fea
Author: Wes McKinney <we...@twosigma.com>
Authored: Tue Sep 19 00:23:03 2017 -0400
Committer: Wes McKinney <we...@twosigma.com>
Committed: Tue Sep 19 00:23:03 2017 -0400
----------------------------------------------------------------------
python/pyarrow/_parquet.pyx | 5 +++++
python/pyarrow/tests/test_parquet.py | 3 ---
2 files changed, 5 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/e1d9c7fc/python/pyarrow/_parquet.pyx
----------------------------------------------------------------------
diff --git a/python/pyarrow/_parquet.pyx b/python/pyarrow/_parquet.pyx
index aea6fb6..b096fa1 100644
--- a/python/pyarrow/_parquet.pyx
+++ b/python/pyarrow/_parquet.pyx
@@ -562,6 +562,7 @@ cdef class ParquetWriter:
cdef:
unique_ptr[FileWriter] writer
shared_ptr[OutputStream] sink
+ bint own_sink
cdef readonly:
object use_dictionary
@@ -588,8 +589,10 @@ cdef class ParquetWriter:
check_status(FileOutputStream.Open(c_where,
&filestream))
self.sink = <shared_ptr[OutputStream]> filestream
+ self.own_sink = True
else:
get_writer(where, &self.sink)
+ self.own_sink = False
self.use_dictionary = use_dictionary
self.compression = compression
@@ -664,6 +667,8 @@ cdef class ParquetWriter:
def close(self):
with nogil:
check_status(self.writer.get().Close())
+ if self.own_sink:
+ check_status(self.sink.get().Close())
def write_table(self, Table table, row_group_size=None):
cdef CTable* ctable = table.table
http://git-wip-us.apache.org/repos/asf/arrow/blob/e1d9c7fc/python/pyarrow/tests/test_parquet.py
----------------------------------------------------------------------
diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py
index 9b5a4bc..790b478 100644
--- a/python/pyarrow/tests/test_parquet.py
+++ b/python/pyarrow/tests/test_parquet.py
@@ -1202,9 +1202,6 @@ def test_write_error_deletes_incomplete_file(tmpdir):
except pa.ArrowException:
pass
- # Ensure that object has been destructed; this causes test failures on
- # Windows
- gc.collect()
assert not os.path.exists(filename)