You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2017/01/13 19:00:19 UTC
arrow git commit: ARROW-239: Test reading remainder of file in HDFS
with read() with no args
Repository: arrow
Updated Branches:
refs/heads/master 281cb9697 -> 876ae8509
ARROW-239: Test reading remainder of file in HDFS with read() with no args
This was already fixed, but adding a unit test to verify.
Author: Wes McKinney <we...@twosigma.com>
Closes #284 from wesm/ARROW-239 and squashes the following commits:
7b3aae7 [Wes McKinney] Test reading remainder of file in HDFS with read() with no args
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/876ae850
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/876ae850
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/876ae850
Branch: refs/heads/master
Commit: 876ae8509d84b8c4dc269250b2fae47cef897054
Parents: 281cb96
Author: Wes McKinney <we...@twosigma.com>
Authored: Fri Jan 13 14:00:12 2017 -0500
Committer: Wes McKinney <we...@twosigma.com>
Committed: Fri Jan 13 14:00:12 2017 -0500
----------------------------------------------------------------------
python/pyarrow/schema.pyx | 8 +++++---
python/pyarrow/tests/test_hdfs.py | 12 ++++++++++++
2 files changed, 17 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/876ae850/python/pyarrow/schema.pyx
----------------------------------------------------------------------
diff --git a/python/pyarrow/schema.pyx b/python/pyarrow/schema.pyx
index f6a1a10..a6aa9d5 100644
--- a/python/pyarrow/schema.pyx
+++ b/python/pyarrow/schema.pyx
@@ -22,6 +22,8 @@
# distutils: language = c++
# cython: embedsignature = True
+from cython.operator cimport dereference as deref
+
from pyarrow.compat import frombytes, tobytes
from pyarrow.includes.libarrow cimport (CDataType, CStructType, CListType,
Type_NA, Type_BOOL,
@@ -31,7 +33,7 @@ from pyarrow.includes.libarrow cimport (CDataType, CStructType, CListType,
Type_UINT64, Type_INT64,
Type_TIMESTAMP, Type_DATE,
Type_FLOAT, Type_DOUBLE,
- Type_STRING, Type_BINARY,
+ Type_STRING, Type_BINARY,
TimeUnit_SECOND, TimeUnit_MILLI,
TimeUnit_MICRO, TimeUnit_NANO,
Type, TimeUnit)
@@ -57,9 +59,9 @@ cdef class DataType:
def __richcmp__(DataType self, DataType other, int op):
if op == cpython.Py_EQ:
- return self.type.Equals(other.sp_type)
+ return self.type.Equals(deref(other.type))
elif op == cpython.Py_NE:
- return not self.type.Equals(other.sp_type)
+ return not self.type.Equals(deref(other.type))
else:
raise TypeError('Invalid comparison')
http://git-wip-us.apache.org/repos/asf/arrow/blob/876ae850/python/pyarrow/tests/test_hdfs.py
----------------------------------------------------------------------
diff --git a/python/pyarrow/tests/test_hdfs.py b/python/pyarrow/tests/test_hdfs.py
index 4ff5a9d..2056f7a 100644
--- a/python/pyarrow/tests/test_hdfs.py
+++ b/python/pyarrow/tests/test_hdfs.py
@@ -126,6 +126,18 @@ class HdfsTestCases(object):
result = f.read(10)
assert result == data
+ def test_hdfs_read_whole_file(self):
+ path = pjoin(self.tmp_path, 'read-whole-file')
+
+ data = b'foo' * 1000
+ with self.hdfs.open(path, 'wb') as f:
+ f.write(data)
+
+ with self.hdfs.open(path, 'rb') as f:
+ result = f.read()
+
+ assert result == data
+
class TestLibHdfs(HdfsTestCases, unittest.TestCase):