You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by uw...@apache.org on 2017/10/05 09:16:46 UTC
arrow git commit: ARROW-1633: [Python] Support NumPy string and
unicode types in pyarrow.array, Array.from_pandas
Repository: arrow
Updated Branches:
refs/heads/master f0873a99a -> 9805ada85
ARROW-1633: [Python] Support NumPy string and unicode types in pyarrow.array, Array.from_pandas
I suppose this could have been worse. If anyone has any better ideas about alternatives to my onion router of UTF32 bytes -> PyUnicode -> PyBytes (UTF8) -> StringBuilder, I'm open to them. Since we support gcc 4.8 we don't have the option of using `std::codecvt`.
Author: Wes McKinney <we...@twosigma.com>
Closes #1167 from wesm/ARROW-1633 and squashes the following commits:
f0840b4 [Wes McKinney] Also test big-endian unicode
ca59ada [Wes McKinney] Implement NumPy unicode to Arrow utf8 conversion, deal with masks, truncated values
9304216 [Wes McKinney] Convert NumPy string (ascii) arrays to arrow binary arrays, handle mask and strided versions
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/9805ada8
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/9805ada8
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/9805ada8
Branch: refs/heads/master
Commit: 9805ada853ae63580c59d765941f1e36a86e554c
Parents: f0873a9
Author: Wes McKinney <we...@twosigma.com>
Authored: Thu Oct 5 11:16:41 2017 +0200
Committer: Uwe L. Korn <uw...@xhochy.com>
Committed: Thu Oct 5 11:16:41 2017 +0200
----------------------------------------------------------------------
cpp/src/arrow/python/numpy_convert.cc | 2 +
cpp/src/arrow/python/numpy_to_arrow.cc | 130 ++++++++++++++++++++++++++--
python/pyarrow/tests/test_array.py | 61 +++++++++++++
3 files changed, 188 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/9805ada8/cpp/src/arrow/python/numpy_convert.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/python/numpy_convert.cc b/cpp/src/arrow/python/numpy_convert.cc
index 4c64cc4..bef8054 100644
--- a/cpp/src/arrow/python/numpy_convert.cc
+++ b/cpp/src/arrow/python/numpy_convert.cc
@@ -167,6 +167,8 @@ Status NumPyDtypeToArrow(PyObject* dtype, std::shared_ptr<DataType>* out) {
#endif
TO_ARROW_TYPE_CASE(FLOAT32, float32);
TO_ARROW_TYPE_CASE(FLOAT64, float64);
+ TO_ARROW_TYPE_CASE(STRING, binary);
+ TO_ARROW_TYPE_CASE(UNICODE, utf8);
case NPY_DATETIME: {
auto date_dtype =
reinterpret_cast<PyArray_DatetimeDTypeMetaData*>(descr->c_metadata);
http://git-wip-us.apache.org/repos/asf/arrow/blob/9805ada8/cpp/src/arrow/python/numpy_to_arrow.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/python/numpy_to_arrow.cc b/cpp/src/arrow/python/numpy_to_arrow.cc
index c0ce61c..e935b45 100644
--- a/cpp/src/arrow/python/numpy_to_arrow.cc
+++ b/cpp/src/arrow/python/numpy_to_arrow.cc
@@ -266,12 +266,11 @@ class NumPyConverter {
mask_ = reinterpret_cast<PyArrayObject*>(mo);
}
length_ = static_cast<int64_t>(PyArray_SIZE(arr_));
+ itemsize_ = static_cast<int>(PyArray_DESCR(arr_)->elsize);
+ stride_ = static_cast<int64_t>(PyArray_STRIDES(arr_)[0]);
}
- bool is_strided() const {
- npy_intp* astrides = PyArray_STRIDES(arr_);
- return astrides[0] != PyArray_DESCR(arr_)->elsize;
- }
+ bool is_strided() const { return itemsize_ != stride_; }
Status Convert();
@@ -293,7 +292,11 @@ class NumPyConverter {
Status Visit(const NullType& type) { return TypeNotImplemented(type.ToString()); }
- Status Visit(const BinaryType& type) { return TypeNotImplemented(type.ToString()); }
+ // NumPy ascii string arrays
+ Status Visit(const BinaryType& type);
+
+ // NumPy unicode arrays
+ Status Visit(const StringType& type);
Status Visit(const FixedSizeBinaryType& type) {
return TypeNotImplemented(type.ToString());
@@ -430,6 +433,8 @@ class NumPyConverter {
PyArrayObject* arr_;
PyArrayObject* mask_;
int64_t length_;
+ int64_t stride_;
+ int itemsize_;
bool use_pandas_null_sentinels_;
@@ -1213,10 +1218,125 @@ Status NumPyConverter::ConvertLists(const std::shared_ptr<DataType>& type) {
return PushBuilderResult(list_builder);
}
+Status NumPyConverter::Visit(const BinaryType& type) {
+ BinaryBuilder builder(pool_);
+
+ auto data = reinterpret_cast<const uint8_t*>(PyArray_DATA(arr_));
+
+ int item_length = 0;
+ if (mask_ != nullptr) {
+ Ndarray1DIndexer<uint8_t> mask_values(mask_);
+ for (int64_t i = 0; i < length_; ++i) {
+ if (mask_values[i]) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else {
+ // This is annoying. NumPy allows strings to have nul-terminators, so
+ // we must check for them here
+ for (item_length = 0; item_length < itemsize_; ++item_length) {
+ if (data[item_length] == 0) {
+ break;
+ }
+ }
+ RETURN_NOT_OK(builder.Append(data, item_length));
+ }
+ data += stride_;
+ }
+ } else {
+ for (int64_t i = 0; i < length_; ++i) {
+ for (item_length = 0; item_length < itemsize_; ++item_length) {
+ // Look for nul-terminator
+ if (data[item_length] == 0) {
+ break;
+ }
+ }
+ RETURN_NOT_OK(builder.Append(data, item_length));
+ data += stride_;
+ }
+ }
+
+ std::shared_ptr<Array> result;
+ RETURN_NOT_OK(builder.Finish(&result));
+ return PushArray(result->data());
+}
+
+namespace {
+
+// NumPy unicode is UCS4/UTF32 always
+constexpr int kNumPyUnicodeSize = 4;
+
+Status AppendUTF32(const char* data, int itemsize, int byteorder,
+ StringBuilder* builder) {
+ // The binary \x00\x00\x00\x00 indicates a nul terminator in NumPy unicode,
+ // so we need to detect that here to truncate if necessary. Yep.
+ int actual_length = 0;
+ for (; actual_length < itemsize / kNumPyUnicodeSize; ++actual_length) {
+ const char* code_point = data + actual_length * kNumPyUnicodeSize;
+ if ((*code_point == '\0') && (*(code_point + 1) == '\0') &&
+ (*(code_point + 2) == '\0') && (*(code_point + 3) == '\0')) {
+ break;
+ }
+ }
+
+ ScopedRef unicode_obj(PyUnicode_DecodeUTF32(data, actual_length * kNumPyUnicodeSize,
+ nullptr, &byteorder));
+ RETURN_IF_PYERROR();
+ ScopedRef utf8_obj(PyUnicode_AsUTF8String(unicode_obj.get()));
+ if (utf8_obj.get() == NULL) {
+ PyErr_Clear();
+ return Status::Invalid("failed converting UTF32 to UTF8");
+ }
+
+ const int32_t length = static_cast<int32_t>(PyBytes_GET_SIZE(utf8_obj.get()));
+ if (builder->value_data_length() + length > kBinaryMemoryLimit) {
+ return Status::Invalid("Encoded string length exceeds maximum size (2GB)");
+ }
+ return builder->Append(PyBytes_AS_STRING(utf8_obj.get()), length);
+}
+
+} // namespace
+
+Status NumPyConverter::Visit(const StringType& type) {
+ StringBuilder builder(pool_);
+
+ auto data = reinterpret_cast<const char*>(PyArray_DATA(arr_));
+
+ char numpy_byteorder = PyArray_DESCR(arr_)->byteorder;
+
+ // For Python C API, -1 is little-endian, 1 is big-endian
+ int byteorder = numpy_byteorder == '>' ? 1 : -1;
+
+ PyAcquireGIL gil_lock;
+
+ if (mask_ != nullptr) {
+ Ndarray1DIndexer<uint8_t> mask_values(mask_);
+ for (int64_t i = 0; i < length_; ++i) {
+ if (mask_values[i]) {
+ RETURN_NOT_OK(builder.AppendNull());
+ } else {
+ RETURN_NOT_OK(AppendUTF32(data, itemsize_, byteorder, &builder));
+ }
+ data += stride_;
+ }
+ } else {
+ for (int64_t i = 0; i < length_; ++i) {
+ RETURN_NOT_OK(AppendUTF32(data, itemsize_, byteorder, &builder));
+ data += stride_;
+ }
+ }
+
+ std::shared_ptr<Array> result;
+ RETURN_NOT_OK(builder.Finish(&result));
+ return PushArray(result->data());
+}
+
Status NdarrayToArrow(MemoryPool* pool, PyObject* ao, PyObject* mo,
bool use_pandas_null_sentinels,
const std::shared_ptr<DataType>& type,
std::shared_ptr<ChunkedArray>* out) {
+ if (!PyArray_Check(ao)) {
+ return Status::Invalid("Input object was not a NumPy array");
+ }
+
NumPyConverter converter(pool, ao, mo, type, use_pandas_null_sentinels);
RETURN_NOT_OK(converter.Convert());
const auto& output_arrays = converter.result();
http://git-wip-us.apache.org/repos/asf/arrow/blob/9805ada8/python/pyarrow/tests/test_array.py
----------------------------------------------------------------------
diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py
index 3bf3926..414a268 100644
--- a/python/pyarrow/tests/test_array.py
+++ b/python/pyarrow/tests/test_array.py
@@ -337,3 +337,64 @@ def test_array_conversions_no_sentinel_values():
type='float32')
assert arr3.type == 'float32'
assert arr3.null_count == 0
+
+
+def test_array_from_numpy_ascii():
+ arr = np.array(['abcde', 'abc', ''], dtype='|S5')
+
+ arrow_arr = pa.array(arr)
+ assert arrow_arr.type == 'binary'
+ expected = pa.array(['abcde', 'abc', ''], type='binary')
+ assert arrow_arr.equals(expected)
+
+ mask = np.array([False, True, False])
+ arrow_arr = pa.array(arr, mask=mask)
+ expected = pa.array(['abcde', None, ''], type='binary')
+ assert arrow_arr.equals(expected)
+
+ # Strided variant
+ arr = np.array(['abcde', 'abc', ''] * 5, dtype='|S5')[::2]
+ mask = np.array([False, True, False] * 5)[::2]
+ arrow_arr = pa.array(arr, mask=mask)
+
+ expected = pa.array(['abcde', '', None, 'abcde', '', None, 'abcde', ''],
+ type='binary')
+ assert arrow_arr.equals(expected)
+
+ # 0 itemsize
+ arr = np.array(['', '', ''], dtype='|S0')
+ arrow_arr = pa.array(arr)
+ expected = pa.array(['', '', ''], type='binary')
+ assert arrow_arr.equals(expected)
+
+
+def test_array_from_numpy_unicode():
+ dtypes = ['<U5', '>U5']
+
+ for dtype in dtypes:
+ arr = np.array(['abcde', 'abc', ''], dtype=dtype)
+
+ arrow_arr = pa.array(arr)
+ assert arrow_arr.type == 'utf8'
+ expected = pa.array(['abcde', 'abc', ''], type='utf8')
+ assert arrow_arr.equals(expected)
+
+ mask = np.array([False, True, False])
+ arrow_arr = pa.array(arr, mask=mask)
+ expected = pa.array(['abcde', None, ''], type='utf8')
+ assert arrow_arr.equals(expected)
+
+ # Strided variant
+ arr = np.array(['abcde', 'abc', ''] * 5, dtype=dtype)[::2]
+ mask = np.array([False, True, False] * 5)[::2]
+ arrow_arr = pa.array(arr, mask=mask)
+
+ expected = pa.array(['abcde', '', None, 'abcde', '', None,
+ 'abcde', ''], type='utf8')
+ assert arrow_arr.equals(expected)
+
+ # 0 itemsize
+ arr = np.array(['', '', ''], dtype='<U0')
+ arrow_arr = pa.array(arr)
+ expected = pa.array(['', '', ''], type='utf8')
+ assert arrow_arr.equals(expected)