You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by jo...@apache.org on 2024/02/07 14:21:44 UTC
(arrow) branch main updated: GH-39852: [Python] Support creating Binary/StringView arrays from python objects (#39853)
This is an automated email from the ASF dual-hosted git repository.
jorisvandenbossche pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new 7e2fe4fe76 GH-39852: [Python] Support creating Binary/StringView arrays from python objects (#39853)
7e2fe4fe76 is described below
commit 7e2fe4fe7634c359017213b79255c9040786fc06
Author: Joris Van den Bossche <jo...@gmail.com>
AuthorDate: Wed Feb 7 15:21:37 2024 +0100
GH-39852: [Python] Support creating Binary/StringView arrays from python objects (#39853)
Next step for Binary/StringView support in Python (https://github.com/apache/arrow/issues/39633), now adding it to the python->arrow conversion code path.
* Closes: #39852
Authored-by: Joris Van den Bossche <jo...@gmail.com>
Signed-off-by: Joris Van den Bossche <jo...@gmail.com>
---
python/pyarrow/src/arrow/python/python_to_arrow.cc | 35 ++++++++++++++--------
python/pyarrow/tests/test_convert_builtin.py | 19 ++++++++++--
python/pyarrow/tests/test_scalars.py | 28 +++--------------
3 files changed, 42 insertions(+), 40 deletions(-)
diff --git a/python/pyarrow/src/arrow/python/python_to_arrow.cc b/python/pyarrow/src/arrow/python/python_to_arrow.cc
index d1d94ac17a..3c4d59d659 100644
--- a/python/pyarrow/src/arrow/python/python_to_arrow.cc
+++ b/python/pyarrow/src/arrow/python/python_to_arrow.cc
@@ -486,6 +486,10 @@ class PyValue {
return view.ParseString(obj);
}
+ static Status Convert(const BinaryViewType*, const O&, I obj, PyBytesView& view) {
+ return view.ParseString(obj);
+ }
+
static Status Convert(const FixedSizeBinaryType* type, const O&, I obj,
PyBytesView& view) {
ARROW_RETURN_NOT_OK(view.ParseString(obj));
@@ -499,8 +503,8 @@ class PyValue {
}
template <typename T>
- static enable_if_string<T, Status> Convert(const T*, const O& options, I obj,
- PyBytesView& view) {
+ static enable_if_t<is_string_type<T>::value || is_string_view_type<T>::value, Status>
+ Convert(const T*, const O& options, I obj, PyBytesView& view) {
if (options.strict) {
// Strict conversion, force output to be unicode / utf8 and validate that
// any binary values are utf8
@@ -570,18 +574,12 @@ struct PyConverterTrait;
template <typename T>
struct PyConverterTrait<
- T,
- enable_if_t<(!is_nested_type<T>::value && !is_interval_type<T>::value &&
- !is_extension_type<T>::value && !is_binary_view_like_type<T>::value) ||
- std::is_same<T, MonthDayNanoIntervalType>::value>> {
+ T, enable_if_t<(!is_nested_type<T>::value && !is_interval_type<T>::value &&
+ !is_extension_type<T>::value) ||
+ std::is_same<T, MonthDayNanoIntervalType>::value>> {
using type = PyPrimitiveConverter<T>;
};
-template <typename T>
-struct PyConverterTrait<T, enable_if_binary_view_like<T>> {
- // not implemented
-};
-
template <typename T>
struct PyConverterTrait<T, enable_if_list_like<T>> {
using type = PyListConverter<T>;
@@ -699,11 +697,22 @@ class PyPrimitiveConverter<T, enable_if_t<std::is_same<T, FixedSizeBinaryType>::
PyBytesView view_;
};
+template <typename T, typename Enable = void>
+struct OffsetTypeTrait {
+ using type = typename T::offset_type;
+};
+
+template <typename T>
+struct OffsetTypeTrait<T, enable_if_binary_view_like<T>> {
+ using type = int64_t;
+};
+
template <typename T>
-class PyPrimitiveConverter<T, enable_if_base_binary<T>>
+class PyPrimitiveConverter<
+ T, enable_if_t<is_base_binary_type<T>::value || is_binary_view_like_type<T>::value>>
: public PrimitiveConverter<T, PyConverter> {
public:
- using OffsetType = typename T::offset_type;
+ using OffsetType = typename OffsetTypeTrait<T>::type;
Status Append(PyObject* value) override {
if (PyValue::IsNull(this->options_, value)) {
diff --git a/python/pyarrow/tests/test_convert_builtin.py b/python/pyarrow/tests/test_convert_builtin.py
index 49c4f1a6e7..55ea28f50f 100644
--- a/python/pyarrow/tests/test_convert_builtin.py
+++ b/python/pyarrow/tests/test_convert_builtin.py
@@ -763,6 +763,16 @@ def test_sequence_unicode():
assert arr.to_pylist() == data
+@pytest.mark.parametrize("ty", [pa.string(), pa.large_string(), pa.string_view()])
+def test_sequence_unicode_explicit_type(ty):
+ data = ['foo', 'bar', None, 'mañana']
+ arr = pa.array(data, type=ty)
+ assert len(arr) == 4
+ assert arr.null_count == 1
+ assert arr.type == ty
+ assert arr.to_pylist() == data
+
+
def check_array_mixed_unicode_bytes(binary_type, string_type):
values = ['qux', b'foo', bytearray(b'barz')]
b_values = [b'qux', b'foo', b'barz']
@@ -787,6 +797,7 @@ def check_array_mixed_unicode_bytes(binary_type, string_type):
def test_array_mixed_unicode_bytes():
check_array_mixed_unicode_bytes(pa.binary(), pa.string())
check_array_mixed_unicode_bytes(pa.large_binary(), pa.large_string())
+ check_array_mixed_unicode_bytes(pa.binary_view(), pa.string_view())
@pytest.mark.large_memory
@@ -818,7 +829,7 @@ def test_large_binary_value(ty):
@pytest.mark.large_memory
-@pytest.mark.parametrize("ty", [pa.binary(), pa.string()])
+@pytest.mark.parametrize("ty", [pa.binary(), pa.string(), pa.string_view()])
def test_string_too_large(ty):
# Construct a binary array with a single value larger than 4GB
s = b"0123456789abcdefghijklmnopqrstuvwxyz"
@@ -836,7 +847,7 @@ def test_sequence_bytes():
u1.decode('utf-8'), # unicode gets encoded,
bytearray(b'bar'),
None]
- for ty in [None, pa.binary(), pa.large_binary()]:
+ for ty in [None, pa.binary(), pa.large_binary(), pa.binary_view()]:
arr = pa.array(data, type=ty)
assert len(arr) == 6
assert arr.null_count == 1
@@ -844,7 +855,7 @@ def test_sequence_bytes():
assert arr.to_pylist() == [b'foo', b'dada', b'data', u1, b'bar', None]
-@pytest.mark.parametrize("ty", [pa.string(), pa.large_string()])
+@pytest.mark.parametrize("ty", [pa.string(), pa.large_string(), pa.string_view()])
def test_sequence_utf8_to_unicode(ty):
# ARROW-1225
data = [b'foo', None, b'bar']
@@ -2431,6 +2442,8 @@ def test_array_from_pylist_offset_overflow():
pa.binary(3)),
([b"a"], [pa.scalar("a", type=pa.large_binary())], pa.large_binary()),
(["a"], [pa.scalar("a", type=pa.large_string())], pa.large_string()),
+ ([b"a"], [pa.scalar("a", type=pa.binary_view())], pa.binary_view()),
+ (["a"], [pa.scalar("a", type=pa.string_view())], pa.string_view()),
(
["a"],
[pa.scalar("a", type=pa.dictionary(pa.int64(), pa.string()))],
diff --git a/python/pyarrow/tests/test_scalars.py b/python/pyarrow/tests/test_scalars.py
index 4a239b23d5..eed5f045be 100644
--- a/python/pyarrow/tests/test_scalars.py
+++ b/python/pyarrow/tests/test_scalars.py
@@ -51,9 +51,8 @@ from pyarrow.tests import util
(b"bytes", None, pa.BinaryScalar),
("largestring", pa.large_string(), pa.LargeStringScalar),
(b"largebytes", pa.large_binary(), pa.LargeBinaryScalar),
- # TODO(GH-39633) pa.scalar(..) requires python->arrow conversion to be implemented
- # ("string_view", pa.string_view(), pa.StringViewScalar),
- # (b"bytes_view", pa.binary_view(), pa.BinaryViewScalar),
+ ("string_view", pa.string_view(), pa.StringViewScalar),
+ (b"bytes_view", pa.binary_view(), pa.BinaryViewScalar),
(b"abc", pa.binary(3), pa.FixedSizeBinaryScalar),
([1, 2, 3], None, pa.ListScalar),
([1, 2, 3, 4], pa.large_list(pa.int8()), pa.LargeListScalar),
@@ -492,7 +491,7 @@ def test_month_day_nano_interval():
@pytest.mark.parametrize(('ty', 'scalar_typ'), [
(pa.string(), pa.StringScalar),
(pa.large_string(), pa.LargeStringScalar),
- # (pa.string_view(), pa.StringViewScalar),
+ (pa.string_view(), pa.StringViewScalar),
])
def test_string(value, ty, scalar_typ):
s = pa.scalar(value, type=ty)
@@ -507,30 +506,11 @@ def test_string(value, ty, scalar_typ):
assert buf.to_pybytes() == value.encode()
-@pytest.mark.parametrize('value', ['foo', 'mañana'])
-def test_string_view(value):
- # TODO: replace with normal scalar construction
- builder = pa.lib.StringViewBuilder()
- builder.append(value)
- arr = builder.finish()
-
- s = arr[0]
- assert isinstance(s, pa.StringViewScalar)
- assert s.as_py() == value
- assert s.as_py() != 'something'
- assert repr(value) in repr(s)
- assert str(s) == str(value)
-
- buf = s.as_buffer()
- assert isinstance(buf, pa.Buffer)
- assert buf.to_pybytes() == value.encode()
-
-
@pytest.mark.parametrize('value', [b'foo', b'bar'])
@pytest.mark.parametrize(('ty', 'scalar_typ'), [
(pa.binary(), pa.BinaryScalar),
(pa.large_binary(), pa.LargeBinaryScalar),
- # (pa.binary_view(), pa.BinaryViewScalar),
+ (pa.binary_view(), pa.BinaryViewScalar),
])
def test_binary(value, ty, scalar_typ):
s = pa.scalar(value, type=ty)