You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by pa...@apache.org on 2024/06/28 02:08:09 UTC
(arrow-nanoarrow) branch main updated: refactor(python): Split type identifier utilities into thier own module (#545)
This is an automated email from the ASF dual-hosted git repository.
paleolimbot pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-nanoarrow.git
The following commit(s) were added to refs/heads/main by this push:
new 721972a6 refactor(python): Split type identifier utilities into thier own module (#545)
721972a6 is described below
commit 721972a6e9b77ecefac99cd2b7e631e0e77ef7f9
Author: Dewey Dunnington <de...@dunnington.ca>
AuthorDate: Fri Jun 28 02:08:04 2024 +0000
refactor(python): Split type identifier utilities into thier own module (#545)
This is another chapter in the saga to split up _lib.pyx to make it
easier to find the piece of code one is looking for. Another advantage
of this is that VSCode's Cython plugin does a much better job when
`cimport`s are explicit: for something like `from nanoarrow_c cimport
*`, this means that every use of a nanoarrow symbol (including
`NANOARROW_TYPE_XXX`) would get a yellow squiggly underline (which
normally highlight things that should be fixed). The setup here enables
both Python and Cython modules to use `_types.UINT8`. In Cython this
compiles into a C symbol reference (i.e., not an attribute lookup, which
might be slow).
---
python/setup.py | 11 ++
python/src/nanoarrow/_lib.pyx | 303 ++++++++-------------------------------
python/src/nanoarrow/_types.pxd | 91 ++++++++++++
python/src/nanoarrow/_types.pyx | 243 +++++++++++++++++++++++++++++++
python/src/nanoarrow/_utils.pyx | 2 +
python/src/nanoarrow/c_array.py | 67 +++++----
python/src/nanoarrow/c_buffer.py | 13 +-
python/src/nanoarrow/iterator.py | 44 +++---
python/src/nanoarrow/schema.py | 130 ++++++++---------
python/src/nanoarrow/visitor.py | 8 +-
10 files changed, 538 insertions(+), 374 deletions(-)
diff --git a/python/setup.py b/python/setup.py
index 557959f1..8e9e4284 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -67,6 +67,17 @@ else:
setup(
ext_modules=[
+ Extension(
+ name="nanoarrow._types",
+ include_dirs=["src/nanoarrow", "vendor"],
+ language="c",
+ sources=[
+ "src/nanoarrow/_types.pyx",
+ ],
+ extra_compile_args=extra_compile_args,
+ extra_link_args=extra_link_args,
+ define_macros=extra_define_macros,
+ ),
Extension(
name="nanoarrow._utils",
include_dirs=["src/nanoarrow", "vendor"],
diff --git a/python/src/nanoarrow/_lib.pyx b/python/src/nanoarrow/_lib.pyx
index a45a3dcd..a98d4907 100644
--- a/python/src/nanoarrow/_lib.pyx
+++ b/python/src/nanoarrow/_lib.pyx
@@ -54,9 +54,10 @@ from nanoarrow_device_c cimport *
from nanoarrow_dlpack cimport *
from enum import Enum
-from sys import byteorder as sys_byteorder
from struct import unpack_from, iter_unpack, calcsize, Struct
from nanoarrow import _repr_utils
+
+from nanoarrow cimport _types
from nanoarrow._utils cimport (
alloc_c_schema,
alloc_c_array,
@@ -93,20 +94,17 @@ cdef void view_dlpack_deleter(DLManagedTensor* tensor) noexcept with gil:
tensor.manager_ctx = NULL
ArrowFree(tensor)
-_uint = (NANOARROW_TYPE_UINT8, NANOARROW_TYPE_UINT16, NANOARROW_TYPE_UINT32, NANOARROW_TYPE_UINT64)
-_int = (NANOARROW_TYPE_INT8, NANOARROW_TYPE_INT16, NANOARROW_TYPE_INT32, NANOARROW_TYPE_INT64, NANOARROW_TYPE_INTERVAL_MONTHS)
-_float = (NANOARROW_TYPE_HALF_FLOAT, NANOARROW_TYPE_FLOAT, NANOARROW_TYPE_DOUBLE)
cdef DLDataType view_to_dlpack_data_type(CBufferView view):
cdef DLDataType dtype
# Define DLDataType struct
- if view.data_type_id in _uint:
+ if _types.is_unsigned_integer(view.data_type_id):
dtype.code = kDLUInt
- elif view.data_type_id in _int:
+ elif _types.is_signed_integer(view.data_type_id):
dtype.code = kDLInt
- elif view.data_type_id in _float:
+ elif _types.is_floating_point(view.data_type_id):
dtype.code = kDLFloat
- elif view.data_type_id == NANOARROW_TYPE_BOOL:
+ elif _types.equal(view.data_type_id, _types.BOOL):
raise ValueError('Bit-packed boolean data type not supported by DLPack.')
else:
raise ValueError('DataType is not compatible with DLPack spec: ' + view.data_type)
@@ -147,9 +145,13 @@ cdef DLDevice view_to_dlpack_device(CBufferView view):
cdef DLDevice device
# Check data type support
- if view.data_type_id == NANOARROW_TYPE_BOOL:
+ if _types.equal(view.data_type_id, _types.BOOL):
raise ValueError('Bit-packed boolean data type not supported by DLPack.')
- elif view.data_type_id not in _uint + _int + _float:
+ elif (
+ not _types.is_unsigned_integer(view.data_type_id)
+ and not _types.is_signed_integer(view.data_type_id)
+ and not _types.is_floating_point(view.data_type_id)
+ ):
raise ValueError('DataType is not compatible with DLPack spec: ' + view.data_type)
# Define DLDevice struct
@@ -161,173 +163,6 @@ cdef DLDevice view_to_dlpack_device(CBufferView view):
return device
-cdef c_arrow_type_from_format(format):
- # PyBuffer_SizeFromFormat() was added in Python 3.9 (potentially faster)
- item_size = calcsize(format)
-
- # Don't allow non-native endian values
- if sys_byteorder == "little" and (">" in format or "!" in format):
- raise ValueError(f"Can't convert format '{format}' to Arrow type")
- elif sys_byteorder == "big" and "<" in format:
- raise ValueError(f"Can't convert format '{format}' to Arrow type")
-
- # Strip system endian specifiers
- format = format.strip("=@")
-
- if format == "c":
- return 0, NANOARROW_TYPE_STRING
- elif format == "e":
- return item_size, NANOARROW_TYPE_HALF_FLOAT
- elif format == "f":
- return item_size, NANOARROW_TYPE_FLOAT
- elif format == "d":
- return item_size, NANOARROW_TYPE_DOUBLE
-
- # Check for signed integers
- if format in ("b", "h", "i", "l", "q", "n"):
- if item_size == 1:
- return item_size, NANOARROW_TYPE_INT8
- elif item_size == 2:
- return item_size, NANOARROW_TYPE_INT16
- elif item_size == 4:
- return item_size, NANOARROW_TYPE_INT32
- elif item_size == 8:
- return item_size, NANOARROW_TYPE_INT64
-
- # Check for unsinged integers
- if format in ("B", "?", "H", "I", "L", "Q", "N"):
- if item_size == 1:
- return item_size, NANOARROW_TYPE_UINT8
- elif item_size == 2:
- return item_size, NANOARROW_TYPE_UINT16
- elif item_size == 4:
- return item_size, NANOARROW_TYPE_UINT32
- elif item_size == 8:
- return item_size, NANOARROW_TYPE_UINT64
-
- # If all else fails, return opaque fixed-size binary
- return item_size, NANOARROW_TYPE_BINARY
-
-
-cdef int c_format_from_arrow_type(ArrowType type_id, int element_size_bits, size_t out_size, char* out):
- if type_id in (NANOARROW_TYPE_BINARY, NANOARROW_TYPE_FIXED_SIZE_BINARY) and element_size_bits > 0:
- snprintf(out, out_size, "%ds", <int>(element_size_bits // 8))
- return element_size_bits
-
- cdef const char* format_const = ""
- cdef int element_size_bits_calc = 0
- if type_id == NANOARROW_TYPE_STRING:
- format_const = "c"
- element_size_bits_calc = 0
- elif type_id == NANOARROW_TYPE_BINARY:
- format_const = "B"
- element_size_bits_calc = 0
- elif type_id == NANOARROW_TYPE_BOOL:
- # Bitmaps export as unspecified binary
- format_const = "B"
- element_size_bits_calc = 1
- elif type_id == NANOARROW_TYPE_INT8:
- format_const = "b"
- element_size_bits_calc = 8
- elif type_id == NANOARROW_TYPE_UINT8:
- format_const = "B"
- element_size_bits_calc = 8
- elif type_id == NANOARROW_TYPE_INT16:
- format_const = "h"
- element_size_bits_calc = 16
- elif type_id == NANOARROW_TYPE_UINT16:
- format_const = "H"
- element_size_bits_calc = 16
- elif type_id in (NANOARROW_TYPE_INT32, NANOARROW_TYPE_INTERVAL_MONTHS):
- format_const = "i"
- element_size_bits_calc = 32
- elif type_id == NANOARROW_TYPE_UINT32:
- format_const = "I"
- element_size_bits_calc = 32
- elif type_id == NANOARROW_TYPE_INT64:
- format_const = "q"
- element_size_bits_calc = 64
- elif type_id == NANOARROW_TYPE_UINT64:
- format_const = "Q"
- element_size_bits_calc = 64
- elif type_id == NANOARROW_TYPE_HALF_FLOAT:
- format_const = "e"
- element_size_bits_calc = 16
- elif type_id == NANOARROW_TYPE_FLOAT:
- format_const = "f"
- element_size_bits_calc = 32
- elif type_id == NANOARROW_TYPE_DOUBLE:
- format_const = "d"
- element_size_bits_calc = 64
- elif type_id == NANOARROW_TYPE_INTERVAL_DAY_TIME:
- format_const = "ii"
- element_size_bits_calc = 64
- elif type_id == NANOARROW_TYPE_INTERVAL_MONTH_DAY_NANO:
- format_const = "iiq"
- element_size_bits_calc = 128
- elif type_id == NANOARROW_TYPE_DECIMAL128:
- format_const = "16s"
- element_size_bits_calc = 128
- elif type_id == NANOARROW_TYPE_DECIMAL256:
- format_const = "32s"
- element_size_bits_calc = 256
- else:
- raise ValueError(f"Unsupported Arrow type_id for format conversion: {type_id}")
-
- snprintf(out, out_size, "%s", format_const)
- return element_size_bits_calc
-
-
-# This could in theory use cpdef enum, but an initial attempt to do so
-# resulted Cython duplicating some function definitions. For now, we resort
-# to a more manual trampoline of values to make them accessible from
-# schema.py.
-cdef class CArrowType:
- """
- Wrapper around ArrowType to provide implementations in Python access
- to the values.
- """
-
- UNINITIALIZED = NANOARROW_TYPE_UNINITIALIZED
- NA = NANOARROW_TYPE_NA
- BOOL = NANOARROW_TYPE_BOOL
- UINT8 = NANOARROW_TYPE_UINT8
- INT8 = NANOARROW_TYPE_INT8
- UINT16 = NANOARROW_TYPE_UINT16
- INT16 = NANOARROW_TYPE_INT16
- UINT32 = NANOARROW_TYPE_UINT32
- INT32 = NANOARROW_TYPE_INT32
- UINT64 = NANOARROW_TYPE_UINT64
- INT64 = NANOARROW_TYPE_INT64
- HALF_FLOAT = NANOARROW_TYPE_HALF_FLOAT
- FLOAT = NANOARROW_TYPE_FLOAT
- DOUBLE = NANOARROW_TYPE_DOUBLE
- STRING = NANOARROW_TYPE_STRING
- BINARY = NANOARROW_TYPE_BINARY
- FIXED_SIZE_BINARY = NANOARROW_TYPE_FIXED_SIZE_BINARY
- DATE32 = NANOARROW_TYPE_DATE32
- DATE64 = NANOARROW_TYPE_DATE64
- TIMESTAMP = NANOARROW_TYPE_TIMESTAMP
- TIME32 = NANOARROW_TYPE_TIME32
- TIME64 = NANOARROW_TYPE_TIME64
- INTERVAL_MONTHS = NANOARROW_TYPE_INTERVAL_MONTHS
- INTERVAL_DAY_TIME = NANOARROW_TYPE_INTERVAL_DAY_TIME
- DECIMAL128 = NANOARROW_TYPE_DECIMAL128
- DECIMAL256 = NANOARROW_TYPE_DECIMAL256
- LIST = NANOARROW_TYPE_LIST
- STRUCT = NANOARROW_TYPE_STRUCT
- SPARSE_UNION = NANOARROW_TYPE_SPARSE_UNION
- DENSE_UNION = NANOARROW_TYPE_DENSE_UNION
- DICTIONARY = NANOARROW_TYPE_DICTIONARY
- MAP = NANOARROW_TYPE_MAP
- EXTENSION = NANOARROW_TYPE_EXTENSION
- FIXED_SIZE_LIST = NANOARROW_TYPE_FIXED_SIZE_LIST
- DURATION = NANOARROW_TYPE_DURATION
- LARGE_STRING = NANOARROW_TYPE_LARGE_STRING
- LARGE_BINARY = NANOARROW_TYPE_LARGE_BINARY
- LARGE_LIST = NANOARROW_TYPE_LARGE_LIST
- INTERVAL_MONTH_DAY_NANO = NANOARROW_TYPE_INTERVAL_MONTH_DAY_NANO
-
cdef class CArrowTimeUnit:
"""
@@ -718,32 +553,10 @@ cdef class CSchemaView:
cdef bint _nullable
cdef bint _map_keys_sorted
- _fixed_size_types = (
- NANOARROW_TYPE_FIXED_SIZE_LIST,
- NANOARROW_TYPE_FIXED_SIZE_BINARY
- )
-
- _decimal_types = (
- NANOARROW_TYPE_DECIMAL128,
- NANOARROW_TYPE_DECIMAL256
- )
-
- _time_unit_types = (
- NANOARROW_TYPE_TIME32,
- NANOARROW_TYPE_TIME64,
- NANOARROW_TYPE_DURATION,
- NANOARROW_TYPE_TIMESTAMP
- )
-
- _union_types = (
- NANOARROW_TYPE_DENSE_UNION,
- NANOARROW_TYPE_SPARSE_UNION
- )
-
def __cinit__(self, CSchema schema):
self._base = schema
- self._schema_view.type = NANOARROW_TYPE_UNINITIALIZED
- self._schema_view.storage_type = NANOARROW_TYPE_UNINITIALIZED
+ self._schema_view.type = <ArrowType>_types.UNINITIALIZED
+ self._schema_view.storage_type = <ArrowType>_types.UNINITIALIZED
cdef Error error = Error()
cdef int code = ArrowSchemaViewInit(&self._schema_view, schema._ptr, &error.c_error)
@@ -769,9 +582,12 @@ cdef class CSchemaView:
def storage_buffer_format(self):
if self.buffer_format is not None:
return self.buffer_format
- elif self._schema_view.type == NANOARROW_TYPE_DATE32:
+ elif _types.equal(self._schema_view.type, _types.DATE32):
return 'i'
- elif self._schema_view.type in (NANOARROW_TYPE_TIMESTAMP, NANOARROW_TYPE_DATE64, NANOARROW_TYPE_DURATION):
+ elif _types.one_of(
+ self._schema_view.type,
+ (_types.TIMESTAMP, _types.DATE64, _types.DURATION)
+ ):
return 'q'
else:
return None
@@ -792,11 +608,11 @@ cdef class CSchemaView:
cdef char out[128]
cdef int element_size_bits = 0
- if self._schema_view.type == NANOARROW_TYPE_FIXED_SIZE_BINARY:
+ if _types.equal(self._schema_view.type, _types.FIXED_SIZE_BINARY):
element_size_bits = self._schema_view.fixed_size * 8
try:
- c_format_from_arrow_type(self._schema_view.type, element_size_bits, sizeof(out), out)
+ _types.to_format(self._schema_view.type, element_size_bits, sizeof(out), out)
return out.decode()
except ValueError:
return None
@@ -819,7 +635,7 @@ cdef class CSchemaView:
@property
def dictionary_ordered(self):
- if self._schema_view.type == NANOARROW_TYPE_DICTIONARY:
+ if _types.equal(self._schema_view.type, _types.DICTIONARY):
return self._dictionary_ordered != 0
else:
return None
@@ -830,63 +646,63 @@ cdef class CSchemaView:
@property
def map_keys_sorted(self):
- if self._schema_view.type == NANOARROW_TYPE_MAP:
+ if _types.equal(self._schema_view.type, _types.MAP):
return self._map_keys_sorted != 0
else:
return None
@property
def fixed_size(self):
- if self._schema_view.type in CSchemaView._fixed_size_types:
+ if _types.is_fixed_size(self._schema_view.type):
return self._schema_view.fixed_size
else:
return None
@property
def decimal_bitwidth(self):
- if self._schema_view.type in CSchemaView._decimal_types:
+ if _types.is_decimal(self._schema_view.type):
return self._schema_view.decimal_bitwidth
else:
return None
@property
def decimal_precision(self):
- if self._schema_view.type in CSchemaView._decimal_types:
+ if _types.is_decimal(self._schema_view.type):
return self._schema_view.decimal_precision
else:
return None
@property
def decimal_scale(self):
- if self._schema_view.type in CSchemaView._decimal_types:
+ if _types.is_decimal(self._schema_view.type):
return self._schema_view.decimal_scale
else:
return None
@property
def time_unit_id(self):
- if self._schema_view.type in CSchemaView._time_unit_types:
+ if _types.has_time_unit(self._schema_view.type):
return self._schema_view.time_unit
else:
return None
@property
def time_unit(self):
- if self._schema_view.type in CSchemaView._time_unit_types:
+ if _types.has_time_unit(self._schema_view.type):
return ArrowTimeUnitString(self._schema_view.time_unit).decode()
else:
return None
@property
def timezone(self):
- if self._schema_view.type == NANOARROW_TYPE_TIMESTAMP:
+ if _types.equal(self._schema_view.type, _types.TIMESTAMP):
return self._schema_view.timezone.decode()
else:
return None
@property
def union_type_ids(self):
- if self._schema_view.type in CSchemaView._union_types:
+ if _types.is_union(self._schema_view.type):
type_ids_str = self._schema_view.union_type_ids.decode().split(',')
return (int(type_id) for type_id in type_ids_str)
else:
@@ -1630,7 +1446,7 @@ cdef class CBufferView:
self._data_type = data_type
self._device = device
self._format[0] = 0
- self._element_size_bits = c_format_from_arrow_type(
+ self._element_size_bits = _types.to_format(
self._data_type,
element_size_bits,
sizeof(self._format),
@@ -1638,7 +1454,7 @@ cdef class CBufferView:
)
self._strides = self._item_size()
self._shape = self._ptr.size_bytes // self._strides
- if self._data_type == NANOARROW_TYPE_BOOL:
+ if _types.equal(self._data_type, _types.BOOL):
self._n_elements = self._shape * 8
else:
self._n_elements = self._shape
@@ -1697,14 +1513,17 @@ cdef class CBufferView:
f"of buffer with length {len(self)}"
)
# memoryview's implementation is very fast but not always possible (half float, fixed-size binary, interval)
- if self._data_type in (
- NANOARROW_TYPE_HALF_FLOAT,
- NANOARROW_TYPE_INTERVAL_DAY_TIME,
- NANOARROW_TYPE_INTERVAL_MONTH_DAY_NANO,
- NANOARROW_TYPE_DECIMAL128,
- NANOARROW_TYPE_DECIMAL256
+ if _types.one_of(
+ self._data_type,
+ (
+ _types.HALF_FLOAT,
+ _types.INTERVAL_DAY_TIME,
+ _types.INTERVAL_MONTH_DAY_NANO,
+ _types.DECIMAL128,
+ _types.DECIMAL256
+ )
) or (
- self._data_type == NANOARROW_TYPE_BINARY and self._element_size_bits != 0
+ _types.equal(self._data_type, _types.BINARY) and self._element_size_bits != 0
):
return self._iter_struct(offset, length)
else:
@@ -1725,7 +1544,7 @@ cdef class CBufferView:
return self._n_elements
def element(self, i):
- if self._data_type == NANOARROW_TYPE_BOOL:
+ if _types.equal(self._data_type, _types.BOOL):
if i < 0 or i >= self.n_elements:
raise IndexError(f"Index {i} out of range")
return ArrowBitGet(self._ptr.data.as_uint8, i)
@@ -1742,7 +1561,7 @@ cdef class CBufferView:
f"of buffer with {self.n_elements} elements"
)
- if self._data_type == NANOARROW_TYPE_BOOL:
+ if _types.equal(self._data_type, _types.BOOL):
return self._iter_bitmap(offset, length)
else:
return self._iter_dispatch(offset, length)
@@ -1775,7 +1594,7 @@ cdef class CBufferView:
return bytes_to_copy
def unpack_bits_into(self, dest, offset=0, length=None, dest_offset=0):
- if self._data_type != NANOARROW_TYPE_BOOL:
+ if not _types.equal(self._data_type, _types.BOOL):
raise ValueError("Can't unpack non-boolean buffer")
if length is None:
@@ -1969,14 +1788,14 @@ cdef class CBuffer:
def __cinit__(self):
self._base = None
self._ptr = NULL
- self._data_type = NANOARROW_TYPE_BINARY
+ self._data_type = <ArrowType>(_types.BINARY)
self._element_size_bits = 0
self._device = DEVICE_CPU
# Set initial format to "B" (Cython makes this hard)
self._format[0] = 66
self._format[1] = 0
self._get_buffer_count = 0
- self._view = CBufferView(None, 0, 0, NANOARROW_TYPE_BINARY, 0, self._device)
+ self._view = CBufferView(None, 0, 0, _types.BINARY, 0, self._device)
cdef _assert_valid(self):
if self._ptr == NULL:
@@ -2017,7 +1836,7 @@ cdef class CBuffer:
def _set_format(self, str format):
self._assert_buffer_count_zero()
- element_size_bytes, data_type = c_arrow_type_from_format(format)
+ element_size_bytes, data_type = _types.from_format(format)
self._data_type = data_type
self._element_size_bits = element_size_bytes * 8
format_bytes = format.encode("UTF-8")
@@ -2028,7 +1847,7 @@ cdef class CBuffer:
def _set_data_type(self, ArrowType type_id, int element_size_bits=0):
self._assert_buffer_count_zero()
- self._element_size_bits = c_format_from_arrow_type(
+ self._element_size_bits = _types.to_format(
type_id,
element_size_bits,
sizeof(self._format),
@@ -2272,7 +2091,7 @@ cdef class CBufferBuilder:
# Boolean arrays need their own writer since Python provides
# no helpers for bitpacking
- if self._buffer._data_type == NANOARROW_TYPE_BOOL:
+ if _types.equal(self._buffer._data_type, _types.BOOL):
return self._write_bits(obj)
cdef int64_t n_values = 0
@@ -2292,8 +2111,10 @@ cdef class CBufferBuilder:
# pack_into() call. if code != NANOARROW_OK is used instead of
# Error.raise_error_not_ok() Cython can avoid the extra function call
# and this is a very tight loop.
- if self._buffer._data_type in (NANOARROW_TYPE_INTERVAL_DAY_TIME,
- NANOARROW_TYPE_INTERVAL_MONTH_DAY_NANO):
+ if _types.one_of(
+ self._buffer._data_type,
+ (_types.INTERVAL_DAY_TIME, _types.INTERVAL_MONTH_DAY_NANO)
+ ):
for item in obj:
write(pack(*item))
n_values += 1
@@ -2384,15 +2205,15 @@ cdef class NoneAwareWrapperIterator:
Error.raise_error_not_ok(self, code)
def _get_value_if_none(self, type_id, item_size_bytes=0):
- if type_id == NANOARROW_TYPE_INTERVAL_DAY_TIME:
+ if _types.equal(type_id, _types.INTERVAL_DAY_TIME):
return (0, 0)
- elif type_id == NANOARROW_TYPE_INTERVAL_MONTH_DAY_NANO:
+ elif _types.equal(type_id, _types.INTERVAL_MONTH_DAY_NANO):
return (0, 0, 0)
- elif type_id == NANOARROW_TYPE_BOOL:
+ elif _types.equal(type_id, _types.BOOL):
return False
- elif type_id in (NANOARROW_TYPE_BINARY, NANOARROW_TYPE_FIXED_SIZE_BINARY):
+ elif _types.one_of(type_id, (_types.BINARY, _types.FIXED_SIZE_BINARY)):
return b"\x00" * item_size_bytes
- elif type_id in (NANOARROW_TYPE_HALF_FLOAT, NANOARROW_TYPE_FLOAT, NANOARROW_TYPE_DOUBLE):
+ elif _types.one_of(type_id, (_types.HALF_FLOAT, _types.FLOAT, _types.DOUBLE)):
return 0.0
else:
return 0
@@ -2440,7 +2261,7 @@ cdef class NoneAwareWrapperIterator:
if null_count > 0:
validity = CBuffer.empty()
ArrowBufferMove(&self._bitmap.buffer, validity._ptr)
- validity._set_data_type(NANOARROW_TYPE_BOOL)
+ validity._set_data_type(<ArrowType>_types.BOOL)
return (
self._item_count,
@@ -2863,7 +2684,7 @@ cdef class CMaterializedArrayStream:
Error.raise_error_not_ok("ArrowBufferAppendInt64()", code)
cdef _finalize(self):
- self._array_ends._set_data_type(NANOARROW_TYPE_INT64)
+ self._array_ends._set_data_type(<ArrowType>_types.INT64)
@property
def schema(self):
diff --git a/python/src/nanoarrow/_types.pxd b/python/src/nanoarrow/_types.pxd
new file mode 100644
index 00000000..81b96900
--- /dev/null
+++ b/python/src/nanoarrow/_types.pxd
@@ -0,0 +1,91 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# cython: language_level = 3
+
+from nanoarrow_c cimport *
+
+
+cpdef enum CArrowType:
+ """ArrowType enumerator
+
+ This enum makes the type identifier constants available to Cython and
+ Python code as (e.g.) ``_types.UNINITIALIZED``. This removes the need
+ for other modules to import type codes, which in previous versions had
+ led to very long ``from nanoarrow_c cimport`` declarations.
+ """
+ UNINITIALIZED = NANOARROW_TYPE_UNINITIALIZED
+ NA = NANOARROW_TYPE_NA
+ BOOL = NANOARROW_TYPE_BOOL
+ UINT8 = NANOARROW_TYPE_UINT8
+ INT8 = NANOARROW_TYPE_INT8
+ UINT16 = NANOARROW_TYPE_UINT16
+ INT16 = NANOARROW_TYPE_INT16
+ UINT32 = NANOARROW_TYPE_UINT32
+ INT32 = NANOARROW_TYPE_INT32
+ UINT64 = NANOARROW_TYPE_UINT64
+ INT64 = NANOARROW_TYPE_INT64
+ HALF_FLOAT = NANOARROW_TYPE_HALF_FLOAT
+ FLOAT = NANOARROW_TYPE_FLOAT
+ DOUBLE = NANOARROW_TYPE_DOUBLE
+ STRING = NANOARROW_TYPE_STRING
+ BINARY = NANOARROW_TYPE_BINARY
+ FIXED_SIZE_BINARY = NANOARROW_TYPE_FIXED_SIZE_BINARY
+ DATE32 = NANOARROW_TYPE_DATE32
+ DATE64 = NANOARROW_TYPE_DATE64
+ TIMESTAMP = NANOARROW_TYPE_TIMESTAMP
+ TIME32 = NANOARROW_TYPE_TIME32
+ TIME64 = NANOARROW_TYPE_TIME64
+ INTERVAL_MONTHS = NANOARROW_TYPE_INTERVAL_MONTHS
+ INTERVAL_DAY_TIME = NANOARROW_TYPE_INTERVAL_DAY_TIME
+ DECIMAL128 = NANOARROW_TYPE_DECIMAL128
+ DECIMAL256 = NANOARROW_TYPE_DECIMAL256
+ LIST = NANOARROW_TYPE_LIST
+ STRUCT = NANOARROW_TYPE_STRUCT
+ SPARSE_UNION = NANOARROW_TYPE_SPARSE_UNION
+ DENSE_UNION = NANOARROW_TYPE_DENSE_UNION
+ DICTIONARY = NANOARROW_TYPE_DICTIONARY
+ MAP = NANOARROW_TYPE_MAP
+ EXTENSION = NANOARROW_TYPE_EXTENSION
+ FIXED_SIZE_LIST = NANOARROW_TYPE_FIXED_SIZE_LIST
+ DURATION = NANOARROW_TYPE_DURATION
+ LARGE_STRING = NANOARROW_TYPE_LARGE_STRING
+ LARGE_BINARY = NANOARROW_TYPE_LARGE_BINARY
+ LARGE_LIST = NANOARROW_TYPE_LARGE_LIST
+ INTERVAL_MONTH_DAY_NANO = NANOARROW_TYPE_INTERVAL_MONTH_DAY_NANO
+
+cdef equal(int type_id1, int type_id2)
+
+cdef one_of(int type_id, tuple type_ids)
+
+cpdef bint is_unsigned_integer(int type_id)
+
+cpdef bint is_signed_integer(int type_id)
+
+cpdef bint is_floating_point(int type_id)
+
+cpdef bint is_fixed_size(int type_id)
+
+cpdef bint is_decimal(int type_id)
+
+cpdef bint has_time_unit(int type_id)
+
+cpdef bint is_union(int type_id)
+
+cdef int to_format(int type_id, int element_size_bits, size_t out_size, char* out)
+
+cdef tuple from_format(format)
diff --git a/python/src/nanoarrow/_types.pyx b/python/src/nanoarrow/_types.pyx
new file mode 100644
index 00000000..c10463b0
--- /dev/null
+++ b/python/src/nanoarrow/_types.pyx
@@ -0,0 +1,243 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# cython: language_level = 3
+
+from libc.stdio cimport snprintf
+from . cimport _types
+
+from struct import calcsize
+from sys import byteorder as sys_byteorder
+
+
+cdef equal(int type_id1, int type_id2):
+ """Check if two type identifiers are equal
+
+ Provided because Cython is picky about comparing enum types and
+ will error for ``_types.UINT8 == NANOARROW_TYPE_UINT8``.
+ """
+ return type_id1 == type_id2
+
+
+cdef one_of(int type_id, tuple type_ids):
+ """Check if type_id is one of several values
+
+ Provided because Cython is picky about comparing enum types and
+ will error for ``_types.UINT8 in (NANOARROW_TYPE_UINT8,)``.
+ """
+ for item in type_ids:
+ if type_id == <int>item:
+ return True
+
+ return False
+
+
+cpdef bint is_unsigned_integer(int type_id):
+ """Check if type_id is an unsigned integral type"""
+ return type_id in (
+ _types.UINT8,
+ _types.UINT16,
+ _types.UINT32,
+ _types.UINT64,
+ )
+
+
+cpdef bint is_signed_integer(int type_id):
+ """Check if type_id is an signed integral type"""
+ return type_id in (
+ _types.INT8,
+ _types.INT16,
+ _types.INT32,
+ _types.INT64,
+ )
+
+
+cpdef bint is_floating_point(int type_id):
+ """Check if type_id is a floating point type"""
+ return type_id in (
+ _types.HALF_FLOAT,
+ _types.FLOAT,
+ _types.DOUBLE,
+ )
+
+
+cpdef bint is_fixed_size(int type_id):
+ """Check if type_id is a fixed-size (binary or list) type"""
+ return type_id in (
+ _types.FIXED_SIZE_LIST,
+ _types.FIXED_SIZE_BINARY,
+ )
+
+
+cpdef bint is_decimal(int type_id):
+ """Check if type_id is a decimal type"""
+ return type_id in (
+ _types.DECIMAL128,
+ _types.DECIMAL256,
+ )
+
+
+cpdef bint has_time_unit(int type_id):
+ """Check if type_id represents a type with a time_unit parameter"""
+ return type_id in (
+ _types.TIME32,
+ _types.TIME64,
+ _types.DURATION,
+ _types.TIMESTAMP,
+ )
+
+
+cpdef bint is_union(int type_id):
+ """Check if type_id is a union type"""
+ return type_id in (
+ _types.DENSE_UNION,
+ _types.SPARSE_UNION,
+ )
+
+
+cdef tuple from_format(format):
+ """Convert a Python buffer protocol format string to a itemsize/type_id tuple
+
+ Returns tuple of item size (in bytes) and the ``_types``. Raises
+ ``ValueError`` if the given format string is cannot be represented
+ (e.g., explicit non-system endian) but will return a fixed-size binary
+ specification for unrecognized format strings. The BOOL type is
+ converted as UINT8.
+ """
+ # PyBuffer_SizeFromFormat() was added in Python 3.9 (potentially faster)
+ item_size = calcsize(format)
+
+ # Don't allow non-native endian values
+ if sys_byteorder == "little" and (">" in format or "!" in format):
+ raise ValueError(f"Can't convert format '{format}' to Arrow type")
+ elif sys_byteorder == "big" and "<" in format:
+ raise ValueError(f"Can't convert format '{format}' to Arrow type")
+
+ # Strip system endian specifiers
+ format = format.strip("=@")
+
+ if format == "c":
+ return 0, _types.STRING
+ elif format == "e":
+ return item_size, _types.HALF_FLOAT
+ elif format == "f":
+ return item_size, _types.FLOAT
+ elif format == "d":
+ return item_size, _types.DOUBLE
+
+ # Check for signed integers
+ if format in ("b", "h", "i", "l", "q", "n"):
+ if item_size == 1:
+ return item_size, _types.INT8
+ elif item_size == 2:
+ return item_size, _types.INT16
+ elif item_size == 4:
+ return item_size, _types.INT32
+ elif item_size == 8:
+ return item_size, _types.INT64
+
+ # Check for unsinged integers
+ if format in ("B", "?", "H", "I", "L", "Q", "N"):
+ if item_size == 1:
+ return item_size, _types.UINT8
+ elif item_size == 2:
+ return item_size, _types.UINT16
+ elif item_size == 4:
+ return item_size, _types.UINT32
+ elif item_size == 8:
+ return item_size, _types.UINT64
+
+ # If all else fails, return opaque fixed-size binary
+ return item_size, _types.BINARY
+
+
+cdef int to_format(int type_id, int element_size_bits, size_t out_size, char* out):
+ """Convert an Arrow type identifier to a Python buffer format string
+
+ Populates a format string describing this type. The populated format string
+ will usually roundtrip a buffer through the Python buffer protocol; however,
+ BOOL exports as ``"B"`` (i.e., unsigned bytes) and fixed-size types with no
+ Python equivalent (e.g., DECIMAL128) export as fixed-size binary. Packed types
+ (e.g., INTERVAL_DAY_TIME) export as packed structs such that their component
+ values are preserved.
+ """
+ if type_id in (_types.BINARY, _types.FIXED_SIZE_BINARY) and element_size_bits > 0:
+ snprintf(out, out_size, "%ds", <int>(element_size_bits // 8))
+ return element_size_bits
+
+ cdef const char* format_const = ""
+ cdef int element_size_bits_calc = 0
+ if type_id == _types.STRING:
+ format_const = "c"
+ element_size_bits_calc = 0
+ elif type_id == _types.BINARY:
+ format_const = "B"
+ element_size_bits_calc = 0
+ elif type_id == _types.BOOL:
+ # Bitmaps export as unspecified binary
+ format_const = "B"
+ element_size_bits_calc = 1
+ elif type_id == _types.INT8:
+ format_const = "b"
+ element_size_bits_calc = 8
+ elif type_id == _types.UINT8:
+ format_const = "B"
+ element_size_bits_calc = 8
+ elif type_id == _types.INT16:
+ format_const = "h"
+ element_size_bits_calc = 16
+ elif type_id == _types.UINT16:
+ format_const = "H"
+ element_size_bits_calc = 16
+ elif type_id in (_types.INT32, _types.INTERVAL_MONTHS):
+ format_const = "i"
+ element_size_bits_calc = 32
+ elif type_id == _types.UINT32:
+ format_const = "I"
+ element_size_bits_calc = 32
+ elif type_id == _types.INT64:
+ format_const = "q"
+ element_size_bits_calc = 64
+ elif type_id == _types.UINT64:
+ format_const = "Q"
+ element_size_bits_calc = 64
+ elif type_id == _types.HALF_FLOAT:
+ format_const = "e"
+ element_size_bits_calc = 16
+ elif type_id == _types.FLOAT:
+ format_const = "f"
+ element_size_bits_calc = 32
+ elif type_id == _types.DOUBLE:
+ format_const = "d"
+ element_size_bits_calc = 64
+ elif type_id == _types.INTERVAL_DAY_TIME:
+ format_const = "ii"
+ element_size_bits_calc = 64
+ elif type_id == _types.INTERVAL_MONTH_DAY_NANO:
+ format_const = "iiq"
+ element_size_bits_calc = 128
+ elif type_id == _types.DECIMAL128:
+ format_const = "16s"
+ element_size_bits_calc = 128
+ elif type_id == _types.DECIMAL256:
+ format_const = "32s"
+ element_size_bits_calc = 256
+ else:
+ raise ValueError(f"Unsupported Arrow type_id for format conversion: {type_id}")
+
+ snprintf(out, out_size, "%s", format_const)
+ return element_size_bits_calc
diff --git a/python/src/nanoarrow/_utils.pyx b/python/src/nanoarrow/_utils.pyx
index f9355c8a..f27e4238 100644
--- a/python/src/nanoarrow/_utils.pyx
+++ b/python/src/nanoarrow/_utils.pyx
@@ -15,6 +15,8 @@
# specific language governing permissions and limitations
# under the License.
+# cython: language_level = 3
+
from libc.stdint cimport uint8_t, int64_t
from libc.string cimport memcpy
from cpython.pycapsule cimport PyCapsule_New, PyCapsule_GetPointer, PyCapsule_IsValid
diff --git a/python/src/nanoarrow/c_array.py b/python/src/nanoarrow/c_array.py
index 1c3e676c..41abede4 100644
--- a/python/src/nanoarrow/c_array.py
+++ b/python/src/nanoarrow/c_array.py
@@ -21,7 +21,6 @@ from nanoarrow._lib import (
CArray,
CArrayBuilder,
CArrayView,
- CArrowType,
CBuffer,
CBufferBuilder,
CSchema,
@@ -32,6 +31,8 @@ from nanoarrow._utils import obj_is_buffer, obj_is_capsule
from nanoarrow.c_buffer import c_buffer
from nanoarrow.c_schema import c_schema, c_schema_view
+from nanoarrow import _types
+
def c_array(obj, schema=None) -> CArray:
"""ArrowArray wrapper
@@ -356,7 +357,7 @@ class EmptyArrayBuilder(ArrayBuilder):
@classmethod
def infer_schema(cls, obj) -> Tuple[Any, CSchema]:
- return obj, CSchemaBuilder.allocate().set_type(CArrowType.NA)
+ return obj, CSchemaBuilder.allocate().set_type(_types.NA)
def start_building(self) -> None:
self._c_builder.start_appending()
@@ -385,18 +386,16 @@ class ArrayFromPyBufferBuilder(ArrayBuilder):
element_size_bits = obj.element_size_bits
# Fixed-size binary needs a schema
- if type_id == CArrowType.BINARY and element_size_bits != 0:
+ if type_id == _types.BINARY and element_size_bits != 0:
schema = (
CSchemaBuilder.allocate()
- .set_type_fixed_size(
- CArrowType.FIXED_SIZE_BINARY, element_size_bits // 8
- )
+ .set_type_fixed_size(_types.FIXED_SIZE_BINARY, element_size_bits // 8)
.finish()
)
- elif type_id == CArrowType.STRING:
- schema = CSchemaBuilder.allocate().set_type(CArrowType.INT8).finish()
- elif type_id == CArrowType.BINARY:
- schema = CSchemaBuilder.allocate().set_type(CArrowType.UINT8).finish()
+ elif type_id == _types.STRING:
+ schema = CSchemaBuilder.allocate().set_type(_types.INT8).finish()
+ elif type_id == _types.BINARY:
+ schema = CSchemaBuilder.allocate().set_type(_types.UINT8).finish()
else:
schema = CSchemaBuilder.allocate().set_type(type_id).finish()
@@ -528,30 +527,30 @@ class ArrayFromIterableBuilder(ArrayBuilder):
_ARRAY_BUILDER_FROM_ITERABLE_METHOD = {
- CArrowType.BOOL: "_append_using_buffer_builder",
- CArrowType.HALF_FLOAT: "_append_using_buffer_builder",
- CArrowType.INTERVAL_MONTH_DAY_NANO: "_append_using_buffer_builder",
- CArrowType.INTERVAL_DAY_TIME: "_append_using_buffer_builder",
- CArrowType.INTERVAL_MONTHS: "_append_using_buffer_builder",
- CArrowType.BINARY: "_append_bytes",
- CArrowType.LARGE_BINARY: "_append_bytes",
- CArrowType.FIXED_SIZE_BINARY: "_append_bytes",
- CArrowType.STRING: "_append_strings",
- CArrowType.LARGE_STRING: "_append_strings",
- CArrowType.INT8: "_append_using_array",
- CArrowType.UINT8: "_append_using_array",
- CArrowType.INT16: "_append_using_array",
- CArrowType.UINT16: "_append_using_array",
- CArrowType.INT32: "_append_using_array",
- CArrowType.UINT32: "_append_using_array",
- CArrowType.INT64: "_append_using_array",
- CArrowType.UINT64: "_append_using_array",
- CArrowType.FLOAT: "_append_using_array",
- CArrowType.DOUBLE: "_append_using_array",
- CArrowType.TIMESTAMP: "_append_using_array",
- CArrowType.DATE32: "_append_using_array",
- CArrowType.DATE64: "_append_using_array",
- CArrowType.DURATION: "_append_using_array",
+ _types.BOOL: "_append_using_buffer_builder",
+ _types.HALF_FLOAT: "_append_using_buffer_builder",
+ _types.INTERVAL_MONTH_DAY_NANO: "_append_using_buffer_builder",
+ _types.INTERVAL_DAY_TIME: "_append_using_buffer_builder",
+ _types.INTERVAL_MONTHS: "_append_using_buffer_builder",
+ _types.BINARY: "_append_bytes",
+ _types.LARGE_BINARY: "_append_bytes",
+ _types.FIXED_SIZE_BINARY: "_append_bytes",
+ _types.STRING: "_append_strings",
+ _types.LARGE_STRING: "_append_strings",
+ _types.INT8: "_append_using_array",
+ _types.UINT8: "_append_using_array",
+ _types.INT16: "_append_using_array",
+ _types.UINT16: "_append_using_array",
+ _types.INT32: "_append_using_array",
+ _types.UINT32: "_append_using_array",
+ _types.INT64: "_append_using_array",
+ _types.UINT64: "_append_using_array",
+ _types.FLOAT: "_append_using_array",
+ _types.DOUBLE: "_append_using_array",
+ _types.TIMESTAMP: "_append_using_array",
+ _types.DATE32: "_append_using_array",
+ _types.DATE64: "_append_using_array",
+ _types.DURATION: "_append_using_array",
}
_ARRAY_BUILDER_FROM_NULLABLE_ITERABLE_METHOD = {
diff --git a/python/src/nanoarrow/c_buffer.py b/python/src/nanoarrow/c_buffer.py
index 78ccdfa2..a0c79ba1 100644
--- a/python/src/nanoarrow/c_buffer.py
+++ b/python/src/nanoarrow/c_buffer.py
@@ -15,10 +15,12 @@
# specific language governing permissions and limitations
# under the License.
-from nanoarrow._lib import CArrowType, CBuffer, CBufferBuilder
+from nanoarrow._lib import CBuffer, CBufferBuilder
from nanoarrow._utils import obj_is_buffer
from nanoarrow.c_schema import c_schema_view
+from nanoarrow import _types
+
def c_buffer(obj, schema=None) -> CBuffer:
"""Owning, read-only ArrowBuffer wrapper
@@ -103,17 +105,14 @@ def _c_buffer_from_iterable(obj, schema=None) -> CBuffer:
builder = CBufferBuilder()
- if schema_view.storage_type_id == CArrowType.FIXED_SIZE_BINARY:
- builder.set_data_type(CArrowType.BINARY, schema_view.fixed_size * 8)
+ if schema_view.storage_type_id == _types.FIXED_SIZE_BINARY:
+ builder.set_data_type(_types.BINARY, schema_view.fixed_size * 8)
else:
builder.set_data_type(schema_view.storage_type_id)
# If we are using a typecode supported by the array module, it has much
# faster implementations of safely building buffers from iterables
- if (
- builder.format in array_typecodes
- and schema_view.storage_type_id != CArrowType.BOOL
- ):
+ if builder.format in array_typecodes and schema_view.storage_type_id != _types.BOOL:
buf = array.array(builder.format, obj)
return CBuffer.from_pybuffer(buf), len(buf)
diff --git a/python/src/nanoarrow/iterator.py b/python/src/nanoarrow/iterator.py
index c1cab2e0..d448adf2 100644
--- a/python/src/nanoarrow/iterator.py
+++ b/python/src/nanoarrow/iterator.py
@@ -20,11 +20,13 @@ from functools import cached_property
from itertools import islice, repeat
from typing import Iterable, Tuple
-from nanoarrow._lib import CArrayView, CArrowType
+from nanoarrow._lib import CArrayView
from nanoarrow.c_array_stream import c_array_stream
from nanoarrow.c_schema import c_schema, c_schema_view
from nanoarrow.schema import Schema
+from nanoarrow import _types
+
def iter_py(obj, schema=None) -> Iterable:
"""Iterate over items in zero or more arrays
@@ -353,7 +355,7 @@ class PyIterator(ArrayViewBaseIterator):
storage = self._primitive_iter(offset, length)
epoch = date(1970, 1, 1)
- if self._schema_view.type_id == CArrowType.DATE32:
+ if self._schema_view.type_id == _types.DATE32:
for item in storage:
if item is None:
yield item
@@ -544,24 +546,24 @@ def _get_tzinfo(tz_string, strategy=None):
_ITEMS_ITER_LOOKUP = {
- CArrowType.NA: "_null_iter",
- CArrowType.BINARY: "_binary_iter",
- CArrowType.LARGE_BINARY: "_binary_iter",
- CArrowType.STRING: "_string_iter",
- CArrowType.LARGE_STRING: "_string_iter",
- CArrowType.STRUCT: "_struct_iter",
- CArrowType.LIST: "_list_iter",
- CArrowType.LARGE_LIST: "_list_iter",
- CArrowType.FIXED_SIZE_LIST: "_fixed_size_list_iter",
- CArrowType.DICTIONARY: "_dictionary_iter",
- CArrowType.DATE32: "_date_iter",
- CArrowType.DATE64: "_date_iter",
- CArrowType.TIME32: "_time_iter",
- CArrowType.TIME64: "_time_iter",
- CArrowType.TIMESTAMP: "_timestamp_iter",
- CArrowType.DURATION: "_duration_iter",
- CArrowType.DECIMAL128: "_decimal_iter",
- CArrowType.DECIMAL256: "_decimal_iter",
+ _types.NA: "_null_iter",
+ _types.BINARY: "_binary_iter",
+ _types.LARGE_BINARY: "_binary_iter",
+ _types.STRING: "_string_iter",
+ _types.LARGE_STRING: "_string_iter",
+ _types.STRUCT: "_struct_iter",
+ _types.LIST: "_list_iter",
+ _types.LARGE_LIST: "_list_iter",
+ _types.FIXED_SIZE_LIST: "_fixed_size_list_iter",
+ _types.DICTIONARY: "_dictionary_iter",
+ _types.DATE32: "_date_iter",
+ _types.DATE64: "_date_iter",
+ _types.TIME32: "_time_iter",
+ _types.TIME64: "_time_iter",
+ _types.TIMESTAMP: "_timestamp_iter",
+ _types.DURATION: "_duration_iter",
+ _types.DECIMAL128: "_decimal_iter",
+ _types.DECIMAL256: "_decimal_iter",
}
_PRIMITIVE_TYPE_NAMES = [
@@ -584,5 +586,5 @@ _PRIMITIVE_TYPE_NAMES = [
]
for type_name in _PRIMITIVE_TYPE_NAMES:
- type_id = getattr(CArrowType, type_name)
+ type_id = getattr(_types, type_name)
_ITEMS_ITER_LOOKUP[type_id] = "_primitive_iter"
diff --git a/python/src/nanoarrow/schema.py b/python/src/nanoarrow/schema.py
index b7e1fe2f..e721c2f1 100644
--- a/python/src/nanoarrow/schema.py
+++ b/python/src/nanoarrow/schema.py
@@ -20,16 +20,10 @@ import reprlib
from functools import cached_property
from typing import List, Mapping, Union
-from nanoarrow._lib import (
- CArrowTimeUnit,
- CArrowType,
- CSchemaBuilder,
- CSchemaView,
- SchemaMetadata,
-)
+from nanoarrow._lib import CArrowTimeUnit, CSchemaBuilder, CSchemaView, SchemaMetadata
from nanoarrow.c_schema import c_schema
-from nanoarrow import _repr_utils
+from nanoarrow import _repr_utils, _types
class Type(enum.Enum):
@@ -38,45 +32,45 @@ class Type(enum.Enum):
:class:`Schema` instances in most places for parameter-free types.
"""
- UNINITIALIZED = CArrowType.UNINITIALIZED
- NULL = CArrowType.NA
- BOOL = CArrowType.BOOL
- UINT8 = CArrowType.UINT8
- INT8 = CArrowType.INT8
- UINT16 = CArrowType.UINT16
- INT16 = CArrowType.INT16
- UINT32 = CArrowType.UINT32
- INT32 = CArrowType.INT32
- UINT64 = CArrowType.UINT64
- INT64 = CArrowType.INT64
- HALF_FLOAT = CArrowType.HALF_FLOAT
- FLOAT = CArrowType.FLOAT
- DOUBLE = CArrowType.DOUBLE
- STRING = CArrowType.STRING
- BINARY = CArrowType.BINARY
- FIXED_SIZE_BINARY = CArrowType.FIXED_SIZE_BINARY
- DATE32 = CArrowType.DATE32
- DATE64 = CArrowType.DATE64
- TIMESTAMP = CArrowType.TIMESTAMP
- TIME32 = CArrowType.TIME32
- TIME64 = CArrowType.TIME64
- INTERVAL_MONTHS = CArrowType.INTERVAL_MONTHS
- INTERVAL_DAY_TIME = CArrowType.INTERVAL_DAY_TIME
- DECIMAL128 = CArrowType.DECIMAL128
- DECIMAL256 = CArrowType.DECIMAL256
- LIST = CArrowType.LIST
- STRUCT = CArrowType.STRUCT
- SPARSE_UNION = CArrowType.SPARSE_UNION
- DENSE_UNION = CArrowType.DENSE_UNION
- DICTIONARY = CArrowType.DICTIONARY
- MAP = CArrowType.MAP
- EXTENSION = CArrowType.EXTENSION
- FIXED_SIZE_LIST = CArrowType.FIXED_SIZE_LIST
- DURATION = CArrowType.DURATION
- LARGE_STRING = CArrowType.LARGE_STRING
- LARGE_BINARY = CArrowType.LARGE_BINARY
- LARGE_LIST = CArrowType.LARGE_LIST
- INTERVAL_MONTH_DAY_NANO = CArrowType.INTERVAL_MONTH_DAY_NANO
+ UNINITIALIZED = int(_types.UNINITIALIZED)
+ NULL = int(_types.NA)
+ BOOL = int(_types.BOOL)
+ UINT8 = int(_types.UINT8)
+ INT8 = int(_types.INT8)
+ UINT16 = int(_types.UINT16)
+ INT16 = int(_types.INT16)
+ UINT32 = int(_types.UINT32)
+ INT32 = int(_types.INT32)
+ UINT64 = int(_types.UINT64)
+ INT64 = int(_types.INT64)
+ HALF_FLOAT = int(_types.HALF_FLOAT)
+ FLOAT = int(_types.FLOAT)
+ DOUBLE = int(_types.DOUBLE)
+ STRING = int(_types.STRING)
+ BINARY = int(_types.BINARY)
+ FIXED_SIZE_BINARY = int(_types.FIXED_SIZE_BINARY)
+ DATE32 = int(_types.DATE32)
+ DATE64 = int(_types.DATE64)
+ TIMESTAMP = int(_types.TIMESTAMP)
+ TIME32 = int(_types.TIME32)
+ TIME64 = int(_types.TIME64)
+ INTERVAL_MONTHS = int(_types.INTERVAL_MONTHS)
+ INTERVAL_DAY_TIME = int(_types.INTERVAL_DAY_TIME)
+ DECIMAL128 = int(_types.DECIMAL128)
+ DECIMAL256 = int(_types.DECIMAL256)
+ LIST = int(_types.LIST)
+ STRUCT = int(_types.STRUCT)
+ SPARSE_UNION = int(_types.SPARSE_UNION)
+ DENSE_UNION = int(_types.DENSE_UNION)
+ DICTIONARY = int(_types.DICTIONARY)
+ MAP = int(_types.MAP)
+ EXTENSION = int(_types.EXTENSION)
+ FIXED_SIZE_LIST = int(_types.FIXED_SIZE_LIST)
+ DURATION = int(_types.DURATION)
+ LARGE_STRING = int(_types.LARGE_STRING)
+ LARGE_BINARY = int(_types.LARGE_BINARY)
+ LARGE_LIST = int(_types.LARGE_LIST)
+ INTERVAL_MONTH_DAY_NANO = int(_types.INTERVAL_MONTH_DAY_NANO)
def __arrow_c_schema__(self):
# This will only work for parameter-free types
@@ -313,7 +307,7 @@ class Schema:
123
"""
- if self._c_schema_view.type_id == CArrowType.FIXED_SIZE_BINARY:
+ if self._c_schema_view.type_id == _types.FIXED_SIZE_BINARY:
return self._c_schema_view.fixed_size
@property
@@ -377,7 +371,7 @@ class Schema:
>>> na.dictionary(na.int32(), na.string()).index_type
<Schema> int32
"""
- if self._c_schema_view.type_id == CArrowType.DICTIONARY:
+ if self._c_schema_view.type_id == _types.DICTIONARY:
index_schema = self._c_schema.modify(
dictionary=False, flags=0, nullable=self.nullable
)
@@ -409,12 +403,12 @@ class Schema:
<Schema> string
"""
if self._c_schema_view.type_id in (
- CArrowType.LIST,
- CArrowType.LARGE_LIST,
- CArrowType.FIXED_SIZE_LIST,
+ _types.LIST,
+ _types.LARGE_LIST,
+ _types.FIXED_SIZE_LIST,
):
return self.field(0)
- elif self._c_schema_view.type_id == CArrowType.DICTIONARY:
+ elif self._c_schema_view.type_id == _types.DICTIONARY:
return Schema(self._c_schema.dictionary)
else:
return None
@@ -427,7 +421,7 @@ class Schema:
>>> na.fixed_size_list(na.int32(), 123).list_size
123
"""
- if self._c_schema_view.type_id == CArrowType.FIXED_SIZE_LIST:
+ if self._c_schema_view.type_id == _types.FIXED_SIZE_LIST:
return self._c_schema_view.fixed_size
else:
return None
@@ -1189,12 +1183,12 @@ def extension_type(
def _c_schema_from_type_and_params(type: Type, params: dict):
factory = CSchemaBuilder.allocate()
- if type.value in CSchemaView._decimal_types:
+ if _types.is_decimal(type.value):
precision = int(params.pop("precision"))
scale = int(params.pop("scale"))
factory.set_type_decimal(type.value, precision, scale)
- elif type.value in CSchemaView._time_unit_types:
+ elif _types.has_time_unit(type.value):
time_unit = params.pop("unit")
if "timezone" in params:
timezone = params.pop("timezone")
@@ -1293,16 +1287,16 @@ def _schema_repr(obj, max_char_width=80, prefix="<Schema> ", include_metadata=Tr
_PARAM_NAMES = {
- CArrowType.FIXED_SIZE_BINARY: ("byte_width",),
- CArrowType.TIMESTAMP: ("unit", "timezone"),
- CArrowType.TIME32: ("unit",),
- CArrowType.TIME64: ("unit",),
- CArrowType.DURATION: ("unit",),
- CArrowType.DECIMAL128: ("precision", "scale"),
- CArrowType.DECIMAL256: ("precision", "scale"),
- CArrowType.STRUCT: ("fields",),
- CArrowType.LIST: ("value_type",),
- CArrowType.LARGE_LIST: ("value_type",),
- CArrowType.FIXED_SIZE_LIST: ("value_type", "list_size"),
- CArrowType.DICTIONARY: ("index_type", "value_type", "dictionary_ordered"),
+ _types.FIXED_SIZE_BINARY: ("byte_width",),
+ _types.TIMESTAMP: ("unit", "timezone"),
+ _types.TIME32: ("unit",),
+ _types.TIME64: ("unit",),
+ _types.DURATION: ("unit",),
+ _types.DECIMAL128: ("precision", "scale"),
+ _types.DECIMAL256: ("precision", "scale"),
+ _types.STRUCT: ("fields",),
+ _types.LIST: ("value_type",),
+ _types.LARGE_LIST: ("value_type",),
+ _types.FIXED_SIZE_LIST: ("value_type", "list_size"),
+ _types.DICTIONARY: ("index_type", "value_type", "dictionary_ordered"),
}
diff --git a/python/src/nanoarrow/visitor.py b/python/src/nanoarrow/visitor.py
index 7adaf471..05aca6ff 100644
--- a/python/src/nanoarrow/visitor.py
+++ b/python/src/nanoarrow/visitor.py
@@ -17,12 +17,14 @@
from typing import Any, Callable, List, Sequence, Tuple, Union
-from nanoarrow._lib import CArrayView, CArrowType, CBuffer, CBufferBuilder
+from nanoarrow._lib import CArrayView, CBuffer, CBufferBuilder
from nanoarrow.c_array_stream import c_array_stream
from nanoarrow.c_schema import c_schema_view
from nanoarrow.iterator import ArrayViewBaseIterator, PyIterator
from nanoarrow.schema import Type
+from nanoarrow import _types
+
class ArrayViewVisitable:
"""Mixin class providing conversion methods based on visitors
@@ -427,7 +429,7 @@ def _resolve_converter_cls(schema, handle_nulls=None):
schema_view = c_schema_view(schema)
if schema_view.nullable:
- if schema_view.type_id == CArrowType.BOOL:
+ if schema_view.type_id == _types.BOOL:
return ToNullableSequenceConverter, {
"converter_cls": ToBooleanBufferConverter,
"handle_nulls": handle_nulls,
@@ -441,7 +443,7 @@ def _resolve_converter_cls(schema, handle_nulls=None):
return ToPyListConverter, {}
else:
- if schema_view.type_id == CArrowType.BOOL:
+ if schema_view.type_id == _types.BOOL:
return ToBooleanBufferConverter, {}
elif schema_view.buffer_format is not None:
return ToPyBufferConverter, {}