You are viewing a plain text version of this content. The canonical link for it is here.
Posted to github@arrow.apache.org by GitBox <gi...@apache.org> on 2020/06/22 20:17:22 UTC
[GitHub] [arrow] kszucs opened a new pull request #7519: ARROW-9153: [C++][Python] Refactor scalar bindings [WIP]
kszucs opened a new pull request #7519:
URL: https://github.com/apache/arrow/pull/7519
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] kszucs commented on a change in pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
kszucs commented on a change in pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#discussion_r447734944
##########
File path: python/pyarrow/scalar.pxi
##########
@@ -16,1198 +16,745 @@
# under the License.
-_NULL = NA = None
+import collections
cdef class Scalar:
"""
- The base class for all array elements.
+ The base class for scalars.
"""
+ def __init__(self):
+ raise TypeError("Do not call {}'s constructor directly, use "
+ "pa.scalar() instead.".format(self.__class__.__name__))
-cdef class NullType(Scalar):
- """
- Singleton for null array elements.
- """
- # TODO rename this NullValue?
+ cdef void init(self, const shared_ptr[CScalar]& wrapped):
+ self.wrapped = wrapped
- def __cinit__(self):
- global NA
- if NA is not None:
- raise Exception('Cannot create multiple NAType instances')
+ @staticmethod
+ cdef wrap(const shared_ptr[CScalar]& wrapped):
+ cdef:
+ Scalar self
+ Type type_id = wrapped.get().type.get().id()
+
+ if type_id == _Type_NA:
+ return _NULL
+
+ typ = _scalar_classes[type_id]
+ self = typ.__new__(typ)
+ self.init(wrapped)
+
+ return self
+
+ cdef inline shared_ptr[CScalar] unwrap(self) nogil:
+ return self.wrapped
+
+ @property
+ def type(self):
+ return pyarrow_wrap_data_type(self.wrapped.get().type)
- self.type = null()
+ @property
+ def is_valid(self):
+ return self.wrapped.get().is_valid
def __repr__(self):
- return 'NULL'
+ return '<pyarrow.{}: {!r}>'.format(
+ self.__class__.__name__, self.as_py()
+ )
- def as_py(self):
- """
- Return None
- """
- return None
+ def __str__(self):
+ return str(self.as_py())
+
+ def equals(self, Scalar other):
+ return self.wrapped.get().Equals(other.unwrap().get()[0])
def __eq__(self, other):
- return NA
+ try:
+ if not isinstance(other, Scalar):
+ other = scalar(other, type=self.type)
+ return self.equals(other)
+ except (TypeError, ValueError, ArrowInvalid):
+ return NotImplemented
+
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+
+ def as_py(self):
+ raise NotImplementedError()
-_NULL = NA = NullType()
+_NULL = NA = None
-cdef class ArrayValue(Scalar):
+cdef class NullScalar(Scalar):
"""
- The base class for non-null array elements.
+ Concrete class for null scalars.
"""
- def __init__(self):
- raise TypeError("Do not call {}'s constructor directly, use array "
- "subscription instead."
- .format(self.__class__.__name__))
+ def __cinit__(self):
+ global NA
+ if NA is not None:
+ raise Exception('Cannot create multiple NAType instances')
+ self.init(shared_ptr[CScalar](new CNullScalar()))
- cdef void init(self, DataType type, const shared_ptr[CArray]& sp_array,
- int64_t index):
- self.type = type
- self.index = index
- self._set_array(sp_array)
+ def __init__(self):
+ pass
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
+ def __eq__(self, other):
+ return NA
- def __repr__(self):
- if hasattr(self, 'as_py'):
- return repr(self.as_py())
- else:
- return super(Scalar, self).__repr__()
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
- def __str__(self):
- if hasattr(self, 'as_py'):
- return str(self.as_py())
- else:
- return super(Scalar, self).__str__()
+ def as_py(self):
+ """
+ Return this value as a Python None.
+ """
+ return None
- def __eq__(self, other):
- if hasattr(self, 'as_py'):
- if isinstance(other, ArrayValue):
- other = other.as_py()
- return self.as_py() == other
- else:
- raise NotImplementedError(
- "Cannot compare Arrow values that don't support as_py()")
- def __hash__(self):
- return hash(self.as_py())
+_NULL = NA = NullScalar()
-cdef class BooleanValue(ArrayValue):
+cdef class BooleanScalar(Scalar):
"""
- Concrete class for boolean array elements.
+ Concrete class for boolean scalars.
"""
def as_py(self):
"""
Return this value as a Python bool.
"""
- cdef CBooleanArray* ap = <CBooleanArray*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CBooleanScalar* sp = <CBooleanScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int8Value(ArrayValue):
+cdef class UInt8Scalar(Scalar):
"""
- Concrete class for int8 array elements.
+ Concrete class for uint8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt8Array* ap = <CInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt8Scalar* sp = <CUInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt8Value(ArrayValue):
+cdef class Int8Scalar(Scalar):
"""
- Concrete class for uint8 array elements.
+ Concrete class for int8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt8Array* ap = <CUInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt8Scalar* sp = <CInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int16Value(ArrayValue):
+cdef class UInt16Scalar(Scalar):
"""
- Concrete class for int16 array elements.
+ Concrete class for uint16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt16Array* ap = <CInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt16Scalar* sp = <CUInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt16Value(ArrayValue):
+cdef class Int16Scalar(Scalar):
"""
- Concrete class for uint16 array elements.
+ Concrete class for int16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt16Array* ap = <CUInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt16Scalar* sp = <CInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int32Value(ArrayValue):
+cdef class UInt32Scalar(Scalar):
"""
- Concrete class for int32 array elements.
+ Concrete class for uint32 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt32Array* ap = <CInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt32Scalar* sp = <CUInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt32Value(ArrayValue):
+cdef class Int32Scalar(Scalar):
"""
- Concrete class for uint32 array elements.
+ Concrete class for int32 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt32Array* ap = <CUInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt32Scalar* sp = <CInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int64Value(ArrayValue):
+cdef class UInt64Scalar(Scalar):
"""
- Concrete class for int64 array elements.
+ Concrete class for uint64 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt64Array* ap = <CInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt64Scalar* sp = <CUInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt64Value(ArrayValue):
+cdef class Int64Scalar(Scalar):
"""
- Concrete class for uint64 array elements.
+ Concrete class for int64 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt64Array* ap = <CUInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt64Scalar* sp = <CInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date32Value(ArrayValue):
+cdef class HalfFloatScalar(Scalar):
"""
- Concrete class for date32 array elements.
+ Concrete class for float scalars.
"""
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+
+ def __eq__(self, other):
+ if hasattr(self, 'as_py'):
+ if isinstance(other, Scalar):
+ other = other.as_py()
+ return self.as_py() == other
+ else:
+ raise NotImplementedError
+
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python float.
"""
- cdef CDate32Array* ap = <CDate32Array*> self.sp_array.get()
+ cdef CHalfFloatScalar* sp = <CHalfFloatScalar*> self.wrapped.get()
+ return PyHalf_FromHalf(sp.value) if sp.is_valid else None
- # Shift to seconds since epoch
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(days=ap.Value(self.index)))
+
+cdef class FloatScalar(Scalar):
+ """
+ Concrete class for float scalars.
+ """
+
+ def as_py(self):
+ """
+ Return this value as a Python float.
+ """
+ cdef CFloatScalar* sp = <CFloatScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date64Value(ArrayValue):
+cdef class DoubleScalar(Scalar):
"""
- Concrete class for date64 array elements.
+ Concrete class for double scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python float.
"""
- cdef CDate64Array* ap = <CDate64Array*> self.sp_array.get()
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(
- days=ap.Value(self.index) / 86400000))
+ cdef CDoubleScalar* sp = <CDoubleScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Time32Value(ArrayValue):
+cdef class Decimal128Scalar(Scalar):
"""
- Concrete class for time32 array elements.
+ Concrete class for decimal128 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.timedelta instance.
+ Return this value as a Python Decimal.
"""
cdef:
- CTime32Array* ap = <CTime32Array*> self.sp_array.get()
- CTime32Type* dtype = <CTime32Type*> ap.type().get()
-
- if dtype.unit() == TimeUnit_SECOND:
- delta = datetime.timedelta(seconds=ap.Value(self.index))
- return (datetime.datetime(1970, 1, 1) + delta).time()
+ CDecimal128Scalar* sp = <CDecimal128Scalar*> self.wrapped.get()
+ CDecimal128Type* dtype = <CDecimal128Type*> sp.type.get()
+ if sp.is_valid:
+ return _pydecimal.Decimal(
Review comment:
`decimal` is imported as `_pydecimal` in [lib.pyx](https://github.com/apache/arrow/blob/master/python/pyarrow/lib.pyx#L23)
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] pitrou commented on a change in pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
pitrou commented on a change in pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#discussion_r447677453
##########
File path: python/pyarrow/lib.pxd
##########
@@ -179,101 +179,18 @@ cdef class Schema:
cdef class Scalar:
- cdef readonly:
- DataType type
-
-
-cdef class NAType(Scalar):
- pass
-
-
-cdef class ArrayValue(Scalar):
cdef:
- shared_ptr[CArray] sp_array
- int64_t index
-
- cdef void init(self, DataType type,
- const shared_ptr[CArray]& sp_array, int64_t index)
-
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array)
-
-cdef class ScalarValue(Scalar):
- cdef:
- shared_ptr[CScalar] sp_scalar
-
- cdef void init(self, const shared_ptr[CScalar]& sp_scalar)
-
-cdef class Int8Value(ArrayValue):
- pass
+ shared_ptr[CScalar] wrapped
+ cdef void init(self, const shared_ptr[CScalar]& wrapped)
-cdef class Int64Value(ArrayValue):
- pass
-
-
-cdef class ListValue(ArrayValue):
- cdef readonly:
- DataType value_type
-
- cdef:
- CListArray* ap
-
- cdef getitem(self, int64_t i)
- cdef int64_t length(self)
-
-
-cdef class LargeListValue(ArrayValue):
- cdef readonly:
- DataType value_type
-
- cdef:
- CLargeListArray* ap
-
- cdef getitem(self, int64_t i)
- cdef int64_t length(self)
-
-
-cdef class MapValue(ArrayValue):
- cdef readonly:
- DataType key_type
- DataType item_type
-
- cdef:
- CMapArray* ap
-
- cdef getitem(self, int64_t i)
- cdef int64_t length(self)
-
-
-cdef class FixedSizeListValue(ArrayValue):
- cdef readonly:
- DataType value_type
-
- cdef:
- CFixedSizeListArray* ap
-
- cdef getitem(self, int64_t i)
- cdef int64_t length(self)
-
-
-cdef class StructValue(ArrayValue):
- cdef:
- CStructArray* ap
-
-
-cdef class UnionValue(ArrayValue):
- cdef:
- CUnionArray* ap
- list value_types
-
- cdef getitem(self, int64_t i)
-
+ @staticmethod
+ cdef wrap(const shared_ptr[CScalar]& wrapped)
-cdef class StringValue(ArrayValue):
- pass
+ cdef inline shared_ptr[CScalar] unwrap(self) nogil
-cdef class FixedSizeBinaryValue(ArrayValue):
+cdef class NAType(Scalar):
Review comment:
`NullScalar` perhaps... also, why does it have its own class?
##########
File path: python/pyarrow/includes/libarrow.pxd
##########
@@ -44,6 +44,11 @@ cdef extern from "arrow/util/key_value_metadata.h" namespace "arrow" nogil:
c_bool Contains(const c_string& key) const
+cdef extern from "arrow/util/decimal.h" namespace "arrow" nogil:
+ cdef cppclass CDecimal128" arrow::Decimal128":
+ c_string ToString(int32_t scale) const
Review comment:
Hmm, at some point we should be able to pretty-print scalars in C++.
##########
File path: python/pyarrow/includes/libarrow.pxd
##########
@@ -877,20 +893,60 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
cdef cppclass CUInt64Scalar" arrow::UInt64Scalar"(CScalar):
uint64_t value
+ cdef cppclass CHalfFloatScalar" arrow::HalfFloatScalar"(CScalar):
+ npy_half value
Review comment:
Ah, cool!
##########
File path: python/pyarrow/scalar.pxi
##########
@@ -16,1198 +16,745 @@
# under the License.
-_NULL = NA = None
+import collections
cdef class Scalar:
"""
- The base class for all array elements.
+ The base class for scalars.
"""
+ def __init__(self):
+ raise TypeError("Do not call {}'s constructor directly, use "
+ "pa.scalar() instead.".format(self.__class__.__name__))
-cdef class NullType(Scalar):
- """
- Singleton for null array elements.
- """
- # TODO rename this NullValue?
+ cdef void init(self, const shared_ptr[CScalar]& wrapped):
+ self.wrapped = wrapped
- def __cinit__(self):
- global NA
- if NA is not None:
- raise Exception('Cannot create multiple NAType instances')
+ @staticmethod
+ cdef wrap(const shared_ptr[CScalar]& wrapped):
+ cdef:
+ Scalar self
+ Type type_id = wrapped.get().type.get().id()
+
+ if type_id == _Type_NA:
+ return _NULL
+
+ typ = _scalar_classes[type_id]
+ self = typ.__new__(typ)
+ self.init(wrapped)
+
+ return self
+
+ cdef inline shared_ptr[CScalar] unwrap(self) nogil:
+ return self.wrapped
+
+ @property
+ def type(self):
+ return pyarrow_wrap_data_type(self.wrapped.get().type)
- self.type = null()
+ @property
+ def is_valid(self):
+ return self.wrapped.get().is_valid
def __repr__(self):
- return 'NULL'
+ return '<pyarrow.{}: {!r}>'.format(
+ self.__class__.__name__, self.as_py()
+ )
- def as_py(self):
- """
- Return None
- """
- return None
+ def __str__(self):
+ return str(self.as_py())
+
+ def equals(self, Scalar other):
+ return self.wrapped.get().Equals(other.unwrap().get()[0])
def __eq__(self, other):
- return NA
+ try:
+ if not isinstance(other, Scalar):
+ other = scalar(other, type=self.type)
+ return self.equals(other)
+ except (TypeError, ValueError, ArrowInvalid):
+ return NotImplemented
+
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+
+ def as_py(self):
+ raise NotImplementedError()
-_NULL = NA = NullType()
+_NULL = NA = None
-cdef class ArrayValue(Scalar):
+cdef class NullScalar(Scalar):
"""
- The base class for non-null array elements.
+ Concrete class for null scalars.
"""
- def __init__(self):
- raise TypeError("Do not call {}'s constructor directly, use array "
- "subscription instead."
- .format(self.__class__.__name__))
+ def __cinit__(self):
+ global NA
+ if NA is not None:
+ raise Exception('Cannot create multiple NAType instances')
+ self.init(shared_ptr[CScalar](new CNullScalar()))
- cdef void init(self, DataType type, const shared_ptr[CArray]& sp_array,
- int64_t index):
- self.type = type
- self.index = index
- self._set_array(sp_array)
+ def __init__(self):
+ pass
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
+ def __eq__(self, other):
+ return NA
- def __repr__(self):
- if hasattr(self, 'as_py'):
- return repr(self.as_py())
- else:
- return super(Scalar, self).__repr__()
+ def __hash__(self):
Review comment:
Why do you need to redefine this method?
##########
File path: python/pyarrow/tests/test_scalars.py
##########
@@ -16,427 +16,443 @@
# under the License.
import datetime
+import decimal
import pytest
-import unittest
import numpy as np
import pyarrow as pa
-class TestScalars(unittest.TestCase):
-
- def test_null_singleton(self):
- with pytest.raises(Exception):
- pa.NAType()
+@pytest.mark.parametrize(['value', 'ty', 'klass', 'deprecated'], [
+ (False, None, pa.BooleanScalar, pa.BooleanValue),
+ (True, None, pa.BooleanScalar, pa.BooleanValue),
+ (1, None, pa.Int64Scalar, pa.Int64Value),
+ (-1, None, pa.Int64Scalar, pa.Int64Value),
+ (1, pa.int8(), pa.Int8Scalar, pa.Int8Value),
+ (1, pa.uint8(), pa.UInt8Scalar, pa.UInt8Value),
+ (1, pa.int16(), pa.Int16Scalar, pa.Int16Value),
+ (1, pa.uint16(), pa.UInt16Scalar, pa.UInt16Value),
+ (1, pa.int32(), pa.Int32Scalar, pa.Int32Value),
+ (1, pa.uint32(), pa.UInt32Scalar, pa.UInt32Value),
+ (1, pa.int64(), pa.Int64Scalar, pa.Int64Value),
+ (1, pa.uint64(), pa.UInt64Scalar, pa.UInt64Value),
+ (1.0, None, pa.DoubleScalar, pa.DoubleValue),
+ (np.float16(1.0), pa.float16(), pa.HalfFloatScalar, pa.HalfFloatValue),
+ (1.0, pa.float32(), pa.FloatScalar, pa.FloatValue),
+ ("string", None, pa.StringScalar, pa.StringValue),
+ (b"bytes", None, pa.BinaryScalar, pa.BinaryValue),
+ ([1, 2, 3], None, pa.ListScalar, pa.ListValue),
+ ([1, 2, 3, 4], pa.large_list(pa.int8()), pa.LargeListScalar,
+ pa.LargeListValue),
+ (datetime.date.today(), None, pa.Date32Scalar, pa.Date64Value),
+ (datetime.datetime.now(), None, pa.TimestampScalar, pa.TimestampValue),
+ ({'a': 1, 'b': [1, 2]}, None, pa.StructScalar, pa.StructValue)
+])
+def test_basics(value, ty, klass, deprecated):
+ s = pa.scalar(value, type=ty)
+ assert isinstance(s, klass)
+ assert s == value
+ assert s == s
+ assert s != "else"
+ assert hash(s) == hash(s)
Review comment:
What's the point of this test? This is going to be trivially true.
##########
File path: python/pyarrow/scalar.pxi
##########
@@ -16,1198 +16,745 @@
# under the License.
-_NULL = NA = None
+import collections
cdef class Scalar:
"""
- The base class for all array elements.
+ The base class for scalars.
"""
+ def __init__(self):
+ raise TypeError("Do not call {}'s constructor directly, use "
+ "pa.scalar() instead.".format(self.__class__.__name__))
-cdef class NullType(Scalar):
- """
- Singleton for null array elements.
- """
- # TODO rename this NullValue?
+ cdef void init(self, const shared_ptr[CScalar]& wrapped):
+ self.wrapped = wrapped
- def __cinit__(self):
- global NA
- if NA is not None:
- raise Exception('Cannot create multiple NAType instances')
+ @staticmethod
+ cdef wrap(const shared_ptr[CScalar]& wrapped):
+ cdef:
+ Scalar self
+ Type type_id = wrapped.get().type.get().id()
+
+ if type_id == _Type_NA:
+ return _NULL
+
+ typ = _scalar_classes[type_id]
+ self = typ.__new__(typ)
+ self.init(wrapped)
+
+ return self
+
+ cdef inline shared_ptr[CScalar] unwrap(self) nogil:
+ return self.wrapped
+
+ @property
+ def type(self):
+ return pyarrow_wrap_data_type(self.wrapped.get().type)
- self.type = null()
+ @property
+ def is_valid(self):
+ return self.wrapped.get().is_valid
def __repr__(self):
- return 'NULL'
+ return '<pyarrow.{}: {!r}>'.format(
+ self.__class__.__name__, self.as_py()
+ )
- def as_py(self):
- """
- Return None
- """
- return None
+ def __str__(self):
+ return str(self.as_py())
+
+ def equals(self, Scalar other):
+ return self.wrapped.get().Equals(other.unwrap().get()[0])
def __eq__(self, other):
- return NA
+ try:
+ if not isinstance(other, Scalar):
+ other = scalar(other, type=self.type)
+ return self.equals(other)
+ except (TypeError, ValueError, ArrowInvalid):
+ return NotImplemented
+
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+
+ def as_py(self):
+ raise NotImplementedError()
-_NULL = NA = NullType()
+_NULL = NA = None
-cdef class ArrayValue(Scalar):
+cdef class NullScalar(Scalar):
"""
- The base class for non-null array elements.
+ Concrete class for null scalars.
"""
- def __init__(self):
- raise TypeError("Do not call {}'s constructor directly, use array "
- "subscription instead."
- .format(self.__class__.__name__))
+ def __cinit__(self):
+ global NA
+ if NA is not None:
+ raise Exception('Cannot create multiple NAType instances')
+ self.init(shared_ptr[CScalar](new CNullScalar()))
- cdef void init(self, DataType type, const shared_ptr[CArray]& sp_array,
- int64_t index):
- self.type = type
- self.index = index
- self._set_array(sp_array)
+ def __init__(self):
+ pass
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
+ def __eq__(self, other):
+ return NA
- def __repr__(self):
- if hasattr(self, 'as_py'):
- return repr(self.as_py())
- else:
- return super(Scalar, self).__repr__()
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
- def __str__(self):
- if hasattr(self, 'as_py'):
- return str(self.as_py())
- else:
- return super(Scalar, self).__str__()
+ def as_py(self):
+ """
+ Return this value as a Python None.
+ """
+ return None
- def __eq__(self, other):
- if hasattr(self, 'as_py'):
- if isinstance(other, ArrayValue):
- other = other.as_py()
- return self.as_py() == other
- else:
- raise NotImplementedError(
- "Cannot compare Arrow values that don't support as_py()")
- def __hash__(self):
- return hash(self.as_py())
+_NULL = NA = NullScalar()
-cdef class BooleanValue(ArrayValue):
+cdef class BooleanScalar(Scalar):
"""
- Concrete class for boolean array elements.
+ Concrete class for boolean scalars.
"""
def as_py(self):
"""
Return this value as a Python bool.
"""
- cdef CBooleanArray* ap = <CBooleanArray*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CBooleanScalar* sp = <CBooleanScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int8Value(ArrayValue):
+cdef class UInt8Scalar(Scalar):
"""
- Concrete class for int8 array elements.
+ Concrete class for uint8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt8Array* ap = <CInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt8Scalar* sp = <CUInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt8Value(ArrayValue):
+cdef class Int8Scalar(Scalar):
"""
- Concrete class for uint8 array elements.
+ Concrete class for int8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt8Array* ap = <CUInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt8Scalar* sp = <CInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int16Value(ArrayValue):
+cdef class UInt16Scalar(Scalar):
"""
- Concrete class for int16 array elements.
+ Concrete class for uint16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt16Array* ap = <CInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt16Scalar* sp = <CUInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt16Value(ArrayValue):
+cdef class Int16Scalar(Scalar):
"""
- Concrete class for uint16 array elements.
+ Concrete class for int16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt16Array* ap = <CUInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt16Scalar* sp = <CInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int32Value(ArrayValue):
+cdef class UInt32Scalar(Scalar):
"""
- Concrete class for int32 array elements.
+ Concrete class for uint32 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt32Array* ap = <CInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt32Scalar* sp = <CUInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt32Value(ArrayValue):
+cdef class Int32Scalar(Scalar):
"""
- Concrete class for uint32 array elements.
+ Concrete class for int32 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt32Array* ap = <CUInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt32Scalar* sp = <CInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int64Value(ArrayValue):
+cdef class UInt64Scalar(Scalar):
"""
- Concrete class for int64 array elements.
+ Concrete class for uint64 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt64Array* ap = <CInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt64Scalar* sp = <CUInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt64Value(ArrayValue):
+cdef class Int64Scalar(Scalar):
"""
- Concrete class for uint64 array elements.
+ Concrete class for int64 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt64Array* ap = <CUInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt64Scalar* sp = <CInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date32Value(ArrayValue):
+cdef class HalfFloatScalar(Scalar):
"""
- Concrete class for date32 array elements.
+ Concrete class for float scalars.
"""
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+
+ def __eq__(self, other):
+ if hasattr(self, 'as_py'):
+ if isinstance(other, Scalar):
+ other = other.as_py()
+ return self.as_py() == other
+ else:
+ raise NotImplementedError
+
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python float.
"""
- cdef CDate32Array* ap = <CDate32Array*> self.sp_array.get()
+ cdef CHalfFloatScalar* sp = <CHalfFloatScalar*> self.wrapped.get()
+ return PyHalf_FromHalf(sp.value) if sp.is_valid else None
- # Shift to seconds since epoch
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(days=ap.Value(self.index)))
+
+cdef class FloatScalar(Scalar):
+ """
+ Concrete class for float scalars.
+ """
+
+ def as_py(self):
+ """
+ Return this value as a Python float.
+ """
+ cdef CFloatScalar* sp = <CFloatScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date64Value(ArrayValue):
+cdef class DoubleScalar(Scalar):
"""
- Concrete class for date64 array elements.
+ Concrete class for double scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python float.
"""
- cdef CDate64Array* ap = <CDate64Array*> self.sp_array.get()
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(
- days=ap.Value(self.index) / 86400000))
+ cdef CDoubleScalar* sp = <CDoubleScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Time32Value(ArrayValue):
+cdef class Decimal128Scalar(Scalar):
"""
- Concrete class for time32 array elements.
+ Concrete class for decimal128 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.timedelta instance.
+ Return this value as a Python Decimal.
"""
cdef:
- CTime32Array* ap = <CTime32Array*> self.sp_array.get()
- CTime32Type* dtype = <CTime32Type*> ap.type().get()
-
- if dtype.unit() == TimeUnit_SECOND:
- delta = datetime.timedelta(seconds=ap.Value(self.index))
- return (datetime.datetime(1970, 1, 1) + delta).time()
+ CDecimal128Scalar* sp = <CDecimal128Scalar*> self.wrapped.get()
+ CDecimal128Type* dtype = <CDecimal128Type*> sp.type.get()
+ if sp.is_valid:
+ return _pydecimal.Decimal(
+ frombytes(sp.value.ToString(dtype.scale()))
+ )
else:
- return _box_time_milli(ap.Value(self.index))
+ return None
-cdef class Time64Value(ArrayValue):
+cdef class Date32Scalar(Scalar):
"""
- Concrete class for time64 array elements.
+ Concrete class for date32 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.timedelta instance.
+ Return this value as a Python datetime.datetime instance.
"""
- cdef:
- CTime64Array* ap = <CTime64Array*> self.sp_array.get()
- CTime64Type* dtype = <CTime64Type*> ap.type().get()
+ cdef CDate32Scalar* sp = <CDate32Scalar*> self.wrapped.get()
- cdef int64_t val = ap.Value(self.index)
- if dtype.unit() == TimeUnit_MICRO:
- return _box_time_micro(val)
+ if sp.is_valid:
+ # shift to seconds since epoch
+ return (
+ datetime.date(1970, 1, 1) + datetime.timedelta(days=sp.value)
+ )
else:
- return (datetime.datetime(1970, 1, 1) +
- datetime.timedelta(microseconds=val / 1000)).time()
-
+ return None
-cpdef _box_time_milli(int64_t val):
- delta = datetime.timedelta(milliseconds=val)
- return (datetime.datetime(1970, 1, 1) + delta).time()
+cdef class Date64Scalar(Scalar):
+ """
+ Concrete class for date64 scalars.
+ """
-cpdef _box_time_micro(int64_t val):
- return (datetime.datetime(1970, 1, 1) +
- datetime.timedelta(microseconds=val)).time()
+ def as_py(self):
+ """
+ Return this value as a Python datetime.datetime instance.
+ """
+ cdef CDate64Scalar* sp = <CDate64Scalar*> self.wrapped.get()
+ if sp.is_valid:
+ return (
+ datetime.date(1970, 1, 1) +
+ datetime.timedelta(days=sp.value / 86400000)
+ )
+ else:
+ return None
-cdef dict _DATETIME_CONVERSION_FUNCTIONS = {}
-cdef c_bool _datetime_conversion_initialized = False
+def _datetime_from_int(int64_t value, TimeUnit unit, tzinfo=None):
+ if unit == TimeUnit_SECOND:
+ delta = datetime.timedelta(seconds=value)
+ elif unit == TimeUnit_MILLI:
+ delta = datetime.timedelta(milliseconds=value)
+ elif unit == TimeUnit_MICRO:
+ delta = datetime.timedelta(microseconds=value)
+ else:
+ # TimeUnit_NANO: prefer pandas timestamps if available
+ if _pandas_api.have_pandas:
+ return _pandas_api.pd.Timestamp(value, tz=tzinfo, unit='ns')
+ # otherwise safely truncate to microsecond resolution datetime
+ if value % 1000 != 0:
+ raise ValueError(
+ "Nanosecond resolution temporal type {} is not safely "
+ "convertible to microseconds to convert to datetime.datetime. "
+ "Install pandas to return as Timestamp with nanosecond "
+ "support or access the .value attribute.".format(value)
+ )
+ delta = datetime.timedelta(microseconds=value // 1000)
-cdef _add_micros_maybe_localize(dt, micros, tzinfo):
- import pytz
- dt = dt.replace(microsecond=micros)
+ dt = datetime.datetime(1970, 1, 1) + delta
+ # adjust timezone if set to the datatype
if tzinfo is not None:
- if not isinstance(tzinfo, datetime.tzinfo):
- tzinfo = string_to_tzinfo(tzinfo)
dt = tzinfo.fromutc(dt)
- return dt
-
-
-cdef _datetime_from_seconds(int64_t v):
- return datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=v)
-
-
-def _nanoseconds_to_datetime_safe(v, tzinfo):
- if v % 1000 != 0:
- raise ValueError("Nanosecond timestamp {} is not safely convertible "
- " to microseconds to convert to datetime.datetime."
- " Install pandas to return as Timestamp with "
- " nanosecond support or access the .value attribute.")
- v = v // 1000
- micros = v % 1_000_000
-
- dt = _datetime_from_seconds(v // 1_000_000)
- return _add_micros_maybe_localize(dt, micros, tzinfo)
-
-
-def _microseconds_to_datetime(v, tzinfo):
- micros = v % 1_000_000
- dt = _datetime_from_seconds(v // 1_000_000)
- return _add_micros_maybe_localize(dt, micros, tzinfo)
+ return dt
-def _millis_to_datetime(v, tzinfo):
- millis = v % 1_000
- dt = _datetime_from_seconds(v // 1000)
- return _add_micros_maybe_localize(dt, millis * 1000, tzinfo)
+cdef class Time32Scalar(Scalar):
+ """
+ Concrete class for time32 scalars.
+ """
-def _seconds_to_datetime(v, tzinfo):
- dt = _datetime_from_seconds(v)
- return _add_micros_maybe_localize(dt, 0, tzinfo)
+ def as_py(self):
+ """
+ Return this value as a Python datetime.timedelta instance.
+ """
+ cdef:
+ CTime32Scalar* sp = <CTime32Scalar*> self.wrapped.get()
+ CTime32Type* dtype = <CTime32Type*> sp.type.get()
+ if sp.is_valid:
+ return _datetime_from_int(sp.value, unit=dtype.unit()).time()
+ else:
+ return None
-def _datetime_conversion_functions():
- global _datetime_conversion_initialized
- if _datetime_conversion_initialized:
- return _DATETIME_CONVERSION_FUNCTIONS
- _DATETIME_CONVERSION_FUNCTIONS.update({
- TimeUnit_SECOND: _seconds_to_datetime,
- TimeUnit_MILLI: _millis_to_datetime,
- TimeUnit_MICRO: _microseconds_to_datetime,
- TimeUnit_NANO: _nanoseconds_to_datetime_safe
- })
+cdef class Time64Scalar(Scalar):
+ """
+ Concrete class for time64 scalars.
+ """
- try:
- import pandas as pd
- _DATETIME_CONVERSION_FUNCTIONS[TimeUnit_NANO] = (
- lambda x, tzinfo: pd.Timestamp(
- x, tz=tzinfo, unit='ns',
- )
- )
- except ImportError:
- pass
+ def as_py(self):
+ """
+ Return this value as a Python datetime.timedelta instance.
+ """
+ cdef:
+ CTime64Scalar* sp = <CTime64Scalar*> self.wrapped.get()
+ CTime64Type* dtype = <CTime64Type*> sp.type.get()
- _datetime_conversion_initialized = True
- return _DATETIME_CONVERSION_FUNCTIONS
+ if sp.is_valid:
+ return _datetime_from_int(sp.value, unit=dtype.unit()).time()
+ else:
+ return None
-cdef class TimestampValue(ArrayValue):
+cdef class TimestampScalar(Scalar):
"""
- Concrete class for timestamp array elements.
+ Concrete class for timestamp scalars.
"""
@property
def value(self):
- cdef CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
- cdef CTimestampType* dtype = <CTimestampType*> ap.type().get()
- return ap.Value(self.index)
+ cdef CTimestampScalar* sp = <CTimestampScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
def as_py(self):
"""
Return this value as a Pandas Timestamp instance (if available),
otherwise as a Python datetime.timedelta instance.
"""
- cdef CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
- cdef CTimestampType* dtype = <CTimestampType*> ap.type().get()
+ cdef:
+ CTimestampScalar* sp = <CTimestampScalar*> self.wrapped.get()
+ CTimestampType* dtype = <CTimestampType*> sp.type.get()
- value = self.value
+ if not sp.is_valid:
+ return None
if not dtype.timezone().empty():
tzinfo = string_to_tzinfo(frombytes(dtype.timezone()))
+ if not isinstance(tzinfo, datetime.tzinfo):
+ tzinfo = string_to_tzinfo(tzinfo)
else:
tzinfo = None
- try:
- converter = _datetime_conversion_functions()[dtype.unit()]
- except KeyError:
- raise ValueError(
- 'Cannot convert nanosecond timestamps without pandas'
- )
- return converter(value, tzinfo=tzinfo)
-
-
-cdef dict _TIMEDELTA_CONVERSION_FUNCTIONS = {}
-
-
-def _nanoseconds_to_timedelta_safe(v):
- if v % 1000 != 0:
- raise ValueError(
- "Nanosecond duration {} is not safely convertible to microseconds "
- "to convert to datetime.timedelta. Install pandas to return as "
- "Timedelta with nanosecond support or access the .value "
- "attribute.".format(v))
- micros = v // 1000
-
- return datetime.timedelta(microseconds=micros)
-
-
-def _timedelta_conversion_functions():
- if _TIMEDELTA_CONVERSION_FUNCTIONS:
- return _TIMEDELTA_CONVERSION_FUNCTIONS
-
- _TIMEDELTA_CONVERSION_FUNCTIONS.update({
- TimeUnit_SECOND: lambda v: datetime.timedelta(seconds=v),
- TimeUnit_MILLI: lambda v: datetime.timedelta(milliseconds=v),
- TimeUnit_MICRO: lambda v: datetime.timedelta(microseconds=v),
- TimeUnit_NANO: _nanoseconds_to_timedelta_safe
- })
-
- try:
- import pandas as pd
- _TIMEDELTA_CONVERSION_FUNCTIONS[TimeUnit_NANO] = (
- lambda v: pd.Timedelta(v, unit='ns')
- )
- except ImportError:
- pass
+ return _datetime_from_int(sp.value, unit=dtype.unit(), tzinfo=tzinfo)
- return _TIMEDELTA_CONVERSION_FUNCTIONS
-
-cdef class DurationValue(ArrayValue):
+cdef class DurationScalar(Scalar):
"""
- Concrete class for duration array elements.
+ Concrete class for duration scalars.
"""
@property
def value(self):
- cdef CDurationArray* ap = <CDurationArray*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CDurationScalar* sp = <CDurationScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
def as_py(self):
"""
Return this value as a Pandas Timestamp instance (if available),
otherwise as a Python datetime.timedelta instance.
"""
- cdef CDurationArray* ap = <CDurationArray*> self.sp_array.get()
- cdef CDurationType* dtype = <CDurationType*> ap.type().get()
+ cdef:
+ CDurationScalar* sp = <CDurationScalar*> self.wrapped.get()
+ CDurationType* dtype = <CDurationType*> sp.type.get()
+ TimeUnit unit = dtype.unit()
+
+ if not sp.is_valid:
+ return None
- cdef int64_t value = ap.Value(self.index)
- converter = _timedelta_conversion_functions()[dtype.unit()]
- return converter(value)
+ if unit == TimeUnit_SECOND:
+ return datetime.timedelta(seconds=sp.value)
+ elif unit == TimeUnit_MILLI:
+ return datetime.timedelta(milliseconds=sp.value)
+ elif unit == TimeUnit_MICRO:
+ return datetime.timedelta(microseconds=sp.value)
+ else:
+ # TimeUnit_NANO: prefer pandas timestamps if available
+ if _pandas_api.have_pandas:
+ return _pandas_api.pd.Timedelta(sp.value, unit='ns')
+ # otherwise safely truncate to microsecond resolution timedelta
+ if sp.value % 1000 != 0:
+ raise ValueError(
+ "Nanosecond duration {} is not safely convertible to "
+ "microseconds to convert to datetime.timedelta. Install "
+ "pandas to return as Timedelta with nanosecond support or "
+ "access the .value attribute.".format(sp.value)
+ )
+ return datetime.timedelta(microseconds=sp.value // 1000)
-cdef class HalfFloatValue(ArrayValue):
+cdef class BinaryScalar(Scalar):
"""
- Concrete class for float16 array elements.
+ Concrete class for binary-like scalars.
"""
- def as_py(self):
+ def as_buffer(self):
"""
- Return this value as a Python float.
+ Return a view over this value as a Buffer object.
"""
- cdef CHalfFloatArray* ap = <CHalfFloatArray*> self.sp_array.get()
- return PyHalf_FromHalf(ap.Value(self.index))
-
-
-cdef class FloatValue(ArrayValue):
- """
- Concrete class for float32 array elements.
- """
+ cdef CBinaryScalar* sp = <CBinaryScalar*> self.wrapped.get()
+ return pyarrow_wrap_buffer(sp.value) if sp.is_valid else None
def as_py(self):
"""
- Return this value as a Python float.
+ Return this value as a Python bytes.
"""
- cdef CFloatArray* ap = <CFloatArray*> self.sp_array.get()
- return ap.Value(self.index)
+ buffer = self.as_buffer()
+ if buffer is not None:
+ return self.as_buffer().to_pybytes()
+ else:
+ return None
-cdef class DoubleValue(ArrayValue):
- """
- Concrete class for float64 array elements.
- """
+cdef class LargeBinaryScalar(BinaryScalar):
+ pass
- def as_py(self):
- """
- Return this value as a Python float.
- """
- cdef CDoubleArray* ap = <CDoubleArray*> self.sp_array.get()
- return ap.Value(self.index)
+
+cdef class FixedSizeBinaryScalar(BinaryScalar):
+ pass
-cdef class DecimalValue(ArrayValue):
+cdef class StringScalar(BinaryScalar):
"""
- Concrete class for decimal128 array elements.
+ Concrete class for string-like (utf8) scalars.
"""
def as_py(self):
"""
- Return this value as a Python Decimal.
+ Return this value as a Python string.
"""
- cdef:
- CDecimal128Array* ap = <CDecimal128Array*> self.sp_array.get()
- c_string s = ap.FormatValue(self.index)
- return _pydecimal.Decimal(s.decode('utf8'))
+ buffer = self.as_buffer()
+ if buffer is not None:
+ return frombytes(self.as_buffer().to_pybytes())
Review comment:
`str(buffer, 'utf8')` should work, since `buffer` supports the buffer protocol:
```python
>>> buf = pa.py_buffer(b'abc')
>>> str(buf, 'utf8')
'abc'
```
##########
File path: python/pyarrow/scalar.pxi
##########
@@ -16,1198 +16,745 @@
# under the License.
-_NULL = NA = None
+import collections
cdef class Scalar:
"""
- The base class for all array elements.
+ The base class for scalars.
"""
+ def __init__(self):
+ raise TypeError("Do not call {}'s constructor directly, use "
+ "pa.scalar() instead.".format(self.__class__.__name__))
-cdef class NullType(Scalar):
- """
- Singleton for null array elements.
- """
- # TODO rename this NullValue?
+ cdef void init(self, const shared_ptr[CScalar]& wrapped):
+ self.wrapped = wrapped
- def __cinit__(self):
- global NA
- if NA is not None:
- raise Exception('Cannot create multiple NAType instances')
+ @staticmethod
+ cdef wrap(const shared_ptr[CScalar]& wrapped):
+ cdef:
+ Scalar self
+ Type type_id = wrapped.get().type.get().id()
+
+ if type_id == _Type_NA:
+ return _NULL
+
+ typ = _scalar_classes[type_id]
+ self = typ.__new__(typ)
+ self.init(wrapped)
+
+ return self
+
+ cdef inline shared_ptr[CScalar] unwrap(self) nogil:
+ return self.wrapped
+
+ @property
+ def type(self):
+ return pyarrow_wrap_data_type(self.wrapped.get().type)
- self.type = null()
+ @property
+ def is_valid(self):
+ return self.wrapped.get().is_valid
def __repr__(self):
- return 'NULL'
+ return '<pyarrow.{}: {!r}>'.format(
+ self.__class__.__name__, self.as_py()
+ )
- def as_py(self):
- """
- Return None
- """
- return None
+ def __str__(self):
+ return str(self.as_py())
+
+ def equals(self, Scalar other):
+ return self.wrapped.get().Equals(other.unwrap().get()[0])
def __eq__(self, other):
- return NA
+ try:
+ if not isinstance(other, Scalar):
+ other = scalar(other, type=self.type)
+ return self.equals(other)
+ except (TypeError, ValueError, ArrowInvalid):
+ return NotImplemented
+
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+
+ def as_py(self):
+ raise NotImplementedError()
-_NULL = NA = NullType()
+_NULL = NA = None
-cdef class ArrayValue(Scalar):
+cdef class NullScalar(Scalar):
"""
- The base class for non-null array elements.
+ Concrete class for null scalars.
"""
- def __init__(self):
- raise TypeError("Do not call {}'s constructor directly, use array "
- "subscription instead."
- .format(self.__class__.__name__))
+ def __cinit__(self):
+ global NA
+ if NA is not None:
+ raise Exception('Cannot create multiple NAType instances')
+ self.init(shared_ptr[CScalar](new CNullScalar()))
- cdef void init(self, DataType type, const shared_ptr[CArray]& sp_array,
- int64_t index):
- self.type = type
- self.index = index
- self._set_array(sp_array)
+ def __init__(self):
+ pass
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
+ def __eq__(self, other):
+ return NA
- def __repr__(self):
- if hasattr(self, 'as_py'):
- return repr(self.as_py())
- else:
- return super(Scalar, self).__repr__()
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
- def __str__(self):
- if hasattr(self, 'as_py'):
- return str(self.as_py())
- else:
- return super(Scalar, self).__str__()
+ def as_py(self):
+ """
+ Return this value as a Python None.
+ """
+ return None
- def __eq__(self, other):
- if hasattr(self, 'as_py'):
- if isinstance(other, ArrayValue):
- other = other.as_py()
- return self.as_py() == other
- else:
- raise NotImplementedError(
- "Cannot compare Arrow values that don't support as_py()")
- def __hash__(self):
- return hash(self.as_py())
+_NULL = NA = NullScalar()
-cdef class BooleanValue(ArrayValue):
+cdef class BooleanScalar(Scalar):
"""
- Concrete class for boolean array elements.
+ Concrete class for boolean scalars.
"""
def as_py(self):
"""
Return this value as a Python bool.
"""
- cdef CBooleanArray* ap = <CBooleanArray*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CBooleanScalar* sp = <CBooleanScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int8Value(ArrayValue):
+cdef class UInt8Scalar(Scalar):
"""
- Concrete class for int8 array elements.
+ Concrete class for uint8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt8Array* ap = <CInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt8Scalar* sp = <CUInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt8Value(ArrayValue):
+cdef class Int8Scalar(Scalar):
"""
- Concrete class for uint8 array elements.
+ Concrete class for int8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt8Array* ap = <CUInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt8Scalar* sp = <CInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int16Value(ArrayValue):
+cdef class UInt16Scalar(Scalar):
"""
- Concrete class for int16 array elements.
+ Concrete class for uint16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt16Array* ap = <CInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt16Scalar* sp = <CUInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt16Value(ArrayValue):
+cdef class Int16Scalar(Scalar):
"""
- Concrete class for uint16 array elements.
+ Concrete class for int16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt16Array* ap = <CUInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt16Scalar* sp = <CInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int32Value(ArrayValue):
+cdef class UInt32Scalar(Scalar):
"""
- Concrete class for int32 array elements.
+ Concrete class for uint32 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt32Array* ap = <CInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt32Scalar* sp = <CUInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt32Value(ArrayValue):
+cdef class Int32Scalar(Scalar):
"""
- Concrete class for uint32 array elements.
+ Concrete class for int32 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt32Array* ap = <CUInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt32Scalar* sp = <CInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int64Value(ArrayValue):
+cdef class UInt64Scalar(Scalar):
"""
- Concrete class for int64 array elements.
+ Concrete class for uint64 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt64Array* ap = <CInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt64Scalar* sp = <CUInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt64Value(ArrayValue):
+cdef class Int64Scalar(Scalar):
"""
- Concrete class for uint64 array elements.
+ Concrete class for int64 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt64Array* ap = <CUInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt64Scalar* sp = <CInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date32Value(ArrayValue):
+cdef class HalfFloatScalar(Scalar):
"""
- Concrete class for date32 array elements.
+ Concrete class for float scalars.
"""
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+
+ def __eq__(self, other):
+ if hasattr(self, 'as_py'):
+ if isinstance(other, Scalar):
+ other = other.as_py()
+ return self.as_py() == other
+ else:
+ raise NotImplementedError
+
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python float.
"""
- cdef CDate32Array* ap = <CDate32Array*> self.sp_array.get()
+ cdef CHalfFloatScalar* sp = <CHalfFloatScalar*> self.wrapped.get()
+ return PyHalf_FromHalf(sp.value) if sp.is_valid else None
- # Shift to seconds since epoch
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(days=ap.Value(self.index)))
+
+cdef class FloatScalar(Scalar):
+ """
+ Concrete class for float scalars.
+ """
+
+ def as_py(self):
+ """
+ Return this value as a Python float.
+ """
+ cdef CFloatScalar* sp = <CFloatScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date64Value(ArrayValue):
+cdef class DoubleScalar(Scalar):
"""
- Concrete class for date64 array elements.
+ Concrete class for double scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python float.
"""
- cdef CDate64Array* ap = <CDate64Array*> self.sp_array.get()
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(
- days=ap.Value(self.index) / 86400000))
+ cdef CDoubleScalar* sp = <CDoubleScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Time32Value(ArrayValue):
+cdef class Decimal128Scalar(Scalar):
"""
- Concrete class for time32 array elements.
+ Concrete class for decimal128 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.timedelta instance.
+ Return this value as a Python Decimal.
"""
cdef:
- CTime32Array* ap = <CTime32Array*> self.sp_array.get()
- CTime32Type* dtype = <CTime32Type*> ap.type().get()
-
- if dtype.unit() == TimeUnit_SECOND:
- delta = datetime.timedelta(seconds=ap.Value(self.index))
- return (datetime.datetime(1970, 1, 1) + delta).time()
+ CDecimal128Scalar* sp = <CDecimal128Scalar*> self.wrapped.get()
+ CDecimal128Type* dtype = <CDecimal128Type*> sp.type.get()
+ if sp.is_valid:
+ return _pydecimal.Decimal(
+ frombytes(sp.value.ToString(dtype.scale()))
+ )
else:
- return _box_time_milli(ap.Value(self.index))
+ return None
-cdef class Time64Value(ArrayValue):
+cdef class Date32Scalar(Scalar):
"""
- Concrete class for time64 array elements.
+ Concrete class for date32 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.timedelta instance.
+ Return this value as a Python datetime.datetime instance.
"""
- cdef:
- CTime64Array* ap = <CTime64Array*> self.sp_array.get()
- CTime64Type* dtype = <CTime64Type*> ap.type().get()
+ cdef CDate32Scalar* sp = <CDate32Scalar*> self.wrapped.get()
- cdef int64_t val = ap.Value(self.index)
- if dtype.unit() == TimeUnit_MICRO:
- return _box_time_micro(val)
+ if sp.is_valid:
+ # shift to seconds since epoch
+ return (
+ datetime.date(1970, 1, 1) + datetime.timedelta(days=sp.value)
+ )
else:
- return (datetime.datetime(1970, 1, 1) +
- datetime.timedelta(microseconds=val / 1000)).time()
-
+ return None
-cpdef _box_time_milli(int64_t val):
- delta = datetime.timedelta(milliseconds=val)
- return (datetime.datetime(1970, 1, 1) + delta).time()
+cdef class Date64Scalar(Scalar):
+ """
+ Concrete class for date64 scalars.
+ """
-cpdef _box_time_micro(int64_t val):
- return (datetime.datetime(1970, 1, 1) +
- datetime.timedelta(microseconds=val)).time()
+ def as_py(self):
+ """
+ Return this value as a Python datetime.datetime instance.
+ """
+ cdef CDate64Scalar* sp = <CDate64Scalar*> self.wrapped.get()
+ if sp.is_valid:
+ return (
+ datetime.date(1970, 1, 1) +
+ datetime.timedelta(days=sp.value / 86400000)
+ )
+ else:
+ return None
-cdef dict _DATETIME_CONVERSION_FUNCTIONS = {}
-cdef c_bool _datetime_conversion_initialized = False
+def _datetime_from_int(int64_t value, TimeUnit unit, tzinfo=None):
+ if unit == TimeUnit_SECOND:
+ delta = datetime.timedelta(seconds=value)
+ elif unit == TimeUnit_MILLI:
+ delta = datetime.timedelta(milliseconds=value)
+ elif unit == TimeUnit_MICRO:
+ delta = datetime.timedelta(microseconds=value)
+ else:
+ # TimeUnit_NANO: prefer pandas timestamps if available
+ if _pandas_api.have_pandas:
+ return _pandas_api.pd.Timestamp(value, tz=tzinfo, unit='ns')
+ # otherwise safely truncate to microsecond resolution datetime
+ if value % 1000 != 0:
+ raise ValueError(
+ "Nanosecond resolution temporal type {} is not safely "
+ "convertible to microseconds to convert to datetime.datetime. "
+ "Install pandas to return as Timestamp with nanosecond "
+ "support or access the .value attribute.".format(value)
+ )
+ delta = datetime.timedelta(microseconds=value // 1000)
-cdef _add_micros_maybe_localize(dt, micros, tzinfo):
- import pytz
- dt = dt.replace(microsecond=micros)
+ dt = datetime.datetime(1970, 1, 1) + delta
+ # adjust timezone if set to the datatype
if tzinfo is not None:
- if not isinstance(tzinfo, datetime.tzinfo):
- tzinfo = string_to_tzinfo(tzinfo)
dt = tzinfo.fromutc(dt)
- return dt
-
-
-cdef _datetime_from_seconds(int64_t v):
- return datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=v)
-
-
-def _nanoseconds_to_datetime_safe(v, tzinfo):
- if v % 1000 != 0:
- raise ValueError("Nanosecond timestamp {} is not safely convertible "
- " to microseconds to convert to datetime.datetime."
- " Install pandas to return as Timestamp with "
- " nanosecond support or access the .value attribute.")
- v = v // 1000
- micros = v % 1_000_000
-
- dt = _datetime_from_seconds(v // 1_000_000)
- return _add_micros_maybe_localize(dt, micros, tzinfo)
-
-
-def _microseconds_to_datetime(v, tzinfo):
- micros = v % 1_000_000
- dt = _datetime_from_seconds(v // 1_000_000)
- return _add_micros_maybe_localize(dt, micros, tzinfo)
+ return dt
-def _millis_to_datetime(v, tzinfo):
- millis = v % 1_000
- dt = _datetime_from_seconds(v // 1000)
- return _add_micros_maybe_localize(dt, millis * 1000, tzinfo)
+cdef class Time32Scalar(Scalar):
+ """
+ Concrete class for time32 scalars.
+ """
-def _seconds_to_datetime(v, tzinfo):
- dt = _datetime_from_seconds(v)
- return _add_micros_maybe_localize(dt, 0, tzinfo)
+ def as_py(self):
+ """
+ Return this value as a Python datetime.timedelta instance.
+ """
+ cdef:
+ CTime32Scalar* sp = <CTime32Scalar*> self.wrapped.get()
+ CTime32Type* dtype = <CTime32Type*> sp.type.get()
+ if sp.is_valid:
+ return _datetime_from_int(sp.value, unit=dtype.unit()).time()
+ else:
+ return None
-def _datetime_conversion_functions():
- global _datetime_conversion_initialized
- if _datetime_conversion_initialized:
- return _DATETIME_CONVERSION_FUNCTIONS
- _DATETIME_CONVERSION_FUNCTIONS.update({
- TimeUnit_SECOND: _seconds_to_datetime,
- TimeUnit_MILLI: _millis_to_datetime,
- TimeUnit_MICRO: _microseconds_to_datetime,
- TimeUnit_NANO: _nanoseconds_to_datetime_safe
- })
+cdef class Time64Scalar(Scalar):
+ """
+ Concrete class for time64 scalars.
+ """
- try:
- import pandas as pd
- _DATETIME_CONVERSION_FUNCTIONS[TimeUnit_NANO] = (
- lambda x, tzinfo: pd.Timestamp(
- x, tz=tzinfo, unit='ns',
- )
- )
- except ImportError:
- pass
+ def as_py(self):
+ """
+ Return this value as a Python datetime.timedelta instance.
+ """
+ cdef:
+ CTime64Scalar* sp = <CTime64Scalar*> self.wrapped.get()
+ CTime64Type* dtype = <CTime64Type*> sp.type.get()
- _datetime_conversion_initialized = True
- return _DATETIME_CONVERSION_FUNCTIONS
+ if sp.is_valid:
+ return _datetime_from_int(sp.value, unit=dtype.unit()).time()
+ else:
+ return None
-cdef class TimestampValue(ArrayValue):
+cdef class TimestampScalar(Scalar):
"""
- Concrete class for timestamp array elements.
+ Concrete class for timestamp scalars.
"""
@property
def value(self):
- cdef CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
- cdef CTimestampType* dtype = <CTimestampType*> ap.type().get()
- return ap.Value(self.index)
+ cdef CTimestampScalar* sp = <CTimestampScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
def as_py(self):
"""
Return this value as a Pandas Timestamp instance (if available),
otherwise as a Python datetime.timedelta instance.
"""
- cdef CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
- cdef CTimestampType* dtype = <CTimestampType*> ap.type().get()
+ cdef:
+ CTimestampScalar* sp = <CTimestampScalar*> self.wrapped.get()
+ CTimestampType* dtype = <CTimestampType*> sp.type.get()
- value = self.value
+ if not sp.is_valid:
+ return None
if not dtype.timezone().empty():
tzinfo = string_to_tzinfo(frombytes(dtype.timezone()))
+ if not isinstance(tzinfo, datetime.tzinfo):
+ tzinfo = string_to_tzinfo(tzinfo)
else:
tzinfo = None
- try:
- converter = _datetime_conversion_functions()[dtype.unit()]
- except KeyError:
- raise ValueError(
- 'Cannot convert nanosecond timestamps without pandas'
- )
- return converter(value, tzinfo=tzinfo)
-
-
-cdef dict _TIMEDELTA_CONVERSION_FUNCTIONS = {}
-
-
-def _nanoseconds_to_timedelta_safe(v):
- if v % 1000 != 0:
- raise ValueError(
- "Nanosecond duration {} is not safely convertible to microseconds "
- "to convert to datetime.timedelta. Install pandas to return as "
- "Timedelta with nanosecond support or access the .value "
- "attribute.".format(v))
- micros = v // 1000
-
- return datetime.timedelta(microseconds=micros)
-
-
-def _timedelta_conversion_functions():
- if _TIMEDELTA_CONVERSION_FUNCTIONS:
- return _TIMEDELTA_CONVERSION_FUNCTIONS
-
- _TIMEDELTA_CONVERSION_FUNCTIONS.update({
- TimeUnit_SECOND: lambda v: datetime.timedelta(seconds=v),
- TimeUnit_MILLI: lambda v: datetime.timedelta(milliseconds=v),
- TimeUnit_MICRO: lambda v: datetime.timedelta(microseconds=v),
- TimeUnit_NANO: _nanoseconds_to_timedelta_safe
- })
-
- try:
- import pandas as pd
- _TIMEDELTA_CONVERSION_FUNCTIONS[TimeUnit_NANO] = (
- lambda v: pd.Timedelta(v, unit='ns')
- )
- except ImportError:
- pass
+ return _datetime_from_int(sp.value, unit=dtype.unit(), tzinfo=tzinfo)
- return _TIMEDELTA_CONVERSION_FUNCTIONS
-
-cdef class DurationValue(ArrayValue):
+cdef class DurationScalar(Scalar):
"""
- Concrete class for duration array elements.
+ Concrete class for duration scalars.
"""
@property
def value(self):
- cdef CDurationArray* ap = <CDurationArray*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CDurationScalar* sp = <CDurationScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
def as_py(self):
"""
Return this value as a Pandas Timestamp instance (if available),
otherwise as a Python datetime.timedelta instance.
"""
- cdef CDurationArray* ap = <CDurationArray*> self.sp_array.get()
- cdef CDurationType* dtype = <CDurationType*> ap.type().get()
+ cdef:
+ CDurationScalar* sp = <CDurationScalar*> self.wrapped.get()
+ CDurationType* dtype = <CDurationType*> sp.type.get()
+ TimeUnit unit = dtype.unit()
+
+ if not sp.is_valid:
+ return None
- cdef int64_t value = ap.Value(self.index)
- converter = _timedelta_conversion_functions()[dtype.unit()]
- return converter(value)
+ if unit == TimeUnit_SECOND:
+ return datetime.timedelta(seconds=sp.value)
+ elif unit == TimeUnit_MILLI:
+ return datetime.timedelta(milliseconds=sp.value)
+ elif unit == TimeUnit_MICRO:
+ return datetime.timedelta(microseconds=sp.value)
+ else:
+ # TimeUnit_NANO: prefer pandas timestamps if available
+ if _pandas_api.have_pandas:
+ return _pandas_api.pd.Timedelta(sp.value, unit='ns')
+ # otherwise safely truncate to microsecond resolution timedelta
+ if sp.value % 1000 != 0:
+ raise ValueError(
+ "Nanosecond duration {} is not safely convertible to "
+ "microseconds to convert to datetime.timedelta. Install "
+ "pandas to return as Timedelta with nanosecond support or "
+ "access the .value attribute.".format(sp.value)
+ )
+ return datetime.timedelta(microseconds=sp.value // 1000)
-cdef class HalfFloatValue(ArrayValue):
+cdef class BinaryScalar(Scalar):
"""
- Concrete class for float16 array elements.
+ Concrete class for binary-like scalars.
"""
- def as_py(self):
+ def as_buffer(self):
"""
- Return this value as a Python float.
+ Return a view over this value as a Buffer object.
"""
- cdef CHalfFloatArray* ap = <CHalfFloatArray*> self.sp_array.get()
- return PyHalf_FromHalf(ap.Value(self.index))
-
-
-cdef class FloatValue(ArrayValue):
- """
- Concrete class for float32 array elements.
- """
+ cdef CBinaryScalar* sp = <CBinaryScalar*> self.wrapped.get()
+ return pyarrow_wrap_buffer(sp.value) if sp.is_valid else None
def as_py(self):
"""
- Return this value as a Python float.
+ Return this value as a Python bytes.
"""
- cdef CFloatArray* ap = <CFloatArray*> self.sp_array.get()
- return ap.Value(self.index)
+ buffer = self.as_buffer()
+ if buffer is not None:
+ return self.as_buffer().to_pybytes()
+ else:
+ return None
-cdef class DoubleValue(ArrayValue):
- """
- Concrete class for float64 array elements.
- """
+cdef class LargeBinaryScalar(BinaryScalar):
+ pass
- def as_py(self):
- """
- Return this value as a Python float.
- """
- cdef CDoubleArray* ap = <CDoubleArray*> self.sp_array.get()
- return ap.Value(self.index)
+
+cdef class FixedSizeBinaryScalar(BinaryScalar):
+ pass
-cdef class DecimalValue(ArrayValue):
+cdef class StringScalar(BinaryScalar):
"""
- Concrete class for decimal128 array elements.
+ Concrete class for string-like (utf8) scalars.
"""
def as_py(self):
"""
- Return this value as a Python Decimal.
+ Return this value as a Python string.
"""
- cdef:
- CDecimal128Array* ap = <CDecimal128Array*> self.sp_array.get()
- c_string s = ap.FormatValue(self.index)
- return _pydecimal.Decimal(s.decode('utf8'))
+ buffer = self.as_buffer()
+ if buffer is not None:
+ return frombytes(self.as_buffer().to_pybytes())
+ else:
+ return None
+
+cdef class LargeStringScalar(StringScalar):
+ pass
-cdef class StringValue(ArrayValue):
+
+cdef class ListScalar(Scalar):
"""
- Concrete class for string (utf8) array elements.
+ Concrete class for list-like scalars.
"""
- def as_py(self):
+ @property
+ def values(self):
+ cdef CListScalar* sp = <CListScalar*> self.wrapped.get()
+ if sp.is_valid:
+ return pyarrow_wrap_array(sp.value)
+ else:
+ return None
+
+ def __len__(self):
"""
- Return this value as a Python unicode string.
+ Return the number of values.
"""
- cdef CStringArray* ap = <CStringArray*> self.sp_array.get()
- return ap.GetString(self.index).decode('utf-8')
+ return len(self.values)
- def as_buffer(self):
+ def __getitem__(self, i):
"""
- Return a view over this value as a Buffer object.
+ Return the value at the given index.
"""
- cdef:
- CStringArray* ap = <CStringArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
-
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
-
+ return self.values[_normalize_index(i, len(self))]
-cdef class LargeStringValue(ArrayValue):
- """
- Concrete class for large string (utf8) array elements.
- """
-
- def as_py(self):
+ def __iter__(self):
"""
- Return this value as a Python unicode string.
+ Iterate over this element's values.
"""
- cdef CLargeStringArray* ap = <CLargeStringArray*> self.sp_array.get()
- return ap.GetString(self.index).decode('utf-8')
+ return iter(self.values)
- def as_buffer(self):
+ def as_py(self):
"""
- Return a view over this value as a Buffer object.
+ Return this value as a Python list.
"""
- cdef:
- CLargeStringArray* ap = <CLargeStringArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
+ arr = self.values
+ return None if arr is None else arr.to_pylist()
+
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
+cdef class FixedSizeListScalar(ListScalar):
+ pass
-cdef class BinaryValue(ArrayValue):
+cdef class LargeListScalar(ListScalar):
+ pass
+
+
+cdef class StructScalar(Scalar, collections.abc.Mapping):
"""
- Concrete class for variable-sized binary array elements.
+ Concrete class for struct scalars.
"""
- def as_py(self):
- """
- Return this value as a Python bytes object.
- """
+ def __len__(self):
+ cdef CStructScalar* sp = <CStructScalar*> self.wrapped.get()
+ return sp.value.size()
+
+ def __iter__(self):
cdef:
- const uint8_t* ptr
- int32_t length
- CBinaryArray* ap = <CBinaryArray*> self.sp_array.get()
+ CStructScalar* sp = <CStructScalar*> self.wrapped.get()
+ CStructType* dtype = <CStructType*> sp.type.get()
+ vector[shared_ptr[CField]] fields = dtype.fields()
- ptr = ap.GetValue(self.index, &length)
- return cp.PyBytes_FromStringAndSize(<const char*>(ptr), length)
+ if sp.is_valid:
+ for i in range(dtype.num_fields()):
+ yield frombytes(fields[i].get().name())
- def as_buffer(self):
+ def __contains__(self, key):
+ try:
+ self[key]
+ except IndexError:
+ return False
+ else:
+ return True
+
+ def __getitem__(self, key):
"""
- Return a view over this value as a Buffer object.
+ Return the child value for the given field.
+
+ Parameters
+ ----------
+ index : Union[int, str]
+ Index / position or name of the field.
+
+ Returns
+ -------
+ result : Scalar
"""
cdef:
- CBinaryArray* ap = <CBinaryArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
+ CFieldRef ref
+ CStructScalar* sp = <CStructScalar*> self.wrapped.get()
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
+ if isinstance(key, (bytes, str)):
+ ref = CFieldRef(<c_string> tobytes(key))
+ elif isinstance(key, int):
+ ref = CFieldRef(<int> key)
+ else:
+ raise TypeError('Expected integer or string index')
-
-cdef class LargeBinaryValue(ArrayValue):
- """
- Concrete class for large variable-sized binary array elements.
- """
-
- def as_py(self):
- """
- Return this value as a Python bytes object.
- """
- cdef:
- const uint8_t* ptr
- int64_t length
- CLargeBinaryArray* ap = <CLargeBinaryArray*> self.sp_array.get()
-
- ptr = ap.GetValue(self.index, &length)
- return cp.PyBytes_FromStringAndSize(<const char*>(ptr), length)
-
- def as_buffer(self):
- """
- Return a view over this value as a Buffer object.
- """
- cdef:
- CLargeBinaryArray* ap = <CLargeBinaryArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
-
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
-
-
-cdef class ListValue(ArrayValue):
- """
- Concrete class for list array elements.
- """
-
- def __len__(self):
- """
- Return the number of values.
- """
- return self.length()
-
- def __getitem__(self, i):
- """
- Return the value at the given index.
- """
- return self.getitem(_normalize_index(i, self.length()))
-
- def __iter__(self):
- """
- Iterate over this element's values.
- """
- for i in range(len(self)):
- yield self.getitem(i)
- raise StopIteration
-
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CListArray*> sp_array.get()
- self.value_type = pyarrow_wrap_data_type(self.ap.value_type())
-
- cdef getitem(self, int64_t i):
- cdef int64_t j = self.ap.value_offset(self.index) + i
- return box_scalar(self.value_type, self.ap.values(), j)
-
- cdef int64_t length(self):
- return self.ap.value_length(self.index)
-
- def as_py(self):
- """
- Return this value as a Python list.
- """
- cdef:
- int64_t j
- list result = []
-
- for j in range(len(self)):
- result.append(self.getitem(j).as_py())
-
- return result
-
-
-cdef class LargeListValue(ArrayValue):
- """
- Concrete class for large list array elements.
- """
-
- def __len__(self):
- """
- Return the number of values.
- """
- return self.length()
-
- def __getitem__(self, i):
- """
- Return the value at the given index.
- """
- return self.getitem(_normalize_index(i, self.length()))
-
- def __iter__(self):
- """
- Iterate over this element's values.
- """
- for i in range(len(self)):
- yield self.getitem(i)
- raise StopIteration
-
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CLargeListArray*> sp_array.get()
- self.value_type = pyarrow_wrap_data_type(self.ap.value_type())
-
- cdef getitem(self, int64_t i):
- cdef int64_t j = self.ap.value_offset(self.index) + i
- return box_scalar(self.value_type, self.ap.values(), j)
-
- cdef int64_t length(self):
- return self.ap.value_length(self.index)
-
- def as_py(self):
- """
- Return this value as a Python list.
- """
- cdef:
- int64_t j
- list result = []
-
- for j in range(len(self)):
- result.append(self.getitem(j).as_py())
-
- return result
-
-
-cdef class MapValue(ArrayValue):
- """
- Concrete class for map array elements.
- """
-
- def __len__(self):
- """
- Return the number of values.
- """
- return self.length()
-
- def __getitem__(self, i):
- """
- Return the value at the given index.
- """
- return self.getitem(_normalize_index(i, self.length()))
-
- def __iter__(self):
- """
- Iterate over this element's values.
- """
- for i in range(len(self)):
- yield self.getitem(i)
- raise StopIteration
-
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CMapArray*> sp_array.get()
- self.key_type = pyarrow_wrap_data_type(self.ap.map_type().key_type())
- self.item_type = pyarrow_wrap_data_type(self.ap.map_type().item_type())
-
- cdef getitem(self, int64_t i):
- cdef int64_t j = self.ap.value_offset(self.index) + i
- return (box_scalar(self.key_type, self.ap.keys(), j),
- box_scalar(self.item_type, self.ap.items(), j))
-
- cdef int64_t length(self):
- return self.ap.value_length(self.index)
+ try:
+ return Scalar.wrap(GetResultValue(sp.field(ref)))
+ except ArrowInvalid:
+ raise IndexError(key)
def as_py(self):
"""
- Return this value as a Python list of tuples, each containing a
- key and item.
+ Return this value as a Python dict.
"""
- cdef:
- int64_t j
- list result = []
-
- for j in range(len(self)):
- key, item = self.getitem(j)
- result.append((key.as_py(), item.as_py()))
-
- return result
+ if self.is_valid:
+ return {k: v.as_py() for k, v in self.items()}
+ else:
+ return None
-cdef class FixedSizeListValue(ArrayValue):
+cdef class MapScalar(ListScalar):
"""
- Concrete class for fixed size list array elements.
+ Concrete class for map scalars.
"""
- def __len__(self):
- """
- Return the number of values.
- """
- return self.length()
-
def __getitem__(self, i):
"""
Return the value at the given index.
"""
- return self.getitem(_normalize_index(i, self.length()))
+ arr = self.values
+ if arr is None:
+ raise IndexError(i)
+ dct = arr[_normalize_index(i, len(arr))]
+ return (dct['key'], dct['value'])
def __iter__(self):
"""
Iterate over this element's values.
"""
- for i in range(len(self)):
- yield self.getitem(i)
- raise StopIteration
-
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CFixedSizeListArray*> sp_array.get()
- self.value_type = pyarrow_wrap_data_type(self.ap.value_type())
-
- cdef getitem(self, int64_t i):
- cdef int64_t j = self.ap.value_offset(self.index) + i
- return box_scalar(self.value_type, self.ap.values(), j)
-
- cdef int64_t length(self):
- return self.ap.value_length(self.index)
+ arr = self.values
+ if arr is None:
+ return iter(zip(arr.field('key'), arr.field('value')))
+ else:
+ raise StopIteration
def as_py(self):
"""
Return this value as a Python list.
"""
- cdef:
- int64_t j
- list result = []
-
- for j in range(len(self)):
- result.append(self.getitem(j).as_py())
-
- return result
-
-
-cdef class UnionValue(ArrayValue):
- """
- Concrete class for union array elements.
- """
-
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CUnionArray*> sp_array.get()
-
- cdef getitem(self, int64_t i):
- cdef int child_id = self.ap.child_id(i)
- cdef shared_ptr[CArray] child = self.ap.field(child_id)
- cdef CDenseUnionArray* dense
- if self.ap.mode() == _UnionMode_SPARSE:
- return box_scalar(self.type[child_id].type, child, i)
+ arr = self.values
+ if arr is not None:
+ return list(zip(arr.field('key'), arr.field('value')))
else:
- dense = <CDenseUnionArray*> self.ap
- return box_scalar(self.type[child_id].type, child,
- dense.value_offset(i))
-
- def as_py(self):
- """
- Return this value as a Python object.
-
- The exact type depends on the underlying union member.
- """
- return self.getitem(self.index).as_py()
+ return None
-cdef class FixedSizeBinaryValue(ArrayValue):
- """
- Concrete class for fixed-size binary array elements.
- """
-
- def as_py(self):
- """
- Return this value as a Python bytes object.
- """
- cdef:
- CFixedSizeBinaryArray* ap
- CFixedSizeBinaryType* ap_type
- int32_t length
- const char* data
- ap = <CFixedSizeBinaryArray*> self.sp_array.get()
- ap_type = <CFixedSizeBinaryType*> ap.type().get()
- length = ap_type.byte_width()
- data = <const char*> ap.GetValue(self.index)
- return cp.PyBytes_FromStringAndSize(data, length)
-
-
-cdef class StructValue(ArrayValue):
+cdef class DictionaryScalar(Scalar):
"""
- Concrete class for struct array elements.
+ Concrete class for dictionary-encoded scalars.
"""
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CStructArray*> sp_array.get()
-
- def __getitem__(self, key):
- """
- Return the child value for the given field name.
- """
- cdef:
- CStructType* type
- int index
-
- type = <CStructType*> self.type.type
- index = type.GetFieldIndex(tobytes(key))
-
- if index < 0:
- raise KeyError(key)
+ # @property
+ # def index(self):
+ # """
+ # Return this value's underlying index as a scalar.
Review comment:
Hmm, why is this disabled? Please add a comment.
##########
File path: python/pyarrow/scalar.pxi
##########
@@ -16,1198 +16,745 @@
# under the License.
-_NULL = NA = None
+import collections
cdef class Scalar:
"""
- The base class for all array elements.
+ The base class for scalars.
"""
+ def __init__(self):
+ raise TypeError("Do not call {}'s constructor directly, use "
+ "pa.scalar() instead.".format(self.__class__.__name__))
-cdef class NullType(Scalar):
- """
- Singleton for null array elements.
- """
- # TODO rename this NullValue?
+ cdef void init(self, const shared_ptr[CScalar]& wrapped):
+ self.wrapped = wrapped
- def __cinit__(self):
- global NA
- if NA is not None:
- raise Exception('Cannot create multiple NAType instances')
+ @staticmethod
+ cdef wrap(const shared_ptr[CScalar]& wrapped):
+ cdef:
+ Scalar self
+ Type type_id = wrapped.get().type.get().id()
+
+ if type_id == _Type_NA:
+ return _NULL
+
+ typ = _scalar_classes[type_id]
+ self = typ.__new__(typ)
+ self.init(wrapped)
+
+ return self
+
+ cdef inline shared_ptr[CScalar] unwrap(self) nogil:
+ return self.wrapped
+
+ @property
+ def type(self):
+ return pyarrow_wrap_data_type(self.wrapped.get().type)
- self.type = null()
+ @property
+ def is_valid(self):
+ return self.wrapped.get().is_valid
def __repr__(self):
- return 'NULL'
+ return '<pyarrow.{}: {!r}>'.format(
+ self.__class__.__name__, self.as_py()
+ )
- def as_py(self):
- """
- Return None
- """
- return None
+ def __str__(self):
+ return str(self.as_py())
+
+ def equals(self, Scalar other):
+ return self.wrapped.get().Equals(other.unwrap().get()[0])
def __eq__(self, other):
- return NA
+ try:
+ if not isinstance(other, Scalar):
+ other = scalar(other, type=self.type)
+ return self.equals(other)
+ except (TypeError, ValueError, ArrowInvalid):
+ return NotImplemented
+
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+
+ def as_py(self):
+ raise NotImplementedError()
-_NULL = NA = NullType()
+_NULL = NA = None
-cdef class ArrayValue(Scalar):
+cdef class NullScalar(Scalar):
"""
- The base class for non-null array elements.
+ Concrete class for null scalars.
"""
- def __init__(self):
- raise TypeError("Do not call {}'s constructor directly, use array "
- "subscription instead."
- .format(self.__class__.__name__))
+ def __cinit__(self):
+ global NA
+ if NA is not None:
+ raise Exception('Cannot create multiple NAType instances')
+ self.init(shared_ptr[CScalar](new CNullScalar()))
- cdef void init(self, DataType type, const shared_ptr[CArray]& sp_array,
- int64_t index):
- self.type = type
- self.index = index
- self._set_array(sp_array)
+ def __init__(self):
+ pass
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
+ def __eq__(self, other):
+ return NA
Review comment:
Hmm, what?
##########
File path: python/pyarrow/scalar.pxi
##########
@@ -16,1198 +16,745 @@
# under the License.
-_NULL = NA = None
+import collections
cdef class Scalar:
"""
- The base class for all array elements.
+ The base class for scalars.
"""
+ def __init__(self):
+ raise TypeError("Do not call {}'s constructor directly, use "
+ "pa.scalar() instead.".format(self.__class__.__name__))
-cdef class NullType(Scalar):
- """
- Singleton for null array elements.
- """
- # TODO rename this NullValue?
+ cdef void init(self, const shared_ptr[CScalar]& wrapped):
+ self.wrapped = wrapped
- def __cinit__(self):
- global NA
- if NA is not None:
- raise Exception('Cannot create multiple NAType instances')
+ @staticmethod
+ cdef wrap(const shared_ptr[CScalar]& wrapped):
+ cdef:
+ Scalar self
+ Type type_id = wrapped.get().type.get().id()
+
+ if type_id == _Type_NA:
+ return _NULL
+
+ typ = _scalar_classes[type_id]
+ self = typ.__new__(typ)
+ self.init(wrapped)
+
+ return self
+
+ cdef inline shared_ptr[CScalar] unwrap(self) nogil:
+ return self.wrapped
+
+ @property
+ def type(self):
+ return pyarrow_wrap_data_type(self.wrapped.get().type)
- self.type = null()
+ @property
+ def is_valid(self):
+ return self.wrapped.get().is_valid
def __repr__(self):
- return 'NULL'
+ return '<pyarrow.{}: {!r}>'.format(
+ self.__class__.__name__, self.as_py()
+ )
- def as_py(self):
- """
- Return None
- """
- return None
+ def __str__(self):
+ return str(self.as_py())
+
+ def equals(self, Scalar other):
+ return self.wrapped.get().Equals(other.unwrap().get()[0])
def __eq__(self, other):
- return NA
+ try:
+ if not isinstance(other, Scalar):
+ other = scalar(other, type=self.type)
+ return self.equals(other)
+ except (TypeError, ValueError, ArrowInvalid):
+ return NotImplemented
+
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+
+ def as_py(self):
+ raise NotImplementedError()
-_NULL = NA = NullType()
+_NULL = NA = None
-cdef class ArrayValue(Scalar):
+cdef class NullScalar(Scalar):
"""
- The base class for non-null array elements.
+ Concrete class for null scalars.
"""
- def __init__(self):
- raise TypeError("Do not call {}'s constructor directly, use array "
- "subscription instead."
- .format(self.__class__.__name__))
+ def __cinit__(self):
+ global NA
+ if NA is not None:
+ raise Exception('Cannot create multiple NAType instances')
+ self.init(shared_ptr[CScalar](new CNullScalar()))
- cdef void init(self, DataType type, const shared_ptr[CArray]& sp_array,
- int64_t index):
- self.type = type
- self.index = index
- self._set_array(sp_array)
+ def __init__(self):
+ pass
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
+ def __eq__(self, other):
+ return NA
- def __repr__(self):
- if hasattr(self, 'as_py'):
- return repr(self.as_py())
- else:
- return super(Scalar, self).__repr__()
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
- def __str__(self):
- if hasattr(self, 'as_py'):
- return str(self.as_py())
- else:
- return super(Scalar, self).__str__()
+ def as_py(self):
+ """
+ Return this value as a Python None.
+ """
+ return None
- def __eq__(self, other):
- if hasattr(self, 'as_py'):
- if isinstance(other, ArrayValue):
- other = other.as_py()
- return self.as_py() == other
- else:
- raise NotImplementedError(
- "Cannot compare Arrow values that don't support as_py()")
- def __hash__(self):
- return hash(self.as_py())
+_NULL = NA = NullScalar()
-cdef class BooleanValue(ArrayValue):
+cdef class BooleanScalar(Scalar):
"""
- Concrete class for boolean array elements.
+ Concrete class for boolean scalars.
"""
def as_py(self):
"""
Return this value as a Python bool.
"""
- cdef CBooleanArray* ap = <CBooleanArray*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CBooleanScalar* sp = <CBooleanScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int8Value(ArrayValue):
+cdef class UInt8Scalar(Scalar):
"""
- Concrete class for int8 array elements.
+ Concrete class for uint8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt8Array* ap = <CInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt8Scalar* sp = <CUInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt8Value(ArrayValue):
+cdef class Int8Scalar(Scalar):
"""
- Concrete class for uint8 array elements.
+ Concrete class for int8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt8Array* ap = <CUInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt8Scalar* sp = <CInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int16Value(ArrayValue):
+cdef class UInt16Scalar(Scalar):
"""
- Concrete class for int16 array elements.
+ Concrete class for uint16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt16Array* ap = <CInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt16Scalar* sp = <CUInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt16Value(ArrayValue):
+cdef class Int16Scalar(Scalar):
"""
- Concrete class for uint16 array elements.
+ Concrete class for int16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt16Array* ap = <CUInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt16Scalar* sp = <CInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int32Value(ArrayValue):
+cdef class UInt32Scalar(Scalar):
"""
- Concrete class for int32 array elements.
+ Concrete class for uint32 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt32Array* ap = <CInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt32Scalar* sp = <CUInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt32Value(ArrayValue):
+cdef class Int32Scalar(Scalar):
"""
- Concrete class for uint32 array elements.
+ Concrete class for int32 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt32Array* ap = <CUInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt32Scalar* sp = <CInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int64Value(ArrayValue):
+cdef class UInt64Scalar(Scalar):
"""
- Concrete class for int64 array elements.
+ Concrete class for uint64 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt64Array* ap = <CInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt64Scalar* sp = <CUInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt64Value(ArrayValue):
+cdef class Int64Scalar(Scalar):
"""
- Concrete class for uint64 array elements.
+ Concrete class for int64 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt64Array* ap = <CUInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt64Scalar* sp = <CInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date32Value(ArrayValue):
+cdef class HalfFloatScalar(Scalar):
"""
- Concrete class for date32 array elements.
+ Concrete class for float scalars.
"""
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+
+ def __eq__(self, other):
+ if hasattr(self, 'as_py'):
Review comment:
This `hasattr` check is pointless as, `self` is a `HalfFloatScalar`.
##########
File path: python/pyarrow/scalar.pxi
##########
@@ -16,1198 +16,745 @@
# under the License.
-_NULL = NA = None
+import collections
cdef class Scalar:
"""
- The base class for all array elements.
+ The base class for scalars.
"""
+ def __init__(self):
+ raise TypeError("Do not call {}'s constructor directly, use "
+ "pa.scalar() instead.".format(self.__class__.__name__))
-cdef class NullType(Scalar):
- """
- Singleton for null array elements.
- """
- # TODO rename this NullValue?
+ cdef void init(self, const shared_ptr[CScalar]& wrapped):
+ self.wrapped = wrapped
- def __cinit__(self):
- global NA
- if NA is not None:
- raise Exception('Cannot create multiple NAType instances')
+ @staticmethod
+ cdef wrap(const shared_ptr[CScalar]& wrapped):
+ cdef:
+ Scalar self
+ Type type_id = wrapped.get().type.get().id()
+
+ if type_id == _Type_NA:
+ return _NULL
+
+ typ = _scalar_classes[type_id]
+ self = typ.__new__(typ)
+ self.init(wrapped)
+
+ return self
+
+ cdef inline shared_ptr[CScalar] unwrap(self) nogil:
+ return self.wrapped
+
+ @property
+ def type(self):
+ return pyarrow_wrap_data_type(self.wrapped.get().type)
- self.type = null()
+ @property
+ def is_valid(self):
+ return self.wrapped.get().is_valid
def __repr__(self):
- return 'NULL'
+ return '<pyarrow.{}: {!r}>'.format(
+ self.__class__.__name__, self.as_py()
+ )
- def as_py(self):
- """
- Return None
- """
- return None
+ def __str__(self):
+ return str(self.as_py())
+
+ def equals(self, Scalar other):
+ return self.wrapped.get().Equals(other.unwrap().get()[0])
def __eq__(self, other):
- return NA
+ try:
+ if not isinstance(other, Scalar):
+ other = scalar(other, type=self.type)
+ return self.equals(other)
+ except (TypeError, ValueError, ArrowInvalid):
+ return NotImplemented
+
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+
+ def as_py(self):
+ raise NotImplementedError()
-_NULL = NA = NullType()
+_NULL = NA = None
-cdef class ArrayValue(Scalar):
+cdef class NullScalar(Scalar):
"""
- The base class for non-null array elements.
+ Concrete class for null scalars.
"""
- def __init__(self):
- raise TypeError("Do not call {}'s constructor directly, use array "
- "subscription instead."
- .format(self.__class__.__name__))
+ def __cinit__(self):
+ global NA
+ if NA is not None:
+ raise Exception('Cannot create multiple NAType instances')
+ self.init(shared_ptr[CScalar](new CNullScalar()))
- cdef void init(self, DataType type, const shared_ptr[CArray]& sp_array,
- int64_t index):
- self.type = type
- self.index = index
- self._set_array(sp_array)
+ def __init__(self):
+ pass
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
+ def __eq__(self, other):
+ return NA
- def __repr__(self):
- if hasattr(self, 'as_py'):
- return repr(self.as_py())
- else:
- return super(Scalar, self).__repr__()
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
- def __str__(self):
- if hasattr(self, 'as_py'):
- return str(self.as_py())
- else:
- return super(Scalar, self).__str__()
+ def as_py(self):
+ """
+ Return this value as a Python None.
+ """
+ return None
- def __eq__(self, other):
- if hasattr(self, 'as_py'):
- if isinstance(other, ArrayValue):
- other = other.as_py()
- return self.as_py() == other
- else:
- raise NotImplementedError(
- "Cannot compare Arrow values that don't support as_py()")
- def __hash__(self):
- return hash(self.as_py())
+_NULL = NA = NullScalar()
-cdef class BooleanValue(ArrayValue):
+cdef class BooleanScalar(Scalar):
"""
- Concrete class for boolean array elements.
+ Concrete class for boolean scalars.
"""
def as_py(self):
"""
Return this value as a Python bool.
"""
- cdef CBooleanArray* ap = <CBooleanArray*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CBooleanScalar* sp = <CBooleanScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int8Value(ArrayValue):
+cdef class UInt8Scalar(Scalar):
"""
- Concrete class for int8 array elements.
+ Concrete class for uint8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt8Array* ap = <CInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt8Scalar* sp = <CUInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt8Value(ArrayValue):
+cdef class Int8Scalar(Scalar):
"""
- Concrete class for uint8 array elements.
+ Concrete class for int8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt8Array* ap = <CUInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt8Scalar* sp = <CInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int16Value(ArrayValue):
+cdef class UInt16Scalar(Scalar):
"""
- Concrete class for int16 array elements.
+ Concrete class for uint16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt16Array* ap = <CInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt16Scalar* sp = <CUInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt16Value(ArrayValue):
+cdef class Int16Scalar(Scalar):
"""
- Concrete class for uint16 array elements.
+ Concrete class for int16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt16Array* ap = <CUInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt16Scalar* sp = <CInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int32Value(ArrayValue):
+cdef class UInt32Scalar(Scalar):
"""
- Concrete class for int32 array elements.
+ Concrete class for uint32 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt32Array* ap = <CInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt32Scalar* sp = <CUInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt32Value(ArrayValue):
+cdef class Int32Scalar(Scalar):
"""
- Concrete class for uint32 array elements.
+ Concrete class for int32 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt32Array* ap = <CUInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt32Scalar* sp = <CInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int64Value(ArrayValue):
+cdef class UInt64Scalar(Scalar):
"""
- Concrete class for int64 array elements.
+ Concrete class for uint64 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt64Array* ap = <CInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt64Scalar* sp = <CUInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt64Value(ArrayValue):
+cdef class Int64Scalar(Scalar):
"""
- Concrete class for uint64 array elements.
+ Concrete class for int64 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt64Array* ap = <CUInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt64Scalar* sp = <CInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date32Value(ArrayValue):
+cdef class HalfFloatScalar(Scalar):
"""
- Concrete class for date32 array elements.
+ Concrete class for float scalars.
"""
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+
+ def __eq__(self, other):
+ if hasattr(self, 'as_py'):
+ if isinstance(other, Scalar):
+ other = other.as_py()
+ return self.as_py() == other
+ else:
+ raise NotImplementedError
+
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python float.
"""
- cdef CDate32Array* ap = <CDate32Array*> self.sp_array.get()
+ cdef CHalfFloatScalar* sp = <CHalfFloatScalar*> self.wrapped.get()
+ return PyHalf_FromHalf(sp.value) if sp.is_valid else None
- # Shift to seconds since epoch
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(days=ap.Value(self.index)))
+
+cdef class FloatScalar(Scalar):
+ """
+ Concrete class for float scalars.
+ """
+
+ def as_py(self):
+ """
+ Return this value as a Python float.
+ """
+ cdef CFloatScalar* sp = <CFloatScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date64Value(ArrayValue):
+cdef class DoubleScalar(Scalar):
"""
- Concrete class for date64 array elements.
+ Concrete class for double scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python float.
"""
- cdef CDate64Array* ap = <CDate64Array*> self.sp_array.get()
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(
- days=ap.Value(self.index) / 86400000))
+ cdef CDoubleScalar* sp = <CDoubleScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Time32Value(ArrayValue):
+cdef class Decimal128Scalar(Scalar):
"""
- Concrete class for time32 array elements.
+ Concrete class for decimal128 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.timedelta instance.
+ Return this value as a Python Decimal.
"""
cdef:
- CTime32Array* ap = <CTime32Array*> self.sp_array.get()
- CTime32Type* dtype = <CTime32Type*> ap.type().get()
-
- if dtype.unit() == TimeUnit_SECOND:
- delta = datetime.timedelta(seconds=ap.Value(self.index))
- return (datetime.datetime(1970, 1, 1) + delta).time()
+ CDecimal128Scalar* sp = <CDecimal128Scalar*> self.wrapped.get()
+ CDecimal128Type* dtype = <CDecimal128Type*> sp.type.get()
+ if sp.is_valid:
+ return _pydecimal.Decimal(
Review comment:
Why `_pydecimal`? You should be using the `decimal` module.
##########
File path: python/pyarrow/scalar.pxi
##########
@@ -16,1198 +16,745 @@
# under the License.
-_NULL = NA = None
+import collections
cdef class Scalar:
"""
- The base class for all array elements.
+ The base class for scalars.
"""
+ def __init__(self):
+ raise TypeError("Do not call {}'s constructor directly, use "
+ "pa.scalar() instead.".format(self.__class__.__name__))
-cdef class NullType(Scalar):
- """
- Singleton for null array elements.
- """
- # TODO rename this NullValue?
+ cdef void init(self, const shared_ptr[CScalar]& wrapped):
+ self.wrapped = wrapped
- def __cinit__(self):
- global NA
- if NA is not None:
- raise Exception('Cannot create multiple NAType instances')
+ @staticmethod
+ cdef wrap(const shared_ptr[CScalar]& wrapped):
+ cdef:
+ Scalar self
+ Type type_id = wrapped.get().type.get().id()
+
+ if type_id == _Type_NA:
+ return _NULL
+
+ typ = _scalar_classes[type_id]
+ self = typ.__new__(typ)
+ self.init(wrapped)
+
+ return self
+
+ cdef inline shared_ptr[CScalar] unwrap(self) nogil:
+ return self.wrapped
+
+ @property
+ def type(self):
+ return pyarrow_wrap_data_type(self.wrapped.get().type)
- self.type = null()
+ @property
+ def is_valid(self):
+ return self.wrapped.get().is_valid
def __repr__(self):
- return 'NULL'
+ return '<pyarrow.{}: {!r}>'.format(
+ self.__class__.__name__, self.as_py()
+ )
- def as_py(self):
- """
- Return None
- """
- return None
+ def __str__(self):
+ return str(self.as_py())
+
+ def equals(self, Scalar other):
+ return self.wrapped.get().Equals(other.unwrap().get()[0])
def __eq__(self, other):
- return NA
+ try:
+ if not isinstance(other, Scalar):
+ other = scalar(other, type=self.type)
+ return self.equals(other)
+ except (TypeError, ValueError, ArrowInvalid):
+ return NotImplemented
+
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+
+ def as_py(self):
+ raise NotImplementedError()
-_NULL = NA = NullType()
+_NULL = NA = None
-cdef class ArrayValue(Scalar):
+cdef class NullScalar(Scalar):
"""
- The base class for non-null array elements.
+ Concrete class for null scalars.
"""
- def __init__(self):
- raise TypeError("Do not call {}'s constructor directly, use array "
- "subscription instead."
- .format(self.__class__.__name__))
+ def __cinit__(self):
+ global NA
+ if NA is not None:
+ raise Exception('Cannot create multiple NAType instances')
+ self.init(shared_ptr[CScalar](new CNullScalar()))
- cdef void init(self, DataType type, const shared_ptr[CArray]& sp_array,
- int64_t index):
- self.type = type
- self.index = index
- self._set_array(sp_array)
+ def __init__(self):
+ pass
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
+ def __eq__(self, other):
+ return NA
- def __repr__(self):
- if hasattr(self, 'as_py'):
- return repr(self.as_py())
- else:
- return super(Scalar, self).__repr__()
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
- def __str__(self):
- if hasattr(self, 'as_py'):
- return str(self.as_py())
- else:
- return super(Scalar, self).__str__()
+ def as_py(self):
+ """
+ Return this value as a Python None.
+ """
+ return None
- def __eq__(self, other):
- if hasattr(self, 'as_py'):
- if isinstance(other, ArrayValue):
- other = other.as_py()
- return self.as_py() == other
- else:
- raise NotImplementedError(
- "Cannot compare Arrow values that don't support as_py()")
- def __hash__(self):
- return hash(self.as_py())
+_NULL = NA = NullScalar()
-cdef class BooleanValue(ArrayValue):
+cdef class BooleanScalar(Scalar):
"""
- Concrete class for boolean array elements.
+ Concrete class for boolean scalars.
"""
def as_py(self):
"""
Return this value as a Python bool.
"""
- cdef CBooleanArray* ap = <CBooleanArray*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CBooleanScalar* sp = <CBooleanScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int8Value(ArrayValue):
+cdef class UInt8Scalar(Scalar):
"""
- Concrete class for int8 array elements.
+ Concrete class for uint8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt8Array* ap = <CInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt8Scalar* sp = <CUInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt8Value(ArrayValue):
+cdef class Int8Scalar(Scalar):
"""
- Concrete class for uint8 array elements.
+ Concrete class for int8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt8Array* ap = <CUInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt8Scalar* sp = <CInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int16Value(ArrayValue):
+cdef class UInt16Scalar(Scalar):
"""
- Concrete class for int16 array elements.
+ Concrete class for uint16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt16Array* ap = <CInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt16Scalar* sp = <CUInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt16Value(ArrayValue):
+cdef class Int16Scalar(Scalar):
"""
- Concrete class for uint16 array elements.
+ Concrete class for int16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt16Array* ap = <CUInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt16Scalar* sp = <CInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int32Value(ArrayValue):
+cdef class UInt32Scalar(Scalar):
"""
- Concrete class for int32 array elements.
+ Concrete class for uint32 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt32Array* ap = <CInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt32Scalar* sp = <CUInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt32Value(ArrayValue):
+cdef class Int32Scalar(Scalar):
"""
- Concrete class for uint32 array elements.
+ Concrete class for int32 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt32Array* ap = <CUInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt32Scalar* sp = <CInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int64Value(ArrayValue):
+cdef class UInt64Scalar(Scalar):
"""
- Concrete class for int64 array elements.
+ Concrete class for uint64 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt64Array* ap = <CInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt64Scalar* sp = <CUInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt64Value(ArrayValue):
+cdef class Int64Scalar(Scalar):
"""
- Concrete class for uint64 array elements.
+ Concrete class for int64 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt64Array* ap = <CUInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt64Scalar* sp = <CInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date32Value(ArrayValue):
+cdef class HalfFloatScalar(Scalar):
"""
- Concrete class for date32 array elements.
+ Concrete class for float scalars.
"""
+ def __hash__(self):
Review comment:
Why do you need to redefine this?
##########
File path: python/pyarrow/scalar.pxi
##########
@@ -16,1198 +16,745 @@
# under the License.
-_NULL = NA = None
+import collections
cdef class Scalar:
"""
- The base class for all array elements.
+ The base class for scalars.
"""
+ def __init__(self):
+ raise TypeError("Do not call {}'s constructor directly, use "
+ "pa.scalar() instead.".format(self.__class__.__name__))
-cdef class NullType(Scalar):
- """
- Singleton for null array elements.
- """
- # TODO rename this NullValue?
+ cdef void init(self, const shared_ptr[CScalar]& wrapped):
+ self.wrapped = wrapped
- def __cinit__(self):
- global NA
- if NA is not None:
- raise Exception('Cannot create multiple NAType instances')
+ @staticmethod
+ cdef wrap(const shared_ptr[CScalar]& wrapped):
+ cdef:
+ Scalar self
+ Type type_id = wrapped.get().type.get().id()
+
+ if type_id == _Type_NA:
+ return _NULL
+
+ typ = _scalar_classes[type_id]
+ self = typ.__new__(typ)
+ self.init(wrapped)
+
+ return self
+
+ cdef inline shared_ptr[CScalar] unwrap(self) nogil:
+ return self.wrapped
+
+ @property
+ def type(self):
Review comment:
Add docstrings here.
##########
File path: python/pyarrow/scalar.pxi
##########
@@ -16,1198 +16,745 @@
# under the License.
-_NULL = NA = None
+import collections
cdef class Scalar:
"""
- The base class for all array elements.
+ The base class for scalars.
"""
+ def __init__(self):
+ raise TypeError("Do not call {}'s constructor directly, use "
+ "pa.scalar() instead.".format(self.__class__.__name__))
-cdef class NullType(Scalar):
- """
- Singleton for null array elements.
- """
- # TODO rename this NullValue?
+ cdef void init(self, const shared_ptr[CScalar]& wrapped):
+ self.wrapped = wrapped
- def __cinit__(self):
- global NA
- if NA is not None:
- raise Exception('Cannot create multiple NAType instances')
+ @staticmethod
+ cdef wrap(const shared_ptr[CScalar]& wrapped):
+ cdef:
+ Scalar self
+ Type type_id = wrapped.get().type.get().id()
+
+ if type_id == _Type_NA:
+ return _NULL
+
+ typ = _scalar_classes[type_id]
+ self = typ.__new__(typ)
+ self.init(wrapped)
+
+ return self
+
+ cdef inline shared_ptr[CScalar] unwrap(self) nogil:
+ return self.wrapped
+
+ @property
+ def type(self):
+ return pyarrow_wrap_data_type(self.wrapped.get().type)
- self.type = null()
+ @property
+ def is_valid(self):
+ return self.wrapped.get().is_valid
def __repr__(self):
- return 'NULL'
+ return '<pyarrow.{}: {!r}>'.format(
+ self.__class__.__name__, self.as_py()
+ )
- def as_py(self):
- """
- Return None
- """
- return None
+ def __str__(self):
+ return str(self.as_py())
+
+ def equals(self, Scalar other):
+ return self.wrapped.get().Equals(other.unwrap().get()[0])
def __eq__(self, other):
- return NA
+ try:
+ if not isinstance(other, Scalar):
+ other = scalar(other, type=self.type)
+ return self.equals(other)
+ except (TypeError, ValueError, ArrowInvalid):
+ return NotImplemented
+
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+
+ def as_py(self):
+ raise NotImplementedError()
-_NULL = NA = NullType()
+_NULL = NA = None
-cdef class ArrayValue(Scalar):
+cdef class NullScalar(Scalar):
"""
- The base class for non-null array elements.
+ Concrete class for null scalars.
"""
- def __init__(self):
- raise TypeError("Do not call {}'s constructor directly, use array "
- "subscription instead."
- .format(self.__class__.__name__))
+ def __cinit__(self):
+ global NA
+ if NA is not None:
+ raise Exception('Cannot create multiple NAType instances')
+ self.init(shared_ptr[CScalar](new CNullScalar()))
- cdef void init(self, DataType type, const shared_ptr[CArray]& sp_array,
- int64_t index):
- self.type = type
- self.index = index
- self._set_array(sp_array)
+ def __init__(self):
+ pass
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
+ def __eq__(self, other):
+ return NA
- def __repr__(self):
- if hasattr(self, 'as_py'):
- return repr(self.as_py())
- else:
- return super(Scalar, self).__repr__()
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
- def __str__(self):
- if hasattr(self, 'as_py'):
- return str(self.as_py())
- else:
- return super(Scalar, self).__str__()
+ def as_py(self):
+ """
+ Return this value as a Python None.
+ """
+ return None
- def __eq__(self, other):
- if hasattr(self, 'as_py'):
- if isinstance(other, ArrayValue):
- other = other.as_py()
- return self.as_py() == other
- else:
- raise NotImplementedError(
- "Cannot compare Arrow values that don't support as_py()")
- def __hash__(self):
- return hash(self.as_py())
+_NULL = NA = NullScalar()
-cdef class BooleanValue(ArrayValue):
+cdef class BooleanScalar(Scalar):
"""
- Concrete class for boolean array elements.
+ Concrete class for boolean scalars.
"""
def as_py(self):
"""
Return this value as a Python bool.
"""
- cdef CBooleanArray* ap = <CBooleanArray*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CBooleanScalar* sp = <CBooleanScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int8Value(ArrayValue):
+cdef class UInt8Scalar(Scalar):
"""
- Concrete class for int8 array elements.
+ Concrete class for uint8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt8Array* ap = <CInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt8Scalar* sp = <CUInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt8Value(ArrayValue):
+cdef class Int8Scalar(Scalar):
"""
- Concrete class for uint8 array elements.
+ Concrete class for int8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt8Array* ap = <CUInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt8Scalar* sp = <CInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int16Value(ArrayValue):
+cdef class UInt16Scalar(Scalar):
"""
- Concrete class for int16 array elements.
+ Concrete class for uint16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt16Array* ap = <CInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt16Scalar* sp = <CUInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt16Value(ArrayValue):
+cdef class Int16Scalar(Scalar):
"""
- Concrete class for uint16 array elements.
+ Concrete class for int16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt16Array* ap = <CUInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt16Scalar* sp = <CInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int32Value(ArrayValue):
+cdef class UInt32Scalar(Scalar):
"""
- Concrete class for int32 array elements.
+ Concrete class for uint32 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt32Array* ap = <CInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt32Scalar* sp = <CUInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt32Value(ArrayValue):
+cdef class Int32Scalar(Scalar):
"""
- Concrete class for uint32 array elements.
+ Concrete class for int32 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt32Array* ap = <CUInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt32Scalar* sp = <CInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int64Value(ArrayValue):
+cdef class UInt64Scalar(Scalar):
"""
- Concrete class for int64 array elements.
+ Concrete class for uint64 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt64Array* ap = <CInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt64Scalar* sp = <CUInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt64Value(ArrayValue):
+cdef class Int64Scalar(Scalar):
"""
- Concrete class for uint64 array elements.
+ Concrete class for int64 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt64Array* ap = <CUInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt64Scalar* sp = <CInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date32Value(ArrayValue):
+cdef class HalfFloatScalar(Scalar):
"""
- Concrete class for date32 array elements.
+ Concrete class for float scalars.
"""
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+
+ def __eq__(self, other):
+ if hasattr(self, 'as_py'):
+ if isinstance(other, Scalar):
+ other = other.as_py()
+ return self.as_py() == other
+ else:
+ raise NotImplementedError
+
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python float.
"""
- cdef CDate32Array* ap = <CDate32Array*> self.sp_array.get()
+ cdef CHalfFloatScalar* sp = <CHalfFloatScalar*> self.wrapped.get()
+ return PyHalf_FromHalf(sp.value) if sp.is_valid else None
- # Shift to seconds since epoch
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(days=ap.Value(self.index)))
+
+cdef class FloatScalar(Scalar):
+ """
+ Concrete class for float scalars.
+ """
+
+ def as_py(self):
+ """
+ Return this value as a Python float.
+ """
+ cdef CFloatScalar* sp = <CFloatScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date64Value(ArrayValue):
+cdef class DoubleScalar(Scalar):
"""
- Concrete class for date64 array elements.
+ Concrete class for double scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python float.
"""
- cdef CDate64Array* ap = <CDate64Array*> self.sp_array.get()
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(
- days=ap.Value(self.index) / 86400000))
+ cdef CDoubleScalar* sp = <CDoubleScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Time32Value(ArrayValue):
+cdef class Decimal128Scalar(Scalar):
"""
- Concrete class for time32 array elements.
+ Concrete class for decimal128 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.timedelta instance.
+ Return this value as a Python Decimal.
"""
cdef:
- CTime32Array* ap = <CTime32Array*> self.sp_array.get()
- CTime32Type* dtype = <CTime32Type*> ap.type().get()
-
- if dtype.unit() == TimeUnit_SECOND:
- delta = datetime.timedelta(seconds=ap.Value(self.index))
- return (datetime.datetime(1970, 1, 1) + delta).time()
+ CDecimal128Scalar* sp = <CDecimal128Scalar*> self.wrapped.get()
+ CDecimal128Type* dtype = <CDecimal128Type*> sp.type.get()
+ if sp.is_valid:
+ return _pydecimal.Decimal(
+ frombytes(sp.value.ToString(dtype.scale()))
+ )
else:
- return _box_time_milli(ap.Value(self.index))
+ return None
-cdef class Time64Value(ArrayValue):
+cdef class Date32Scalar(Scalar):
"""
- Concrete class for time64 array elements.
+ Concrete class for date32 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.timedelta instance.
+ Return this value as a Python datetime.datetime instance.
"""
- cdef:
- CTime64Array* ap = <CTime64Array*> self.sp_array.get()
- CTime64Type* dtype = <CTime64Type*> ap.type().get()
+ cdef CDate32Scalar* sp = <CDate32Scalar*> self.wrapped.get()
- cdef int64_t val = ap.Value(self.index)
- if dtype.unit() == TimeUnit_MICRO:
- return _box_time_micro(val)
+ if sp.is_valid:
+ # shift to seconds since epoch
+ return (
+ datetime.date(1970, 1, 1) + datetime.timedelta(days=sp.value)
+ )
else:
- return (datetime.datetime(1970, 1, 1) +
- datetime.timedelta(microseconds=val / 1000)).time()
-
+ return None
-cpdef _box_time_milli(int64_t val):
- delta = datetime.timedelta(milliseconds=val)
- return (datetime.datetime(1970, 1, 1) + delta).time()
+cdef class Date64Scalar(Scalar):
+ """
+ Concrete class for date64 scalars.
+ """
-cpdef _box_time_micro(int64_t val):
- return (datetime.datetime(1970, 1, 1) +
- datetime.timedelta(microseconds=val)).time()
+ def as_py(self):
+ """
+ Return this value as a Python datetime.datetime instance.
+ """
+ cdef CDate64Scalar* sp = <CDate64Scalar*> self.wrapped.get()
+ if sp.is_valid:
+ return (
+ datetime.date(1970, 1, 1) +
+ datetime.timedelta(days=sp.value / 86400000)
+ )
+ else:
+ return None
-cdef dict _DATETIME_CONVERSION_FUNCTIONS = {}
-cdef c_bool _datetime_conversion_initialized = False
+def _datetime_from_int(int64_t value, TimeUnit unit, tzinfo=None):
+ if unit == TimeUnit_SECOND:
+ delta = datetime.timedelta(seconds=value)
+ elif unit == TimeUnit_MILLI:
+ delta = datetime.timedelta(milliseconds=value)
+ elif unit == TimeUnit_MICRO:
+ delta = datetime.timedelta(microseconds=value)
+ else:
+ # TimeUnit_NANO: prefer pandas timestamps if available
+ if _pandas_api.have_pandas:
+ return _pandas_api.pd.Timestamp(value, tz=tzinfo, unit='ns')
+ # otherwise safely truncate to microsecond resolution datetime
+ if value % 1000 != 0:
+ raise ValueError(
+ "Nanosecond resolution temporal type {} is not safely "
+ "convertible to microseconds to convert to datetime.datetime. "
+ "Install pandas to return as Timestamp with nanosecond "
+ "support or access the .value attribute.".format(value)
+ )
+ delta = datetime.timedelta(microseconds=value // 1000)
-cdef _add_micros_maybe_localize(dt, micros, tzinfo):
- import pytz
- dt = dt.replace(microsecond=micros)
+ dt = datetime.datetime(1970, 1, 1) + delta
+ # adjust timezone if set to the datatype
if tzinfo is not None:
- if not isinstance(tzinfo, datetime.tzinfo):
- tzinfo = string_to_tzinfo(tzinfo)
dt = tzinfo.fromutc(dt)
- return dt
-
-
-cdef _datetime_from_seconds(int64_t v):
- return datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=v)
-
-
-def _nanoseconds_to_datetime_safe(v, tzinfo):
- if v % 1000 != 0:
- raise ValueError("Nanosecond timestamp {} is not safely convertible "
- " to microseconds to convert to datetime.datetime."
- " Install pandas to return as Timestamp with "
- " nanosecond support or access the .value attribute.")
- v = v // 1000
- micros = v % 1_000_000
-
- dt = _datetime_from_seconds(v // 1_000_000)
- return _add_micros_maybe_localize(dt, micros, tzinfo)
-
-
-def _microseconds_to_datetime(v, tzinfo):
- micros = v % 1_000_000
- dt = _datetime_from_seconds(v // 1_000_000)
- return _add_micros_maybe_localize(dt, micros, tzinfo)
+ return dt
-def _millis_to_datetime(v, tzinfo):
- millis = v % 1_000
- dt = _datetime_from_seconds(v // 1000)
- return _add_micros_maybe_localize(dt, millis * 1000, tzinfo)
+cdef class Time32Scalar(Scalar):
+ """
+ Concrete class for time32 scalars.
+ """
-def _seconds_to_datetime(v, tzinfo):
- dt = _datetime_from_seconds(v)
- return _add_micros_maybe_localize(dt, 0, tzinfo)
+ def as_py(self):
+ """
+ Return this value as a Python datetime.timedelta instance.
+ """
+ cdef:
+ CTime32Scalar* sp = <CTime32Scalar*> self.wrapped.get()
+ CTime32Type* dtype = <CTime32Type*> sp.type.get()
+ if sp.is_valid:
+ return _datetime_from_int(sp.value, unit=dtype.unit()).time()
+ else:
+ return None
-def _datetime_conversion_functions():
- global _datetime_conversion_initialized
- if _datetime_conversion_initialized:
- return _DATETIME_CONVERSION_FUNCTIONS
- _DATETIME_CONVERSION_FUNCTIONS.update({
- TimeUnit_SECOND: _seconds_to_datetime,
- TimeUnit_MILLI: _millis_to_datetime,
- TimeUnit_MICRO: _microseconds_to_datetime,
- TimeUnit_NANO: _nanoseconds_to_datetime_safe
- })
+cdef class Time64Scalar(Scalar):
+ """
+ Concrete class for time64 scalars.
+ """
- try:
- import pandas as pd
- _DATETIME_CONVERSION_FUNCTIONS[TimeUnit_NANO] = (
- lambda x, tzinfo: pd.Timestamp(
- x, tz=tzinfo, unit='ns',
- )
- )
- except ImportError:
- pass
+ def as_py(self):
+ """
+ Return this value as a Python datetime.timedelta instance.
+ """
+ cdef:
+ CTime64Scalar* sp = <CTime64Scalar*> self.wrapped.get()
+ CTime64Type* dtype = <CTime64Type*> sp.type.get()
- _datetime_conversion_initialized = True
- return _DATETIME_CONVERSION_FUNCTIONS
+ if sp.is_valid:
+ return _datetime_from_int(sp.value, unit=dtype.unit()).time()
+ else:
+ return None
-cdef class TimestampValue(ArrayValue):
+cdef class TimestampScalar(Scalar):
"""
- Concrete class for timestamp array elements.
+ Concrete class for timestamp scalars.
"""
@property
def value(self):
- cdef CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
- cdef CTimestampType* dtype = <CTimestampType*> ap.type().get()
- return ap.Value(self.index)
+ cdef CTimestampScalar* sp = <CTimestampScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
def as_py(self):
"""
Return this value as a Pandas Timestamp instance (if available),
otherwise as a Python datetime.timedelta instance.
"""
- cdef CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
- cdef CTimestampType* dtype = <CTimestampType*> ap.type().get()
+ cdef:
+ CTimestampScalar* sp = <CTimestampScalar*> self.wrapped.get()
+ CTimestampType* dtype = <CTimestampType*> sp.type.get()
- value = self.value
+ if not sp.is_valid:
+ return None
if not dtype.timezone().empty():
tzinfo = string_to_tzinfo(frombytes(dtype.timezone()))
+ if not isinstance(tzinfo, datetime.tzinfo):
+ tzinfo = string_to_tzinfo(tzinfo)
else:
tzinfo = None
- try:
- converter = _datetime_conversion_functions()[dtype.unit()]
- except KeyError:
- raise ValueError(
- 'Cannot convert nanosecond timestamps without pandas'
- )
- return converter(value, tzinfo=tzinfo)
-
-
-cdef dict _TIMEDELTA_CONVERSION_FUNCTIONS = {}
-
-
-def _nanoseconds_to_timedelta_safe(v):
- if v % 1000 != 0:
- raise ValueError(
- "Nanosecond duration {} is not safely convertible to microseconds "
- "to convert to datetime.timedelta. Install pandas to return as "
- "Timedelta with nanosecond support or access the .value "
- "attribute.".format(v))
- micros = v // 1000
-
- return datetime.timedelta(microseconds=micros)
-
-
-def _timedelta_conversion_functions():
- if _TIMEDELTA_CONVERSION_FUNCTIONS:
- return _TIMEDELTA_CONVERSION_FUNCTIONS
-
- _TIMEDELTA_CONVERSION_FUNCTIONS.update({
- TimeUnit_SECOND: lambda v: datetime.timedelta(seconds=v),
- TimeUnit_MILLI: lambda v: datetime.timedelta(milliseconds=v),
- TimeUnit_MICRO: lambda v: datetime.timedelta(microseconds=v),
- TimeUnit_NANO: _nanoseconds_to_timedelta_safe
- })
-
- try:
- import pandas as pd
- _TIMEDELTA_CONVERSION_FUNCTIONS[TimeUnit_NANO] = (
- lambda v: pd.Timedelta(v, unit='ns')
- )
- except ImportError:
- pass
+ return _datetime_from_int(sp.value, unit=dtype.unit(), tzinfo=tzinfo)
- return _TIMEDELTA_CONVERSION_FUNCTIONS
-
-cdef class DurationValue(ArrayValue):
+cdef class DurationScalar(Scalar):
"""
- Concrete class for duration array elements.
+ Concrete class for duration scalars.
"""
@property
def value(self):
- cdef CDurationArray* ap = <CDurationArray*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CDurationScalar* sp = <CDurationScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
def as_py(self):
"""
Return this value as a Pandas Timestamp instance (if available),
otherwise as a Python datetime.timedelta instance.
"""
- cdef CDurationArray* ap = <CDurationArray*> self.sp_array.get()
- cdef CDurationType* dtype = <CDurationType*> ap.type().get()
+ cdef:
+ CDurationScalar* sp = <CDurationScalar*> self.wrapped.get()
+ CDurationType* dtype = <CDurationType*> sp.type.get()
+ TimeUnit unit = dtype.unit()
+
+ if not sp.is_valid:
+ return None
- cdef int64_t value = ap.Value(self.index)
- converter = _timedelta_conversion_functions()[dtype.unit()]
- return converter(value)
+ if unit == TimeUnit_SECOND:
+ return datetime.timedelta(seconds=sp.value)
+ elif unit == TimeUnit_MILLI:
+ return datetime.timedelta(milliseconds=sp.value)
+ elif unit == TimeUnit_MICRO:
+ return datetime.timedelta(microseconds=sp.value)
+ else:
+ # TimeUnit_NANO: prefer pandas timestamps if available
+ if _pandas_api.have_pandas:
+ return _pandas_api.pd.Timedelta(sp.value, unit='ns')
+ # otherwise safely truncate to microsecond resolution timedelta
+ if sp.value % 1000 != 0:
+ raise ValueError(
+ "Nanosecond duration {} is not safely convertible to "
+ "microseconds to convert to datetime.timedelta. Install "
+ "pandas to return as Timedelta with nanosecond support or "
+ "access the .value attribute.".format(sp.value)
+ )
+ return datetime.timedelta(microseconds=sp.value // 1000)
-cdef class HalfFloatValue(ArrayValue):
+cdef class BinaryScalar(Scalar):
"""
- Concrete class for float16 array elements.
+ Concrete class for binary-like scalars.
"""
- def as_py(self):
+ def as_buffer(self):
"""
- Return this value as a Python float.
+ Return a view over this value as a Buffer object.
"""
- cdef CHalfFloatArray* ap = <CHalfFloatArray*> self.sp_array.get()
- return PyHalf_FromHalf(ap.Value(self.index))
-
-
-cdef class FloatValue(ArrayValue):
- """
- Concrete class for float32 array elements.
- """
+ cdef CBinaryScalar* sp = <CBinaryScalar*> self.wrapped.get()
+ return pyarrow_wrap_buffer(sp.value) if sp.is_valid else None
def as_py(self):
"""
- Return this value as a Python float.
+ Return this value as a Python bytes.
"""
- cdef CFloatArray* ap = <CFloatArray*> self.sp_array.get()
- return ap.Value(self.index)
+ buffer = self.as_buffer()
+ if buffer is not None:
+ return self.as_buffer().to_pybytes()
+ else:
+ return None
-cdef class DoubleValue(ArrayValue):
- """
- Concrete class for float64 array elements.
- """
+cdef class LargeBinaryScalar(BinaryScalar):
Review comment:
It is not safe to inherit `BinaryScalar.as_buffer`, because it is casting the C++ scalar to `CBinaryScalar`.
Instead you should redefine that method.
##########
File path: python/pyarrow/scalar.pxi
##########
@@ -16,1198 +16,745 @@
# under the License.
-_NULL = NA = None
+import collections
cdef class Scalar:
"""
- The base class for all array elements.
+ The base class for scalars.
"""
+ def __init__(self):
+ raise TypeError("Do not call {}'s constructor directly, use "
+ "pa.scalar() instead.".format(self.__class__.__name__))
-cdef class NullType(Scalar):
- """
- Singleton for null array elements.
- """
- # TODO rename this NullValue?
+ cdef void init(self, const shared_ptr[CScalar]& wrapped):
+ self.wrapped = wrapped
- def __cinit__(self):
- global NA
- if NA is not None:
- raise Exception('Cannot create multiple NAType instances')
+ @staticmethod
+ cdef wrap(const shared_ptr[CScalar]& wrapped):
+ cdef:
+ Scalar self
+ Type type_id = wrapped.get().type.get().id()
+
+ if type_id == _Type_NA:
+ return _NULL
+
+ typ = _scalar_classes[type_id]
+ self = typ.__new__(typ)
+ self.init(wrapped)
+
+ return self
+
+ cdef inline shared_ptr[CScalar] unwrap(self) nogil:
+ return self.wrapped
+
+ @property
+ def type(self):
+ return pyarrow_wrap_data_type(self.wrapped.get().type)
- self.type = null()
+ @property
+ def is_valid(self):
+ return self.wrapped.get().is_valid
def __repr__(self):
- return 'NULL'
+ return '<pyarrow.{}: {!r}>'.format(
+ self.__class__.__name__, self.as_py()
+ )
- def as_py(self):
- """
- Return None
- """
- return None
+ def __str__(self):
+ return str(self.as_py())
+
+ def equals(self, Scalar other):
+ return self.wrapped.get().Equals(other.unwrap().get()[0])
def __eq__(self, other):
- return NA
+ try:
+ if not isinstance(other, Scalar):
+ other = scalar(other, type=self.type)
+ return self.equals(other)
+ except (TypeError, ValueError, ArrowInvalid):
+ return NotImplemented
+
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+
+ def as_py(self):
+ raise NotImplementedError()
-_NULL = NA = NullType()
+_NULL = NA = None
-cdef class ArrayValue(Scalar):
+cdef class NullScalar(Scalar):
"""
- The base class for non-null array elements.
+ Concrete class for null scalars.
"""
- def __init__(self):
- raise TypeError("Do not call {}'s constructor directly, use array "
- "subscription instead."
- .format(self.__class__.__name__))
+ def __cinit__(self):
+ global NA
+ if NA is not None:
+ raise Exception('Cannot create multiple NAType instances')
+ self.init(shared_ptr[CScalar](new CNullScalar()))
- cdef void init(self, DataType type, const shared_ptr[CArray]& sp_array,
- int64_t index):
- self.type = type
- self.index = index
- self._set_array(sp_array)
+ def __init__(self):
+ pass
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
+ def __eq__(self, other):
+ return NA
- def __repr__(self):
- if hasattr(self, 'as_py'):
- return repr(self.as_py())
- else:
- return super(Scalar, self).__repr__()
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
- def __str__(self):
- if hasattr(self, 'as_py'):
- return str(self.as_py())
- else:
- return super(Scalar, self).__str__()
+ def as_py(self):
+ """
+ Return this value as a Python None.
+ """
+ return None
- def __eq__(self, other):
- if hasattr(self, 'as_py'):
- if isinstance(other, ArrayValue):
- other = other.as_py()
- return self.as_py() == other
- else:
- raise NotImplementedError(
- "Cannot compare Arrow values that don't support as_py()")
- def __hash__(self):
- return hash(self.as_py())
+_NULL = NA = NullScalar()
-cdef class BooleanValue(ArrayValue):
+cdef class BooleanScalar(Scalar):
"""
- Concrete class for boolean array elements.
+ Concrete class for boolean scalars.
"""
def as_py(self):
"""
Return this value as a Python bool.
"""
- cdef CBooleanArray* ap = <CBooleanArray*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CBooleanScalar* sp = <CBooleanScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int8Value(ArrayValue):
+cdef class UInt8Scalar(Scalar):
"""
- Concrete class for int8 array elements.
+ Concrete class for uint8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt8Array* ap = <CInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt8Scalar* sp = <CUInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt8Value(ArrayValue):
+cdef class Int8Scalar(Scalar):
"""
- Concrete class for uint8 array elements.
+ Concrete class for int8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt8Array* ap = <CUInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt8Scalar* sp = <CInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int16Value(ArrayValue):
+cdef class UInt16Scalar(Scalar):
"""
- Concrete class for int16 array elements.
+ Concrete class for uint16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt16Array* ap = <CInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt16Scalar* sp = <CUInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt16Value(ArrayValue):
+cdef class Int16Scalar(Scalar):
"""
- Concrete class for uint16 array elements.
+ Concrete class for int16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt16Array* ap = <CUInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt16Scalar* sp = <CInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int32Value(ArrayValue):
+cdef class UInt32Scalar(Scalar):
"""
- Concrete class for int32 array elements.
+ Concrete class for uint32 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt32Array* ap = <CInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt32Scalar* sp = <CUInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt32Value(ArrayValue):
+cdef class Int32Scalar(Scalar):
"""
- Concrete class for uint32 array elements.
+ Concrete class for int32 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt32Array* ap = <CUInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt32Scalar* sp = <CInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int64Value(ArrayValue):
+cdef class UInt64Scalar(Scalar):
"""
- Concrete class for int64 array elements.
+ Concrete class for uint64 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt64Array* ap = <CInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt64Scalar* sp = <CUInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt64Value(ArrayValue):
+cdef class Int64Scalar(Scalar):
"""
- Concrete class for uint64 array elements.
+ Concrete class for int64 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt64Array* ap = <CUInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt64Scalar* sp = <CInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date32Value(ArrayValue):
+cdef class HalfFloatScalar(Scalar):
"""
- Concrete class for date32 array elements.
+ Concrete class for float scalars.
"""
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+
+ def __eq__(self, other):
+ if hasattr(self, 'as_py'):
+ if isinstance(other, Scalar):
+ other = other.as_py()
+ return self.as_py() == other
+ else:
+ raise NotImplementedError
+
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python float.
"""
- cdef CDate32Array* ap = <CDate32Array*> self.sp_array.get()
+ cdef CHalfFloatScalar* sp = <CHalfFloatScalar*> self.wrapped.get()
+ return PyHalf_FromHalf(sp.value) if sp.is_valid else None
- # Shift to seconds since epoch
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(days=ap.Value(self.index)))
+
+cdef class FloatScalar(Scalar):
+ """
+ Concrete class for float scalars.
+ """
+
+ def as_py(self):
+ """
+ Return this value as a Python float.
+ """
+ cdef CFloatScalar* sp = <CFloatScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date64Value(ArrayValue):
+cdef class DoubleScalar(Scalar):
"""
- Concrete class for date64 array elements.
+ Concrete class for double scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python float.
"""
- cdef CDate64Array* ap = <CDate64Array*> self.sp_array.get()
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(
- days=ap.Value(self.index) / 86400000))
+ cdef CDoubleScalar* sp = <CDoubleScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Time32Value(ArrayValue):
+cdef class Decimal128Scalar(Scalar):
"""
- Concrete class for time32 array elements.
+ Concrete class for decimal128 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.timedelta instance.
+ Return this value as a Python Decimal.
"""
cdef:
- CTime32Array* ap = <CTime32Array*> self.sp_array.get()
- CTime32Type* dtype = <CTime32Type*> ap.type().get()
-
- if dtype.unit() == TimeUnit_SECOND:
- delta = datetime.timedelta(seconds=ap.Value(self.index))
- return (datetime.datetime(1970, 1, 1) + delta).time()
+ CDecimal128Scalar* sp = <CDecimal128Scalar*> self.wrapped.get()
+ CDecimal128Type* dtype = <CDecimal128Type*> sp.type.get()
+ if sp.is_valid:
+ return _pydecimal.Decimal(
+ frombytes(sp.value.ToString(dtype.scale()))
+ )
else:
- return _box_time_milli(ap.Value(self.index))
+ return None
-cdef class Time64Value(ArrayValue):
+cdef class Date32Scalar(Scalar):
"""
- Concrete class for time64 array elements.
+ Concrete class for date32 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.timedelta instance.
+ Return this value as a Python datetime.datetime instance.
"""
- cdef:
- CTime64Array* ap = <CTime64Array*> self.sp_array.get()
- CTime64Type* dtype = <CTime64Type*> ap.type().get()
+ cdef CDate32Scalar* sp = <CDate32Scalar*> self.wrapped.get()
- cdef int64_t val = ap.Value(self.index)
- if dtype.unit() == TimeUnit_MICRO:
- return _box_time_micro(val)
+ if sp.is_valid:
+ # shift to seconds since epoch
+ return (
+ datetime.date(1970, 1, 1) + datetime.timedelta(days=sp.value)
+ )
else:
- return (datetime.datetime(1970, 1, 1) +
- datetime.timedelta(microseconds=val / 1000)).time()
-
+ return None
-cpdef _box_time_milli(int64_t val):
- delta = datetime.timedelta(milliseconds=val)
- return (datetime.datetime(1970, 1, 1) + delta).time()
+cdef class Date64Scalar(Scalar):
+ """
+ Concrete class for date64 scalars.
+ """
-cpdef _box_time_micro(int64_t val):
- return (datetime.datetime(1970, 1, 1) +
- datetime.timedelta(microseconds=val)).time()
+ def as_py(self):
+ """
+ Return this value as a Python datetime.datetime instance.
+ """
+ cdef CDate64Scalar* sp = <CDate64Scalar*> self.wrapped.get()
+ if sp.is_valid:
+ return (
+ datetime.date(1970, 1, 1) +
+ datetime.timedelta(days=sp.value / 86400000)
+ )
+ else:
+ return None
-cdef dict _DATETIME_CONVERSION_FUNCTIONS = {}
-cdef c_bool _datetime_conversion_initialized = False
+def _datetime_from_int(int64_t value, TimeUnit unit, tzinfo=None):
+ if unit == TimeUnit_SECOND:
+ delta = datetime.timedelta(seconds=value)
+ elif unit == TimeUnit_MILLI:
+ delta = datetime.timedelta(milliseconds=value)
+ elif unit == TimeUnit_MICRO:
+ delta = datetime.timedelta(microseconds=value)
+ else:
+ # TimeUnit_NANO: prefer pandas timestamps if available
+ if _pandas_api.have_pandas:
+ return _pandas_api.pd.Timestamp(value, tz=tzinfo, unit='ns')
+ # otherwise safely truncate to microsecond resolution datetime
+ if value % 1000 != 0:
+ raise ValueError(
+ "Nanosecond resolution temporal type {} is not safely "
+ "convertible to microseconds to convert to datetime.datetime. "
+ "Install pandas to return as Timestamp with nanosecond "
+ "support or access the .value attribute.".format(value)
+ )
+ delta = datetime.timedelta(microseconds=value // 1000)
-cdef _add_micros_maybe_localize(dt, micros, tzinfo):
- import pytz
- dt = dt.replace(microsecond=micros)
+ dt = datetime.datetime(1970, 1, 1) + delta
+ # adjust timezone if set to the datatype
if tzinfo is not None:
- if not isinstance(tzinfo, datetime.tzinfo):
- tzinfo = string_to_tzinfo(tzinfo)
dt = tzinfo.fromutc(dt)
- return dt
-
-
-cdef _datetime_from_seconds(int64_t v):
- return datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=v)
-
-
-def _nanoseconds_to_datetime_safe(v, tzinfo):
- if v % 1000 != 0:
- raise ValueError("Nanosecond timestamp {} is not safely convertible "
- " to microseconds to convert to datetime.datetime."
- " Install pandas to return as Timestamp with "
- " nanosecond support or access the .value attribute.")
- v = v // 1000
- micros = v % 1_000_000
-
- dt = _datetime_from_seconds(v // 1_000_000)
- return _add_micros_maybe_localize(dt, micros, tzinfo)
-
-
-def _microseconds_to_datetime(v, tzinfo):
- micros = v % 1_000_000
- dt = _datetime_from_seconds(v // 1_000_000)
- return _add_micros_maybe_localize(dt, micros, tzinfo)
+ return dt
-def _millis_to_datetime(v, tzinfo):
- millis = v % 1_000
- dt = _datetime_from_seconds(v // 1000)
- return _add_micros_maybe_localize(dt, millis * 1000, tzinfo)
+cdef class Time32Scalar(Scalar):
+ """
+ Concrete class for time32 scalars.
+ """
-def _seconds_to_datetime(v, tzinfo):
- dt = _datetime_from_seconds(v)
- return _add_micros_maybe_localize(dt, 0, tzinfo)
+ def as_py(self):
+ """
+ Return this value as a Python datetime.timedelta instance.
+ """
+ cdef:
+ CTime32Scalar* sp = <CTime32Scalar*> self.wrapped.get()
+ CTime32Type* dtype = <CTime32Type*> sp.type.get()
+ if sp.is_valid:
+ return _datetime_from_int(sp.value, unit=dtype.unit()).time()
+ else:
+ return None
-def _datetime_conversion_functions():
- global _datetime_conversion_initialized
- if _datetime_conversion_initialized:
- return _DATETIME_CONVERSION_FUNCTIONS
- _DATETIME_CONVERSION_FUNCTIONS.update({
- TimeUnit_SECOND: _seconds_to_datetime,
- TimeUnit_MILLI: _millis_to_datetime,
- TimeUnit_MICRO: _microseconds_to_datetime,
- TimeUnit_NANO: _nanoseconds_to_datetime_safe
- })
+cdef class Time64Scalar(Scalar):
+ """
+ Concrete class for time64 scalars.
+ """
- try:
- import pandas as pd
- _DATETIME_CONVERSION_FUNCTIONS[TimeUnit_NANO] = (
- lambda x, tzinfo: pd.Timestamp(
- x, tz=tzinfo, unit='ns',
- )
- )
- except ImportError:
- pass
+ def as_py(self):
+ """
+ Return this value as a Python datetime.timedelta instance.
+ """
+ cdef:
+ CTime64Scalar* sp = <CTime64Scalar*> self.wrapped.get()
+ CTime64Type* dtype = <CTime64Type*> sp.type.get()
- _datetime_conversion_initialized = True
- return _DATETIME_CONVERSION_FUNCTIONS
+ if sp.is_valid:
+ return _datetime_from_int(sp.value, unit=dtype.unit()).time()
+ else:
+ return None
-cdef class TimestampValue(ArrayValue):
+cdef class TimestampScalar(Scalar):
"""
- Concrete class for timestamp array elements.
+ Concrete class for timestamp scalars.
"""
@property
def value(self):
- cdef CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
- cdef CTimestampType* dtype = <CTimestampType*> ap.type().get()
- return ap.Value(self.index)
+ cdef CTimestampScalar* sp = <CTimestampScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
def as_py(self):
"""
Return this value as a Pandas Timestamp instance (if available),
otherwise as a Python datetime.timedelta instance.
"""
- cdef CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
- cdef CTimestampType* dtype = <CTimestampType*> ap.type().get()
+ cdef:
+ CTimestampScalar* sp = <CTimestampScalar*> self.wrapped.get()
+ CTimestampType* dtype = <CTimestampType*> sp.type.get()
- value = self.value
+ if not sp.is_valid:
+ return None
if not dtype.timezone().empty():
tzinfo = string_to_tzinfo(frombytes(dtype.timezone()))
+ if not isinstance(tzinfo, datetime.tzinfo):
+ tzinfo = string_to_tzinfo(tzinfo)
Review comment:
Why do you have to do this twice? If `string_to_tzinfo` is broken, it should be fixed IMO.
##########
File path: python/pyarrow/tests/test_dataset.py
##########
@@ -385,10 +385,12 @@ def test_partitioning():
def test_expression_serialization():
a = ds.scalar(1)
+ a_ = ds.scalar(pa.scalar(1))
Review comment:
Uh, can you use another name? Why not `g`?
##########
File path: python/pyarrow/scalar.pxi
##########
@@ -16,1198 +16,745 @@
# under the License.
-_NULL = NA = None
+import collections
cdef class Scalar:
"""
- The base class for all array elements.
+ The base class for scalars.
"""
+ def __init__(self):
+ raise TypeError("Do not call {}'s constructor directly, use "
+ "pa.scalar() instead.".format(self.__class__.__name__))
-cdef class NullType(Scalar):
- """
- Singleton for null array elements.
- """
- # TODO rename this NullValue?
+ cdef void init(self, const shared_ptr[CScalar]& wrapped):
+ self.wrapped = wrapped
- def __cinit__(self):
- global NA
- if NA is not None:
- raise Exception('Cannot create multiple NAType instances')
+ @staticmethod
+ cdef wrap(const shared_ptr[CScalar]& wrapped):
+ cdef:
+ Scalar self
+ Type type_id = wrapped.get().type.get().id()
+
+ if type_id == _Type_NA:
+ return _NULL
+
+ typ = _scalar_classes[type_id]
+ self = typ.__new__(typ)
+ self.init(wrapped)
+
+ return self
+
+ cdef inline shared_ptr[CScalar] unwrap(self) nogil:
+ return self.wrapped
+
+ @property
+ def type(self):
+ return pyarrow_wrap_data_type(self.wrapped.get().type)
- self.type = null()
+ @property
+ def is_valid(self):
+ return self.wrapped.get().is_valid
def __repr__(self):
- return 'NULL'
+ return '<pyarrow.{}: {!r}>'.format(
+ self.__class__.__name__, self.as_py()
+ )
- def as_py(self):
- """
- Return None
- """
- return None
+ def __str__(self):
+ return str(self.as_py())
+
+ def equals(self, Scalar other):
+ return self.wrapped.get().Equals(other.unwrap().get()[0])
def __eq__(self, other):
- return NA
+ try:
+ if not isinstance(other, Scalar):
+ other = scalar(other, type=self.type)
+ return self.equals(other)
+ except (TypeError, ValueError, ArrowInvalid):
+ return NotImplemented
+
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+
+ def as_py(self):
+ raise NotImplementedError()
-_NULL = NA = NullType()
+_NULL = NA = None
-cdef class ArrayValue(Scalar):
+cdef class NullScalar(Scalar):
"""
- The base class for non-null array elements.
+ Concrete class for null scalars.
"""
- def __init__(self):
- raise TypeError("Do not call {}'s constructor directly, use array "
- "subscription instead."
- .format(self.__class__.__name__))
+ def __cinit__(self):
+ global NA
+ if NA is not None:
+ raise Exception('Cannot create multiple NAType instances')
+ self.init(shared_ptr[CScalar](new CNullScalar()))
- cdef void init(self, DataType type, const shared_ptr[CArray]& sp_array,
- int64_t index):
- self.type = type
- self.index = index
- self._set_array(sp_array)
+ def __init__(self):
+ pass
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
+ def __eq__(self, other):
+ return NA
- def __repr__(self):
- if hasattr(self, 'as_py'):
- return repr(self.as_py())
- else:
- return super(Scalar, self).__repr__()
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
- def __str__(self):
- if hasattr(self, 'as_py'):
- return str(self.as_py())
- else:
- return super(Scalar, self).__str__()
+ def as_py(self):
+ """
+ Return this value as a Python None.
+ """
+ return None
- def __eq__(self, other):
- if hasattr(self, 'as_py'):
- if isinstance(other, ArrayValue):
- other = other.as_py()
- return self.as_py() == other
- else:
- raise NotImplementedError(
- "Cannot compare Arrow values that don't support as_py()")
- def __hash__(self):
- return hash(self.as_py())
+_NULL = NA = NullScalar()
-cdef class BooleanValue(ArrayValue):
+cdef class BooleanScalar(Scalar):
"""
- Concrete class for boolean array elements.
+ Concrete class for boolean scalars.
"""
def as_py(self):
"""
Return this value as a Python bool.
"""
- cdef CBooleanArray* ap = <CBooleanArray*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CBooleanScalar* sp = <CBooleanScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int8Value(ArrayValue):
+cdef class UInt8Scalar(Scalar):
"""
- Concrete class for int8 array elements.
+ Concrete class for uint8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt8Array* ap = <CInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt8Scalar* sp = <CUInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt8Value(ArrayValue):
+cdef class Int8Scalar(Scalar):
"""
- Concrete class for uint8 array elements.
+ Concrete class for int8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt8Array* ap = <CUInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt8Scalar* sp = <CInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int16Value(ArrayValue):
+cdef class UInt16Scalar(Scalar):
"""
- Concrete class for int16 array elements.
+ Concrete class for uint16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt16Array* ap = <CInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt16Scalar* sp = <CUInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt16Value(ArrayValue):
+cdef class Int16Scalar(Scalar):
"""
- Concrete class for uint16 array elements.
+ Concrete class for int16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt16Array* ap = <CUInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt16Scalar* sp = <CInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int32Value(ArrayValue):
+cdef class UInt32Scalar(Scalar):
"""
- Concrete class for int32 array elements.
+ Concrete class for uint32 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt32Array* ap = <CInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt32Scalar* sp = <CUInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt32Value(ArrayValue):
+cdef class Int32Scalar(Scalar):
"""
- Concrete class for uint32 array elements.
+ Concrete class for int32 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt32Array* ap = <CUInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt32Scalar* sp = <CInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int64Value(ArrayValue):
+cdef class UInt64Scalar(Scalar):
"""
- Concrete class for int64 array elements.
+ Concrete class for uint64 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt64Array* ap = <CInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt64Scalar* sp = <CUInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt64Value(ArrayValue):
+cdef class Int64Scalar(Scalar):
"""
- Concrete class for uint64 array elements.
+ Concrete class for int64 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt64Array* ap = <CUInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt64Scalar* sp = <CInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date32Value(ArrayValue):
+cdef class HalfFloatScalar(Scalar):
"""
- Concrete class for date32 array elements.
+ Concrete class for float scalars.
"""
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+
+ def __eq__(self, other):
+ if hasattr(self, 'as_py'):
+ if isinstance(other, Scalar):
+ other = other.as_py()
+ return self.as_py() == other
+ else:
+ raise NotImplementedError
+
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python float.
"""
- cdef CDate32Array* ap = <CDate32Array*> self.sp_array.get()
+ cdef CHalfFloatScalar* sp = <CHalfFloatScalar*> self.wrapped.get()
+ return PyHalf_FromHalf(sp.value) if sp.is_valid else None
- # Shift to seconds since epoch
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(days=ap.Value(self.index)))
+
+cdef class FloatScalar(Scalar):
+ """
+ Concrete class for float scalars.
+ """
+
+ def as_py(self):
+ """
+ Return this value as a Python float.
+ """
+ cdef CFloatScalar* sp = <CFloatScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date64Value(ArrayValue):
+cdef class DoubleScalar(Scalar):
"""
- Concrete class for date64 array elements.
+ Concrete class for double scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python float.
"""
- cdef CDate64Array* ap = <CDate64Array*> self.sp_array.get()
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(
- days=ap.Value(self.index) / 86400000))
+ cdef CDoubleScalar* sp = <CDoubleScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Time32Value(ArrayValue):
+cdef class Decimal128Scalar(Scalar):
"""
- Concrete class for time32 array elements.
+ Concrete class for decimal128 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.timedelta instance.
+ Return this value as a Python Decimal.
"""
cdef:
- CTime32Array* ap = <CTime32Array*> self.sp_array.get()
- CTime32Type* dtype = <CTime32Type*> ap.type().get()
-
- if dtype.unit() == TimeUnit_SECOND:
- delta = datetime.timedelta(seconds=ap.Value(self.index))
- return (datetime.datetime(1970, 1, 1) + delta).time()
+ CDecimal128Scalar* sp = <CDecimal128Scalar*> self.wrapped.get()
+ CDecimal128Type* dtype = <CDecimal128Type*> sp.type.get()
+ if sp.is_valid:
+ return _pydecimal.Decimal(
+ frombytes(sp.value.ToString(dtype.scale()))
+ )
else:
- return _box_time_milli(ap.Value(self.index))
+ return None
-cdef class Time64Value(ArrayValue):
+cdef class Date32Scalar(Scalar):
"""
- Concrete class for time64 array elements.
+ Concrete class for date32 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.timedelta instance.
+ Return this value as a Python datetime.datetime instance.
"""
- cdef:
- CTime64Array* ap = <CTime64Array*> self.sp_array.get()
- CTime64Type* dtype = <CTime64Type*> ap.type().get()
+ cdef CDate32Scalar* sp = <CDate32Scalar*> self.wrapped.get()
- cdef int64_t val = ap.Value(self.index)
- if dtype.unit() == TimeUnit_MICRO:
- return _box_time_micro(val)
+ if sp.is_valid:
+ # shift to seconds since epoch
+ return (
+ datetime.date(1970, 1, 1) + datetime.timedelta(days=sp.value)
+ )
else:
- return (datetime.datetime(1970, 1, 1) +
- datetime.timedelta(microseconds=val / 1000)).time()
-
+ return None
-cpdef _box_time_milli(int64_t val):
- delta = datetime.timedelta(milliseconds=val)
- return (datetime.datetime(1970, 1, 1) + delta).time()
+cdef class Date64Scalar(Scalar):
+ """
+ Concrete class for date64 scalars.
+ """
-cpdef _box_time_micro(int64_t val):
- return (datetime.datetime(1970, 1, 1) +
- datetime.timedelta(microseconds=val)).time()
+ def as_py(self):
+ """
+ Return this value as a Python datetime.datetime instance.
+ """
+ cdef CDate64Scalar* sp = <CDate64Scalar*> self.wrapped.get()
+ if sp.is_valid:
+ return (
+ datetime.date(1970, 1, 1) +
+ datetime.timedelta(days=sp.value / 86400000)
+ )
+ else:
+ return None
-cdef dict _DATETIME_CONVERSION_FUNCTIONS = {}
-cdef c_bool _datetime_conversion_initialized = False
+def _datetime_from_int(int64_t value, TimeUnit unit, tzinfo=None):
+ if unit == TimeUnit_SECOND:
+ delta = datetime.timedelta(seconds=value)
+ elif unit == TimeUnit_MILLI:
+ delta = datetime.timedelta(milliseconds=value)
+ elif unit == TimeUnit_MICRO:
+ delta = datetime.timedelta(microseconds=value)
+ else:
+ # TimeUnit_NANO: prefer pandas timestamps if available
+ if _pandas_api.have_pandas:
+ return _pandas_api.pd.Timestamp(value, tz=tzinfo, unit='ns')
+ # otherwise safely truncate to microsecond resolution datetime
+ if value % 1000 != 0:
+ raise ValueError(
+ "Nanosecond resolution temporal type {} is not safely "
+ "convertible to microseconds to convert to datetime.datetime. "
+ "Install pandas to return as Timestamp with nanosecond "
+ "support or access the .value attribute.".format(value)
+ )
+ delta = datetime.timedelta(microseconds=value // 1000)
-cdef _add_micros_maybe_localize(dt, micros, tzinfo):
- import pytz
- dt = dt.replace(microsecond=micros)
+ dt = datetime.datetime(1970, 1, 1) + delta
+ # adjust timezone if set to the datatype
if tzinfo is not None:
- if not isinstance(tzinfo, datetime.tzinfo):
- tzinfo = string_to_tzinfo(tzinfo)
dt = tzinfo.fromutc(dt)
- return dt
-
-
-cdef _datetime_from_seconds(int64_t v):
- return datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=v)
-
-
-def _nanoseconds_to_datetime_safe(v, tzinfo):
- if v % 1000 != 0:
- raise ValueError("Nanosecond timestamp {} is not safely convertible "
- " to microseconds to convert to datetime.datetime."
- " Install pandas to return as Timestamp with "
- " nanosecond support or access the .value attribute.")
- v = v // 1000
- micros = v % 1_000_000
-
- dt = _datetime_from_seconds(v // 1_000_000)
- return _add_micros_maybe_localize(dt, micros, tzinfo)
-
-
-def _microseconds_to_datetime(v, tzinfo):
- micros = v % 1_000_000
- dt = _datetime_from_seconds(v // 1_000_000)
- return _add_micros_maybe_localize(dt, micros, tzinfo)
+ return dt
-def _millis_to_datetime(v, tzinfo):
- millis = v % 1_000
- dt = _datetime_from_seconds(v // 1000)
- return _add_micros_maybe_localize(dt, millis * 1000, tzinfo)
+cdef class Time32Scalar(Scalar):
+ """
+ Concrete class for time32 scalars.
+ """
-def _seconds_to_datetime(v, tzinfo):
- dt = _datetime_from_seconds(v)
- return _add_micros_maybe_localize(dt, 0, tzinfo)
+ def as_py(self):
+ """
+ Return this value as a Python datetime.timedelta instance.
+ """
+ cdef:
+ CTime32Scalar* sp = <CTime32Scalar*> self.wrapped.get()
+ CTime32Type* dtype = <CTime32Type*> sp.type.get()
+ if sp.is_valid:
+ return _datetime_from_int(sp.value, unit=dtype.unit()).time()
+ else:
+ return None
-def _datetime_conversion_functions():
- global _datetime_conversion_initialized
- if _datetime_conversion_initialized:
- return _DATETIME_CONVERSION_FUNCTIONS
- _DATETIME_CONVERSION_FUNCTIONS.update({
- TimeUnit_SECOND: _seconds_to_datetime,
- TimeUnit_MILLI: _millis_to_datetime,
- TimeUnit_MICRO: _microseconds_to_datetime,
- TimeUnit_NANO: _nanoseconds_to_datetime_safe
- })
+cdef class Time64Scalar(Scalar):
+ """
+ Concrete class for time64 scalars.
+ """
- try:
- import pandas as pd
- _DATETIME_CONVERSION_FUNCTIONS[TimeUnit_NANO] = (
- lambda x, tzinfo: pd.Timestamp(
- x, tz=tzinfo, unit='ns',
- )
- )
- except ImportError:
- pass
+ def as_py(self):
+ """
+ Return this value as a Python datetime.timedelta instance.
+ """
+ cdef:
+ CTime64Scalar* sp = <CTime64Scalar*> self.wrapped.get()
+ CTime64Type* dtype = <CTime64Type*> sp.type.get()
- _datetime_conversion_initialized = True
- return _DATETIME_CONVERSION_FUNCTIONS
+ if sp.is_valid:
+ return _datetime_from_int(sp.value, unit=dtype.unit()).time()
+ else:
+ return None
-cdef class TimestampValue(ArrayValue):
+cdef class TimestampScalar(Scalar):
"""
- Concrete class for timestamp array elements.
+ Concrete class for timestamp scalars.
"""
@property
def value(self):
- cdef CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
- cdef CTimestampType* dtype = <CTimestampType*> ap.type().get()
- return ap.Value(self.index)
+ cdef CTimestampScalar* sp = <CTimestampScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
def as_py(self):
"""
Return this value as a Pandas Timestamp instance (if available),
otherwise as a Python datetime.timedelta instance.
"""
- cdef CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
- cdef CTimestampType* dtype = <CTimestampType*> ap.type().get()
+ cdef:
+ CTimestampScalar* sp = <CTimestampScalar*> self.wrapped.get()
+ CTimestampType* dtype = <CTimestampType*> sp.type.get()
- value = self.value
+ if not sp.is_valid:
+ return None
if not dtype.timezone().empty():
tzinfo = string_to_tzinfo(frombytes(dtype.timezone()))
+ if not isinstance(tzinfo, datetime.tzinfo):
+ tzinfo = string_to_tzinfo(tzinfo)
else:
tzinfo = None
- try:
- converter = _datetime_conversion_functions()[dtype.unit()]
- except KeyError:
- raise ValueError(
- 'Cannot convert nanosecond timestamps without pandas'
- )
- return converter(value, tzinfo=tzinfo)
-
-
-cdef dict _TIMEDELTA_CONVERSION_FUNCTIONS = {}
-
-
-def _nanoseconds_to_timedelta_safe(v):
- if v % 1000 != 0:
- raise ValueError(
- "Nanosecond duration {} is not safely convertible to microseconds "
- "to convert to datetime.timedelta. Install pandas to return as "
- "Timedelta with nanosecond support or access the .value "
- "attribute.".format(v))
- micros = v // 1000
-
- return datetime.timedelta(microseconds=micros)
-
-
-def _timedelta_conversion_functions():
- if _TIMEDELTA_CONVERSION_FUNCTIONS:
- return _TIMEDELTA_CONVERSION_FUNCTIONS
-
- _TIMEDELTA_CONVERSION_FUNCTIONS.update({
- TimeUnit_SECOND: lambda v: datetime.timedelta(seconds=v),
- TimeUnit_MILLI: lambda v: datetime.timedelta(milliseconds=v),
- TimeUnit_MICRO: lambda v: datetime.timedelta(microseconds=v),
- TimeUnit_NANO: _nanoseconds_to_timedelta_safe
- })
-
- try:
- import pandas as pd
- _TIMEDELTA_CONVERSION_FUNCTIONS[TimeUnit_NANO] = (
- lambda v: pd.Timedelta(v, unit='ns')
- )
- except ImportError:
- pass
+ return _datetime_from_int(sp.value, unit=dtype.unit(), tzinfo=tzinfo)
- return _TIMEDELTA_CONVERSION_FUNCTIONS
-
-cdef class DurationValue(ArrayValue):
+cdef class DurationScalar(Scalar):
"""
- Concrete class for duration array elements.
+ Concrete class for duration scalars.
"""
@property
def value(self):
- cdef CDurationArray* ap = <CDurationArray*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CDurationScalar* sp = <CDurationScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
def as_py(self):
"""
Return this value as a Pandas Timestamp instance (if available),
otherwise as a Python datetime.timedelta instance.
"""
- cdef CDurationArray* ap = <CDurationArray*> self.sp_array.get()
- cdef CDurationType* dtype = <CDurationType*> ap.type().get()
+ cdef:
+ CDurationScalar* sp = <CDurationScalar*> self.wrapped.get()
+ CDurationType* dtype = <CDurationType*> sp.type.get()
+ TimeUnit unit = dtype.unit()
+
+ if not sp.is_valid:
+ return None
- cdef int64_t value = ap.Value(self.index)
- converter = _timedelta_conversion_functions()[dtype.unit()]
- return converter(value)
+ if unit == TimeUnit_SECOND:
+ return datetime.timedelta(seconds=sp.value)
+ elif unit == TimeUnit_MILLI:
+ return datetime.timedelta(milliseconds=sp.value)
+ elif unit == TimeUnit_MICRO:
+ return datetime.timedelta(microseconds=sp.value)
+ else:
+ # TimeUnit_NANO: prefer pandas timestamps if available
+ if _pandas_api.have_pandas:
+ return _pandas_api.pd.Timedelta(sp.value, unit='ns')
+ # otherwise safely truncate to microsecond resolution timedelta
+ if sp.value % 1000 != 0:
+ raise ValueError(
+ "Nanosecond duration {} is not safely convertible to "
+ "microseconds to convert to datetime.timedelta. Install "
+ "pandas to return as Timedelta with nanosecond support or "
+ "access the .value attribute.".format(sp.value)
+ )
+ return datetime.timedelta(microseconds=sp.value // 1000)
-cdef class HalfFloatValue(ArrayValue):
+cdef class BinaryScalar(Scalar):
"""
- Concrete class for float16 array elements.
+ Concrete class for binary-like scalars.
"""
- def as_py(self):
+ def as_buffer(self):
"""
- Return this value as a Python float.
+ Return a view over this value as a Buffer object.
"""
- cdef CHalfFloatArray* ap = <CHalfFloatArray*> self.sp_array.get()
- return PyHalf_FromHalf(ap.Value(self.index))
-
-
-cdef class FloatValue(ArrayValue):
- """
- Concrete class for float32 array elements.
- """
+ cdef CBinaryScalar* sp = <CBinaryScalar*> self.wrapped.get()
+ return pyarrow_wrap_buffer(sp.value) if sp.is_valid else None
def as_py(self):
"""
- Return this value as a Python float.
+ Return this value as a Python bytes.
"""
- cdef CFloatArray* ap = <CFloatArray*> self.sp_array.get()
- return ap.Value(self.index)
+ buffer = self.as_buffer()
+ if buffer is not None:
+ return self.as_buffer().to_pybytes()
+ else:
+ return None
-cdef class DoubleValue(ArrayValue):
- """
- Concrete class for float64 array elements.
- """
+cdef class LargeBinaryScalar(BinaryScalar):
+ pass
- def as_py(self):
- """
- Return this value as a Python float.
- """
- cdef CDoubleArray* ap = <CDoubleArray*> self.sp_array.get()
- return ap.Value(self.index)
+
+cdef class FixedSizeBinaryScalar(BinaryScalar):
+ pass
-cdef class DecimalValue(ArrayValue):
+cdef class StringScalar(BinaryScalar):
"""
- Concrete class for decimal128 array elements.
+ Concrete class for string-like (utf8) scalars.
"""
def as_py(self):
"""
- Return this value as a Python Decimal.
+ Return this value as a Python string.
"""
- cdef:
- CDecimal128Array* ap = <CDecimal128Array*> self.sp_array.get()
- c_string s = ap.FormatValue(self.index)
- return _pydecimal.Decimal(s.decode('utf8'))
+ buffer = self.as_buffer()
+ if buffer is not None:
+ return frombytes(self.as_buffer().to_pybytes())
+ else:
+ return None
+
+cdef class LargeStringScalar(StringScalar):
+ pass
-cdef class StringValue(ArrayValue):
+
+cdef class ListScalar(Scalar):
"""
- Concrete class for string (utf8) array elements.
+ Concrete class for list-like scalars.
"""
- def as_py(self):
+ @property
+ def values(self):
+ cdef CListScalar* sp = <CListScalar*> self.wrapped.get()
+ if sp.is_valid:
+ return pyarrow_wrap_array(sp.value)
+ else:
+ return None
+
+ def __len__(self):
"""
- Return this value as a Python unicode string.
+ Return the number of values.
"""
- cdef CStringArray* ap = <CStringArray*> self.sp_array.get()
- return ap.GetString(self.index).decode('utf-8')
+ return len(self.values)
- def as_buffer(self):
+ def __getitem__(self, i):
"""
- Return a view over this value as a Buffer object.
+ Return the value at the given index.
"""
- cdef:
- CStringArray* ap = <CStringArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
-
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
-
+ return self.values[_normalize_index(i, len(self))]
-cdef class LargeStringValue(ArrayValue):
- """
- Concrete class for large string (utf8) array elements.
- """
-
- def as_py(self):
+ def __iter__(self):
"""
- Return this value as a Python unicode string.
+ Iterate over this element's values.
"""
- cdef CLargeStringArray* ap = <CLargeStringArray*> self.sp_array.get()
- return ap.GetString(self.index).decode('utf-8')
+ return iter(self.values)
- def as_buffer(self):
+ def as_py(self):
"""
- Return a view over this value as a Buffer object.
+ Return this value as a Python list.
"""
- cdef:
- CLargeStringArray* ap = <CLargeStringArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
+ arr = self.values
+ return None if arr is None else arr.to_pylist()
+
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
+cdef class FixedSizeListScalar(ListScalar):
+ pass
-cdef class BinaryValue(ArrayValue):
+cdef class LargeListScalar(ListScalar):
+ pass
+
+
+cdef class StructScalar(Scalar, collections.abc.Mapping):
"""
- Concrete class for variable-sized binary array elements.
+ Concrete class for struct scalars.
"""
- def as_py(self):
- """
- Return this value as a Python bytes object.
- """
+ def __len__(self):
+ cdef CStructScalar* sp = <CStructScalar*> self.wrapped.get()
+ return sp.value.size()
+
+ def __iter__(self):
cdef:
- const uint8_t* ptr
- int32_t length
- CBinaryArray* ap = <CBinaryArray*> self.sp_array.get()
+ CStructScalar* sp = <CStructScalar*> self.wrapped.get()
+ CStructType* dtype = <CStructType*> sp.type.get()
+ vector[shared_ptr[CField]] fields = dtype.fields()
- ptr = ap.GetValue(self.index, &length)
- return cp.PyBytes_FromStringAndSize(<const char*>(ptr), length)
+ if sp.is_valid:
+ for i in range(dtype.num_fields()):
+ yield frombytes(fields[i].get().name())
- def as_buffer(self):
+ def __contains__(self, key):
+ try:
+ self[key]
+ except IndexError:
+ return False
+ else:
+ return True
+
+ def __getitem__(self, key):
"""
- Return a view over this value as a Buffer object.
+ Return the child value for the given field.
+
+ Parameters
+ ----------
+ index : Union[int, str]
+ Index / position or name of the field.
+
+ Returns
+ -------
+ result : Scalar
"""
cdef:
- CBinaryArray* ap = <CBinaryArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
+ CFieldRef ref
+ CStructScalar* sp = <CStructScalar*> self.wrapped.get()
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
+ if isinstance(key, (bytes, str)):
+ ref = CFieldRef(<c_string> tobytes(key))
+ elif isinstance(key, int):
+ ref = CFieldRef(<int> key)
+ else:
+ raise TypeError('Expected integer or string index')
-
-cdef class LargeBinaryValue(ArrayValue):
- """
- Concrete class for large variable-sized binary array elements.
- """
-
- def as_py(self):
- """
- Return this value as a Python bytes object.
- """
- cdef:
- const uint8_t* ptr
- int64_t length
- CLargeBinaryArray* ap = <CLargeBinaryArray*> self.sp_array.get()
-
- ptr = ap.GetValue(self.index, &length)
- return cp.PyBytes_FromStringAndSize(<const char*>(ptr), length)
-
- def as_buffer(self):
- """
- Return a view over this value as a Buffer object.
- """
- cdef:
- CLargeBinaryArray* ap = <CLargeBinaryArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
-
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
-
-
-cdef class ListValue(ArrayValue):
- """
- Concrete class for list array elements.
- """
-
- def __len__(self):
- """
- Return the number of values.
- """
- return self.length()
-
- def __getitem__(self, i):
- """
- Return the value at the given index.
- """
- return self.getitem(_normalize_index(i, self.length()))
-
- def __iter__(self):
- """
- Iterate over this element's values.
- """
- for i in range(len(self)):
- yield self.getitem(i)
- raise StopIteration
-
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CListArray*> sp_array.get()
- self.value_type = pyarrow_wrap_data_type(self.ap.value_type())
-
- cdef getitem(self, int64_t i):
- cdef int64_t j = self.ap.value_offset(self.index) + i
- return box_scalar(self.value_type, self.ap.values(), j)
-
- cdef int64_t length(self):
- return self.ap.value_length(self.index)
-
- def as_py(self):
- """
- Return this value as a Python list.
- """
- cdef:
- int64_t j
- list result = []
-
- for j in range(len(self)):
- result.append(self.getitem(j).as_py())
-
- return result
-
-
-cdef class LargeListValue(ArrayValue):
- """
- Concrete class for large list array elements.
- """
-
- def __len__(self):
- """
- Return the number of values.
- """
- return self.length()
-
- def __getitem__(self, i):
- """
- Return the value at the given index.
- """
- return self.getitem(_normalize_index(i, self.length()))
-
- def __iter__(self):
- """
- Iterate over this element's values.
- """
- for i in range(len(self)):
- yield self.getitem(i)
- raise StopIteration
-
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CLargeListArray*> sp_array.get()
- self.value_type = pyarrow_wrap_data_type(self.ap.value_type())
-
- cdef getitem(self, int64_t i):
- cdef int64_t j = self.ap.value_offset(self.index) + i
- return box_scalar(self.value_type, self.ap.values(), j)
-
- cdef int64_t length(self):
- return self.ap.value_length(self.index)
-
- def as_py(self):
- """
- Return this value as a Python list.
- """
- cdef:
- int64_t j
- list result = []
-
- for j in range(len(self)):
- result.append(self.getitem(j).as_py())
-
- return result
-
-
-cdef class MapValue(ArrayValue):
- """
- Concrete class for map array elements.
- """
-
- def __len__(self):
- """
- Return the number of values.
- """
- return self.length()
-
- def __getitem__(self, i):
- """
- Return the value at the given index.
- """
- return self.getitem(_normalize_index(i, self.length()))
-
- def __iter__(self):
- """
- Iterate over this element's values.
- """
- for i in range(len(self)):
- yield self.getitem(i)
- raise StopIteration
-
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CMapArray*> sp_array.get()
- self.key_type = pyarrow_wrap_data_type(self.ap.map_type().key_type())
- self.item_type = pyarrow_wrap_data_type(self.ap.map_type().item_type())
-
- cdef getitem(self, int64_t i):
- cdef int64_t j = self.ap.value_offset(self.index) + i
- return (box_scalar(self.key_type, self.ap.keys(), j),
- box_scalar(self.item_type, self.ap.items(), j))
-
- cdef int64_t length(self):
- return self.ap.value_length(self.index)
+ try:
+ return Scalar.wrap(GetResultValue(sp.field(ref)))
+ except ArrowInvalid:
+ raise IndexError(key)
def as_py(self):
"""
- Return this value as a Python list of tuples, each containing a
- key and item.
+ Return this value as a Python dict.
"""
- cdef:
- int64_t j
- list result = []
-
- for j in range(len(self)):
- key, item = self.getitem(j)
- result.append((key.as_py(), item.as_py()))
-
- return result
+ if self.is_valid:
+ return {k: v.as_py() for k, v in self.items()}
+ else:
+ return None
-cdef class FixedSizeListValue(ArrayValue):
+cdef class MapScalar(ListScalar):
Review comment:
Same inheritance problem.
##########
File path: python/pyarrow/scalar.pxi
##########
@@ -16,1198 +16,745 @@
# under the License.
-_NULL = NA = None
+import collections
cdef class Scalar:
"""
- The base class for all array elements.
+ The base class for scalars.
"""
+ def __init__(self):
+ raise TypeError("Do not call {}'s constructor directly, use "
+ "pa.scalar() instead.".format(self.__class__.__name__))
-cdef class NullType(Scalar):
- """
- Singleton for null array elements.
- """
- # TODO rename this NullValue?
+ cdef void init(self, const shared_ptr[CScalar]& wrapped):
+ self.wrapped = wrapped
- def __cinit__(self):
- global NA
- if NA is not None:
- raise Exception('Cannot create multiple NAType instances')
+ @staticmethod
+ cdef wrap(const shared_ptr[CScalar]& wrapped):
+ cdef:
+ Scalar self
+ Type type_id = wrapped.get().type.get().id()
+
+ if type_id == _Type_NA:
+ return _NULL
+
+ typ = _scalar_classes[type_id]
+ self = typ.__new__(typ)
+ self.init(wrapped)
+
+ return self
+
+ cdef inline shared_ptr[CScalar] unwrap(self) nogil:
+ return self.wrapped
+
+ @property
+ def type(self):
+ return pyarrow_wrap_data_type(self.wrapped.get().type)
- self.type = null()
+ @property
+ def is_valid(self):
+ return self.wrapped.get().is_valid
def __repr__(self):
- return 'NULL'
+ return '<pyarrow.{}: {!r}>'.format(
+ self.__class__.__name__, self.as_py()
+ )
- def as_py(self):
- """
- Return None
- """
- return None
+ def __str__(self):
+ return str(self.as_py())
+
+ def equals(self, Scalar other):
+ return self.wrapped.get().Equals(other.unwrap().get()[0])
def __eq__(self, other):
- return NA
+ try:
+ if not isinstance(other, Scalar):
+ other = scalar(other, type=self.type)
+ return self.equals(other)
+ except (TypeError, ValueError, ArrowInvalid):
+ return NotImplemented
+
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+
+ def as_py(self):
+ raise NotImplementedError()
-_NULL = NA = NullType()
+_NULL = NA = None
-cdef class ArrayValue(Scalar):
+cdef class NullScalar(Scalar):
"""
- The base class for non-null array elements.
+ Concrete class for null scalars.
"""
- def __init__(self):
- raise TypeError("Do not call {}'s constructor directly, use array "
- "subscription instead."
- .format(self.__class__.__name__))
+ def __cinit__(self):
+ global NA
+ if NA is not None:
+ raise Exception('Cannot create multiple NAType instances')
+ self.init(shared_ptr[CScalar](new CNullScalar()))
- cdef void init(self, DataType type, const shared_ptr[CArray]& sp_array,
- int64_t index):
- self.type = type
- self.index = index
- self._set_array(sp_array)
+ def __init__(self):
+ pass
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
+ def __eq__(self, other):
+ return NA
- def __repr__(self):
- if hasattr(self, 'as_py'):
- return repr(self.as_py())
- else:
- return super(Scalar, self).__repr__()
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
- def __str__(self):
- if hasattr(self, 'as_py'):
- return str(self.as_py())
- else:
- return super(Scalar, self).__str__()
+ def as_py(self):
+ """
+ Return this value as a Python None.
+ """
+ return None
- def __eq__(self, other):
- if hasattr(self, 'as_py'):
- if isinstance(other, ArrayValue):
- other = other.as_py()
- return self.as_py() == other
- else:
- raise NotImplementedError(
- "Cannot compare Arrow values that don't support as_py()")
- def __hash__(self):
- return hash(self.as_py())
+_NULL = NA = NullScalar()
-cdef class BooleanValue(ArrayValue):
+cdef class BooleanScalar(Scalar):
"""
- Concrete class for boolean array elements.
+ Concrete class for boolean scalars.
"""
def as_py(self):
"""
Return this value as a Python bool.
"""
- cdef CBooleanArray* ap = <CBooleanArray*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CBooleanScalar* sp = <CBooleanScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int8Value(ArrayValue):
+cdef class UInt8Scalar(Scalar):
"""
- Concrete class for int8 array elements.
+ Concrete class for uint8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt8Array* ap = <CInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt8Scalar* sp = <CUInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt8Value(ArrayValue):
+cdef class Int8Scalar(Scalar):
"""
- Concrete class for uint8 array elements.
+ Concrete class for int8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt8Array* ap = <CUInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt8Scalar* sp = <CInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int16Value(ArrayValue):
+cdef class UInt16Scalar(Scalar):
"""
- Concrete class for int16 array elements.
+ Concrete class for uint16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt16Array* ap = <CInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt16Scalar* sp = <CUInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt16Value(ArrayValue):
+cdef class Int16Scalar(Scalar):
"""
- Concrete class for uint16 array elements.
+ Concrete class for int16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt16Array* ap = <CUInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt16Scalar* sp = <CInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int32Value(ArrayValue):
+cdef class UInt32Scalar(Scalar):
"""
- Concrete class for int32 array elements.
+ Concrete class for uint32 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt32Array* ap = <CInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt32Scalar* sp = <CUInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt32Value(ArrayValue):
+cdef class Int32Scalar(Scalar):
"""
- Concrete class for uint32 array elements.
+ Concrete class for int32 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt32Array* ap = <CUInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt32Scalar* sp = <CInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int64Value(ArrayValue):
+cdef class UInt64Scalar(Scalar):
"""
- Concrete class for int64 array elements.
+ Concrete class for uint64 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt64Array* ap = <CInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt64Scalar* sp = <CUInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt64Value(ArrayValue):
+cdef class Int64Scalar(Scalar):
"""
- Concrete class for uint64 array elements.
+ Concrete class for int64 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt64Array* ap = <CUInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt64Scalar* sp = <CInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date32Value(ArrayValue):
+cdef class HalfFloatScalar(Scalar):
"""
- Concrete class for date32 array elements.
+ Concrete class for float scalars.
"""
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+
+ def __eq__(self, other):
+ if hasattr(self, 'as_py'):
+ if isinstance(other, Scalar):
+ other = other.as_py()
+ return self.as_py() == other
+ else:
+ raise NotImplementedError
+
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python float.
"""
- cdef CDate32Array* ap = <CDate32Array*> self.sp_array.get()
+ cdef CHalfFloatScalar* sp = <CHalfFloatScalar*> self.wrapped.get()
+ return PyHalf_FromHalf(sp.value) if sp.is_valid else None
- # Shift to seconds since epoch
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(days=ap.Value(self.index)))
+
+cdef class FloatScalar(Scalar):
+ """
+ Concrete class for float scalars.
+ """
+
+ def as_py(self):
+ """
+ Return this value as a Python float.
+ """
+ cdef CFloatScalar* sp = <CFloatScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date64Value(ArrayValue):
+cdef class DoubleScalar(Scalar):
"""
- Concrete class for date64 array elements.
+ Concrete class for double scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python float.
"""
- cdef CDate64Array* ap = <CDate64Array*> self.sp_array.get()
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(
- days=ap.Value(self.index) / 86400000))
+ cdef CDoubleScalar* sp = <CDoubleScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Time32Value(ArrayValue):
+cdef class Decimal128Scalar(Scalar):
"""
- Concrete class for time32 array elements.
+ Concrete class for decimal128 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.timedelta instance.
+ Return this value as a Python Decimal.
"""
cdef:
- CTime32Array* ap = <CTime32Array*> self.sp_array.get()
- CTime32Type* dtype = <CTime32Type*> ap.type().get()
-
- if dtype.unit() == TimeUnit_SECOND:
- delta = datetime.timedelta(seconds=ap.Value(self.index))
- return (datetime.datetime(1970, 1, 1) + delta).time()
+ CDecimal128Scalar* sp = <CDecimal128Scalar*> self.wrapped.get()
+ CDecimal128Type* dtype = <CDecimal128Type*> sp.type.get()
+ if sp.is_valid:
+ return _pydecimal.Decimal(
+ frombytes(sp.value.ToString(dtype.scale()))
+ )
else:
- return _box_time_milli(ap.Value(self.index))
+ return None
-cdef class Time64Value(ArrayValue):
+cdef class Date32Scalar(Scalar):
"""
- Concrete class for time64 array elements.
+ Concrete class for date32 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.timedelta instance.
+ Return this value as a Python datetime.datetime instance.
"""
- cdef:
- CTime64Array* ap = <CTime64Array*> self.sp_array.get()
- CTime64Type* dtype = <CTime64Type*> ap.type().get()
+ cdef CDate32Scalar* sp = <CDate32Scalar*> self.wrapped.get()
- cdef int64_t val = ap.Value(self.index)
- if dtype.unit() == TimeUnit_MICRO:
- return _box_time_micro(val)
+ if sp.is_valid:
+ # shift to seconds since epoch
+ return (
+ datetime.date(1970, 1, 1) + datetime.timedelta(days=sp.value)
+ )
else:
- return (datetime.datetime(1970, 1, 1) +
- datetime.timedelta(microseconds=val / 1000)).time()
-
+ return None
-cpdef _box_time_milli(int64_t val):
- delta = datetime.timedelta(milliseconds=val)
- return (datetime.datetime(1970, 1, 1) + delta).time()
+cdef class Date64Scalar(Scalar):
+ """
+ Concrete class for date64 scalars.
+ """
-cpdef _box_time_micro(int64_t val):
- return (datetime.datetime(1970, 1, 1) +
- datetime.timedelta(microseconds=val)).time()
+ def as_py(self):
+ """
+ Return this value as a Python datetime.datetime instance.
+ """
+ cdef CDate64Scalar* sp = <CDate64Scalar*> self.wrapped.get()
+ if sp.is_valid:
+ return (
+ datetime.date(1970, 1, 1) +
+ datetime.timedelta(days=sp.value / 86400000)
+ )
+ else:
+ return None
-cdef dict _DATETIME_CONVERSION_FUNCTIONS = {}
-cdef c_bool _datetime_conversion_initialized = False
+def _datetime_from_int(int64_t value, TimeUnit unit, tzinfo=None):
+ if unit == TimeUnit_SECOND:
+ delta = datetime.timedelta(seconds=value)
+ elif unit == TimeUnit_MILLI:
+ delta = datetime.timedelta(milliseconds=value)
+ elif unit == TimeUnit_MICRO:
+ delta = datetime.timedelta(microseconds=value)
+ else:
+ # TimeUnit_NANO: prefer pandas timestamps if available
+ if _pandas_api.have_pandas:
+ return _pandas_api.pd.Timestamp(value, tz=tzinfo, unit='ns')
+ # otherwise safely truncate to microsecond resolution datetime
+ if value % 1000 != 0:
+ raise ValueError(
+ "Nanosecond resolution temporal type {} is not safely "
+ "convertible to microseconds to convert to datetime.datetime. "
+ "Install pandas to return as Timestamp with nanosecond "
+ "support or access the .value attribute.".format(value)
+ )
+ delta = datetime.timedelta(microseconds=value // 1000)
-cdef _add_micros_maybe_localize(dt, micros, tzinfo):
- import pytz
- dt = dt.replace(microsecond=micros)
+ dt = datetime.datetime(1970, 1, 1) + delta
+ # adjust timezone if set to the datatype
if tzinfo is not None:
- if not isinstance(tzinfo, datetime.tzinfo):
- tzinfo = string_to_tzinfo(tzinfo)
dt = tzinfo.fromutc(dt)
- return dt
-
-
-cdef _datetime_from_seconds(int64_t v):
- return datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=v)
-
-
-def _nanoseconds_to_datetime_safe(v, tzinfo):
- if v % 1000 != 0:
- raise ValueError("Nanosecond timestamp {} is not safely convertible "
- " to microseconds to convert to datetime.datetime."
- " Install pandas to return as Timestamp with "
- " nanosecond support or access the .value attribute.")
- v = v // 1000
- micros = v % 1_000_000
-
- dt = _datetime_from_seconds(v // 1_000_000)
- return _add_micros_maybe_localize(dt, micros, tzinfo)
-
-
-def _microseconds_to_datetime(v, tzinfo):
- micros = v % 1_000_000
- dt = _datetime_from_seconds(v // 1_000_000)
- return _add_micros_maybe_localize(dt, micros, tzinfo)
+ return dt
-def _millis_to_datetime(v, tzinfo):
- millis = v % 1_000
- dt = _datetime_from_seconds(v // 1000)
- return _add_micros_maybe_localize(dt, millis * 1000, tzinfo)
+cdef class Time32Scalar(Scalar):
+ """
+ Concrete class for time32 scalars.
+ """
-def _seconds_to_datetime(v, tzinfo):
- dt = _datetime_from_seconds(v)
- return _add_micros_maybe_localize(dt, 0, tzinfo)
+ def as_py(self):
+ """
+ Return this value as a Python datetime.timedelta instance.
+ """
+ cdef:
+ CTime32Scalar* sp = <CTime32Scalar*> self.wrapped.get()
+ CTime32Type* dtype = <CTime32Type*> sp.type.get()
+ if sp.is_valid:
+ return _datetime_from_int(sp.value, unit=dtype.unit()).time()
+ else:
+ return None
-def _datetime_conversion_functions():
- global _datetime_conversion_initialized
- if _datetime_conversion_initialized:
- return _DATETIME_CONVERSION_FUNCTIONS
- _DATETIME_CONVERSION_FUNCTIONS.update({
- TimeUnit_SECOND: _seconds_to_datetime,
- TimeUnit_MILLI: _millis_to_datetime,
- TimeUnit_MICRO: _microseconds_to_datetime,
- TimeUnit_NANO: _nanoseconds_to_datetime_safe
- })
+cdef class Time64Scalar(Scalar):
+ """
+ Concrete class for time64 scalars.
+ """
- try:
- import pandas as pd
- _DATETIME_CONVERSION_FUNCTIONS[TimeUnit_NANO] = (
- lambda x, tzinfo: pd.Timestamp(
- x, tz=tzinfo, unit='ns',
- )
- )
- except ImportError:
- pass
+ def as_py(self):
+ """
+ Return this value as a Python datetime.timedelta instance.
+ """
+ cdef:
+ CTime64Scalar* sp = <CTime64Scalar*> self.wrapped.get()
+ CTime64Type* dtype = <CTime64Type*> sp.type.get()
- _datetime_conversion_initialized = True
- return _DATETIME_CONVERSION_FUNCTIONS
+ if sp.is_valid:
+ return _datetime_from_int(sp.value, unit=dtype.unit()).time()
+ else:
+ return None
-cdef class TimestampValue(ArrayValue):
+cdef class TimestampScalar(Scalar):
"""
- Concrete class for timestamp array elements.
+ Concrete class for timestamp scalars.
"""
@property
def value(self):
- cdef CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
- cdef CTimestampType* dtype = <CTimestampType*> ap.type().get()
- return ap.Value(self.index)
+ cdef CTimestampScalar* sp = <CTimestampScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
def as_py(self):
"""
Return this value as a Pandas Timestamp instance (if available),
otherwise as a Python datetime.timedelta instance.
"""
- cdef CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
- cdef CTimestampType* dtype = <CTimestampType*> ap.type().get()
+ cdef:
+ CTimestampScalar* sp = <CTimestampScalar*> self.wrapped.get()
+ CTimestampType* dtype = <CTimestampType*> sp.type.get()
- value = self.value
+ if not sp.is_valid:
+ return None
if not dtype.timezone().empty():
tzinfo = string_to_tzinfo(frombytes(dtype.timezone()))
+ if not isinstance(tzinfo, datetime.tzinfo):
+ tzinfo = string_to_tzinfo(tzinfo)
else:
tzinfo = None
- try:
- converter = _datetime_conversion_functions()[dtype.unit()]
- except KeyError:
- raise ValueError(
- 'Cannot convert nanosecond timestamps without pandas'
- )
- return converter(value, tzinfo=tzinfo)
-
-
-cdef dict _TIMEDELTA_CONVERSION_FUNCTIONS = {}
-
-
-def _nanoseconds_to_timedelta_safe(v):
- if v % 1000 != 0:
- raise ValueError(
- "Nanosecond duration {} is not safely convertible to microseconds "
- "to convert to datetime.timedelta. Install pandas to return as "
- "Timedelta with nanosecond support or access the .value "
- "attribute.".format(v))
- micros = v // 1000
-
- return datetime.timedelta(microseconds=micros)
-
-
-def _timedelta_conversion_functions():
- if _TIMEDELTA_CONVERSION_FUNCTIONS:
- return _TIMEDELTA_CONVERSION_FUNCTIONS
-
- _TIMEDELTA_CONVERSION_FUNCTIONS.update({
- TimeUnit_SECOND: lambda v: datetime.timedelta(seconds=v),
- TimeUnit_MILLI: lambda v: datetime.timedelta(milliseconds=v),
- TimeUnit_MICRO: lambda v: datetime.timedelta(microseconds=v),
- TimeUnit_NANO: _nanoseconds_to_timedelta_safe
- })
-
- try:
- import pandas as pd
- _TIMEDELTA_CONVERSION_FUNCTIONS[TimeUnit_NANO] = (
- lambda v: pd.Timedelta(v, unit='ns')
- )
- except ImportError:
- pass
+ return _datetime_from_int(sp.value, unit=dtype.unit(), tzinfo=tzinfo)
- return _TIMEDELTA_CONVERSION_FUNCTIONS
-
-cdef class DurationValue(ArrayValue):
+cdef class DurationScalar(Scalar):
"""
- Concrete class for duration array elements.
+ Concrete class for duration scalars.
"""
@property
def value(self):
- cdef CDurationArray* ap = <CDurationArray*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CDurationScalar* sp = <CDurationScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
def as_py(self):
"""
Return this value as a Pandas Timestamp instance (if available),
otherwise as a Python datetime.timedelta instance.
"""
- cdef CDurationArray* ap = <CDurationArray*> self.sp_array.get()
- cdef CDurationType* dtype = <CDurationType*> ap.type().get()
+ cdef:
+ CDurationScalar* sp = <CDurationScalar*> self.wrapped.get()
+ CDurationType* dtype = <CDurationType*> sp.type.get()
+ TimeUnit unit = dtype.unit()
+
+ if not sp.is_valid:
+ return None
- cdef int64_t value = ap.Value(self.index)
- converter = _timedelta_conversion_functions()[dtype.unit()]
- return converter(value)
+ if unit == TimeUnit_SECOND:
+ return datetime.timedelta(seconds=sp.value)
+ elif unit == TimeUnit_MILLI:
+ return datetime.timedelta(milliseconds=sp.value)
+ elif unit == TimeUnit_MICRO:
+ return datetime.timedelta(microseconds=sp.value)
+ else:
+ # TimeUnit_NANO: prefer pandas timestamps if available
+ if _pandas_api.have_pandas:
+ return _pandas_api.pd.Timedelta(sp.value, unit='ns')
+ # otherwise safely truncate to microsecond resolution timedelta
+ if sp.value % 1000 != 0:
+ raise ValueError(
+ "Nanosecond duration {} is not safely convertible to "
+ "microseconds to convert to datetime.timedelta. Install "
+ "pandas to return as Timedelta with nanosecond support or "
+ "access the .value attribute.".format(sp.value)
+ )
+ return datetime.timedelta(microseconds=sp.value // 1000)
-cdef class HalfFloatValue(ArrayValue):
+cdef class BinaryScalar(Scalar):
"""
- Concrete class for float16 array elements.
+ Concrete class for binary-like scalars.
"""
- def as_py(self):
+ def as_buffer(self):
"""
- Return this value as a Python float.
+ Return a view over this value as a Buffer object.
"""
- cdef CHalfFloatArray* ap = <CHalfFloatArray*> self.sp_array.get()
- return PyHalf_FromHalf(ap.Value(self.index))
-
-
-cdef class FloatValue(ArrayValue):
- """
- Concrete class for float32 array elements.
- """
+ cdef CBinaryScalar* sp = <CBinaryScalar*> self.wrapped.get()
+ return pyarrow_wrap_buffer(sp.value) if sp.is_valid else None
def as_py(self):
"""
- Return this value as a Python float.
+ Return this value as a Python bytes.
"""
- cdef CFloatArray* ap = <CFloatArray*> self.sp_array.get()
- return ap.Value(self.index)
+ buffer = self.as_buffer()
+ if buffer is not None:
+ return self.as_buffer().to_pybytes()
+ else:
+ return None
-cdef class DoubleValue(ArrayValue):
- """
- Concrete class for float64 array elements.
- """
+cdef class LargeBinaryScalar(BinaryScalar):
+ pass
- def as_py(self):
- """
- Return this value as a Python float.
- """
- cdef CDoubleArray* ap = <CDoubleArray*> self.sp_array.get()
- return ap.Value(self.index)
+
+cdef class FixedSizeBinaryScalar(BinaryScalar):
+ pass
-cdef class DecimalValue(ArrayValue):
+cdef class StringScalar(BinaryScalar):
Review comment:
Same here.
##########
File path: python/pyarrow/scalar.pxi
##########
@@ -1217,21 +764,50 @@ cdef dict _scalar_classes = {
_Type_INT16: Int16Scalar,
_Type_INT32: Int32Scalar,
_Type_INT64: Int64Scalar,
+ _Type_HALF_FLOAT: HalfFloatScalar,
_Type_FLOAT: FloatScalar,
_Type_DOUBLE: DoubleScalar,
+ _Type_DECIMAL: Decimal128Scalar,
+ _Type_DATE32: Date32Scalar,
+ _Type_DATE64: Date64Scalar,
+ _Type_TIME32: Time32Scalar,
+ _Type_TIME64: Time64Scalar,
+ _Type_TIMESTAMP: TimestampScalar,
+ _Type_DURATION: DurationScalar,
+ _Type_BINARY: BinaryScalar,
+ _Type_LARGE_BINARY: LargeBinaryScalar,
+ _Type_FIXED_SIZE_BINARY: FixedSizeBinaryScalar,
_Type_STRING: StringScalar,
+ _Type_LARGE_STRING: LargeStringScalar,
+ _Type_LIST: ListScalar,
+ _Type_LARGE_LIST: LargeListScalar,
+ _Type_FIXED_SIZE_LIST: FixedSizeListScalar,
+ _Type_STRUCT: StructScalar,
+ _Type_MAP: MapScalar,
+ _Type_DICTIONARY: DictionaryScalar,
+ _Type_SPARSE_UNION: UnionScalar,
+ _Type_DENSE_UNION: UnionScalar,
}
-cdef object box_scalar(DataType type, const shared_ptr[CArray]& sp_array,
- int64_t index):
- cdef ArrayValue value
- if type.type.id() == _Type_NA:
- return _NULL
- elif sp_array.get().IsNull(index):
- return _NULL
- else:
- klass = _array_value_classes[type.type.id()]
- value = klass.__new__(klass)
- value.init(type, sp_array, index)
- return value
+def scalar(value, DataType type=None, bint safe=True,
Review comment:
It seems `safe` isn't used btw.
##########
File path: python/pyarrow/tests/test_scalars.py
##########
@@ -16,427 +16,443 @@
# under the License.
import datetime
+import decimal
import pytest
-import unittest
import numpy as np
import pyarrow as pa
-class TestScalars(unittest.TestCase):
-
- def test_null_singleton(self):
- with pytest.raises(Exception):
- pa.NAType()
+@pytest.mark.parametrize(['value', 'ty', 'klass', 'deprecated'], [
+ (False, None, pa.BooleanScalar, pa.BooleanValue),
+ (True, None, pa.BooleanScalar, pa.BooleanValue),
+ (1, None, pa.Int64Scalar, pa.Int64Value),
+ (-1, None, pa.Int64Scalar, pa.Int64Value),
+ (1, pa.int8(), pa.Int8Scalar, pa.Int8Value),
+ (1, pa.uint8(), pa.UInt8Scalar, pa.UInt8Value),
+ (1, pa.int16(), pa.Int16Scalar, pa.Int16Value),
+ (1, pa.uint16(), pa.UInt16Scalar, pa.UInt16Value),
+ (1, pa.int32(), pa.Int32Scalar, pa.Int32Value),
+ (1, pa.uint32(), pa.UInt32Scalar, pa.UInt32Value),
+ (1, pa.int64(), pa.Int64Scalar, pa.Int64Value),
+ (1, pa.uint64(), pa.UInt64Scalar, pa.UInt64Value),
+ (1.0, None, pa.DoubleScalar, pa.DoubleValue),
+ (np.float16(1.0), pa.float16(), pa.HalfFloatScalar, pa.HalfFloatValue),
+ (1.0, pa.float32(), pa.FloatScalar, pa.FloatValue),
+ ("string", None, pa.StringScalar, pa.StringValue),
+ (b"bytes", None, pa.BinaryScalar, pa.BinaryValue),
+ ([1, 2, 3], None, pa.ListScalar, pa.ListValue),
+ ([1, 2, 3, 4], pa.large_list(pa.int8()), pa.LargeListScalar,
+ pa.LargeListValue),
+ (datetime.date.today(), None, pa.Date32Scalar, pa.Date64Value),
+ (datetime.datetime.now(), None, pa.TimestampScalar, pa.TimestampValue),
+ ({'a': 1, 'b': [1, 2]}, None, pa.StructScalar, pa.StructValue)
+])
+def test_basics(value, ty, klass, deprecated):
+ s = pa.scalar(value, type=ty)
+ assert isinstance(s, klass)
+ assert s == value
+ assert s == s
+ assert s != "else"
+ assert hash(s) == hash(s)
+ assert s.is_valid is True
+ with pytest.warns(FutureWarning):
+ isinstance(s, deprecated)
+
+ s = pa.scalar(None, type=s.type)
+ assert s.is_valid is False
+ assert s.as_py() is None
+
+
+def test_null_singleton():
+ with pytest.raises(Exception):
+ pa.NullScalar()
+
+
+def test_nulls():
+ null = pa.scalar(None)
+ assert null is pa.NA
+ assert null.as_py() is None
+ assert (null == "something") is pa.NA
Review comment:
Also `assert null == pa.scalar(None)`
##########
File path: python/pyarrow/tests/test_scalars.py
##########
@@ -16,427 +16,443 @@
# under the License.
import datetime
+import decimal
import pytest
-import unittest
import numpy as np
import pyarrow as pa
-class TestScalars(unittest.TestCase):
-
- def test_null_singleton(self):
- with pytest.raises(Exception):
- pa.NAType()
+@pytest.mark.parametrize(['value', 'ty', 'klass', 'deprecated'], [
+ (False, None, pa.BooleanScalar, pa.BooleanValue),
+ (True, None, pa.BooleanScalar, pa.BooleanValue),
+ (1, None, pa.Int64Scalar, pa.Int64Value),
+ (-1, None, pa.Int64Scalar, pa.Int64Value),
+ (1, pa.int8(), pa.Int8Scalar, pa.Int8Value),
+ (1, pa.uint8(), pa.UInt8Scalar, pa.UInt8Value),
+ (1, pa.int16(), pa.Int16Scalar, pa.Int16Value),
+ (1, pa.uint16(), pa.UInt16Scalar, pa.UInt16Value),
+ (1, pa.int32(), pa.Int32Scalar, pa.Int32Value),
+ (1, pa.uint32(), pa.UInt32Scalar, pa.UInt32Value),
+ (1, pa.int64(), pa.Int64Scalar, pa.Int64Value),
+ (1, pa.uint64(), pa.UInt64Scalar, pa.UInt64Value),
+ (1.0, None, pa.DoubleScalar, pa.DoubleValue),
+ (np.float16(1.0), pa.float16(), pa.HalfFloatScalar, pa.HalfFloatValue),
+ (1.0, pa.float32(), pa.FloatScalar, pa.FloatValue),
+ ("string", None, pa.StringScalar, pa.StringValue),
+ (b"bytes", None, pa.BinaryScalar, pa.BinaryValue),
+ ([1, 2, 3], None, pa.ListScalar, pa.ListValue),
+ ([1, 2, 3, 4], pa.large_list(pa.int8()), pa.LargeListScalar,
+ pa.LargeListValue),
+ (datetime.date.today(), None, pa.Date32Scalar, pa.Date64Value),
+ (datetime.datetime.now(), None, pa.TimestampScalar, pa.TimestampValue),
+ ({'a': 1, 'b': [1, 2]}, None, pa.StructScalar, pa.StructValue)
+])
+def test_basics(value, ty, klass, deprecated):
+ s = pa.scalar(value, type=ty)
+ assert isinstance(s, klass)
+ assert s == value
+ assert s == s
+ assert s != "else"
+ assert hash(s) == hash(s)
+ assert s.is_valid is True
+ with pytest.warns(FutureWarning):
+ isinstance(s, deprecated)
+
+ s = pa.scalar(None, type=s.type)
+ assert s.is_valid is False
+ assert s.as_py() is None
+
+
+def test_null_singleton():
+ with pytest.raises(Exception):
Review comment:
You should a test more precise exception.
##########
File path: python/pyarrow/scalar.pxi
##########
@@ -1217,21 +764,50 @@ cdef dict _scalar_classes = {
_Type_INT16: Int16Scalar,
_Type_INT32: Int32Scalar,
_Type_INT64: Int64Scalar,
+ _Type_HALF_FLOAT: HalfFloatScalar,
_Type_FLOAT: FloatScalar,
_Type_DOUBLE: DoubleScalar,
+ _Type_DECIMAL: Decimal128Scalar,
+ _Type_DATE32: Date32Scalar,
+ _Type_DATE64: Date64Scalar,
+ _Type_TIME32: Time32Scalar,
+ _Type_TIME64: Time64Scalar,
+ _Type_TIMESTAMP: TimestampScalar,
+ _Type_DURATION: DurationScalar,
+ _Type_BINARY: BinaryScalar,
+ _Type_LARGE_BINARY: LargeBinaryScalar,
+ _Type_FIXED_SIZE_BINARY: FixedSizeBinaryScalar,
_Type_STRING: StringScalar,
+ _Type_LARGE_STRING: LargeStringScalar,
+ _Type_LIST: ListScalar,
+ _Type_LARGE_LIST: LargeListScalar,
+ _Type_FIXED_SIZE_LIST: FixedSizeListScalar,
+ _Type_STRUCT: StructScalar,
+ _Type_MAP: MapScalar,
+ _Type_DICTIONARY: DictionaryScalar,
+ _Type_SPARSE_UNION: UnionScalar,
+ _Type_DENSE_UNION: UnionScalar,
}
-cdef object box_scalar(DataType type, const shared_ptr[CArray]& sp_array,
- int64_t index):
- cdef ArrayValue value
- if type.type.id() == _Type_NA:
- return _NULL
- elif sp_array.get().IsNull(index):
- return _NULL
- else:
- klass = _array_value_classes[type.type.id()]
- value = klass.__new__(klass)
- value.init(type, sp_array, index)
- return value
+def scalar(value, DataType type=None, bint safe=True,
+ MemoryPool memory_pool=None):
+ cdef:
+ PyConversionOptions options
+ shared_ptr[CScalar] scalar
+ shared_ptr[CArray] array
+ shared_ptr[CChunkedArray] chunked
+
+ options.size = 1
+ options.pool = maybe_unbox_memory_pool(memory_pool)
+ # options.from_pandas = from_pandas
Review comment:
What?
##########
File path: python/pyarrow/tests/test_scalars.py
##########
@@ -16,427 +16,443 @@
# under the License.
import datetime
+import decimal
import pytest
-import unittest
import numpy as np
import pyarrow as pa
-class TestScalars(unittest.TestCase):
-
- def test_null_singleton(self):
- with pytest.raises(Exception):
- pa.NAType()
+@pytest.mark.parametrize(['value', 'ty', 'klass', 'deprecated'], [
+ (False, None, pa.BooleanScalar, pa.BooleanValue),
+ (True, None, pa.BooleanScalar, pa.BooleanValue),
+ (1, None, pa.Int64Scalar, pa.Int64Value),
+ (-1, None, pa.Int64Scalar, pa.Int64Value),
+ (1, pa.int8(), pa.Int8Scalar, pa.Int8Value),
+ (1, pa.uint8(), pa.UInt8Scalar, pa.UInt8Value),
+ (1, pa.int16(), pa.Int16Scalar, pa.Int16Value),
+ (1, pa.uint16(), pa.UInt16Scalar, pa.UInt16Value),
+ (1, pa.int32(), pa.Int32Scalar, pa.Int32Value),
+ (1, pa.uint32(), pa.UInt32Scalar, pa.UInt32Value),
+ (1, pa.int64(), pa.Int64Scalar, pa.Int64Value),
+ (1, pa.uint64(), pa.UInt64Scalar, pa.UInt64Value),
+ (1.0, None, pa.DoubleScalar, pa.DoubleValue),
+ (np.float16(1.0), pa.float16(), pa.HalfFloatScalar, pa.HalfFloatValue),
+ (1.0, pa.float32(), pa.FloatScalar, pa.FloatValue),
+ ("string", None, pa.StringScalar, pa.StringValue),
+ (b"bytes", None, pa.BinaryScalar, pa.BinaryValue),
+ ([1, 2, 3], None, pa.ListScalar, pa.ListValue),
+ ([1, 2, 3, 4], pa.large_list(pa.int8()), pa.LargeListScalar,
+ pa.LargeListValue),
+ (datetime.date.today(), None, pa.Date32Scalar, pa.Date64Value),
+ (datetime.datetime.now(), None, pa.TimestampScalar, pa.TimestampValue),
+ ({'a': 1, 'b': [1, 2]}, None, pa.StructScalar, pa.StructValue)
+])
+def test_basics(value, ty, klass, deprecated):
+ s = pa.scalar(value, type=ty)
+ assert isinstance(s, klass)
+ assert s == value
+ assert s == s
+ assert s != "else"
+ assert hash(s) == hash(s)
+ assert s.is_valid is True
+ with pytest.warns(FutureWarning):
+ isinstance(s, deprecated)
+
+ s = pa.scalar(None, type=s.type)
+ assert s.is_valid is False
+ assert s.as_py() is None
+
+
+def test_null_singleton():
+ with pytest.raises(Exception):
+ pa.NullScalar()
+
+
+def test_nulls():
+ null = pa.scalar(None)
+ assert null is pa.NA
+ assert null.as_py() is None
+ assert (null == "something") is pa.NA
Review comment:
This looks incorrect and should return `False`.
##########
File path: python/pyarrow/scalar.pxi
##########
@@ -16,1198 +16,745 @@
# under the License.
-_NULL = NA = None
+import collections
cdef class Scalar:
"""
- The base class for all array elements.
+ The base class for scalars.
"""
+ def __init__(self):
+ raise TypeError("Do not call {}'s constructor directly, use "
+ "pa.scalar() instead.".format(self.__class__.__name__))
-cdef class NullType(Scalar):
- """
- Singleton for null array elements.
- """
- # TODO rename this NullValue?
+ cdef void init(self, const shared_ptr[CScalar]& wrapped):
+ self.wrapped = wrapped
- def __cinit__(self):
- global NA
- if NA is not None:
- raise Exception('Cannot create multiple NAType instances')
+ @staticmethod
+ cdef wrap(const shared_ptr[CScalar]& wrapped):
+ cdef:
+ Scalar self
+ Type type_id = wrapped.get().type.get().id()
+
+ if type_id == _Type_NA:
+ return _NULL
+
+ typ = _scalar_classes[type_id]
+ self = typ.__new__(typ)
+ self.init(wrapped)
+
+ return self
+
+ cdef inline shared_ptr[CScalar] unwrap(self) nogil:
+ return self.wrapped
+
+ @property
+ def type(self):
+ return pyarrow_wrap_data_type(self.wrapped.get().type)
- self.type = null()
+ @property
+ def is_valid(self):
+ return self.wrapped.get().is_valid
def __repr__(self):
- return 'NULL'
+ return '<pyarrow.{}: {!r}>'.format(
+ self.__class__.__name__, self.as_py()
+ )
- def as_py(self):
- """
- Return None
- """
- return None
+ def __str__(self):
+ return str(self.as_py())
+
+ def equals(self, Scalar other):
+ return self.wrapped.get().Equals(other.unwrap().get()[0])
def __eq__(self, other):
- return NA
+ try:
+ if not isinstance(other, Scalar):
+ other = scalar(other, type=self.type)
+ return self.equals(other)
+ except (TypeError, ValueError, ArrowInvalid):
+ return NotImplemented
+
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+
+ def as_py(self):
+ raise NotImplementedError()
-_NULL = NA = NullType()
+_NULL = NA = None
-cdef class ArrayValue(Scalar):
+cdef class NullScalar(Scalar):
"""
- The base class for non-null array elements.
+ Concrete class for null scalars.
"""
- def __init__(self):
- raise TypeError("Do not call {}'s constructor directly, use array "
- "subscription instead."
- .format(self.__class__.__name__))
+ def __cinit__(self):
+ global NA
+ if NA is not None:
+ raise Exception('Cannot create multiple NAType instances')
+ self.init(shared_ptr[CScalar](new CNullScalar()))
- cdef void init(self, DataType type, const shared_ptr[CArray]& sp_array,
- int64_t index):
- self.type = type
- self.index = index
- self._set_array(sp_array)
+ def __init__(self):
+ pass
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
+ def __eq__(self, other):
+ return NA
- def __repr__(self):
- if hasattr(self, 'as_py'):
- return repr(self.as_py())
- else:
- return super(Scalar, self).__repr__()
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
- def __str__(self):
- if hasattr(self, 'as_py'):
- return str(self.as_py())
- else:
- return super(Scalar, self).__str__()
+ def as_py(self):
+ """
+ Return this value as a Python None.
+ """
+ return None
- def __eq__(self, other):
- if hasattr(self, 'as_py'):
- if isinstance(other, ArrayValue):
- other = other.as_py()
- return self.as_py() == other
- else:
- raise NotImplementedError(
- "Cannot compare Arrow values that don't support as_py()")
- def __hash__(self):
- return hash(self.as_py())
+_NULL = NA = NullScalar()
-cdef class BooleanValue(ArrayValue):
+cdef class BooleanScalar(Scalar):
"""
- Concrete class for boolean array elements.
+ Concrete class for boolean scalars.
"""
def as_py(self):
"""
Return this value as a Python bool.
"""
- cdef CBooleanArray* ap = <CBooleanArray*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CBooleanScalar* sp = <CBooleanScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int8Value(ArrayValue):
+cdef class UInt8Scalar(Scalar):
"""
- Concrete class for int8 array elements.
+ Concrete class for uint8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt8Array* ap = <CInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt8Scalar* sp = <CUInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt8Value(ArrayValue):
+cdef class Int8Scalar(Scalar):
"""
- Concrete class for uint8 array elements.
+ Concrete class for int8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt8Array* ap = <CUInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt8Scalar* sp = <CInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int16Value(ArrayValue):
+cdef class UInt16Scalar(Scalar):
"""
- Concrete class for int16 array elements.
+ Concrete class for uint16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt16Array* ap = <CInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt16Scalar* sp = <CUInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt16Value(ArrayValue):
+cdef class Int16Scalar(Scalar):
"""
- Concrete class for uint16 array elements.
+ Concrete class for int16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt16Array* ap = <CUInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt16Scalar* sp = <CInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int32Value(ArrayValue):
+cdef class UInt32Scalar(Scalar):
"""
- Concrete class for int32 array elements.
+ Concrete class for uint32 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt32Array* ap = <CInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt32Scalar* sp = <CUInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt32Value(ArrayValue):
+cdef class Int32Scalar(Scalar):
"""
- Concrete class for uint32 array elements.
+ Concrete class for int32 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt32Array* ap = <CUInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt32Scalar* sp = <CInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int64Value(ArrayValue):
+cdef class UInt64Scalar(Scalar):
"""
- Concrete class for int64 array elements.
+ Concrete class for uint64 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt64Array* ap = <CInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt64Scalar* sp = <CUInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt64Value(ArrayValue):
+cdef class Int64Scalar(Scalar):
"""
- Concrete class for uint64 array elements.
+ Concrete class for int64 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt64Array* ap = <CUInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt64Scalar* sp = <CInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date32Value(ArrayValue):
+cdef class HalfFloatScalar(Scalar):
"""
- Concrete class for date32 array elements.
+ Concrete class for float scalars.
"""
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+
+ def __eq__(self, other):
+ if hasattr(self, 'as_py'):
+ if isinstance(other, Scalar):
+ other = other.as_py()
+ return self.as_py() == other
+ else:
+ raise NotImplementedError
+
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python float.
"""
- cdef CDate32Array* ap = <CDate32Array*> self.sp_array.get()
+ cdef CHalfFloatScalar* sp = <CHalfFloatScalar*> self.wrapped.get()
+ return PyHalf_FromHalf(sp.value) if sp.is_valid else None
- # Shift to seconds since epoch
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(days=ap.Value(self.index)))
+
+cdef class FloatScalar(Scalar):
+ """
+ Concrete class for float scalars.
+ """
+
+ def as_py(self):
+ """
+ Return this value as a Python float.
+ """
+ cdef CFloatScalar* sp = <CFloatScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date64Value(ArrayValue):
+cdef class DoubleScalar(Scalar):
"""
- Concrete class for date64 array elements.
+ Concrete class for double scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python float.
"""
- cdef CDate64Array* ap = <CDate64Array*> self.sp_array.get()
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(
- days=ap.Value(self.index) / 86400000))
+ cdef CDoubleScalar* sp = <CDoubleScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Time32Value(ArrayValue):
+cdef class Decimal128Scalar(Scalar):
"""
- Concrete class for time32 array elements.
+ Concrete class for decimal128 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.timedelta instance.
+ Return this value as a Python Decimal.
"""
cdef:
- CTime32Array* ap = <CTime32Array*> self.sp_array.get()
- CTime32Type* dtype = <CTime32Type*> ap.type().get()
-
- if dtype.unit() == TimeUnit_SECOND:
- delta = datetime.timedelta(seconds=ap.Value(self.index))
- return (datetime.datetime(1970, 1, 1) + delta).time()
+ CDecimal128Scalar* sp = <CDecimal128Scalar*> self.wrapped.get()
+ CDecimal128Type* dtype = <CDecimal128Type*> sp.type.get()
+ if sp.is_valid:
+ return _pydecimal.Decimal(
+ frombytes(sp.value.ToString(dtype.scale()))
+ )
else:
- return _box_time_milli(ap.Value(self.index))
+ return None
-cdef class Time64Value(ArrayValue):
+cdef class Date32Scalar(Scalar):
"""
- Concrete class for time64 array elements.
+ Concrete class for date32 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.timedelta instance.
+ Return this value as a Python datetime.datetime instance.
"""
- cdef:
- CTime64Array* ap = <CTime64Array*> self.sp_array.get()
- CTime64Type* dtype = <CTime64Type*> ap.type().get()
+ cdef CDate32Scalar* sp = <CDate32Scalar*> self.wrapped.get()
- cdef int64_t val = ap.Value(self.index)
- if dtype.unit() == TimeUnit_MICRO:
- return _box_time_micro(val)
+ if sp.is_valid:
+ # shift to seconds since epoch
+ return (
+ datetime.date(1970, 1, 1) + datetime.timedelta(days=sp.value)
+ )
else:
- return (datetime.datetime(1970, 1, 1) +
- datetime.timedelta(microseconds=val / 1000)).time()
-
+ return None
-cpdef _box_time_milli(int64_t val):
- delta = datetime.timedelta(milliseconds=val)
- return (datetime.datetime(1970, 1, 1) + delta).time()
+cdef class Date64Scalar(Scalar):
+ """
+ Concrete class for date64 scalars.
+ """
-cpdef _box_time_micro(int64_t val):
- return (datetime.datetime(1970, 1, 1) +
- datetime.timedelta(microseconds=val)).time()
+ def as_py(self):
+ """
+ Return this value as a Python datetime.datetime instance.
+ """
+ cdef CDate64Scalar* sp = <CDate64Scalar*> self.wrapped.get()
+ if sp.is_valid:
+ return (
+ datetime.date(1970, 1, 1) +
+ datetime.timedelta(days=sp.value / 86400000)
+ )
+ else:
+ return None
-cdef dict _DATETIME_CONVERSION_FUNCTIONS = {}
-cdef c_bool _datetime_conversion_initialized = False
+def _datetime_from_int(int64_t value, TimeUnit unit, tzinfo=None):
+ if unit == TimeUnit_SECOND:
+ delta = datetime.timedelta(seconds=value)
+ elif unit == TimeUnit_MILLI:
+ delta = datetime.timedelta(milliseconds=value)
+ elif unit == TimeUnit_MICRO:
+ delta = datetime.timedelta(microseconds=value)
+ else:
+ # TimeUnit_NANO: prefer pandas timestamps if available
+ if _pandas_api.have_pandas:
+ return _pandas_api.pd.Timestamp(value, tz=tzinfo, unit='ns')
+ # otherwise safely truncate to microsecond resolution datetime
+ if value % 1000 != 0:
+ raise ValueError(
+ "Nanosecond resolution temporal type {} is not safely "
+ "convertible to microseconds to convert to datetime.datetime. "
+ "Install pandas to return as Timestamp with nanosecond "
+ "support or access the .value attribute.".format(value)
+ )
+ delta = datetime.timedelta(microseconds=value // 1000)
-cdef _add_micros_maybe_localize(dt, micros, tzinfo):
- import pytz
- dt = dt.replace(microsecond=micros)
+ dt = datetime.datetime(1970, 1, 1) + delta
+ # adjust timezone if set to the datatype
if tzinfo is not None:
- if not isinstance(tzinfo, datetime.tzinfo):
- tzinfo = string_to_tzinfo(tzinfo)
dt = tzinfo.fromutc(dt)
- return dt
-
-
-cdef _datetime_from_seconds(int64_t v):
- return datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=v)
-
-
-def _nanoseconds_to_datetime_safe(v, tzinfo):
- if v % 1000 != 0:
- raise ValueError("Nanosecond timestamp {} is not safely convertible "
- " to microseconds to convert to datetime.datetime."
- " Install pandas to return as Timestamp with "
- " nanosecond support or access the .value attribute.")
- v = v // 1000
- micros = v % 1_000_000
-
- dt = _datetime_from_seconds(v // 1_000_000)
- return _add_micros_maybe_localize(dt, micros, tzinfo)
-
-
-def _microseconds_to_datetime(v, tzinfo):
- micros = v % 1_000_000
- dt = _datetime_from_seconds(v // 1_000_000)
- return _add_micros_maybe_localize(dt, micros, tzinfo)
+ return dt
-def _millis_to_datetime(v, tzinfo):
- millis = v % 1_000
- dt = _datetime_from_seconds(v // 1000)
- return _add_micros_maybe_localize(dt, millis * 1000, tzinfo)
+cdef class Time32Scalar(Scalar):
+ """
+ Concrete class for time32 scalars.
+ """
-def _seconds_to_datetime(v, tzinfo):
- dt = _datetime_from_seconds(v)
- return _add_micros_maybe_localize(dt, 0, tzinfo)
+ def as_py(self):
+ """
+ Return this value as a Python datetime.timedelta instance.
+ """
+ cdef:
+ CTime32Scalar* sp = <CTime32Scalar*> self.wrapped.get()
+ CTime32Type* dtype = <CTime32Type*> sp.type.get()
+ if sp.is_valid:
+ return _datetime_from_int(sp.value, unit=dtype.unit()).time()
+ else:
+ return None
-def _datetime_conversion_functions():
- global _datetime_conversion_initialized
- if _datetime_conversion_initialized:
- return _DATETIME_CONVERSION_FUNCTIONS
- _DATETIME_CONVERSION_FUNCTIONS.update({
- TimeUnit_SECOND: _seconds_to_datetime,
- TimeUnit_MILLI: _millis_to_datetime,
- TimeUnit_MICRO: _microseconds_to_datetime,
- TimeUnit_NANO: _nanoseconds_to_datetime_safe
- })
+cdef class Time64Scalar(Scalar):
+ """
+ Concrete class for time64 scalars.
+ """
- try:
- import pandas as pd
- _DATETIME_CONVERSION_FUNCTIONS[TimeUnit_NANO] = (
- lambda x, tzinfo: pd.Timestamp(
- x, tz=tzinfo, unit='ns',
- )
- )
- except ImportError:
- pass
+ def as_py(self):
+ """
+ Return this value as a Python datetime.timedelta instance.
+ """
+ cdef:
+ CTime64Scalar* sp = <CTime64Scalar*> self.wrapped.get()
+ CTime64Type* dtype = <CTime64Type*> sp.type.get()
- _datetime_conversion_initialized = True
- return _DATETIME_CONVERSION_FUNCTIONS
+ if sp.is_valid:
+ return _datetime_from_int(sp.value, unit=dtype.unit()).time()
+ else:
+ return None
-cdef class TimestampValue(ArrayValue):
+cdef class TimestampScalar(Scalar):
"""
- Concrete class for timestamp array elements.
+ Concrete class for timestamp scalars.
"""
@property
def value(self):
- cdef CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
- cdef CTimestampType* dtype = <CTimestampType*> ap.type().get()
- return ap.Value(self.index)
+ cdef CTimestampScalar* sp = <CTimestampScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
def as_py(self):
"""
Return this value as a Pandas Timestamp instance (if available),
otherwise as a Python datetime.timedelta instance.
"""
- cdef CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
- cdef CTimestampType* dtype = <CTimestampType*> ap.type().get()
+ cdef:
+ CTimestampScalar* sp = <CTimestampScalar*> self.wrapped.get()
+ CTimestampType* dtype = <CTimestampType*> sp.type.get()
- value = self.value
+ if not sp.is_valid:
+ return None
if not dtype.timezone().empty():
tzinfo = string_to_tzinfo(frombytes(dtype.timezone()))
+ if not isinstance(tzinfo, datetime.tzinfo):
+ tzinfo = string_to_tzinfo(tzinfo)
else:
tzinfo = None
- try:
- converter = _datetime_conversion_functions()[dtype.unit()]
- except KeyError:
- raise ValueError(
- 'Cannot convert nanosecond timestamps without pandas'
- )
- return converter(value, tzinfo=tzinfo)
-
-
-cdef dict _TIMEDELTA_CONVERSION_FUNCTIONS = {}
-
-
-def _nanoseconds_to_timedelta_safe(v):
- if v % 1000 != 0:
- raise ValueError(
- "Nanosecond duration {} is not safely convertible to microseconds "
- "to convert to datetime.timedelta. Install pandas to return as "
- "Timedelta with nanosecond support or access the .value "
- "attribute.".format(v))
- micros = v // 1000
-
- return datetime.timedelta(microseconds=micros)
-
-
-def _timedelta_conversion_functions():
- if _TIMEDELTA_CONVERSION_FUNCTIONS:
- return _TIMEDELTA_CONVERSION_FUNCTIONS
-
- _TIMEDELTA_CONVERSION_FUNCTIONS.update({
- TimeUnit_SECOND: lambda v: datetime.timedelta(seconds=v),
- TimeUnit_MILLI: lambda v: datetime.timedelta(milliseconds=v),
- TimeUnit_MICRO: lambda v: datetime.timedelta(microseconds=v),
- TimeUnit_NANO: _nanoseconds_to_timedelta_safe
- })
-
- try:
- import pandas as pd
- _TIMEDELTA_CONVERSION_FUNCTIONS[TimeUnit_NANO] = (
- lambda v: pd.Timedelta(v, unit='ns')
- )
- except ImportError:
- pass
+ return _datetime_from_int(sp.value, unit=dtype.unit(), tzinfo=tzinfo)
- return _TIMEDELTA_CONVERSION_FUNCTIONS
-
-cdef class DurationValue(ArrayValue):
+cdef class DurationScalar(Scalar):
"""
- Concrete class for duration array elements.
+ Concrete class for duration scalars.
"""
@property
def value(self):
- cdef CDurationArray* ap = <CDurationArray*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CDurationScalar* sp = <CDurationScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
def as_py(self):
"""
Return this value as a Pandas Timestamp instance (if available),
otherwise as a Python datetime.timedelta instance.
"""
- cdef CDurationArray* ap = <CDurationArray*> self.sp_array.get()
- cdef CDurationType* dtype = <CDurationType*> ap.type().get()
+ cdef:
+ CDurationScalar* sp = <CDurationScalar*> self.wrapped.get()
+ CDurationType* dtype = <CDurationType*> sp.type.get()
+ TimeUnit unit = dtype.unit()
+
+ if not sp.is_valid:
+ return None
- cdef int64_t value = ap.Value(self.index)
- converter = _timedelta_conversion_functions()[dtype.unit()]
- return converter(value)
+ if unit == TimeUnit_SECOND:
+ return datetime.timedelta(seconds=sp.value)
+ elif unit == TimeUnit_MILLI:
+ return datetime.timedelta(milliseconds=sp.value)
+ elif unit == TimeUnit_MICRO:
+ return datetime.timedelta(microseconds=sp.value)
+ else:
+ # TimeUnit_NANO: prefer pandas timestamps if available
+ if _pandas_api.have_pandas:
+ return _pandas_api.pd.Timedelta(sp.value, unit='ns')
+ # otherwise safely truncate to microsecond resolution timedelta
+ if sp.value % 1000 != 0:
+ raise ValueError(
+ "Nanosecond duration {} is not safely convertible to "
+ "microseconds to convert to datetime.timedelta. Install "
+ "pandas to return as Timedelta with nanosecond support or "
+ "access the .value attribute.".format(sp.value)
+ )
+ return datetime.timedelta(microseconds=sp.value // 1000)
-cdef class HalfFloatValue(ArrayValue):
+cdef class BinaryScalar(Scalar):
"""
- Concrete class for float16 array elements.
+ Concrete class for binary-like scalars.
"""
- def as_py(self):
+ def as_buffer(self):
"""
- Return this value as a Python float.
+ Return a view over this value as a Buffer object.
"""
- cdef CHalfFloatArray* ap = <CHalfFloatArray*> self.sp_array.get()
- return PyHalf_FromHalf(ap.Value(self.index))
-
-
-cdef class FloatValue(ArrayValue):
- """
- Concrete class for float32 array elements.
- """
+ cdef CBinaryScalar* sp = <CBinaryScalar*> self.wrapped.get()
+ return pyarrow_wrap_buffer(sp.value) if sp.is_valid else None
def as_py(self):
"""
- Return this value as a Python float.
+ Return this value as a Python bytes.
"""
- cdef CFloatArray* ap = <CFloatArray*> self.sp_array.get()
- return ap.Value(self.index)
+ buffer = self.as_buffer()
+ if buffer is not None:
+ return self.as_buffer().to_pybytes()
+ else:
+ return None
-cdef class DoubleValue(ArrayValue):
- """
- Concrete class for float64 array elements.
- """
+cdef class LargeBinaryScalar(BinaryScalar):
+ pass
- def as_py(self):
- """
- Return this value as a Python float.
- """
- cdef CDoubleArray* ap = <CDoubleArray*> self.sp_array.get()
- return ap.Value(self.index)
+
+cdef class FixedSizeBinaryScalar(BinaryScalar):
+ pass
-cdef class DecimalValue(ArrayValue):
+cdef class StringScalar(BinaryScalar):
"""
- Concrete class for decimal128 array elements.
+ Concrete class for string-like (utf8) scalars.
"""
def as_py(self):
"""
- Return this value as a Python Decimal.
+ Return this value as a Python string.
"""
- cdef:
- CDecimal128Array* ap = <CDecimal128Array*> self.sp_array.get()
- c_string s = ap.FormatValue(self.index)
- return _pydecimal.Decimal(s.decode('utf8'))
+ buffer = self.as_buffer()
+ if buffer is not None:
+ return frombytes(self.as_buffer().to_pybytes())
+ else:
+ return None
+
+cdef class LargeStringScalar(StringScalar):
Review comment:
Same here.
##########
File path: python/pyarrow/scalar.pxi
##########
@@ -16,1198 +16,745 @@
# under the License.
-_NULL = NA = None
+import collections
cdef class Scalar:
"""
- The base class for all array elements.
+ The base class for scalars.
"""
+ def __init__(self):
+ raise TypeError("Do not call {}'s constructor directly, use "
+ "pa.scalar() instead.".format(self.__class__.__name__))
-cdef class NullType(Scalar):
- """
- Singleton for null array elements.
- """
- # TODO rename this NullValue?
+ cdef void init(self, const shared_ptr[CScalar]& wrapped):
+ self.wrapped = wrapped
- def __cinit__(self):
- global NA
- if NA is not None:
- raise Exception('Cannot create multiple NAType instances')
+ @staticmethod
+ cdef wrap(const shared_ptr[CScalar]& wrapped):
+ cdef:
+ Scalar self
+ Type type_id = wrapped.get().type.get().id()
+
+ if type_id == _Type_NA:
+ return _NULL
+
+ typ = _scalar_classes[type_id]
+ self = typ.__new__(typ)
+ self.init(wrapped)
+
+ return self
+
+ cdef inline shared_ptr[CScalar] unwrap(self) nogil:
+ return self.wrapped
+
+ @property
+ def type(self):
+ return pyarrow_wrap_data_type(self.wrapped.get().type)
- self.type = null()
+ @property
+ def is_valid(self):
+ return self.wrapped.get().is_valid
def __repr__(self):
- return 'NULL'
+ return '<pyarrow.{}: {!r}>'.format(
+ self.__class__.__name__, self.as_py()
+ )
- def as_py(self):
- """
- Return None
- """
- return None
+ def __str__(self):
+ return str(self.as_py())
+
+ def equals(self, Scalar other):
+ return self.wrapped.get().Equals(other.unwrap().get()[0])
def __eq__(self, other):
- return NA
+ try:
+ if not isinstance(other, Scalar):
+ other = scalar(other, type=self.type)
+ return self.equals(other)
+ except (TypeError, ValueError, ArrowInvalid):
+ return NotImplemented
+
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+
+ def as_py(self):
+ raise NotImplementedError()
-_NULL = NA = NullType()
+_NULL = NA = None
-cdef class ArrayValue(Scalar):
+cdef class NullScalar(Scalar):
"""
- The base class for non-null array elements.
+ Concrete class for null scalars.
"""
- def __init__(self):
- raise TypeError("Do not call {}'s constructor directly, use array "
- "subscription instead."
- .format(self.__class__.__name__))
+ def __cinit__(self):
+ global NA
+ if NA is not None:
+ raise Exception('Cannot create multiple NAType instances')
+ self.init(shared_ptr[CScalar](new CNullScalar()))
- cdef void init(self, DataType type, const shared_ptr[CArray]& sp_array,
- int64_t index):
- self.type = type
- self.index = index
- self._set_array(sp_array)
+ def __init__(self):
+ pass
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
+ def __eq__(self, other):
+ return NA
- def __repr__(self):
- if hasattr(self, 'as_py'):
- return repr(self.as_py())
- else:
- return super(Scalar, self).__repr__()
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
- def __str__(self):
- if hasattr(self, 'as_py'):
- return str(self.as_py())
- else:
- return super(Scalar, self).__str__()
+ def as_py(self):
+ """
+ Return this value as a Python None.
+ """
+ return None
- def __eq__(self, other):
- if hasattr(self, 'as_py'):
- if isinstance(other, ArrayValue):
- other = other.as_py()
- return self.as_py() == other
- else:
- raise NotImplementedError(
- "Cannot compare Arrow values that don't support as_py()")
- def __hash__(self):
- return hash(self.as_py())
+_NULL = NA = NullScalar()
-cdef class BooleanValue(ArrayValue):
+cdef class BooleanScalar(Scalar):
"""
- Concrete class for boolean array elements.
+ Concrete class for boolean scalars.
"""
def as_py(self):
"""
Return this value as a Python bool.
"""
- cdef CBooleanArray* ap = <CBooleanArray*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CBooleanScalar* sp = <CBooleanScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int8Value(ArrayValue):
+cdef class UInt8Scalar(Scalar):
"""
- Concrete class for int8 array elements.
+ Concrete class for uint8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt8Array* ap = <CInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt8Scalar* sp = <CUInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt8Value(ArrayValue):
+cdef class Int8Scalar(Scalar):
"""
- Concrete class for uint8 array elements.
+ Concrete class for int8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt8Array* ap = <CUInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt8Scalar* sp = <CInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int16Value(ArrayValue):
+cdef class UInt16Scalar(Scalar):
"""
- Concrete class for int16 array elements.
+ Concrete class for uint16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt16Array* ap = <CInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt16Scalar* sp = <CUInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt16Value(ArrayValue):
+cdef class Int16Scalar(Scalar):
"""
- Concrete class for uint16 array elements.
+ Concrete class for int16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt16Array* ap = <CUInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt16Scalar* sp = <CInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int32Value(ArrayValue):
+cdef class UInt32Scalar(Scalar):
"""
- Concrete class for int32 array elements.
+ Concrete class for uint32 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt32Array* ap = <CInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt32Scalar* sp = <CUInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt32Value(ArrayValue):
+cdef class Int32Scalar(Scalar):
"""
- Concrete class for uint32 array elements.
+ Concrete class for int32 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt32Array* ap = <CUInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt32Scalar* sp = <CInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int64Value(ArrayValue):
+cdef class UInt64Scalar(Scalar):
"""
- Concrete class for int64 array elements.
+ Concrete class for uint64 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt64Array* ap = <CInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt64Scalar* sp = <CUInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt64Value(ArrayValue):
+cdef class Int64Scalar(Scalar):
"""
- Concrete class for uint64 array elements.
+ Concrete class for int64 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt64Array* ap = <CUInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt64Scalar* sp = <CInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date32Value(ArrayValue):
+cdef class HalfFloatScalar(Scalar):
"""
- Concrete class for date32 array elements.
+ Concrete class for float scalars.
"""
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+
+ def __eq__(self, other):
+ if hasattr(self, 'as_py'):
+ if isinstance(other, Scalar):
+ other = other.as_py()
+ return self.as_py() == other
+ else:
+ raise NotImplementedError
+
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python float.
"""
- cdef CDate32Array* ap = <CDate32Array*> self.sp_array.get()
+ cdef CHalfFloatScalar* sp = <CHalfFloatScalar*> self.wrapped.get()
+ return PyHalf_FromHalf(sp.value) if sp.is_valid else None
- # Shift to seconds since epoch
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(days=ap.Value(self.index)))
+
+cdef class FloatScalar(Scalar):
+ """
+ Concrete class for float scalars.
+ """
+
+ def as_py(self):
+ """
+ Return this value as a Python float.
+ """
+ cdef CFloatScalar* sp = <CFloatScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date64Value(ArrayValue):
+cdef class DoubleScalar(Scalar):
"""
- Concrete class for date64 array elements.
+ Concrete class for double scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python float.
"""
- cdef CDate64Array* ap = <CDate64Array*> self.sp_array.get()
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(
- days=ap.Value(self.index) / 86400000))
+ cdef CDoubleScalar* sp = <CDoubleScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Time32Value(ArrayValue):
+cdef class Decimal128Scalar(Scalar):
"""
- Concrete class for time32 array elements.
+ Concrete class for decimal128 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.timedelta instance.
+ Return this value as a Python Decimal.
"""
cdef:
- CTime32Array* ap = <CTime32Array*> self.sp_array.get()
- CTime32Type* dtype = <CTime32Type*> ap.type().get()
-
- if dtype.unit() == TimeUnit_SECOND:
- delta = datetime.timedelta(seconds=ap.Value(self.index))
- return (datetime.datetime(1970, 1, 1) + delta).time()
+ CDecimal128Scalar* sp = <CDecimal128Scalar*> self.wrapped.get()
+ CDecimal128Type* dtype = <CDecimal128Type*> sp.type.get()
+ if sp.is_valid:
+ return _pydecimal.Decimal(
+ frombytes(sp.value.ToString(dtype.scale()))
+ )
else:
- return _box_time_milli(ap.Value(self.index))
+ return None
-cdef class Time64Value(ArrayValue):
+cdef class Date32Scalar(Scalar):
"""
- Concrete class for time64 array elements.
+ Concrete class for date32 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.timedelta instance.
+ Return this value as a Python datetime.datetime instance.
"""
- cdef:
- CTime64Array* ap = <CTime64Array*> self.sp_array.get()
- CTime64Type* dtype = <CTime64Type*> ap.type().get()
+ cdef CDate32Scalar* sp = <CDate32Scalar*> self.wrapped.get()
- cdef int64_t val = ap.Value(self.index)
- if dtype.unit() == TimeUnit_MICRO:
- return _box_time_micro(val)
+ if sp.is_valid:
+ # shift to seconds since epoch
+ return (
+ datetime.date(1970, 1, 1) + datetime.timedelta(days=sp.value)
+ )
else:
- return (datetime.datetime(1970, 1, 1) +
- datetime.timedelta(microseconds=val / 1000)).time()
-
+ return None
-cpdef _box_time_milli(int64_t val):
- delta = datetime.timedelta(milliseconds=val)
- return (datetime.datetime(1970, 1, 1) + delta).time()
+cdef class Date64Scalar(Scalar):
+ """
+ Concrete class for date64 scalars.
+ """
-cpdef _box_time_micro(int64_t val):
- return (datetime.datetime(1970, 1, 1) +
- datetime.timedelta(microseconds=val)).time()
+ def as_py(self):
+ """
+ Return this value as a Python datetime.datetime instance.
+ """
+ cdef CDate64Scalar* sp = <CDate64Scalar*> self.wrapped.get()
+ if sp.is_valid:
+ return (
+ datetime.date(1970, 1, 1) +
+ datetime.timedelta(days=sp.value / 86400000)
+ )
+ else:
+ return None
-cdef dict _DATETIME_CONVERSION_FUNCTIONS = {}
-cdef c_bool _datetime_conversion_initialized = False
+def _datetime_from_int(int64_t value, TimeUnit unit, tzinfo=None):
+ if unit == TimeUnit_SECOND:
+ delta = datetime.timedelta(seconds=value)
+ elif unit == TimeUnit_MILLI:
+ delta = datetime.timedelta(milliseconds=value)
+ elif unit == TimeUnit_MICRO:
+ delta = datetime.timedelta(microseconds=value)
+ else:
+ # TimeUnit_NANO: prefer pandas timestamps if available
+ if _pandas_api.have_pandas:
+ return _pandas_api.pd.Timestamp(value, tz=tzinfo, unit='ns')
+ # otherwise safely truncate to microsecond resolution datetime
+ if value % 1000 != 0:
+ raise ValueError(
+ "Nanosecond resolution temporal type {} is not safely "
+ "convertible to microseconds to convert to datetime.datetime. "
+ "Install pandas to return as Timestamp with nanosecond "
+ "support or access the .value attribute.".format(value)
+ )
+ delta = datetime.timedelta(microseconds=value // 1000)
-cdef _add_micros_maybe_localize(dt, micros, tzinfo):
- import pytz
- dt = dt.replace(microsecond=micros)
+ dt = datetime.datetime(1970, 1, 1) + delta
+ # adjust timezone if set to the datatype
if tzinfo is not None:
- if not isinstance(tzinfo, datetime.tzinfo):
- tzinfo = string_to_tzinfo(tzinfo)
dt = tzinfo.fromutc(dt)
- return dt
-
-
-cdef _datetime_from_seconds(int64_t v):
- return datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=v)
-
-
-def _nanoseconds_to_datetime_safe(v, tzinfo):
- if v % 1000 != 0:
- raise ValueError("Nanosecond timestamp {} is not safely convertible "
- " to microseconds to convert to datetime.datetime."
- " Install pandas to return as Timestamp with "
- " nanosecond support or access the .value attribute.")
- v = v // 1000
- micros = v % 1_000_000
-
- dt = _datetime_from_seconds(v // 1_000_000)
- return _add_micros_maybe_localize(dt, micros, tzinfo)
-
-
-def _microseconds_to_datetime(v, tzinfo):
- micros = v % 1_000_000
- dt = _datetime_from_seconds(v // 1_000_000)
- return _add_micros_maybe_localize(dt, micros, tzinfo)
+ return dt
-def _millis_to_datetime(v, tzinfo):
- millis = v % 1_000
- dt = _datetime_from_seconds(v // 1000)
- return _add_micros_maybe_localize(dt, millis * 1000, tzinfo)
+cdef class Time32Scalar(Scalar):
+ """
+ Concrete class for time32 scalars.
+ """
-def _seconds_to_datetime(v, tzinfo):
- dt = _datetime_from_seconds(v)
- return _add_micros_maybe_localize(dt, 0, tzinfo)
+ def as_py(self):
+ """
+ Return this value as a Python datetime.timedelta instance.
+ """
+ cdef:
+ CTime32Scalar* sp = <CTime32Scalar*> self.wrapped.get()
+ CTime32Type* dtype = <CTime32Type*> sp.type.get()
+ if sp.is_valid:
+ return _datetime_from_int(sp.value, unit=dtype.unit()).time()
+ else:
+ return None
-def _datetime_conversion_functions():
- global _datetime_conversion_initialized
- if _datetime_conversion_initialized:
- return _DATETIME_CONVERSION_FUNCTIONS
- _DATETIME_CONVERSION_FUNCTIONS.update({
- TimeUnit_SECOND: _seconds_to_datetime,
- TimeUnit_MILLI: _millis_to_datetime,
- TimeUnit_MICRO: _microseconds_to_datetime,
- TimeUnit_NANO: _nanoseconds_to_datetime_safe
- })
+cdef class Time64Scalar(Scalar):
+ """
+ Concrete class for time64 scalars.
+ """
- try:
- import pandas as pd
- _DATETIME_CONVERSION_FUNCTIONS[TimeUnit_NANO] = (
- lambda x, tzinfo: pd.Timestamp(
- x, tz=tzinfo, unit='ns',
- )
- )
- except ImportError:
- pass
+ def as_py(self):
+ """
+ Return this value as a Python datetime.timedelta instance.
+ """
+ cdef:
+ CTime64Scalar* sp = <CTime64Scalar*> self.wrapped.get()
+ CTime64Type* dtype = <CTime64Type*> sp.type.get()
- _datetime_conversion_initialized = True
- return _DATETIME_CONVERSION_FUNCTIONS
+ if sp.is_valid:
+ return _datetime_from_int(sp.value, unit=dtype.unit()).time()
+ else:
+ return None
-cdef class TimestampValue(ArrayValue):
+cdef class TimestampScalar(Scalar):
"""
- Concrete class for timestamp array elements.
+ Concrete class for timestamp scalars.
"""
@property
def value(self):
- cdef CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
- cdef CTimestampType* dtype = <CTimestampType*> ap.type().get()
- return ap.Value(self.index)
+ cdef CTimestampScalar* sp = <CTimestampScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
def as_py(self):
"""
Return this value as a Pandas Timestamp instance (if available),
otherwise as a Python datetime.timedelta instance.
"""
- cdef CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
- cdef CTimestampType* dtype = <CTimestampType*> ap.type().get()
+ cdef:
+ CTimestampScalar* sp = <CTimestampScalar*> self.wrapped.get()
+ CTimestampType* dtype = <CTimestampType*> sp.type.get()
- value = self.value
+ if not sp.is_valid:
+ return None
if not dtype.timezone().empty():
tzinfo = string_to_tzinfo(frombytes(dtype.timezone()))
+ if not isinstance(tzinfo, datetime.tzinfo):
+ tzinfo = string_to_tzinfo(tzinfo)
else:
tzinfo = None
- try:
- converter = _datetime_conversion_functions()[dtype.unit()]
- except KeyError:
- raise ValueError(
- 'Cannot convert nanosecond timestamps without pandas'
- )
- return converter(value, tzinfo=tzinfo)
-
-
-cdef dict _TIMEDELTA_CONVERSION_FUNCTIONS = {}
-
-
-def _nanoseconds_to_timedelta_safe(v):
- if v % 1000 != 0:
- raise ValueError(
- "Nanosecond duration {} is not safely convertible to microseconds "
- "to convert to datetime.timedelta. Install pandas to return as "
- "Timedelta with nanosecond support or access the .value "
- "attribute.".format(v))
- micros = v // 1000
-
- return datetime.timedelta(microseconds=micros)
-
-
-def _timedelta_conversion_functions():
- if _TIMEDELTA_CONVERSION_FUNCTIONS:
- return _TIMEDELTA_CONVERSION_FUNCTIONS
-
- _TIMEDELTA_CONVERSION_FUNCTIONS.update({
- TimeUnit_SECOND: lambda v: datetime.timedelta(seconds=v),
- TimeUnit_MILLI: lambda v: datetime.timedelta(milliseconds=v),
- TimeUnit_MICRO: lambda v: datetime.timedelta(microseconds=v),
- TimeUnit_NANO: _nanoseconds_to_timedelta_safe
- })
-
- try:
- import pandas as pd
- _TIMEDELTA_CONVERSION_FUNCTIONS[TimeUnit_NANO] = (
- lambda v: pd.Timedelta(v, unit='ns')
- )
- except ImportError:
- pass
+ return _datetime_from_int(sp.value, unit=dtype.unit(), tzinfo=tzinfo)
- return _TIMEDELTA_CONVERSION_FUNCTIONS
-
-cdef class DurationValue(ArrayValue):
+cdef class DurationScalar(Scalar):
"""
- Concrete class for duration array elements.
+ Concrete class for duration scalars.
"""
@property
def value(self):
- cdef CDurationArray* ap = <CDurationArray*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CDurationScalar* sp = <CDurationScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
def as_py(self):
"""
Return this value as a Pandas Timestamp instance (if available),
otherwise as a Python datetime.timedelta instance.
"""
- cdef CDurationArray* ap = <CDurationArray*> self.sp_array.get()
- cdef CDurationType* dtype = <CDurationType*> ap.type().get()
+ cdef:
+ CDurationScalar* sp = <CDurationScalar*> self.wrapped.get()
+ CDurationType* dtype = <CDurationType*> sp.type.get()
+ TimeUnit unit = dtype.unit()
+
+ if not sp.is_valid:
+ return None
- cdef int64_t value = ap.Value(self.index)
- converter = _timedelta_conversion_functions()[dtype.unit()]
- return converter(value)
+ if unit == TimeUnit_SECOND:
+ return datetime.timedelta(seconds=sp.value)
+ elif unit == TimeUnit_MILLI:
+ return datetime.timedelta(milliseconds=sp.value)
+ elif unit == TimeUnit_MICRO:
+ return datetime.timedelta(microseconds=sp.value)
+ else:
+ # TimeUnit_NANO: prefer pandas timestamps if available
+ if _pandas_api.have_pandas:
+ return _pandas_api.pd.Timedelta(sp.value, unit='ns')
+ # otherwise safely truncate to microsecond resolution timedelta
+ if sp.value % 1000 != 0:
+ raise ValueError(
+ "Nanosecond duration {} is not safely convertible to "
+ "microseconds to convert to datetime.timedelta. Install "
+ "pandas to return as Timedelta with nanosecond support or "
+ "access the .value attribute.".format(sp.value)
+ )
+ return datetime.timedelta(microseconds=sp.value // 1000)
-cdef class HalfFloatValue(ArrayValue):
+cdef class BinaryScalar(Scalar):
"""
- Concrete class for float16 array elements.
+ Concrete class for binary-like scalars.
"""
- def as_py(self):
+ def as_buffer(self):
"""
- Return this value as a Python float.
+ Return a view over this value as a Buffer object.
"""
- cdef CHalfFloatArray* ap = <CHalfFloatArray*> self.sp_array.get()
- return PyHalf_FromHalf(ap.Value(self.index))
-
-
-cdef class FloatValue(ArrayValue):
- """
- Concrete class for float32 array elements.
- """
+ cdef CBinaryScalar* sp = <CBinaryScalar*> self.wrapped.get()
+ return pyarrow_wrap_buffer(sp.value) if sp.is_valid else None
def as_py(self):
"""
- Return this value as a Python float.
+ Return this value as a Python bytes.
"""
- cdef CFloatArray* ap = <CFloatArray*> self.sp_array.get()
- return ap.Value(self.index)
+ buffer = self.as_buffer()
+ if buffer is not None:
+ return self.as_buffer().to_pybytes()
Review comment:
`buffer.to_pybytes()`
##########
File path: python/pyarrow/tests/test_scalars.py
##########
@@ -16,427 +16,443 @@
# under the License.
import datetime
+import decimal
import pytest
-import unittest
import numpy as np
import pyarrow as pa
-class TestScalars(unittest.TestCase):
-
- def test_null_singleton(self):
- with pytest.raises(Exception):
- pa.NAType()
+@pytest.mark.parametrize(['value', 'ty', 'klass', 'deprecated'], [
+ (False, None, pa.BooleanScalar, pa.BooleanValue),
+ (True, None, pa.BooleanScalar, pa.BooleanValue),
+ (1, None, pa.Int64Scalar, pa.Int64Value),
+ (-1, None, pa.Int64Scalar, pa.Int64Value),
+ (1, pa.int8(), pa.Int8Scalar, pa.Int8Value),
+ (1, pa.uint8(), pa.UInt8Scalar, pa.UInt8Value),
+ (1, pa.int16(), pa.Int16Scalar, pa.Int16Value),
+ (1, pa.uint16(), pa.UInt16Scalar, pa.UInt16Value),
+ (1, pa.int32(), pa.Int32Scalar, pa.Int32Value),
+ (1, pa.uint32(), pa.UInt32Scalar, pa.UInt32Value),
+ (1, pa.int64(), pa.Int64Scalar, pa.Int64Value),
+ (1, pa.uint64(), pa.UInt64Scalar, pa.UInt64Value),
+ (1.0, None, pa.DoubleScalar, pa.DoubleValue),
+ (np.float16(1.0), pa.float16(), pa.HalfFloatScalar, pa.HalfFloatValue),
+ (1.0, pa.float32(), pa.FloatScalar, pa.FloatValue),
+ ("string", None, pa.StringScalar, pa.StringValue),
+ (b"bytes", None, pa.BinaryScalar, pa.BinaryValue),
+ ([1, 2, 3], None, pa.ListScalar, pa.ListValue),
+ ([1, 2, 3, 4], pa.large_list(pa.int8()), pa.LargeListScalar,
+ pa.LargeListValue),
+ (datetime.date.today(), None, pa.Date32Scalar, pa.Date64Value),
+ (datetime.datetime.now(), None, pa.TimestampScalar, pa.TimestampValue),
+ ({'a': 1, 'b': [1, 2]}, None, pa.StructScalar, pa.StructValue)
+])
+def test_basics(value, ty, klass, deprecated):
+ s = pa.scalar(value, type=ty)
+ assert isinstance(s, klass)
+ assert s == value
+ assert s == s
Review comment:
It would be more interesting to instantiate `s` a second time, e.g. `assert s == pa.scalar(value, type=ty)`
##########
File path: python/pyarrow/scalar.pxi
##########
@@ -1217,21 +764,50 @@ cdef dict _scalar_classes = {
_Type_INT16: Int16Scalar,
_Type_INT32: Int32Scalar,
_Type_INT64: Int64Scalar,
+ _Type_HALF_FLOAT: HalfFloatScalar,
_Type_FLOAT: FloatScalar,
_Type_DOUBLE: DoubleScalar,
+ _Type_DECIMAL: Decimal128Scalar,
+ _Type_DATE32: Date32Scalar,
+ _Type_DATE64: Date64Scalar,
+ _Type_TIME32: Time32Scalar,
+ _Type_TIME64: Time64Scalar,
+ _Type_TIMESTAMP: TimestampScalar,
+ _Type_DURATION: DurationScalar,
+ _Type_BINARY: BinaryScalar,
+ _Type_LARGE_BINARY: LargeBinaryScalar,
+ _Type_FIXED_SIZE_BINARY: FixedSizeBinaryScalar,
_Type_STRING: StringScalar,
+ _Type_LARGE_STRING: LargeStringScalar,
+ _Type_LIST: ListScalar,
+ _Type_LARGE_LIST: LargeListScalar,
+ _Type_FIXED_SIZE_LIST: FixedSizeListScalar,
+ _Type_STRUCT: StructScalar,
+ _Type_MAP: MapScalar,
+ _Type_DICTIONARY: DictionaryScalar,
+ _Type_SPARSE_UNION: UnionScalar,
+ _Type_DENSE_UNION: UnionScalar,
}
-cdef object box_scalar(DataType type, const shared_ptr[CArray]& sp_array,
- int64_t index):
- cdef ArrayValue value
- if type.type.id() == _Type_NA:
- return _NULL
- elif sp_array.get().IsNull(index):
- return _NULL
- else:
- klass = _array_value_classes[type.type.id()]
- value = klass.__new__(klass)
- value.init(type, sp_array, index)
- return value
+def scalar(value, DataType type=None, bint safe=True,
Review comment:
Can you add a docstring? Also, make `safe` and `memory_pool` keyword-only.
##########
File path: python/pyarrow/tests/test_scalars.py
##########
@@ -16,427 +16,443 @@
# under the License.
import datetime
+import decimal
import pytest
-import unittest
import numpy as np
import pyarrow as pa
-class TestScalars(unittest.TestCase):
-
- def test_null_singleton(self):
- with pytest.raises(Exception):
- pa.NAType()
+@pytest.mark.parametrize(['value', 'ty', 'klass', 'deprecated'], [
+ (False, None, pa.BooleanScalar, pa.BooleanValue),
+ (True, None, pa.BooleanScalar, pa.BooleanValue),
+ (1, None, pa.Int64Scalar, pa.Int64Value),
+ (-1, None, pa.Int64Scalar, pa.Int64Value),
+ (1, pa.int8(), pa.Int8Scalar, pa.Int8Value),
+ (1, pa.uint8(), pa.UInt8Scalar, pa.UInt8Value),
+ (1, pa.int16(), pa.Int16Scalar, pa.Int16Value),
+ (1, pa.uint16(), pa.UInt16Scalar, pa.UInt16Value),
+ (1, pa.int32(), pa.Int32Scalar, pa.Int32Value),
+ (1, pa.uint32(), pa.UInt32Scalar, pa.UInt32Value),
+ (1, pa.int64(), pa.Int64Scalar, pa.Int64Value),
+ (1, pa.uint64(), pa.UInt64Scalar, pa.UInt64Value),
+ (1.0, None, pa.DoubleScalar, pa.DoubleValue),
+ (np.float16(1.0), pa.float16(), pa.HalfFloatScalar, pa.HalfFloatValue),
+ (1.0, pa.float32(), pa.FloatScalar, pa.FloatValue),
+ ("string", None, pa.StringScalar, pa.StringValue),
+ (b"bytes", None, pa.BinaryScalar, pa.BinaryValue),
+ ([1, 2, 3], None, pa.ListScalar, pa.ListValue),
+ ([1, 2, 3, 4], pa.large_list(pa.int8()), pa.LargeListScalar,
+ pa.LargeListValue),
+ (datetime.date.today(), None, pa.Date32Scalar, pa.Date64Value),
+ (datetime.datetime.now(), None, pa.TimestampScalar, pa.TimestampValue),
+ ({'a': 1, 'b': [1, 2]}, None, pa.StructScalar, pa.StructValue)
+])
+def test_basics(value, ty, klass, deprecated):
+ s = pa.scalar(value, type=ty)
+ assert isinstance(s, klass)
+ assert s == value
+ assert s == s
+ assert s != "else"
+ assert hash(s) == hash(s)
+ assert s.is_valid is True
+ with pytest.warns(FutureWarning):
+ isinstance(s, deprecated)
+
+ s = pa.scalar(None, type=s.type)
+ assert s.is_valid is False
+ assert s.as_py() is None
Review comment:
Also `assert s != pa.scalar(value, type=ty)`
##########
File path: python/pyarrow/tests/test_parquet.py
##########
@@ -2028,7 +2028,7 @@ def test_filters_invalid_pred_op(tempdir, use_legacy_dataset):
use_legacy_dataset=use_legacy_dataset)
assert dataset.read().num_rows == 0
- with pytest.raises(ValueError if use_legacy_dataset else TypeError):
+ with pytest.raises(ValueError):
# dataset API returns TypeError when trying create invalid comparison
Review comment:
Comment seems obsolete?
##########
File path: python/pyarrow/scalar.pxi
##########
@@ -16,1198 +16,745 @@
# under the License.
-_NULL = NA = None
+import collections
cdef class Scalar:
"""
- The base class for all array elements.
+ The base class for scalars.
"""
+ def __init__(self):
+ raise TypeError("Do not call {}'s constructor directly, use "
+ "pa.scalar() instead.".format(self.__class__.__name__))
-cdef class NullType(Scalar):
- """
- Singleton for null array elements.
- """
- # TODO rename this NullValue?
+ cdef void init(self, const shared_ptr[CScalar]& wrapped):
+ self.wrapped = wrapped
- def __cinit__(self):
- global NA
- if NA is not None:
- raise Exception('Cannot create multiple NAType instances')
+ @staticmethod
+ cdef wrap(const shared_ptr[CScalar]& wrapped):
+ cdef:
+ Scalar self
+ Type type_id = wrapped.get().type.get().id()
+
+ if type_id == _Type_NA:
+ return _NULL
+
+ typ = _scalar_classes[type_id]
+ self = typ.__new__(typ)
+ self.init(wrapped)
+
+ return self
+
+ cdef inline shared_ptr[CScalar] unwrap(self) nogil:
+ return self.wrapped
+
+ @property
+ def type(self):
+ return pyarrow_wrap_data_type(self.wrapped.get().type)
- self.type = null()
+ @property
+ def is_valid(self):
+ return self.wrapped.get().is_valid
def __repr__(self):
- return 'NULL'
+ return '<pyarrow.{}: {!r}>'.format(
+ self.__class__.__name__, self.as_py()
+ )
- def as_py(self):
- """
- Return None
- """
- return None
+ def __str__(self):
+ return str(self.as_py())
+
+ def equals(self, Scalar other):
+ return self.wrapped.get().Equals(other.unwrap().get()[0])
def __eq__(self, other):
- return NA
+ try:
+ if not isinstance(other, Scalar):
+ other = scalar(other, type=self.type)
+ return self.equals(other)
+ except (TypeError, ValueError, ArrowInvalid):
+ return NotImplemented
+
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+
+ def as_py(self):
+ raise NotImplementedError()
-_NULL = NA = NullType()
+_NULL = NA = None
-cdef class ArrayValue(Scalar):
+cdef class NullScalar(Scalar):
"""
- The base class for non-null array elements.
+ Concrete class for null scalars.
"""
- def __init__(self):
- raise TypeError("Do not call {}'s constructor directly, use array "
- "subscription instead."
- .format(self.__class__.__name__))
+ def __cinit__(self):
+ global NA
+ if NA is not None:
+ raise Exception('Cannot create multiple NAType instances')
+ self.init(shared_ptr[CScalar](new CNullScalar()))
- cdef void init(self, DataType type, const shared_ptr[CArray]& sp_array,
- int64_t index):
- self.type = type
- self.index = index
- self._set_array(sp_array)
+ def __init__(self):
+ pass
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
+ def __eq__(self, other):
+ return NA
- def __repr__(self):
- if hasattr(self, 'as_py'):
- return repr(self.as_py())
- else:
- return super(Scalar, self).__repr__()
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
- def __str__(self):
- if hasattr(self, 'as_py'):
- return str(self.as_py())
- else:
- return super(Scalar, self).__str__()
+ def as_py(self):
+ """
+ Return this value as a Python None.
+ """
+ return None
- def __eq__(self, other):
- if hasattr(self, 'as_py'):
- if isinstance(other, ArrayValue):
- other = other.as_py()
- return self.as_py() == other
- else:
- raise NotImplementedError(
- "Cannot compare Arrow values that don't support as_py()")
- def __hash__(self):
- return hash(self.as_py())
+_NULL = NA = NullScalar()
-cdef class BooleanValue(ArrayValue):
+cdef class BooleanScalar(Scalar):
"""
- Concrete class for boolean array elements.
+ Concrete class for boolean scalars.
"""
def as_py(self):
"""
Return this value as a Python bool.
"""
- cdef CBooleanArray* ap = <CBooleanArray*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CBooleanScalar* sp = <CBooleanScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int8Value(ArrayValue):
+cdef class UInt8Scalar(Scalar):
"""
- Concrete class for int8 array elements.
+ Concrete class for uint8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt8Array* ap = <CInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt8Scalar* sp = <CUInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt8Value(ArrayValue):
+cdef class Int8Scalar(Scalar):
"""
- Concrete class for uint8 array elements.
+ Concrete class for int8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt8Array* ap = <CUInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt8Scalar* sp = <CInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int16Value(ArrayValue):
+cdef class UInt16Scalar(Scalar):
"""
- Concrete class for int16 array elements.
+ Concrete class for uint16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt16Array* ap = <CInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt16Scalar* sp = <CUInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt16Value(ArrayValue):
+cdef class Int16Scalar(Scalar):
"""
- Concrete class for uint16 array elements.
+ Concrete class for int16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt16Array* ap = <CUInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt16Scalar* sp = <CInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int32Value(ArrayValue):
+cdef class UInt32Scalar(Scalar):
"""
- Concrete class for int32 array elements.
+ Concrete class for uint32 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt32Array* ap = <CInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt32Scalar* sp = <CUInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt32Value(ArrayValue):
+cdef class Int32Scalar(Scalar):
"""
- Concrete class for uint32 array elements.
+ Concrete class for int32 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt32Array* ap = <CUInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt32Scalar* sp = <CInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int64Value(ArrayValue):
+cdef class UInt64Scalar(Scalar):
"""
- Concrete class for int64 array elements.
+ Concrete class for uint64 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt64Array* ap = <CInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt64Scalar* sp = <CUInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt64Value(ArrayValue):
+cdef class Int64Scalar(Scalar):
"""
- Concrete class for uint64 array elements.
+ Concrete class for int64 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt64Array* ap = <CUInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt64Scalar* sp = <CInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date32Value(ArrayValue):
+cdef class HalfFloatScalar(Scalar):
"""
- Concrete class for date32 array elements.
+ Concrete class for float scalars.
"""
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+
+ def __eq__(self, other):
+ if hasattr(self, 'as_py'):
+ if isinstance(other, Scalar):
+ other = other.as_py()
+ return self.as_py() == other
+ else:
+ raise NotImplementedError
+
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python float.
"""
- cdef CDate32Array* ap = <CDate32Array*> self.sp_array.get()
+ cdef CHalfFloatScalar* sp = <CHalfFloatScalar*> self.wrapped.get()
+ return PyHalf_FromHalf(sp.value) if sp.is_valid else None
- # Shift to seconds since epoch
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(days=ap.Value(self.index)))
+
+cdef class FloatScalar(Scalar):
+ """
+ Concrete class for float scalars.
+ """
+
+ def as_py(self):
+ """
+ Return this value as a Python float.
+ """
+ cdef CFloatScalar* sp = <CFloatScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date64Value(ArrayValue):
+cdef class DoubleScalar(Scalar):
"""
- Concrete class for date64 array elements.
+ Concrete class for double scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python float.
"""
- cdef CDate64Array* ap = <CDate64Array*> self.sp_array.get()
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(
- days=ap.Value(self.index) / 86400000))
+ cdef CDoubleScalar* sp = <CDoubleScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Time32Value(ArrayValue):
+cdef class Decimal128Scalar(Scalar):
"""
- Concrete class for time32 array elements.
+ Concrete class for decimal128 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.timedelta instance.
+ Return this value as a Python Decimal.
"""
cdef:
- CTime32Array* ap = <CTime32Array*> self.sp_array.get()
- CTime32Type* dtype = <CTime32Type*> ap.type().get()
-
- if dtype.unit() == TimeUnit_SECOND:
- delta = datetime.timedelta(seconds=ap.Value(self.index))
- return (datetime.datetime(1970, 1, 1) + delta).time()
+ CDecimal128Scalar* sp = <CDecimal128Scalar*> self.wrapped.get()
+ CDecimal128Type* dtype = <CDecimal128Type*> sp.type.get()
+ if sp.is_valid:
+ return _pydecimal.Decimal(
+ frombytes(sp.value.ToString(dtype.scale()))
+ )
else:
- return _box_time_milli(ap.Value(self.index))
+ return None
-cdef class Time64Value(ArrayValue):
+cdef class Date32Scalar(Scalar):
"""
- Concrete class for time64 array elements.
+ Concrete class for date32 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.timedelta instance.
+ Return this value as a Python datetime.datetime instance.
"""
- cdef:
- CTime64Array* ap = <CTime64Array*> self.sp_array.get()
- CTime64Type* dtype = <CTime64Type*> ap.type().get()
+ cdef CDate32Scalar* sp = <CDate32Scalar*> self.wrapped.get()
- cdef int64_t val = ap.Value(self.index)
- if dtype.unit() == TimeUnit_MICRO:
- return _box_time_micro(val)
+ if sp.is_valid:
+ # shift to seconds since epoch
+ return (
+ datetime.date(1970, 1, 1) + datetime.timedelta(days=sp.value)
+ )
else:
- return (datetime.datetime(1970, 1, 1) +
- datetime.timedelta(microseconds=val / 1000)).time()
-
+ return None
-cpdef _box_time_milli(int64_t val):
- delta = datetime.timedelta(milliseconds=val)
- return (datetime.datetime(1970, 1, 1) + delta).time()
+cdef class Date64Scalar(Scalar):
+ """
+ Concrete class for date64 scalars.
+ """
-cpdef _box_time_micro(int64_t val):
- return (datetime.datetime(1970, 1, 1) +
- datetime.timedelta(microseconds=val)).time()
+ def as_py(self):
+ """
+ Return this value as a Python datetime.datetime instance.
+ """
+ cdef CDate64Scalar* sp = <CDate64Scalar*> self.wrapped.get()
+ if sp.is_valid:
+ return (
+ datetime.date(1970, 1, 1) +
+ datetime.timedelta(days=sp.value / 86400000)
+ )
+ else:
+ return None
-cdef dict _DATETIME_CONVERSION_FUNCTIONS = {}
-cdef c_bool _datetime_conversion_initialized = False
+def _datetime_from_int(int64_t value, TimeUnit unit, tzinfo=None):
+ if unit == TimeUnit_SECOND:
+ delta = datetime.timedelta(seconds=value)
+ elif unit == TimeUnit_MILLI:
+ delta = datetime.timedelta(milliseconds=value)
+ elif unit == TimeUnit_MICRO:
+ delta = datetime.timedelta(microseconds=value)
+ else:
+ # TimeUnit_NANO: prefer pandas timestamps if available
+ if _pandas_api.have_pandas:
+ return _pandas_api.pd.Timestamp(value, tz=tzinfo, unit='ns')
+ # otherwise safely truncate to microsecond resolution datetime
+ if value % 1000 != 0:
+ raise ValueError(
+ "Nanosecond resolution temporal type {} is not safely "
+ "convertible to microseconds to convert to datetime.datetime. "
+ "Install pandas to return as Timestamp with nanosecond "
+ "support or access the .value attribute.".format(value)
+ )
+ delta = datetime.timedelta(microseconds=value // 1000)
-cdef _add_micros_maybe_localize(dt, micros, tzinfo):
- import pytz
- dt = dt.replace(microsecond=micros)
+ dt = datetime.datetime(1970, 1, 1) + delta
+ # adjust timezone if set to the datatype
if tzinfo is not None:
- if not isinstance(tzinfo, datetime.tzinfo):
- tzinfo = string_to_tzinfo(tzinfo)
dt = tzinfo.fromutc(dt)
- return dt
-
-
-cdef _datetime_from_seconds(int64_t v):
- return datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=v)
-
-
-def _nanoseconds_to_datetime_safe(v, tzinfo):
- if v % 1000 != 0:
- raise ValueError("Nanosecond timestamp {} is not safely convertible "
- " to microseconds to convert to datetime.datetime."
- " Install pandas to return as Timestamp with "
- " nanosecond support or access the .value attribute.")
- v = v // 1000
- micros = v % 1_000_000
-
- dt = _datetime_from_seconds(v // 1_000_000)
- return _add_micros_maybe_localize(dt, micros, tzinfo)
-
-
-def _microseconds_to_datetime(v, tzinfo):
- micros = v % 1_000_000
- dt = _datetime_from_seconds(v // 1_000_000)
- return _add_micros_maybe_localize(dt, micros, tzinfo)
+ return dt
-def _millis_to_datetime(v, tzinfo):
- millis = v % 1_000
- dt = _datetime_from_seconds(v // 1000)
- return _add_micros_maybe_localize(dt, millis * 1000, tzinfo)
+cdef class Time32Scalar(Scalar):
+ """
+ Concrete class for time32 scalars.
+ """
-def _seconds_to_datetime(v, tzinfo):
- dt = _datetime_from_seconds(v)
- return _add_micros_maybe_localize(dt, 0, tzinfo)
+ def as_py(self):
+ """
+ Return this value as a Python datetime.timedelta instance.
+ """
+ cdef:
+ CTime32Scalar* sp = <CTime32Scalar*> self.wrapped.get()
+ CTime32Type* dtype = <CTime32Type*> sp.type.get()
+ if sp.is_valid:
+ return _datetime_from_int(sp.value, unit=dtype.unit()).time()
+ else:
+ return None
-def _datetime_conversion_functions():
- global _datetime_conversion_initialized
- if _datetime_conversion_initialized:
- return _DATETIME_CONVERSION_FUNCTIONS
- _DATETIME_CONVERSION_FUNCTIONS.update({
- TimeUnit_SECOND: _seconds_to_datetime,
- TimeUnit_MILLI: _millis_to_datetime,
- TimeUnit_MICRO: _microseconds_to_datetime,
- TimeUnit_NANO: _nanoseconds_to_datetime_safe
- })
+cdef class Time64Scalar(Scalar):
+ """
+ Concrete class for time64 scalars.
+ """
- try:
- import pandas as pd
- _DATETIME_CONVERSION_FUNCTIONS[TimeUnit_NANO] = (
- lambda x, tzinfo: pd.Timestamp(
- x, tz=tzinfo, unit='ns',
- )
- )
- except ImportError:
- pass
+ def as_py(self):
+ """
+ Return this value as a Python datetime.timedelta instance.
+ """
+ cdef:
+ CTime64Scalar* sp = <CTime64Scalar*> self.wrapped.get()
+ CTime64Type* dtype = <CTime64Type*> sp.type.get()
- _datetime_conversion_initialized = True
- return _DATETIME_CONVERSION_FUNCTIONS
+ if sp.is_valid:
+ return _datetime_from_int(sp.value, unit=dtype.unit()).time()
+ else:
+ return None
-cdef class TimestampValue(ArrayValue):
+cdef class TimestampScalar(Scalar):
"""
- Concrete class for timestamp array elements.
+ Concrete class for timestamp scalars.
"""
@property
def value(self):
- cdef CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
- cdef CTimestampType* dtype = <CTimestampType*> ap.type().get()
- return ap.Value(self.index)
+ cdef CTimestampScalar* sp = <CTimestampScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
def as_py(self):
"""
Return this value as a Pandas Timestamp instance (if available),
otherwise as a Python datetime.timedelta instance.
"""
- cdef CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
- cdef CTimestampType* dtype = <CTimestampType*> ap.type().get()
+ cdef:
+ CTimestampScalar* sp = <CTimestampScalar*> self.wrapped.get()
+ CTimestampType* dtype = <CTimestampType*> sp.type.get()
- value = self.value
+ if not sp.is_valid:
+ return None
if not dtype.timezone().empty():
tzinfo = string_to_tzinfo(frombytes(dtype.timezone()))
+ if not isinstance(tzinfo, datetime.tzinfo):
+ tzinfo = string_to_tzinfo(tzinfo)
else:
tzinfo = None
- try:
- converter = _datetime_conversion_functions()[dtype.unit()]
- except KeyError:
- raise ValueError(
- 'Cannot convert nanosecond timestamps without pandas'
- )
- return converter(value, tzinfo=tzinfo)
-
-
-cdef dict _TIMEDELTA_CONVERSION_FUNCTIONS = {}
-
-
-def _nanoseconds_to_timedelta_safe(v):
- if v % 1000 != 0:
- raise ValueError(
- "Nanosecond duration {} is not safely convertible to microseconds "
- "to convert to datetime.timedelta. Install pandas to return as "
- "Timedelta with nanosecond support or access the .value "
- "attribute.".format(v))
- micros = v // 1000
-
- return datetime.timedelta(microseconds=micros)
-
-
-def _timedelta_conversion_functions():
- if _TIMEDELTA_CONVERSION_FUNCTIONS:
- return _TIMEDELTA_CONVERSION_FUNCTIONS
-
- _TIMEDELTA_CONVERSION_FUNCTIONS.update({
- TimeUnit_SECOND: lambda v: datetime.timedelta(seconds=v),
- TimeUnit_MILLI: lambda v: datetime.timedelta(milliseconds=v),
- TimeUnit_MICRO: lambda v: datetime.timedelta(microseconds=v),
- TimeUnit_NANO: _nanoseconds_to_timedelta_safe
- })
-
- try:
- import pandas as pd
- _TIMEDELTA_CONVERSION_FUNCTIONS[TimeUnit_NANO] = (
- lambda v: pd.Timedelta(v, unit='ns')
- )
- except ImportError:
- pass
+ return _datetime_from_int(sp.value, unit=dtype.unit(), tzinfo=tzinfo)
- return _TIMEDELTA_CONVERSION_FUNCTIONS
-
-cdef class DurationValue(ArrayValue):
+cdef class DurationScalar(Scalar):
"""
- Concrete class for duration array elements.
+ Concrete class for duration scalars.
"""
@property
def value(self):
- cdef CDurationArray* ap = <CDurationArray*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CDurationScalar* sp = <CDurationScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
def as_py(self):
"""
Return this value as a Pandas Timestamp instance (if available),
otherwise as a Python datetime.timedelta instance.
"""
- cdef CDurationArray* ap = <CDurationArray*> self.sp_array.get()
- cdef CDurationType* dtype = <CDurationType*> ap.type().get()
+ cdef:
+ CDurationScalar* sp = <CDurationScalar*> self.wrapped.get()
+ CDurationType* dtype = <CDurationType*> sp.type.get()
+ TimeUnit unit = dtype.unit()
+
+ if not sp.is_valid:
+ return None
- cdef int64_t value = ap.Value(self.index)
- converter = _timedelta_conversion_functions()[dtype.unit()]
- return converter(value)
+ if unit == TimeUnit_SECOND:
+ return datetime.timedelta(seconds=sp.value)
+ elif unit == TimeUnit_MILLI:
+ return datetime.timedelta(milliseconds=sp.value)
+ elif unit == TimeUnit_MICRO:
+ return datetime.timedelta(microseconds=sp.value)
+ else:
+ # TimeUnit_NANO: prefer pandas timestamps if available
+ if _pandas_api.have_pandas:
+ return _pandas_api.pd.Timedelta(sp.value, unit='ns')
+ # otherwise safely truncate to microsecond resolution timedelta
+ if sp.value % 1000 != 0:
+ raise ValueError(
+ "Nanosecond duration {} is not safely convertible to "
+ "microseconds to convert to datetime.timedelta. Install "
+ "pandas to return as Timedelta with nanosecond support or "
+ "access the .value attribute.".format(sp.value)
+ )
+ return datetime.timedelta(microseconds=sp.value // 1000)
-cdef class HalfFloatValue(ArrayValue):
+cdef class BinaryScalar(Scalar):
"""
- Concrete class for float16 array elements.
+ Concrete class for binary-like scalars.
"""
- def as_py(self):
+ def as_buffer(self):
"""
- Return this value as a Python float.
+ Return a view over this value as a Buffer object.
"""
- cdef CHalfFloatArray* ap = <CHalfFloatArray*> self.sp_array.get()
- return PyHalf_FromHalf(ap.Value(self.index))
-
-
-cdef class FloatValue(ArrayValue):
- """
- Concrete class for float32 array elements.
- """
+ cdef CBinaryScalar* sp = <CBinaryScalar*> self.wrapped.get()
+ return pyarrow_wrap_buffer(sp.value) if sp.is_valid else None
def as_py(self):
"""
- Return this value as a Python float.
+ Return this value as a Python bytes.
"""
- cdef CFloatArray* ap = <CFloatArray*> self.sp_array.get()
- return ap.Value(self.index)
+ buffer = self.as_buffer()
+ if buffer is not None:
+ return self.as_buffer().to_pybytes()
+ else:
+ return None
-cdef class DoubleValue(ArrayValue):
- """
- Concrete class for float64 array elements.
- """
+cdef class LargeBinaryScalar(BinaryScalar):
+ pass
- def as_py(self):
- """
- Return this value as a Python float.
- """
- cdef CDoubleArray* ap = <CDoubleArray*> self.sp_array.get()
- return ap.Value(self.index)
+
+cdef class FixedSizeBinaryScalar(BinaryScalar):
Review comment:
Same here.
##########
File path: python/pyarrow/tests/test_convert_builtin.py
##########
@@ -968,25 +968,31 @@ def test_sequence_timestamp_from_int_with_unit():
arr_s = pa.array(data, type=s)
assert len(arr_s) == 1
assert arr_s.type == s
- assert repr(arr_s[0]) == "datetime.datetime(1970, 1, 1, 0, 0, 1)"
+ assert repr(arr_s[0].as_py()) == "datetime.datetime(1970, 1, 1, 0, 0, 1)"
Review comment:
What is being tested here? If you're testing the `as_py()` result, then `repr` isn't needed. Otherwise, you may instead call `str(arr_s[0])`.
##########
File path: python/pyarrow/tests/test_scalars.py
##########
@@ -16,427 +16,443 @@
# under the License.
import datetime
+import decimal
import pytest
-import unittest
import numpy as np
import pyarrow as pa
-class TestScalars(unittest.TestCase):
-
- def test_null_singleton(self):
- with pytest.raises(Exception):
- pa.NAType()
+@pytest.mark.parametrize(['value', 'ty', 'klass', 'deprecated'], [
+ (False, None, pa.BooleanScalar, pa.BooleanValue),
+ (True, None, pa.BooleanScalar, pa.BooleanValue),
+ (1, None, pa.Int64Scalar, pa.Int64Value),
+ (-1, None, pa.Int64Scalar, pa.Int64Value),
+ (1, pa.int8(), pa.Int8Scalar, pa.Int8Value),
+ (1, pa.uint8(), pa.UInt8Scalar, pa.UInt8Value),
+ (1, pa.int16(), pa.Int16Scalar, pa.Int16Value),
+ (1, pa.uint16(), pa.UInt16Scalar, pa.UInt16Value),
+ (1, pa.int32(), pa.Int32Scalar, pa.Int32Value),
+ (1, pa.uint32(), pa.UInt32Scalar, pa.UInt32Value),
+ (1, pa.int64(), pa.Int64Scalar, pa.Int64Value),
+ (1, pa.uint64(), pa.UInt64Scalar, pa.UInt64Value),
+ (1.0, None, pa.DoubleScalar, pa.DoubleValue),
+ (np.float16(1.0), pa.float16(), pa.HalfFloatScalar, pa.HalfFloatValue),
+ (1.0, pa.float32(), pa.FloatScalar, pa.FloatValue),
+ ("string", None, pa.StringScalar, pa.StringValue),
+ (b"bytes", None, pa.BinaryScalar, pa.BinaryValue),
+ ([1, 2, 3], None, pa.ListScalar, pa.ListValue),
+ ([1, 2, 3, 4], pa.large_list(pa.int8()), pa.LargeListScalar,
+ pa.LargeListValue),
+ (datetime.date.today(), None, pa.Date32Scalar, pa.Date64Value),
+ (datetime.datetime.now(), None, pa.TimestampScalar, pa.TimestampValue),
+ ({'a': 1, 'b': [1, 2]}, None, pa.StructScalar, pa.StructValue)
+])
+def test_basics(value, ty, klass, deprecated):
+ s = pa.scalar(value, type=ty)
+ assert isinstance(s, klass)
+ assert s == value
+ assert s == s
+ assert s != "else"
+ assert hash(s) == hash(s)
+ assert s.is_valid is True
+ with pytest.warns(FutureWarning):
+ isinstance(s, deprecated)
Review comment:
`assert isinstance` perhaps?
##########
File path: python/pyarrow/scalar.pxi
##########
@@ -16,1198 +16,745 @@
# under the License.
-_NULL = NA = None
+import collections
cdef class Scalar:
"""
- The base class for all array elements.
+ The base class for scalars.
"""
+ def __init__(self):
+ raise TypeError("Do not call {}'s constructor directly, use "
+ "pa.scalar() instead.".format(self.__class__.__name__))
-cdef class NullType(Scalar):
- """
- Singleton for null array elements.
- """
- # TODO rename this NullValue?
+ cdef void init(self, const shared_ptr[CScalar]& wrapped):
+ self.wrapped = wrapped
- def __cinit__(self):
- global NA
- if NA is not None:
- raise Exception('Cannot create multiple NAType instances')
+ @staticmethod
+ cdef wrap(const shared_ptr[CScalar]& wrapped):
+ cdef:
+ Scalar self
+ Type type_id = wrapped.get().type.get().id()
+
+ if type_id == _Type_NA:
+ return _NULL
+
+ typ = _scalar_classes[type_id]
+ self = typ.__new__(typ)
+ self.init(wrapped)
+
+ return self
+
+ cdef inline shared_ptr[CScalar] unwrap(self) nogil:
+ return self.wrapped
+
+ @property
+ def type(self):
+ return pyarrow_wrap_data_type(self.wrapped.get().type)
- self.type = null()
+ @property
+ def is_valid(self):
+ return self.wrapped.get().is_valid
def __repr__(self):
- return 'NULL'
+ return '<pyarrow.{}: {!r}>'.format(
+ self.__class__.__name__, self.as_py()
+ )
- def as_py(self):
- """
- Return None
- """
- return None
+ def __str__(self):
+ return str(self.as_py())
+
+ def equals(self, Scalar other):
+ return self.wrapped.get().Equals(other.unwrap().get()[0])
def __eq__(self, other):
- return NA
+ try:
+ if not isinstance(other, Scalar):
+ other = scalar(other, type=self.type)
+ return self.equals(other)
+ except (TypeError, ValueError, ArrowInvalid):
+ return NotImplemented
+
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+
+ def as_py(self):
+ raise NotImplementedError()
-_NULL = NA = NullType()
+_NULL = NA = None
-cdef class ArrayValue(Scalar):
+cdef class NullScalar(Scalar):
"""
- The base class for non-null array elements.
+ Concrete class for null scalars.
"""
- def __init__(self):
- raise TypeError("Do not call {}'s constructor directly, use array "
- "subscription instead."
- .format(self.__class__.__name__))
+ def __cinit__(self):
+ global NA
+ if NA is not None:
+ raise Exception('Cannot create multiple NAType instances')
+ self.init(shared_ptr[CScalar](new CNullScalar()))
- cdef void init(self, DataType type, const shared_ptr[CArray]& sp_array,
- int64_t index):
- self.type = type
- self.index = index
- self._set_array(sp_array)
+ def __init__(self):
+ pass
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
+ def __eq__(self, other):
+ return NA
- def __repr__(self):
- if hasattr(self, 'as_py'):
- return repr(self.as_py())
- else:
- return super(Scalar, self).__repr__()
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
- def __str__(self):
- if hasattr(self, 'as_py'):
- return str(self.as_py())
- else:
- return super(Scalar, self).__str__()
+ def as_py(self):
+ """
+ Return this value as a Python None.
+ """
+ return None
- def __eq__(self, other):
- if hasattr(self, 'as_py'):
- if isinstance(other, ArrayValue):
- other = other.as_py()
- return self.as_py() == other
- else:
- raise NotImplementedError(
- "Cannot compare Arrow values that don't support as_py()")
- def __hash__(self):
- return hash(self.as_py())
+_NULL = NA = NullScalar()
-cdef class BooleanValue(ArrayValue):
+cdef class BooleanScalar(Scalar):
"""
- Concrete class for boolean array elements.
+ Concrete class for boolean scalars.
"""
def as_py(self):
"""
Return this value as a Python bool.
"""
- cdef CBooleanArray* ap = <CBooleanArray*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CBooleanScalar* sp = <CBooleanScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int8Value(ArrayValue):
+cdef class UInt8Scalar(Scalar):
"""
- Concrete class for int8 array elements.
+ Concrete class for uint8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt8Array* ap = <CInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt8Scalar* sp = <CUInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt8Value(ArrayValue):
+cdef class Int8Scalar(Scalar):
"""
- Concrete class for uint8 array elements.
+ Concrete class for int8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt8Array* ap = <CUInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt8Scalar* sp = <CInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int16Value(ArrayValue):
+cdef class UInt16Scalar(Scalar):
"""
- Concrete class for int16 array elements.
+ Concrete class for uint16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt16Array* ap = <CInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt16Scalar* sp = <CUInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt16Value(ArrayValue):
+cdef class Int16Scalar(Scalar):
"""
- Concrete class for uint16 array elements.
+ Concrete class for int16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt16Array* ap = <CUInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt16Scalar* sp = <CInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int32Value(ArrayValue):
+cdef class UInt32Scalar(Scalar):
"""
- Concrete class for int32 array elements.
+ Concrete class for uint32 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt32Array* ap = <CInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt32Scalar* sp = <CUInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt32Value(ArrayValue):
+cdef class Int32Scalar(Scalar):
"""
- Concrete class for uint32 array elements.
+ Concrete class for int32 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt32Array* ap = <CUInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt32Scalar* sp = <CInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int64Value(ArrayValue):
+cdef class UInt64Scalar(Scalar):
"""
- Concrete class for int64 array elements.
+ Concrete class for uint64 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt64Array* ap = <CInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt64Scalar* sp = <CUInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt64Value(ArrayValue):
+cdef class Int64Scalar(Scalar):
"""
- Concrete class for uint64 array elements.
+ Concrete class for int64 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt64Array* ap = <CUInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt64Scalar* sp = <CInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date32Value(ArrayValue):
+cdef class HalfFloatScalar(Scalar):
"""
- Concrete class for date32 array elements.
+ Concrete class for float scalars.
"""
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+
+ def __eq__(self, other):
+ if hasattr(self, 'as_py'):
+ if isinstance(other, Scalar):
+ other = other.as_py()
+ return self.as_py() == other
+ else:
+ raise NotImplementedError
+
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python float.
"""
- cdef CDate32Array* ap = <CDate32Array*> self.sp_array.get()
+ cdef CHalfFloatScalar* sp = <CHalfFloatScalar*> self.wrapped.get()
+ return PyHalf_FromHalf(sp.value) if sp.is_valid else None
- # Shift to seconds since epoch
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(days=ap.Value(self.index)))
+
+cdef class FloatScalar(Scalar):
+ """
+ Concrete class for float scalars.
+ """
+
+ def as_py(self):
+ """
+ Return this value as a Python float.
+ """
+ cdef CFloatScalar* sp = <CFloatScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date64Value(ArrayValue):
+cdef class DoubleScalar(Scalar):
"""
- Concrete class for date64 array elements.
+ Concrete class for double scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python float.
"""
- cdef CDate64Array* ap = <CDate64Array*> self.sp_array.get()
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(
- days=ap.Value(self.index) / 86400000))
+ cdef CDoubleScalar* sp = <CDoubleScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Time32Value(ArrayValue):
+cdef class Decimal128Scalar(Scalar):
"""
- Concrete class for time32 array elements.
+ Concrete class for decimal128 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.timedelta instance.
+ Return this value as a Python Decimal.
"""
cdef:
- CTime32Array* ap = <CTime32Array*> self.sp_array.get()
- CTime32Type* dtype = <CTime32Type*> ap.type().get()
-
- if dtype.unit() == TimeUnit_SECOND:
- delta = datetime.timedelta(seconds=ap.Value(self.index))
- return (datetime.datetime(1970, 1, 1) + delta).time()
+ CDecimal128Scalar* sp = <CDecimal128Scalar*> self.wrapped.get()
+ CDecimal128Type* dtype = <CDecimal128Type*> sp.type.get()
+ if sp.is_valid:
+ return _pydecimal.Decimal(
+ frombytes(sp.value.ToString(dtype.scale()))
+ )
else:
- return _box_time_milli(ap.Value(self.index))
+ return None
-cdef class Time64Value(ArrayValue):
+cdef class Date32Scalar(Scalar):
"""
- Concrete class for time64 array elements.
+ Concrete class for date32 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.timedelta instance.
+ Return this value as a Python datetime.datetime instance.
"""
- cdef:
- CTime64Array* ap = <CTime64Array*> self.sp_array.get()
- CTime64Type* dtype = <CTime64Type*> ap.type().get()
+ cdef CDate32Scalar* sp = <CDate32Scalar*> self.wrapped.get()
- cdef int64_t val = ap.Value(self.index)
- if dtype.unit() == TimeUnit_MICRO:
- return _box_time_micro(val)
+ if sp.is_valid:
+ # shift to seconds since epoch
+ return (
+ datetime.date(1970, 1, 1) + datetime.timedelta(days=sp.value)
+ )
else:
- return (datetime.datetime(1970, 1, 1) +
- datetime.timedelta(microseconds=val / 1000)).time()
-
+ return None
-cpdef _box_time_milli(int64_t val):
- delta = datetime.timedelta(milliseconds=val)
- return (datetime.datetime(1970, 1, 1) + delta).time()
+cdef class Date64Scalar(Scalar):
+ """
+ Concrete class for date64 scalars.
+ """
-cpdef _box_time_micro(int64_t val):
- return (datetime.datetime(1970, 1, 1) +
- datetime.timedelta(microseconds=val)).time()
+ def as_py(self):
+ """
+ Return this value as a Python datetime.datetime instance.
+ """
+ cdef CDate64Scalar* sp = <CDate64Scalar*> self.wrapped.get()
+ if sp.is_valid:
+ return (
+ datetime.date(1970, 1, 1) +
+ datetime.timedelta(days=sp.value / 86400000)
+ )
+ else:
+ return None
-cdef dict _DATETIME_CONVERSION_FUNCTIONS = {}
-cdef c_bool _datetime_conversion_initialized = False
+def _datetime_from_int(int64_t value, TimeUnit unit, tzinfo=None):
+ if unit == TimeUnit_SECOND:
+ delta = datetime.timedelta(seconds=value)
+ elif unit == TimeUnit_MILLI:
+ delta = datetime.timedelta(milliseconds=value)
+ elif unit == TimeUnit_MICRO:
+ delta = datetime.timedelta(microseconds=value)
+ else:
+ # TimeUnit_NANO: prefer pandas timestamps if available
+ if _pandas_api.have_pandas:
+ return _pandas_api.pd.Timestamp(value, tz=tzinfo, unit='ns')
+ # otherwise safely truncate to microsecond resolution datetime
+ if value % 1000 != 0:
+ raise ValueError(
+ "Nanosecond resolution temporal type {} is not safely "
+ "convertible to microseconds to convert to datetime.datetime. "
+ "Install pandas to return as Timestamp with nanosecond "
+ "support or access the .value attribute.".format(value)
+ )
+ delta = datetime.timedelta(microseconds=value // 1000)
-cdef _add_micros_maybe_localize(dt, micros, tzinfo):
- import pytz
- dt = dt.replace(microsecond=micros)
+ dt = datetime.datetime(1970, 1, 1) + delta
+ # adjust timezone if set to the datatype
if tzinfo is not None:
- if not isinstance(tzinfo, datetime.tzinfo):
- tzinfo = string_to_tzinfo(tzinfo)
dt = tzinfo.fromutc(dt)
- return dt
-
-
-cdef _datetime_from_seconds(int64_t v):
- return datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=v)
-
-
-def _nanoseconds_to_datetime_safe(v, tzinfo):
- if v % 1000 != 0:
- raise ValueError("Nanosecond timestamp {} is not safely convertible "
- " to microseconds to convert to datetime.datetime."
- " Install pandas to return as Timestamp with "
- " nanosecond support or access the .value attribute.")
- v = v // 1000
- micros = v % 1_000_000
-
- dt = _datetime_from_seconds(v // 1_000_000)
- return _add_micros_maybe_localize(dt, micros, tzinfo)
-
-
-def _microseconds_to_datetime(v, tzinfo):
- micros = v % 1_000_000
- dt = _datetime_from_seconds(v // 1_000_000)
- return _add_micros_maybe_localize(dt, micros, tzinfo)
+ return dt
-def _millis_to_datetime(v, tzinfo):
- millis = v % 1_000
- dt = _datetime_from_seconds(v // 1000)
- return _add_micros_maybe_localize(dt, millis * 1000, tzinfo)
+cdef class Time32Scalar(Scalar):
+ """
+ Concrete class for time32 scalars.
+ """
-def _seconds_to_datetime(v, tzinfo):
- dt = _datetime_from_seconds(v)
- return _add_micros_maybe_localize(dt, 0, tzinfo)
+ def as_py(self):
+ """
+ Return this value as a Python datetime.timedelta instance.
+ """
+ cdef:
+ CTime32Scalar* sp = <CTime32Scalar*> self.wrapped.get()
+ CTime32Type* dtype = <CTime32Type*> sp.type.get()
+ if sp.is_valid:
+ return _datetime_from_int(sp.value, unit=dtype.unit()).time()
+ else:
+ return None
-def _datetime_conversion_functions():
- global _datetime_conversion_initialized
- if _datetime_conversion_initialized:
- return _DATETIME_CONVERSION_FUNCTIONS
- _DATETIME_CONVERSION_FUNCTIONS.update({
- TimeUnit_SECOND: _seconds_to_datetime,
- TimeUnit_MILLI: _millis_to_datetime,
- TimeUnit_MICRO: _microseconds_to_datetime,
- TimeUnit_NANO: _nanoseconds_to_datetime_safe
- })
+cdef class Time64Scalar(Scalar):
+ """
+ Concrete class for time64 scalars.
+ """
- try:
- import pandas as pd
- _DATETIME_CONVERSION_FUNCTIONS[TimeUnit_NANO] = (
- lambda x, tzinfo: pd.Timestamp(
- x, tz=tzinfo, unit='ns',
- )
- )
- except ImportError:
- pass
+ def as_py(self):
+ """
+ Return this value as a Python datetime.timedelta instance.
+ """
+ cdef:
+ CTime64Scalar* sp = <CTime64Scalar*> self.wrapped.get()
+ CTime64Type* dtype = <CTime64Type*> sp.type.get()
- _datetime_conversion_initialized = True
- return _DATETIME_CONVERSION_FUNCTIONS
+ if sp.is_valid:
+ return _datetime_from_int(sp.value, unit=dtype.unit()).time()
+ else:
+ return None
-cdef class TimestampValue(ArrayValue):
+cdef class TimestampScalar(Scalar):
"""
- Concrete class for timestamp array elements.
+ Concrete class for timestamp scalars.
"""
@property
def value(self):
- cdef CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
- cdef CTimestampType* dtype = <CTimestampType*> ap.type().get()
- return ap.Value(self.index)
+ cdef CTimestampScalar* sp = <CTimestampScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
def as_py(self):
"""
Return this value as a Pandas Timestamp instance (if available),
otherwise as a Python datetime.timedelta instance.
"""
- cdef CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
- cdef CTimestampType* dtype = <CTimestampType*> ap.type().get()
+ cdef:
+ CTimestampScalar* sp = <CTimestampScalar*> self.wrapped.get()
+ CTimestampType* dtype = <CTimestampType*> sp.type.get()
- value = self.value
+ if not sp.is_valid:
+ return None
if not dtype.timezone().empty():
tzinfo = string_to_tzinfo(frombytes(dtype.timezone()))
+ if not isinstance(tzinfo, datetime.tzinfo):
+ tzinfo = string_to_tzinfo(tzinfo)
else:
tzinfo = None
- try:
- converter = _datetime_conversion_functions()[dtype.unit()]
- except KeyError:
- raise ValueError(
- 'Cannot convert nanosecond timestamps without pandas'
- )
- return converter(value, tzinfo=tzinfo)
-
-
-cdef dict _TIMEDELTA_CONVERSION_FUNCTIONS = {}
-
-
-def _nanoseconds_to_timedelta_safe(v):
- if v % 1000 != 0:
- raise ValueError(
- "Nanosecond duration {} is not safely convertible to microseconds "
- "to convert to datetime.timedelta. Install pandas to return as "
- "Timedelta with nanosecond support or access the .value "
- "attribute.".format(v))
- micros = v // 1000
-
- return datetime.timedelta(microseconds=micros)
-
-
-def _timedelta_conversion_functions():
- if _TIMEDELTA_CONVERSION_FUNCTIONS:
- return _TIMEDELTA_CONVERSION_FUNCTIONS
-
- _TIMEDELTA_CONVERSION_FUNCTIONS.update({
- TimeUnit_SECOND: lambda v: datetime.timedelta(seconds=v),
- TimeUnit_MILLI: lambda v: datetime.timedelta(milliseconds=v),
- TimeUnit_MICRO: lambda v: datetime.timedelta(microseconds=v),
- TimeUnit_NANO: _nanoseconds_to_timedelta_safe
- })
-
- try:
- import pandas as pd
- _TIMEDELTA_CONVERSION_FUNCTIONS[TimeUnit_NANO] = (
- lambda v: pd.Timedelta(v, unit='ns')
- )
- except ImportError:
- pass
+ return _datetime_from_int(sp.value, unit=dtype.unit(), tzinfo=tzinfo)
- return _TIMEDELTA_CONVERSION_FUNCTIONS
-
-cdef class DurationValue(ArrayValue):
+cdef class DurationScalar(Scalar):
"""
- Concrete class for duration array elements.
+ Concrete class for duration scalars.
"""
@property
def value(self):
- cdef CDurationArray* ap = <CDurationArray*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CDurationScalar* sp = <CDurationScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
def as_py(self):
"""
Return this value as a Pandas Timestamp instance (if available),
otherwise as a Python datetime.timedelta instance.
"""
- cdef CDurationArray* ap = <CDurationArray*> self.sp_array.get()
- cdef CDurationType* dtype = <CDurationType*> ap.type().get()
+ cdef:
+ CDurationScalar* sp = <CDurationScalar*> self.wrapped.get()
+ CDurationType* dtype = <CDurationType*> sp.type.get()
+ TimeUnit unit = dtype.unit()
+
+ if not sp.is_valid:
+ return None
- cdef int64_t value = ap.Value(self.index)
- converter = _timedelta_conversion_functions()[dtype.unit()]
- return converter(value)
+ if unit == TimeUnit_SECOND:
+ return datetime.timedelta(seconds=sp.value)
+ elif unit == TimeUnit_MILLI:
+ return datetime.timedelta(milliseconds=sp.value)
+ elif unit == TimeUnit_MICRO:
+ return datetime.timedelta(microseconds=sp.value)
+ else:
+ # TimeUnit_NANO: prefer pandas timestamps if available
+ if _pandas_api.have_pandas:
+ return _pandas_api.pd.Timedelta(sp.value, unit='ns')
+ # otherwise safely truncate to microsecond resolution timedelta
+ if sp.value % 1000 != 0:
+ raise ValueError(
+ "Nanosecond duration {} is not safely convertible to "
+ "microseconds to convert to datetime.timedelta. Install "
+ "pandas to return as Timedelta with nanosecond support or "
+ "access the .value attribute.".format(sp.value)
+ )
+ return datetime.timedelta(microseconds=sp.value // 1000)
-cdef class HalfFloatValue(ArrayValue):
+cdef class BinaryScalar(Scalar):
"""
- Concrete class for float16 array elements.
+ Concrete class for binary-like scalars.
"""
- def as_py(self):
+ def as_buffer(self):
"""
- Return this value as a Python float.
+ Return a view over this value as a Buffer object.
"""
- cdef CHalfFloatArray* ap = <CHalfFloatArray*> self.sp_array.get()
- return PyHalf_FromHalf(ap.Value(self.index))
-
-
-cdef class FloatValue(ArrayValue):
- """
- Concrete class for float32 array elements.
- """
+ cdef CBinaryScalar* sp = <CBinaryScalar*> self.wrapped.get()
+ return pyarrow_wrap_buffer(sp.value) if sp.is_valid else None
def as_py(self):
"""
- Return this value as a Python float.
+ Return this value as a Python bytes.
"""
- cdef CFloatArray* ap = <CFloatArray*> self.sp_array.get()
- return ap.Value(self.index)
+ buffer = self.as_buffer()
+ if buffer is not None:
+ return self.as_buffer().to_pybytes()
+ else:
+ return None
-cdef class DoubleValue(ArrayValue):
- """
- Concrete class for float64 array elements.
- """
+cdef class LargeBinaryScalar(BinaryScalar):
+ pass
- def as_py(self):
- """
- Return this value as a Python float.
- """
- cdef CDoubleArray* ap = <CDoubleArray*> self.sp_array.get()
- return ap.Value(self.index)
+
+cdef class FixedSizeBinaryScalar(BinaryScalar):
+ pass
-cdef class DecimalValue(ArrayValue):
+cdef class StringScalar(BinaryScalar):
"""
- Concrete class for decimal128 array elements.
+ Concrete class for string-like (utf8) scalars.
"""
def as_py(self):
"""
- Return this value as a Python Decimal.
+ Return this value as a Python string.
"""
- cdef:
- CDecimal128Array* ap = <CDecimal128Array*> self.sp_array.get()
- c_string s = ap.FormatValue(self.index)
- return _pydecimal.Decimal(s.decode('utf8'))
+ buffer = self.as_buffer()
+ if buffer is not None:
+ return frombytes(self.as_buffer().to_pybytes())
+ else:
+ return None
+
+cdef class LargeStringScalar(StringScalar):
+ pass
-cdef class StringValue(ArrayValue):
+
+cdef class ListScalar(Scalar):
"""
- Concrete class for string (utf8) array elements.
+ Concrete class for list-like scalars.
"""
- def as_py(self):
+ @property
+ def values(self):
+ cdef CListScalar* sp = <CListScalar*> self.wrapped.get()
+ if sp.is_valid:
+ return pyarrow_wrap_array(sp.value)
+ else:
+ return None
+
+ def __len__(self):
"""
- Return this value as a Python unicode string.
+ Return the number of values.
"""
- cdef CStringArray* ap = <CStringArray*> self.sp_array.get()
- return ap.GetString(self.index).decode('utf-8')
+ return len(self.values)
- def as_buffer(self):
+ def __getitem__(self, i):
"""
- Return a view over this value as a Buffer object.
+ Return the value at the given index.
"""
- cdef:
- CStringArray* ap = <CStringArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
-
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
-
+ return self.values[_normalize_index(i, len(self))]
-cdef class LargeStringValue(ArrayValue):
- """
- Concrete class for large string (utf8) array elements.
- """
-
- def as_py(self):
+ def __iter__(self):
"""
- Return this value as a Python unicode string.
+ Iterate over this element's values.
"""
- cdef CLargeStringArray* ap = <CLargeStringArray*> self.sp_array.get()
- return ap.GetString(self.index).decode('utf-8')
+ return iter(self.values)
- def as_buffer(self):
+ def as_py(self):
"""
- Return a view over this value as a Buffer object.
+ Return this value as a Python list.
"""
- cdef:
- CLargeStringArray* ap = <CLargeStringArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
+ arr = self.values
+ return None if arr is None else arr.to_pylist()
+
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
+cdef class FixedSizeListScalar(ListScalar):
+ pass
-cdef class BinaryValue(ArrayValue):
+cdef class LargeListScalar(ListScalar):
+ pass
+
+
+cdef class StructScalar(Scalar, collections.abc.Mapping):
"""
- Concrete class for variable-sized binary array elements.
+ Concrete class for struct scalars.
"""
- def as_py(self):
- """
- Return this value as a Python bytes object.
- """
+ def __len__(self):
+ cdef CStructScalar* sp = <CStructScalar*> self.wrapped.get()
+ return sp.value.size()
+
+ def __iter__(self):
cdef:
- const uint8_t* ptr
- int32_t length
- CBinaryArray* ap = <CBinaryArray*> self.sp_array.get()
+ CStructScalar* sp = <CStructScalar*> self.wrapped.get()
+ CStructType* dtype = <CStructType*> sp.type.get()
+ vector[shared_ptr[CField]] fields = dtype.fields()
- ptr = ap.GetValue(self.index, &length)
- return cp.PyBytes_FromStringAndSize(<const char*>(ptr), length)
+ if sp.is_valid:
+ for i in range(dtype.num_fields()):
+ yield frombytes(fields[i].get().name())
- def as_buffer(self):
+ def __contains__(self, key):
+ try:
+ self[key]
+ except IndexError:
+ return False
+ else:
+ return True
+
+ def __getitem__(self, key):
"""
- Return a view over this value as a Buffer object.
+ Return the child value for the given field.
+
+ Parameters
+ ----------
+ index : Union[int, str]
+ Index / position or name of the field.
+
+ Returns
+ -------
+ result : Scalar
"""
cdef:
- CBinaryArray* ap = <CBinaryArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
+ CFieldRef ref
+ CStructScalar* sp = <CStructScalar*> self.wrapped.get()
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
+ if isinstance(key, (bytes, str)):
+ ref = CFieldRef(<c_string> tobytes(key))
+ elif isinstance(key, int):
+ ref = CFieldRef(<int> key)
+ else:
+ raise TypeError('Expected integer or string index')
-
-cdef class LargeBinaryValue(ArrayValue):
- """
- Concrete class for large variable-sized binary array elements.
- """
-
- def as_py(self):
- """
- Return this value as a Python bytes object.
- """
- cdef:
- const uint8_t* ptr
- int64_t length
- CLargeBinaryArray* ap = <CLargeBinaryArray*> self.sp_array.get()
-
- ptr = ap.GetValue(self.index, &length)
- return cp.PyBytes_FromStringAndSize(<const char*>(ptr), length)
-
- def as_buffer(self):
- """
- Return a view over this value as a Buffer object.
- """
- cdef:
- CLargeBinaryArray* ap = <CLargeBinaryArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
-
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
-
-
-cdef class ListValue(ArrayValue):
- """
- Concrete class for list array elements.
- """
-
- def __len__(self):
- """
- Return the number of values.
- """
- return self.length()
-
- def __getitem__(self, i):
- """
- Return the value at the given index.
- """
- return self.getitem(_normalize_index(i, self.length()))
-
- def __iter__(self):
- """
- Iterate over this element's values.
- """
- for i in range(len(self)):
- yield self.getitem(i)
- raise StopIteration
-
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CListArray*> sp_array.get()
- self.value_type = pyarrow_wrap_data_type(self.ap.value_type())
-
- cdef getitem(self, int64_t i):
- cdef int64_t j = self.ap.value_offset(self.index) + i
- return box_scalar(self.value_type, self.ap.values(), j)
-
- cdef int64_t length(self):
- return self.ap.value_length(self.index)
-
- def as_py(self):
- """
- Return this value as a Python list.
- """
- cdef:
- int64_t j
- list result = []
-
- for j in range(len(self)):
- result.append(self.getitem(j).as_py())
-
- return result
-
-
-cdef class LargeListValue(ArrayValue):
- """
- Concrete class for large list array elements.
- """
-
- def __len__(self):
- """
- Return the number of values.
- """
- return self.length()
-
- def __getitem__(self, i):
- """
- Return the value at the given index.
- """
- return self.getitem(_normalize_index(i, self.length()))
-
- def __iter__(self):
- """
- Iterate over this element's values.
- """
- for i in range(len(self)):
- yield self.getitem(i)
- raise StopIteration
-
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CLargeListArray*> sp_array.get()
- self.value_type = pyarrow_wrap_data_type(self.ap.value_type())
-
- cdef getitem(self, int64_t i):
- cdef int64_t j = self.ap.value_offset(self.index) + i
- return box_scalar(self.value_type, self.ap.values(), j)
-
- cdef int64_t length(self):
- return self.ap.value_length(self.index)
-
- def as_py(self):
- """
- Return this value as a Python list.
- """
- cdef:
- int64_t j
- list result = []
-
- for j in range(len(self)):
- result.append(self.getitem(j).as_py())
-
- return result
-
-
-cdef class MapValue(ArrayValue):
- """
- Concrete class for map array elements.
- """
-
- def __len__(self):
- """
- Return the number of values.
- """
- return self.length()
-
- def __getitem__(self, i):
- """
- Return the value at the given index.
- """
- return self.getitem(_normalize_index(i, self.length()))
-
- def __iter__(self):
- """
- Iterate over this element's values.
- """
- for i in range(len(self)):
- yield self.getitem(i)
- raise StopIteration
-
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CMapArray*> sp_array.get()
- self.key_type = pyarrow_wrap_data_type(self.ap.map_type().key_type())
- self.item_type = pyarrow_wrap_data_type(self.ap.map_type().item_type())
-
- cdef getitem(self, int64_t i):
- cdef int64_t j = self.ap.value_offset(self.index) + i
- return (box_scalar(self.key_type, self.ap.keys(), j),
- box_scalar(self.item_type, self.ap.items(), j))
-
- cdef int64_t length(self):
- return self.ap.value_length(self.index)
+ try:
+ return Scalar.wrap(GetResultValue(sp.field(ref)))
+ except ArrowInvalid:
+ raise IndexError(key)
Review comment:
Ideally, this should be `KeyError` for string keys.
##########
File path: python/pyarrow/scalar.pxi
##########
@@ -16,1198 +16,745 @@
# under the License.
-_NULL = NA = None
+import collections
cdef class Scalar:
"""
- The base class for all array elements.
+ The base class for scalars.
"""
+ def __init__(self):
+ raise TypeError("Do not call {}'s constructor directly, use "
+ "pa.scalar() instead.".format(self.__class__.__name__))
-cdef class NullType(Scalar):
- """
- Singleton for null array elements.
- """
- # TODO rename this NullValue?
+ cdef void init(self, const shared_ptr[CScalar]& wrapped):
+ self.wrapped = wrapped
- def __cinit__(self):
- global NA
- if NA is not None:
- raise Exception('Cannot create multiple NAType instances')
+ @staticmethod
+ cdef wrap(const shared_ptr[CScalar]& wrapped):
+ cdef:
+ Scalar self
+ Type type_id = wrapped.get().type.get().id()
+
+ if type_id == _Type_NA:
+ return _NULL
+
+ typ = _scalar_classes[type_id]
+ self = typ.__new__(typ)
+ self.init(wrapped)
+
+ return self
+
+ cdef inline shared_ptr[CScalar] unwrap(self) nogil:
+ return self.wrapped
+
+ @property
+ def type(self):
+ return pyarrow_wrap_data_type(self.wrapped.get().type)
- self.type = null()
+ @property
+ def is_valid(self):
+ return self.wrapped.get().is_valid
def __repr__(self):
- return 'NULL'
+ return '<pyarrow.{}: {!r}>'.format(
+ self.__class__.__name__, self.as_py()
+ )
- def as_py(self):
- """
- Return None
- """
- return None
+ def __str__(self):
+ return str(self.as_py())
+
+ def equals(self, Scalar other):
+ return self.wrapped.get().Equals(other.unwrap().get()[0])
def __eq__(self, other):
- return NA
+ try:
+ if not isinstance(other, Scalar):
+ other = scalar(other, type=self.type)
+ return self.equals(other)
+ except (TypeError, ValueError, ArrowInvalid):
+ return NotImplemented
+
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+
+ def as_py(self):
+ raise NotImplementedError()
-_NULL = NA = NullType()
+_NULL = NA = None
-cdef class ArrayValue(Scalar):
+cdef class NullScalar(Scalar):
"""
- The base class for non-null array elements.
+ Concrete class for null scalars.
"""
- def __init__(self):
- raise TypeError("Do not call {}'s constructor directly, use array "
- "subscription instead."
- .format(self.__class__.__name__))
+ def __cinit__(self):
+ global NA
+ if NA is not None:
+ raise Exception('Cannot create multiple NAType instances')
+ self.init(shared_ptr[CScalar](new CNullScalar()))
- cdef void init(self, DataType type, const shared_ptr[CArray]& sp_array,
- int64_t index):
- self.type = type
- self.index = index
- self._set_array(sp_array)
+ def __init__(self):
+ pass
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
+ def __eq__(self, other):
+ return NA
- def __repr__(self):
- if hasattr(self, 'as_py'):
- return repr(self.as_py())
- else:
- return super(Scalar, self).__repr__()
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
- def __str__(self):
- if hasattr(self, 'as_py'):
- return str(self.as_py())
- else:
- return super(Scalar, self).__str__()
+ def as_py(self):
+ """
+ Return this value as a Python None.
+ """
+ return None
- def __eq__(self, other):
- if hasattr(self, 'as_py'):
- if isinstance(other, ArrayValue):
- other = other.as_py()
- return self.as_py() == other
- else:
- raise NotImplementedError(
- "Cannot compare Arrow values that don't support as_py()")
- def __hash__(self):
- return hash(self.as_py())
+_NULL = NA = NullScalar()
-cdef class BooleanValue(ArrayValue):
+cdef class BooleanScalar(Scalar):
"""
- Concrete class for boolean array elements.
+ Concrete class for boolean scalars.
"""
def as_py(self):
"""
Return this value as a Python bool.
"""
- cdef CBooleanArray* ap = <CBooleanArray*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CBooleanScalar* sp = <CBooleanScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int8Value(ArrayValue):
+cdef class UInt8Scalar(Scalar):
"""
- Concrete class for int8 array elements.
+ Concrete class for uint8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt8Array* ap = <CInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt8Scalar* sp = <CUInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt8Value(ArrayValue):
+cdef class Int8Scalar(Scalar):
"""
- Concrete class for uint8 array elements.
+ Concrete class for int8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt8Array* ap = <CUInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt8Scalar* sp = <CInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int16Value(ArrayValue):
+cdef class UInt16Scalar(Scalar):
"""
- Concrete class for int16 array elements.
+ Concrete class for uint16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt16Array* ap = <CInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt16Scalar* sp = <CUInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt16Value(ArrayValue):
+cdef class Int16Scalar(Scalar):
"""
- Concrete class for uint16 array elements.
+ Concrete class for int16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt16Array* ap = <CUInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt16Scalar* sp = <CInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int32Value(ArrayValue):
+cdef class UInt32Scalar(Scalar):
"""
- Concrete class for int32 array elements.
+ Concrete class for uint32 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt32Array* ap = <CInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt32Scalar* sp = <CUInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt32Value(ArrayValue):
+cdef class Int32Scalar(Scalar):
"""
- Concrete class for uint32 array elements.
+ Concrete class for int32 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt32Array* ap = <CUInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt32Scalar* sp = <CInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int64Value(ArrayValue):
+cdef class UInt64Scalar(Scalar):
"""
- Concrete class for int64 array elements.
+ Concrete class for uint64 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt64Array* ap = <CInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt64Scalar* sp = <CUInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt64Value(ArrayValue):
+cdef class Int64Scalar(Scalar):
"""
- Concrete class for uint64 array elements.
+ Concrete class for int64 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt64Array* ap = <CUInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt64Scalar* sp = <CInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date32Value(ArrayValue):
+cdef class HalfFloatScalar(Scalar):
"""
- Concrete class for date32 array elements.
+ Concrete class for float scalars.
"""
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+
+ def __eq__(self, other):
+ if hasattr(self, 'as_py'):
+ if isinstance(other, Scalar):
+ other = other.as_py()
+ return self.as_py() == other
+ else:
+ raise NotImplementedError
+
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python float.
"""
- cdef CDate32Array* ap = <CDate32Array*> self.sp_array.get()
+ cdef CHalfFloatScalar* sp = <CHalfFloatScalar*> self.wrapped.get()
+ return PyHalf_FromHalf(sp.value) if sp.is_valid else None
- # Shift to seconds since epoch
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(days=ap.Value(self.index)))
+
+cdef class FloatScalar(Scalar):
+ """
+ Concrete class for float scalars.
+ """
+
+ def as_py(self):
+ """
+ Return this value as a Python float.
+ """
+ cdef CFloatScalar* sp = <CFloatScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date64Value(ArrayValue):
+cdef class DoubleScalar(Scalar):
"""
- Concrete class for date64 array elements.
+ Concrete class for double scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python float.
"""
- cdef CDate64Array* ap = <CDate64Array*> self.sp_array.get()
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(
- days=ap.Value(self.index) / 86400000))
+ cdef CDoubleScalar* sp = <CDoubleScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Time32Value(ArrayValue):
+cdef class Decimal128Scalar(Scalar):
"""
- Concrete class for time32 array elements.
+ Concrete class for decimal128 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.timedelta instance.
+ Return this value as a Python Decimal.
"""
cdef:
- CTime32Array* ap = <CTime32Array*> self.sp_array.get()
- CTime32Type* dtype = <CTime32Type*> ap.type().get()
-
- if dtype.unit() == TimeUnit_SECOND:
- delta = datetime.timedelta(seconds=ap.Value(self.index))
- return (datetime.datetime(1970, 1, 1) + delta).time()
+ CDecimal128Scalar* sp = <CDecimal128Scalar*> self.wrapped.get()
+ CDecimal128Type* dtype = <CDecimal128Type*> sp.type.get()
+ if sp.is_valid:
+ return _pydecimal.Decimal(
+ frombytes(sp.value.ToString(dtype.scale()))
+ )
else:
- return _box_time_milli(ap.Value(self.index))
+ return None
-cdef class Time64Value(ArrayValue):
+cdef class Date32Scalar(Scalar):
"""
- Concrete class for time64 array elements.
+ Concrete class for date32 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.timedelta instance.
+ Return this value as a Python datetime.datetime instance.
"""
- cdef:
- CTime64Array* ap = <CTime64Array*> self.sp_array.get()
- CTime64Type* dtype = <CTime64Type*> ap.type().get()
+ cdef CDate32Scalar* sp = <CDate32Scalar*> self.wrapped.get()
- cdef int64_t val = ap.Value(self.index)
- if dtype.unit() == TimeUnit_MICRO:
- return _box_time_micro(val)
+ if sp.is_valid:
+ # shift to seconds since epoch
+ return (
+ datetime.date(1970, 1, 1) + datetime.timedelta(days=sp.value)
+ )
else:
- return (datetime.datetime(1970, 1, 1) +
- datetime.timedelta(microseconds=val / 1000)).time()
-
+ return None
-cpdef _box_time_milli(int64_t val):
- delta = datetime.timedelta(milliseconds=val)
- return (datetime.datetime(1970, 1, 1) + delta).time()
+cdef class Date64Scalar(Scalar):
+ """
+ Concrete class for date64 scalars.
+ """
-cpdef _box_time_micro(int64_t val):
- return (datetime.datetime(1970, 1, 1) +
- datetime.timedelta(microseconds=val)).time()
+ def as_py(self):
+ """
+ Return this value as a Python datetime.datetime instance.
+ """
+ cdef CDate64Scalar* sp = <CDate64Scalar*> self.wrapped.get()
+ if sp.is_valid:
+ return (
+ datetime.date(1970, 1, 1) +
+ datetime.timedelta(days=sp.value / 86400000)
+ )
+ else:
+ return None
-cdef dict _DATETIME_CONVERSION_FUNCTIONS = {}
-cdef c_bool _datetime_conversion_initialized = False
+def _datetime_from_int(int64_t value, TimeUnit unit, tzinfo=None):
+ if unit == TimeUnit_SECOND:
+ delta = datetime.timedelta(seconds=value)
+ elif unit == TimeUnit_MILLI:
+ delta = datetime.timedelta(milliseconds=value)
+ elif unit == TimeUnit_MICRO:
+ delta = datetime.timedelta(microseconds=value)
+ else:
+ # TimeUnit_NANO: prefer pandas timestamps if available
+ if _pandas_api.have_pandas:
+ return _pandas_api.pd.Timestamp(value, tz=tzinfo, unit='ns')
+ # otherwise safely truncate to microsecond resolution datetime
+ if value % 1000 != 0:
+ raise ValueError(
+ "Nanosecond resolution temporal type {} is not safely "
+ "convertible to microseconds to convert to datetime.datetime. "
+ "Install pandas to return as Timestamp with nanosecond "
+ "support or access the .value attribute.".format(value)
+ )
+ delta = datetime.timedelta(microseconds=value // 1000)
-cdef _add_micros_maybe_localize(dt, micros, tzinfo):
- import pytz
- dt = dt.replace(microsecond=micros)
+ dt = datetime.datetime(1970, 1, 1) + delta
+ # adjust timezone if set to the datatype
if tzinfo is not None:
- if not isinstance(tzinfo, datetime.tzinfo):
- tzinfo = string_to_tzinfo(tzinfo)
dt = tzinfo.fromutc(dt)
- return dt
-
-
-cdef _datetime_from_seconds(int64_t v):
- return datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=v)
-
-
-def _nanoseconds_to_datetime_safe(v, tzinfo):
- if v % 1000 != 0:
- raise ValueError("Nanosecond timestamp {} is not safely convertible "
- " to microseconds to convert to datetime.datetime."
- " Install pandas to return as Timestamp with "
- " nanosecond support or access the .value attribute.")
- v = v // 1000
- micros = v % 1_000_000
-
- dt = _datetime_from_seconds(v // 1_000_000)
- return _add_micros_maybe_localize(dt, micros, tzinfo)
-
-
-def _microseconds_to_datetime(v, tzinfo):
- micros = v % 1_000_000
- dt = _datetime_from_seconds(v // 1_000_000)
- return _add_micros_maybe_localize(dt, micros, tzinfo)
+ return dt
-def _millis_to_datetime(v, tzinfo):
- millis = v % 1_000
- dt = _datetime_from_seconds(v // 1000)
- return _add_micros_maybe_localize(dt, millis * 1000, tzinfo)
+cdef class Time32Scalar(Scalar):
+ """
+ Concrete class for time32 scalars.
+ """
-def _seconds_to_datetime(v, tzinfo):
- dt = _datetime_from_seconds(v)
- return _add_micros_maybe_localize(dt, 0, tzinfo)
+ def as_py(self):
+ """
+ Return this value as a Python datetime.timedelta instance.
+ """
+ cdef:
+ CTime32Scalar* sp = <CTime32Scalar*> self.wrapped.get()
+ CTime32Type* dtype = <CTime32Type*> sp.type.get()
+ if sp.is_valid:
+ return _datetime_from_int(sp.value, unit=dtype.unit()).time()
+ else:
+ return None
-def _datetime_conversion_functions():
- global _datetime_conversion_initialized
- if _datetime_conversion_initialized:
- return _DATETIME_CONVERSION_FUNCTIONS
- _DATETIME_CONVERSION_FUNCTIONS.update({
- TimeUnit_SECOND: _seconds_to_datetime,
- TimeUnit_MILLI: _millis_to_datetime,
- TimeUnit_MICRO: _microseconds_to_datetime,
- TimeUnit_NANO: _nanoseconds_to_datetime_safe
- })
+cdef class Time64Scalar(Scalar):
+ """
+ Concrete class for time64 scalars.
+ """
- try:
- import pandas as pd
- _DATETIME_CONVERSION_FUNCTIONS[TimeUnit_NANO] = (
- lambda x, tzinfo: pd.Timestamp(
- x, tz=tzinfo, unit='ns',
- )
- )
- except ImportError:
- pass
+ def as_py(self):
+ """
+ Return this value as a Python datetime.timedelta instance.
+ """
+ cdef:
+ CTime64Scalar* sp = <CTime64Scalar*> self.wrapped.get()
+ CTime64Type* dtype = <CTime64Type*> sp.type.get()
- _datetime_conversion_initialized = True
- return _DATETIME_CONVERSION_FUNCTIONS
+ if sp.is_valid:
+ return _datetime_from_int(sp.value, unit=dtype.unit()).time()
+ else:
+ return None
-cdef class TimestampValue(ArrayValue):
+cdef class TimestampScalar(Scalar):
"""
- Concrete class for timestamp array elements.
+ Concrete class for timestamp scalars.
"""
@property
def value(self):
- cdef CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
- cdef CTimestampType* dtype = <CTimestampType*> ap.type().get()
- return ap.Value(self.index)
+ cdef CTimestampScalar* sp = <CTimestampScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
def as_py(self):
"""
Return this value as a Pandas Timestamp instance (if available),
otherwise as a Python datetime.timedelta instance.
"""
- cdef CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
- cdef CTimestampType* dtype = <CTimestampType*> ap.type().get()
+ cdef:
+ CTimestampScalar* sp = <CTimestampScalar*> self.wrapped.get()
+ CTimestampType* dtype = <CTimestampType*> sp.type.get()
- value = self.value
+ if not sp.is_valid:
+ return None
if not dtype.timezone().empty():
tzinfo = string_to_tzinfo(frombytes(dtype.timezone()))
+ if not isinstance(tzinfo, datetime.tzinfo):
+ tzinfo = string_to_tzinfo(tzinfo)
else:
tzinfo = None
- try:
- converter = _datetime_conversion_functions()[dtype.unit()]
- except KeyError:
- raise ValueError(
- 'Cannot convert nanosecond timestamps without pandas'
- )
- return converter(value, tzinfo=tzinfo)
-
-
-cdef dict _TIMEDELTA_CONVERSION_FUNCTIONS = {}
-
-
-def _nanoseconds_to_timedelta_safe(v):
- if v % 1000 != 0:
- raise ValueError(
- "Nanosecond duration {} is not safely convertible to microseconds "
- "to convert to datetime.timedelta. Install pandas to return as "
- "Timedelta with nanosecond support or access the .value "
- "attribute.".format(v))
- micros = v // 1000
-
- return datetime.timedelta(microseconds=micros)
-
-
-def _timedelta_conversion_functions():
- if _TIMEDELTA_CONVERSION_FUNCTIONS:
- return _TIMEDELTA_CONVERSION_FUNCTIONS
-
- _TIMEDELTA_CONVERSION_FUNCTIONS.update({
- TimeUnit_SECOND: lambda v: datetime.timedelta(seconds=v),
- TimeUnit_MILLI: lambda v: datetime.timedelta(milliseconds=v),
- TimeUnit_MICRO: lambda v: datetime.timedelta(microseconds=v),
- TimeUnit_NANO: _nanoseconds_to_timedelta_safe
- })
-
- try:
- import pandas as pd
- _TIMEDELTA_CONVERSION_FUNCTIONS[TimeUnit_NANO] = (
- lambda v: pd.Timedelta(v, unit='ns')
- )
- except ImportError:
- pass
+ return _datetime_from_int(sp.value, unit=dtype.unit(), tzinfo=tzinfo)
- return _TIMEDELTA_CONVERSION_FUNCTIONS
-
-cdef class DurationValue(ArrayValue):
+cdef class DurationScalar(Scalar):
"""
- Concrete class for duration array elements.
+ Concrete class for duration scalars.
"""
@property
def value(self):
- cdef CDurationArray* ap = <CDurationArray*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CDurationScalar* sp = <CDurationScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
def as_py(self):
"""
Return this value as a Pandas Timestamp instance (if available),
otherwise as a Python datetime.timedelta instance.
"""
- cdef CDurationArray* ap = <CDurationArray*> self.sp_array.get()
- cdef CDurationType* dtype = <CDurationType*> ap.type().get()
+ cdef:
+ CDurationScalar* sp = <CDurationScalar*> self.wrapped.get()
+ CDurationType* dtype = <CDurationType*> sp.type.get()
+ TimeUnit unit = dtype.unit()
+
+ if not sp.is_valid:
+ return None
- cdef int64_t value = ap.Value(self.index)
- converter = _timedelta_conversion_functions()[dtype.unit()]
- return converter(value)
+ if unit == TimeUnit_SECOND:
+ return datetime.timedelta(seconds=sp.value)
+ elif unit == TimeUnit_MILLI:
+ return datetime.timedelta(milliseconds=sp.value)
+ elif unit == TimeUnit_MICRO:
+ return datetime.timedelta(microseconds=sp.value)
+ else:
+ # TimeUnit_NANO: prefer pandas timestamps if available
+ if _pandas_api.have_pandas:
+ return _pandas_api.pd.Timedelta(sp.value, unit='ns')
+ # otherwise safely truncate to microsecond resolution timedelta
+ if sp.value % 1000 != 0:
+ raise ValueError(
+ "Nanosecond duration {} is not safely convertible to "
+ "microseconds to convert to datetime.timedelta. Install "
+ "pandas to return as Timedelta with nanosecond support or "
+ "access the .value attribute.".format(sp.value)
+ )
+ return datetime.timedelta(microseconds=sp.value // 1000)
-cdef class HalfFloatValue(ArrayValue):
+cdef class BinaryScalar(Scalar):
"""
- Concrete class for float16 array elements.
+ Concrete class for binary-like scalars.
"""
- def as_py(self):
+ def as_buffer(self):
"""
- Return this value as a Python float.
+ Return a view over this value as a Buffer object.
"""
- cdef CHalfFloatArray* ap = <CHalfFloatArray*> self.sp_array.get()
- return PyHalf_FromHalf(ap.Value(self.index))
-
-
-cdef class FloatValue(ArrayValue):
- """
- Concrete class for float32 array elements.
- """
+ cdef CBinaryScalar* sp = <CBinaryScalar*> self.wrapped.get()
+ return pyarrow_wrap_buffer(sp.value) if sp.is_valid else None
def as_py(self):
"""
- Return this value as a Python float.
+ Return this value as a Python bytes.
"""
- cdef CFloatArray* ap = <CFloatArray*> self.sp_array.get()
- return ap.Value(self.index)
+ buffer = self.as_buffer()
+ if buffer is not None:
+ return self.as_buffer().to_pybytes()
+ else:
+ return None
-cdef class DoubleValue(ArrayValue):
- """
- Concrete class for float64 array elements.
- """
+cdef class LargeBinaryScalar(BinaryScalar):
+ pass
- def as_py(self):
- """
- Return this value as a Python float.
- """
- cdef CDoubleArray* ap = <CDoubleArray*> self.sp_array.get()
- return ap.Value(self.index)
+
+cdef class FixedSizeBinaryScalar(BinaryScalar):
+ pass
-cdef class DecimalValue(ArrayValue):
+cdef class StringScalar(BinaryScalar):
"""
- Concrete class for decimal128 array elements.
+ Concrete class for string-like (utf8) scalars.
"""
def as_py(self):
"""
- Return this value as a Python Decimal.
+ Return this value as a Python string.
"""
- cdef:
- CDecimal128Array* ap = <CDecimal128Array*> self.sp_array.get()
- c_string s = ap.FormatValue(self.index)
- return _pydecimal.Decimal(s.decode('utf8'))
+ buffer = self.as_buffer()
+ if buffer is not None:
+ return frombytes(self.as_buffer().to_pybytes())
+ else:
+ return None
+
+cdef class LargeStringScalar(StringScalar):
+ pass
-cdef class StringValue(ArrayValue):
+
+cdef class ListScalar(Scalar):
"""
- Concrete class for string (utf8) array elements.
+ Concrete class for list-like scalars.
"""
- def as_py(self):
+ @property
+ def values(self):
+ cdef CListScalar* sp = <CListScalar*> self.wrapped.get()
+ if sp.is_valid:
+ return pyarrow_wrap_array(sp.value)
+ else:
+ return None
+
+ def __len__(self):
"""
- Return this value as a Python unicode string.
+ Return the number of values.
"""
- cdef CStringArray* ap = <CStringArray*> self.sp_array.get()
- return ap.GetString(self.index).decode('utf-8')
+ return len(self.values)
- def as_buffer(self):
+ def __getitem__(self, i):
"""
- Return a view over this value as a Buffer object.
+ Return the value at the given index.
"""
- cdef:
- CStringArray* ap = <CStringArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
-
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
-
+ return self.values[_normalize_index(i, len(self))]
-cdef class LargeStringValue(ArrayValue):
- """
- Concrete class for large string (utf8) array elements.
- """
-
- def as_py(self):
+ def __iter__(self):
"""
- Return this value as a Python unicode string.
+ Iterate over this element's values.
"""
- cdef CLargeStringArray* ap = <CLargeStringArray*> self.sp_array.get()
- return ap.GetString(self.index).decode('utf-8')
+ return iter(self.values)
- def as_buffer(self):
+ def as_py(self):
"""
- Return a view over this value as a Buffer object.
+ Return this value as a Python list.
"""
- cdef:
- CLargeStringArray* ap = <CLargeStringArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
+ arr = self.values
+ return None if arr is None else arr.to_pylist()
+
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
+cdef class FixedSizeListScalar(ListScalar):
+ pass
-cdef class BinaryValue(ArrayValue):
+cdef class LargeListScalar(ListScalar):
+ pass
+
+
+cdef class StructScalar(Scalar, collections.abc.Mapping):
"""
- Concrete class for variable-sized binary array elements.
+ Concrete class for struct scalars.
"""
- def as_py(self):
- """
- Return this value as a Python bytes object.
- """
+ def __len__(self):
+ cdef CStructScalar* sp = <CStructScalar*> self.wrapped.get()
+ return sp.value.size()
+
+ def __iter__(self):
cdef:
- const uint8_t* ptr
- int32_t length
- CBinaryArray* ap = <CBinaryArray*> self.sp_array.get()
+ CStructScalar* sp = <CStructScalar*> self.wrapped.get()
+ CStructType* dtype = <CStructType*> sp.type.get()
+ vector[shared_ptr[CField]] fields = dtype.fields()
- ptr = ap.GetValue(self.index, &length)
- return cp.PyBytes_FromStringAndSize(<const char*>(ptr), length)
+ if sp.is_valid:
+ for i in range(dtype.num_fields()):
+ yield frombytes(fields[i].get().name())
- def as_buffer(self):
+ def __contains__(self, key):
+ try:
+ self[key]
+ except IndexError:
+ return False
+ else:
+ return True
+
+ def __getitem__(self, key):
"""
- Return a view over this value as a Buffer object.
+ Return the child value for the given field.
+
+ Parameters
+ ----------
+ index : Union[int, str]
+ Index / position or name of the field.
+
+ Returns
+ -------
+ result : Scalar
"""
cdef:
- CBinaryArray* ap = <CBinaryArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
+ CFieldRef ref
+ CStructScalar* sp = <CStructScalar*> self.wrapped.get()
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
+ if isinstance(key, (bytes, str)):
+ ref = CFieldRef(<c_string> tobytes(key))
+ elif isinstance(key, int):
+ ref = CFieldRef(<int> key)
+ else:
+ raise TypeError('Expected integer or string index')
-
-cdef class LargeBinaryValue(ArrayValue):
- """
- Concrete class for large variable-sized binary array elements.
- """
-
- def as_py(self):
- """
- Return this value as a Python bytes object.
- """
- cdef:
- const uint8_t* ptr
- int64_t length
- CLargeBinaryArray* ap = <CLargeBinaryArray*> self.sp_array.get()
-
- ptr = ap.GetValue(self.index, &length)
- return cp.PyBytes_FromStringAndSize(<const char*>(ptr), length)
-
- def as_buffer(self):
- """
- Return a view over this value as a Buffer object.
- """
- cdef:
- CLargeBinaryArray* ap = <CLargeBinaryArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
-
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
-
-
-cdef class ListValue(ArrayValue):
- """
- Concrete class for list array elements.
- """
-
- def __len__(self):
- """
- Return the number of values.
- """
- return self.length()
-
- def __getitem__(self, i):
- """
- Return the value at the given index.
- """
- return self.getitem(_normalize_index(i, self.length()))
-
- def __iter__(self):
- """
- Iterate over this element's values.
- """
- for i in range(len(self)):
- yield self.getitem(i)
- raise StopIteration
-
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CListArray*> sp_array.get()
- self.value_type = pyarrow_wrap_data_type(self.ap.value_type())
-
- cdef getitem(self, int64_t i):
- cdef int64_t j = self.ap.value_offset(self.index) + i
- return box_scalar(self.value_type, self.ap.values(), j)
-
- cdef int64_t length(self):
- return self.ap.value_length(self.index)
-
- def as_py(self):
- """
- Return this value as a Python list.
- """
- cdef:
- int64_t j
- list result = []
-
- for j in range(len(self)):
- result.append(self.getitem(j).as_py())
-
- return result
-
-
-cdef class LargeListValue(ArrayValue):
- """
- Concrete class for large list array elements.
- """
-
- def __len__(self):
- """
- Return the number of values.
- """
- return self.length()
-
- def __getitem__(self, i):
- """
- Return the value at the given index.
- """
- return self.getitem(_normalize_index(i, self.length()))
-
- def __iter__(self):
- """
- Iterate over this element's values.
- """
- for i in range(len(self)):
- yield self.getitem(i)
- raise StopIteration
-
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CLargeListArray*> sp_array.get()
- self.value_type = pyarrow_wrap_data_type(self.ap.value_type())
-
- cdef getitem(self, int64_t i):
- cdef int64_t j = self.ap.value_offset(self.index) + i
- return box_scalar(self.value_type, self.ap.values(), j)
-
- cdef int64_t length(self):
- return self.ap.value_length(self.index)
-
- def as_py(self):
- """
- Return this value as a Python list.
- """
- cdef:
- int64_t j
- list result = []
-
- for j in range(len(self)):
- result.append(self.getitem(j).as_py())
-
- return result
-
-
-cdef class MapValue(ArrayValue):
- """
- Concrete class for map array elements.
- """
-
- def __len__(self):
- """
- Return the number of values.
- """
- return self.length()
-
- def __getitem__(self, i):
- """
- Return the value at the given index.
- """
- return self.getitem(_normalize_index(i, self.length()))
-
- def __iter__(self):
- """
- Iterate over this element's values.
- """
- for i in range(len(self)):
- yield self.getitem(i)
- raise StopIteration
-
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CMapArray*> sp_array.get()
- self.key_type = pyarrow_wrap_data_type(self.ap.map_type().key_type())
- self.item_type = pyarrow_wrap_data_type(self.ap.map_type().item_type())
-
- cdef getitem(self, int64_t i):
- cdef int64_t j = self.ap.value_offset(self.index) + i
- return (box_scalar(self.key_type, self.ap.keys(), j),
- box_scalar(self.item_type, self.ap.items(), j))
-
- cdef int64_t length(self):
- return self.ap.value_length(self.index)
+ try:
+ return Scalar.wrap(GetResultValue(sp.field(ref)))
+ except ArrowInvalid:
+ raise IndexError(key)
def as_py(self):
"""
- Return this value as a Python list of tuples, each containing a
- key and item.
+ Return this value as a Python dict.
"""
- cdef:
- int64_t j
- list result = []
-
- for j in range(len(self)):
- key, item = self.getitem(j)
- result.append((key.as_py(), item.as_py()))
-
- return result
+ if self.is_valid:
+ return {k: v.as_py() for k, v in self.items()}
+ else:
+ return None
-cdef class FixedSizeListValue(ArrayValue):
+cdef class MapScalar(ListScalar):
"""
- Concrete class for fixed size list array elements.
+ Concrete class for map scalars.
"""
- def __len__(self):
- """
- Return the number of values.
- """
- return self.length()
-
def __getitem__(self, i):
"""
Return the value at the given index.
"""
- return self.getitem(_normalize_index(i, self.length()))
+ arr = self.values
+ if arr is None:
+ raise IndexError(i)
+ dct = arr[_normalize_index(i, len(arr))]
+ return (dct['key'], dct['value'])
def __iter__(self):
"""
Iterate over this element's values.
"""
- for i in range(len(self)):
- yield self.getitem(i)
- raise StopIteration
-
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CFixedSizeListArray*> sp_array.get()
- self.value_type = pyarrow_wrap_data_type(self.ap.value_type())
-
- cdef getitem(self, int64_t i):
- cdef int64_t j = self.ap.value_offset(self.index) + i
- return box_scalar(self.value_type, self.ap.values(), j)
-
- cdef int64_t length(self):
- return self.ap.value_length(self.index)
+ arr = self.values
+ if arr is None:
+ return iter(zip(arr.field('key'), arr.field('value')))
+ else:
+ raise StopIteration
def as_py(self):
"""
Return this value as a Python list.
"""
- cdef:
- int64_t j
- list result = []
-
- for j in range(len(self)):
- result.append(self.getitem(j).as_py())
-
- return result
-
-
-cdef class UnionValue(ArrayValue):
- """
- Concrete class for union array elements.
- """
-
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CUnionArray*> sp_array.get()
-
- cdef getitem(self, int64_t i):
- cdef int child_id = self.ap.child_id(i)
- cdef shared_ptr[CArray] child = self.ap.field(child_id)
- cdef CDenseUnionArray* dense
- if self.ap.mode() == _UnionMode_SPARSE:
- return box_scalar(self.type[child_id].type, child, i)
+ arr = self.values
+ if arr is not None:
+ return list(zip(arr.field('key'), arr.field('value')))
else:
- dense = <CDenseUnionArray*> self.ap
- return box_scalar(self.type[child_id].type, child,
- dense.value_offset(i))
-
- def as_py(self):
- """
- Return this value as a Python object.
-
- The exact type depends on the underlying union member.
- """
- return self.getitem(self.index).as_py()
+ return None
-cdef class FixedSizeBinaryValue(ArrayValue):
- """
- Concrete class for fixed-size binary array elements.
- """
-
- def as_py(self):
- """
- Return this value as a Python bytes object.
- """
- cdef:
- CFixedSizeBinaryArray* ap
- CFixedSizeBinaryType* ap_type
- int32_t length
- const char* data
- ap = <CFixedSizeBinaryArray*> self.sp_array.get()
- ap_type = <CFixedSizeBinaryType*> ap.type().get()
- length = ap_type.byte_width()
- data = <const char*> ap.GetValue(self.index)
- return cp.PyBytes_FromStringAndSize(data, length)
-
-
-cdef class StructValue(ArrayValue):
+cdef class DictionaryScalar(Scalar):
"""
- Concrete class for struct array elements.
+ Concrete class for dictionary-encoded scalars.
"""
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CStructArray*> sp_array.get()
-
- def __getitem__(self, key):
- """
- Return the child value for the given field name.
- """
- cdef:
- CStructType* type
- int index
-
- type = <CStructType*> self.type.type
- index = type.GetFieldIndex(tobytes(key))
-
- if index < 0:
- raise KeyError(key)
+ # @property
+ # def index(self):
+ # """
+ # Return this value's underlying index as a scalar.
+ # """
+ # cdef CDictionaryScalar* sp = <CDictionaryScalar*> self.wrapped.get()
+ # return Scalar.wrap(sp.index)
- return pyarrow_wrap_array(self.ap.field(index))[self.index]
-
- def as_py(self):
+ @property
+ def value(self):
"""
- Return this value as a Python dict.
+ Return this value's underlying dictionary value as a scalar.
"""
- cdef:
- vector[shared_ptr[CField]] child_fields = self.type.type.fields()
-
- wrapped_arrays = [pyarrow_wrap_array(self.ap.field(i))
- for i in range(self.ap.num_fields())]
- child_names = [child.get().name() for child in child_fields]
- # Return the struct as a dict
- return {
- frombytes(name): child_array[self.index].as_py()
- for name, child_array in zip(child_names, wrapped_arrays)
- }
-
-
-cdef class DictionaryValue(ArrayValue):
- """
- Concrete class for dictionary-encoded array elements.
- """
+ cdef CDictionaryScalar* sp = <CDictionaryScalar*> self.wrapped.get()
+ return Scalar.wrap(sp.value)
def as_py(self):
"""
Return this value as a Python object.
The exact type depends on the dictionary value type.
"""
- return self.dictionary_value.as_py()
+ value = self.value
+ return None if value is None else value.as_py()
- @property
- def index_value(self):
- """
- Return this value's underlying index as a ArrayValue of the right
- signed integer type.
- """
- cdef CDictionaryArray* darr = <CDictionaryArray*>(self.sp_array.get())
- indices = pyarrow_wrap_array(darr.indices())
- return indices[self.index]
+ # TODO(kszucs): deprecate these
+ # @property
+ # def index_value(self):
+ # index = self.index
+ # return None if index is None else self.index
@property
def dictionary_value(self):
- """
- Return this value's underlying dictionary value as a ArrayValue.
- """
- cdef CDictionaryArray* darr = <CDictionaryArray*>(self.sp_array.get())
- dictionary = pyarrow_wrap_array(darr.dictionary())
- return dictionary[self.index_value.as_py()]
-
-
-cdef dict _array_value_classes = {
- _Type_BOOL: BooleanValue,
- _Type_UINT8: UInt8Value,
- _Type_UINT16: UInt16Value,
- _Type_UINT32: UInt32Value,
- _Type_UINT64: UInt64Value,
- _Type_INT8: Int8Value,
- _Type_INT16: Int16Value,
- _Type_INT32: Int32Value,
- _Type_INT64: Int64Value,
- _Type_DATE32: Date32Value,
- _Type_DATE64: Date64Value,
- _Type_TIME32: Time32Value,
- _Type_TIME64: Time64Value,
- _Type_TIMESTAMP: TimestampValue,
- _Type_DURATION: DurationValue,
- _Type_HALF_FLOAT: HalfFloatValue,
- _Type_FLOAT: FloatValue,
- _Type_DOUBLE: DoubleValue,
- _Type_LIST: ListValue,
- _Type_LARGE_LIST: LargeListValue,
- _Type_MAP: MapValue,
- _Type_FIXED_SIZE_LIST: FixedSizeListValue,
- _Type_SPARSE_UNION: UnionValue,
- _Type_DENSE_UNION: UnionValue,
- _Type_BINARY: BinaryValue,
- _Type_STRING: StringValue,
- _Type_LARGE_BINARY: LargeBinaryValue,
- _Type_LARGE_STRING: LargeStringValue,
- _Type_FIXED_SIZE_BINARY: FixedSizeBinaryValue,
- _Type_DECIMAL: DecimalValue,
- _Type_STRUCT: StructValue,
- _Type_DICTIONARY: DictionaryValue,
-}
-
-cdef class ScalarValue(Scalar):
- """
- The base class for scalars.
- """
-
- def __init__(self):
- raise TypeError("Do not call {}'s constructor directly."
- .format(self.__class__.__name__))
-
- cdef void init(self, const shared_ptr[CScalar]& sp_scalar):
- self.sp_scalar = sp_scalar
-
- def __repr__(self):
- if hasattr(self, 'as_py'):
- return repr(self.as_py())
- else:
- return super(Scalar, self).__repr__()
-
- def __str__(self):
- if hasattr(self, 'as_py'):
- return str(self.as_py())
- else:
- return super(Scalar, self).__str__()
-
- def __eq__(self, other):
- if hasattr(self, 'as_py'):
- if isinstance(other, ScalarValue):
- other = other.as_py()
- return self.as_py() == other
- else:
- raise NotImplemented(
- "Cannot compare scalars that don't support as_py()")
-
- def __hash__(self):
- return hash(self.as_py())
-
-
-cdef class NullScalar(ScalarValue):
- """
- Concrete class for null scalars.
- """
-
- def as_py(self):
- """
- Return this value as a Python None.
- """
- return None
-
-
-cdef class BooleanScalar(ScalarValue):
- """
- Concrete class for boolean scalars.
- """
-
- def as_py(self):
- """
- Return this value as a Python bool.
- """
- cdef CBooleanScalar* sp = <CBooleanScalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
-
-
-cdef class UInt8Scalar(ScalarValue):
- """
- Concrete class for uint8 scalars.
- """
-
- def as_py(self):
- """
- Return this value as a Python int.
- """
- cdef CUInt8Scalar* sp = <CUInt8Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
-
-
-cdef class Int8Scalar(ScalarValue):
- """
- Concrete class for int8 scalars.
- """
-
- def as_py(self):
- """
- Return this value as a Python int.
- """
- cdef CInt8Scalar* sp = <CInt8Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
-
-
-cdef class UInt16Scalar(ScalarValue):
- """
- Concrete class for uint16 scalars.
- """
-
- def as_py(self):
- """
- Return this value as a Python int.
- """
- cdef CUInt16Scalar* sp = <CUInt16Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
-
-
-cdef class Int16Scalar(ScalarValue):
- """
- Concrete class for int16 scalars.
- """
-
- def as_py(self):
- """
- Return this value as a Python int.
- """
- cdef CInt16Scalar* sp = <CInt16Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
-
-
-cdef class UInt32Scalar(ScalarValue):
- """
- Concrete class for uint32 scalars.
- """
-
- def as_py(self):
- """
- Return this value as a Python int.
- """
- cdef CUInt32Scalar* sp = <CUInt32Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
-
-
-cdef class Int32Scalar(ScalarValue):
- """
- Concrete class for int32 scalars.
- """
-
- def as_py(self):
- """
- Return this value as a Python int.
- """
- cdef CInt32Scalar* sp = <CInt32Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
-
-
-cdef class UInt64Scalar(ScalarValue):
- """
- Concrete class for uint64 scalars.
- """
-
- def as_py(self):
- """
- Return this value as a Python int.
- """
- cdef CUInt64Scalar* sp = <CUInt64Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
-
-
-cdef class Int64Scalar(ScalarValue):
- """
- Concrete class for int64 scalars.
- """
-
- def as_py(self):
- """
- Return this value as a Python int.
- """
- cdef CInt64Scalar* sp = <CInt64Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
+ value = self.value
+ return None if value is None else self.value
-cdef class FloatScalar(ScalarValue):
+cdef class UnionScalar(Scalar):
"""
- Concrete class for float scalars.
+ Concrete class for Union scalars.
"""
- def as_py(self):
+ @property
+ def value(self):
"""
- Return this value as a Python float.
+ Return this value's underlying dictionary value as a scalar.
"""
- cdef CFloatScalar* sp = <CFloatScalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
-
-
-cdef class DoubleScalar(ScalarValue):
- """
- Concrete class for double scalars.
- """
+ cdef CDictionaryScalar* sp = <CDictionaryScalar*> self.wrapped.get()
Review comment:
UnionScalar
##########
File path: python/pyarrow/scalar.pxi
##########
@@ -16,1198 +16,745 @@
# under the License.
-_NULL = NA = None
+import collections
cdef class Scalar:
"""
- The base class for all array elements.
+ The base class for scalars.
"""
+ def __init__(self):
+ raise TypeError("Do not call {}'s constructor directly, use "
+ "pa.scalar() instead.".format(self.__class__.__name__))
-cdef class NullType(Scalar):
- """
- Singleton for null array elements.
- """
- # TODO rename this NullValue?
+ cdef void init(self, const shared_ptr[CScalar]& wrapped):
+ self.wrapped = wrapped
- def __cinit__(self):
- global NA
- if NA is not None:
- raise Exception('Cannot create multiple NAType instances')
+ @staticmethod
+ cdef wrap(const shared_ptr[CScalar]& wrapped):
+ cdef:
+ Scalar self
+ Type type_id = wrapped.get().type.get().id()
+
+ if type_id == _Type_NA:
+ return _NULL
+
+ typ = _scalar_classes[type_id]
+ self = typ.__new__(typ)
+ self.init(wrapped)
+
+ return self
+
+ cdef inline shared_ptr[CScalar] unwrap(self) nogil:
+ return self.wrapped
+
+ @property
+ def type(self):
+ return pyarrow_wrap_data_type(self.wrapped.get().type)
- self.type = null()
+ @property
+ def is_valid(self):
+ return self.wrapped.get().is_valid
def __repr__(self):
- return 'NULL'
+ return '<pyarrow.{}: {!r}>'.format(
+ self.__class__.__name__, self.as_py()
+ )
- def as_py(self):
- """
- Return None
- """
- return None
+ def __str__(self):
+ return str(self.as_py())
+
+ def equals(self, Scalar other):
+ return self.wrapped.get().Equals(other.unwrap().get()[0])
def __eq__(self, other):
- return NA
+ try:
+ if not isinstance(other, Scalar):
+ other = scalar(other, type=self.type)
+ return self.equals(other)
+ except (TypeError, ValueError, ArrowInvalid):
+ return NotImplemented
+
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+
+ def as_py(self):
+ raise NotImplementedError()
-_NULL = NA = NullType()
+_NULL = NA = None
-cdef class ArrayValue(Scalar):
+cdef class NullScalar(Scalar):
"""
- The base class for non-null array elements.
+ Concrete class for null scalars.
"""
- def __init__(self):
- raise TypeError("Do not call {}'s constructor directly, use array "
- "subscription instead."
- .format(self.__class__.__name__))
+ def __cinit__(self):
+ global NA
+ if NA is not None:
+ raise Exception('Cannot create multiple NAType instances')
+ self.init(shared_ptr[CScalar](new CNullScalar()))
- cdef void init(self, DataType type, const shared_ptr[CArray]& sp_array,
- int64_t index):
- self.type = type
- self.index = index
- self._set_array(sp_array)
+ def __init__(self):
+ pass
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
+ def __eq__(self, other):
+ return NA
- def __repr__(self):
- if hasattr(self, 'as_py'):
- return repr(self.as_py())
- else:
- return super(Scalar, self).__repr__()
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
- def __str__(self):
- if hasattr(self, 'as_py'):
- return str(self.as_py())
- else:
- return super(Scalar, self).__str__()
+ def as_py(self):
+ """
+ Return this value as a Python None.
+ """
+ return None
- def __eq__(self, other):
- if hasattr(self, 'as_py'):
- if isinstance(other, ArrayValue):
- other = other.as_py()
- return self.as_py() == other
- else:
- raise NotImplementedError(
- "Cannot compare Arrow values that don't support as_py()")
- def __hash__(self):
- return hash(self.as_py())
+_NULL = NA = NullScalar()
-cdef class BooleanValue(ArrayValue):
+cdef class BooleanScalar(Scalar):
"""
- Concrete class for boolean array elements.
+ Concrete class for boolean scalars.
"""
def as_py(self):
"""
Return this value as a Python bool.
"""
- cdef CBooleanArray* ap = <CBooleanArray*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CBooleanScalar* sp = <CBooleanScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int8Value(ArrayValue):
+cdef class UInt8Scalar(Scalar):
"""
- Concrete class for int8 array elements.
+ Concrete class for uint8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt8Array* ap = <CInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt8Scalar* sp = <CUInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt8Value(ArrayValue):
+cdef class Int8Scalar(Scalar):
"""
- Concrete class for uint8 array elements.
+ Concrete class for int8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt8Array* ap = <CUInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt8Scalar* sp = <CInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int16Value(ArrayValue):
+cdef class UInt16Scalar(Scalar):
"""
- Concrete class for int16 array elements.
+ Concrete class for uint16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt16Array* ap = <CInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt16Scalar* sp = <CUInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt16Value(ArrayValue):
+cdef class Int16Scalar(Scalar):
"""
- Concrete class for uint16 array elements.
+ Concrete class for int16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt16Array* ap = <CUInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt16Scalar* sp = <CInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int32Value(ArrayValue):
+cdef class UInt32Scalar(Scalar):
"""
- Concrete class for int32 array elements.
+ Concrete class for uint32 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt32Array* ap = <CInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt32Scalar* sp = <CUInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt32Value(ArrayValue):
+cdef class Int32Scalar(Scalar):
"""
- Concrete class for uint32 array elements.
+ Concrete class for int32 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt32Array* ap = <CUInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt32Scalar* sp = <CInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int64Value(ArrayValue):
+cdef class UInt64Scalar(Scalar):
"""
- Concrete class for int64 array elements.
+ Concrete class for uint64 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt64Array* ap = <CInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt64Scalar* sp = <CUInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt64Value(ArrayValue):
+cdef class Int64Scalar(Scalar):
"""
- Concrete class for uint64 array elements.
+ Concrete class for int64 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt64Array* ap = <CUInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt64Scalar* sp = <CInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date32Value(ArrayValue):
+cdef class HalfFloatScalar(Scalar):
"""
- Concrete class for date32 array elements.
+ Concrete class for float scalars.
"""
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+
+ def __eq__(self, other):
+ if hasattr(self, 'as_py'):
+ if isinstance(other, Scalar):
+ other = other.as_py()
+ return self.as_py() == other
+ else:
+ raise NotImplementedError
+
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python float.
"""
- cdef CDate32Array* ap = <CDate32Array*> self.sp_array.get()
+ cdef CHalfFloatScalar* sp = <CHalfFloatScalar*> self.wrapped.get()
+ return PyHalf_FromHalf(sp.value) if sp.is_valid else None
- # Shift to seconds since epoch
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(days=ap.Value(self.index)))
+
+cdef class FloatScalar(Scalar):
+ """
+ Concrete class for float scalars.
+ """
+
+ def as_py(self):
+ """
+ Return this value as a Python float.
+ """
+ cdef CFloatScalar* sp = <CFloatScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date64Value(ArrayValue):
+cdef class DoubleScalar(Scalar):
"""
- Concrete class for date64 array elements.
+ Concrete class for double scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python float.
"""
- cdef CDate64Array* ap = <CDate64Array*> self.sp_array.get()
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(
- days=ap.Value(self.index) / 86400000))
+ cdef CDoubleScalar* sp = <CDoubleScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Time32Value(ArrayValue):
+cdef class Decimal128Scalar(Scalar):
"""
- Concrete class for time32 array elements.
+ Concrete class for decimal128 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.timedelta instance.
+ Return this value as a Python Decimal.
"""
cdef:
- CTime32Array* ap = <CTime32Array*> self.sp_array.get()
- CTime32Type* dtype = <CTime32Type*> ap.type().get()
-
- if dtype.unit() == TimeUnit_SECOND:
- delta = datetime.timedelta(seconds=ap.Value(self.index))
- return (datetime.datetime(1970, 1, 1) + delta).time()
+ CDecimal128Scalar* sp = <CDecimal128Scalar*> self.wrapped.get()
+ CDecimal128Type* dtype = <CDecimal128Type*> sp.type.get()
+ if sp.is_valid:
+ return _pydecimal.Decimal(
+ frombytes(sp.value.ToString(dtype.scale()))
+ )
else:
- return _box_time_milli(ap.Value(self.index))
+ return None
-cdef class Time64Value(ArrayValue):
+cdef class Date32Scalar(Scalar):
"""
- Concrete class for time64 array elements.
+ Concrete class for date32 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.timedelta instance.
+ Return this value as a Python datetime.datetime instance.
"""
- cdef:
- CTime64Array* ap = <CTime64Array*> self.sp_array.get()
- CTime64Type* dtype = <CTime64Type*> ap.type().get()
+ cdef CDate32Scalar* sp = <CDate32Scalar*> self.wrapped.get()
- cdef int64_t val = ap.Value(self.index)
- if dtype.unit() == TimeUnit_MICRO:
- return _box_time_micro(val)
+ if sp.is_valid:
+ # shift to seconds since epoch
+ return (
+ datetime.date(1970, 1, 1) + datetime.timedelta(days=sp.value)
+ )
else:
- return (datetime.datetime(1970, 1, 1) +
- datetime.timedelta(microseconds=val / 1000)).time()
-
+ return None
-cpdef _box_time_milli(int64_t val):
- delta = datetime.timedelta(milliseconds=val)
- return (datetime.datetime(1970, 1, 1) + delta).time()
+cdef class Date64Scalar(Scalar):
+ """
+ Concrete class for date64 scalars.
+ """
-cpdef _box_time_micro(int64_t val):
- return (datetime.datetime(1970, 1, 1) +
- datetime.timedelta(microseconds=val)).time()
+ def as_py(self):
+ """
+ Return this value as a Python datetime.datetime instance.
+ """
+ cdef CDate64Scalar* sp = <CDate64Scalar*> self.wrapped.get()
+ if sp.is_valid:
+ return (
+ datetime.date(1970, 1, 1) +
+ datetime.timedelta(days=sp.value / 86400000)
+ )
+ else:
+ return None
-cdef dict _DATETIME_CONVERSION_FUNCTIONS = {}
-cdef c_bool _datetime_conversion_initialized = False
+def _datetime_from_int(int64_t value, TimeUnit unit, tzinfo=None):
+ if unit == TimeUnit_SECOND:
+ delta = datetime.timedelta(seconds=value)
+ elif unit == TimeUnit_MILLI:
+ delta = datetime.timedelta(milliseconds=value)
+ elif unit == TimeUnit_MICRO:
+ delta = datetime.timedelta(microseconds=value)
+ else:
+ # TimeUnit_NANO: prefer pandas timestamps if available
+ if _pandas_api.have_pandas:
+ return _pandas_api.pd.Timestamp(value, tz=tzinfo, unit='ns')
+ # otherwise safely truncate to microsecond resolution datetime
+ if value % 1000 != 0:
+ raise ValueError(
+ "Nanosecond resolution temporal type {} is not safely "
+ "convertible to microseconds to convert to datetime.datetime. "
+ "Install pandas to return as Timestamp with nanosecond "
+ "support or access the .value attribute.".format(value)
+ )
+ delta = datetime.timedelta(microseconds=value // 1000)
-cdef _add_micros_maybe_localize(dt, micros, tzinfo):
- import pytz
- dt = dt.replace(microsecond=micros)
+ dt = datetime.datetime(1970, 1, 1) + delta
+ # adjust timezone if set to the datatype
if tzinfo is not None:
- if not isinstance(tzinfo, datetime.tzinfo):
- tzinfo = string_to_tzinfo(tzinfo)
dt = tzinfo.fromutc(dt)
- return dt
-
-
-cdef _datetime_from_seconds(int64_t v):
- return datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=v)
-
-
-def _nanoseconds_to_datetime_safe(v, tzinfo):
- if v % 1000 != 0:
- raise ValueError("Nanosecond timestamp {} is not safely convertible "
- " to microseconds to convert to datetime.datetime."
- " Install pandas to return as Timestamp with "
- " nanosecond support or access the .value attribute.")
- v = v // 1000
- micros = v % 1_000_000
-
- dt = _datetime_from_seconds(v // 1_000_000)
- return _add_micros_maybe_localize(dt, micros, tzinfo)
-
-
-def _microseconds_to_datetime(v, tzinfo):
- micros = v % 1_000_000
- dt = _datetime_from_seconds(v // 1_000_000)
- return _add_micros_maybe_localize(dt, micros, tzinfo)
+ return dt
-def _millis_to_datetime(v, tzinfo):
- millis = v % 1_000
- dt = _datetime_from_seconds(v // 1000)
- return _add_micros_maybe_localize(dt, millis * 1000, tzinfo)
+cdef class Time32Scalar(Scalar):
+ """
+ Concrete class for time32 scalars.
+ """
-def _seconds_to_datetime(v, tzinfo):
- dt = _datetime_from_seconds(v)
- return _add_micros_maybe_localize(dt, 0, tzinfo)
+ def as_py(self):
+ """
+ Return this value as a Python datetime.timedelta instance.
+ """
+ cdef:
+ CTime32Scalar* sp = <CTime32Scalar*> self.wrapped.get()
+ CTime32Type* dtype = <CTime32Type*> sp.type.get()
+ if sp.is_valid:
+ return _datetime_from_int(sp.value, unit=dtype.unit()).time()
+ else:
+ return None
-def _datetime_conversion_functions():
- global _datetime_conversion_initialized
- if _datetime_conversion_initialized:
- return _DATETIME_CONVERSION_FUNCTIONS
- _DATETIME_CONVERSION_FUNCTIONS.update({
- TimeUnit_SECOND: _seconds_to_datetime,
- TimeUnit_MILLI: _millis_to_datetime,
- TimeUnit_MICRO: _microseconds_to_datetime,
- TimeUnit_NANO: _nanoseconds_to_datetime_safe
- })
+cdef class Time64Scalar(Scalar):
+ """
+ Concrete class for time64 scalars.
+ """
- try:
- import pandas as pd
- _DATETIME_CONVERSION_FUNCTIONS[TimeUnit_NANO] = (
- lambda x, tzinfo: pd.Timestamp(
- x, tz=tzinfo, unit='ns',
- )
- )
- except ImportError:
- pass
+ def as_py(self):
+ """
+ Return this value as a Python datetime.timedelta instance.
+ """
+ cdef:
+ CTime64Scalar* sp = <CTime64Scalar*> self.wrapped.get()
+ CTime64Type* dtype = <CTime64Type*> sp.type.get()
- _datetime_conversion_initialized = True
- return _DATETIME_CONVERSION_FUNCTIONS
+ if sp.is_valid:
+ return _datetime_from_int(sp.value, unit=dtype.unit()).time()
+ else:
+ return None
-cdef class TimestampValue(ArrayValue):
+cdef class TimestampScalar(Scalar):
"""
- Concrete class for timestamp array elements.
+ Concrete class for timestamp scalars.
"""
@property
def value(self):
- cdef CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
- cdef CTimestampType* dtype = <CTimestampType*> ap.type().get()
- return ap.Value(self.index)
+ cdef CTimestampScalar* sp = <CTimestampScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
def as_py(self):
"""
Return this value as a Pandas Timestamp instance (if available),
otherwise as a Python datetime.timedelta instance.
"""
- cdef CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
- cdef CTimestampType* dtype = <CTimestampType*> ap.type().get()
+ cdef:
+ CTimestampScalar* sp = <CTimestampScalar*> self.wrapped.get()
+ CTimestampType* dtype = <CTimestampType*> sp.type.get()
- value = self.value
+ if not sp.is_valid:
+ return None
if not dtype.timezone().empty():
tzinfo = string_to_tzinfo(frombytes(dtype.timezone()))
+ if not isinstance(tzinfo, datetime.tzinfo):
+ tzinfo = string_to_tzinfo(tzinfo)
else:
tzinfo = None
- try:
- converter = _datetime_conversion_functions()[dtype.unit()]
- except KeyError:
- raise ValueError(
- 'Cannot convert nanosecond timestamps without pandas'
- )
- return converter(value, tzinfo=tzinfo)
-
-
-cdef dict _TIMEDELTA_CONVERSION_FUNCTIONS = {}
-
-
-def _nanoseconds_to_timedelta_safe(v):
- if v % 1000 != 0:
- raise ValueError(
- "Nanosecond duration {} is not safely convertible to microseconds "
- "to convert to datetime.timedelta. Install pandas to return as "
- "Timedelta with nanosecond support or access the .value "
- "attribute.".format(v))
- micros = v // 1000
-
- return datetime.timedelta(microseconds=micros)
-
-
-def _timedelta_conversion_functions():
- if _TIMEDELTA_CONVERSION_FUNCTIONS:
- return _TIMEDELTA_CONVERSION_FUNCTIONS
-
- _TIMEDELTA_CONVERSION_FUNCTIONS.update({
- TimeUnit_SECOND: lambda v: datetime.timedelta(seconds=v),
- TimeUnit_MILLI: lambda v: datetime.timedelta(milliseconds=v),
- TimeUnit_MICRO: lambda v: datetime.timedelta(microseconds=v),
- TimeUnit_NANO: _nanoseconds_to_timedelta_safe
- })
-
- try:
- import pandas as pd
- _TIMEDELTA_CONVERSION_FUNCTIONS[TimeUnit_NANO] = (
- lambda v: pd.Timedelta(v, unit='ns')
- )
- except ImportError:
- pass
+ return _datetime_from_int(sp.value, unit=dtype.unit(), tzinfo=tzinfo)
- return _TIMEDELTA_CONVERSION_FUNCTIONS
-
-cdef class DurationValue(ArrayValue):
+cdef class DurationScalar(Scalar):
"""
- Concrete class for duration array elements.
+ Concrete class for duration scalars.
"""
@property
def value(self):
- cdef CDurationArray* ap = <CDurationArray*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CDurationScalar* sp = <CDurationScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
def as_py(self):
"""
Return this value as a Pandas Timestamp instance (if available),
otherwise as a Python datetime.timedelta instance.
"""
- cdef CDurationArray* ap = <CDurationArray*> self.sp_array.get()
- cdef CDurationType* dtype = <CDurationType*> ap.type().get()
+ cdef:
+ CDurationScalar* sp = <CDurationScalar*> self.wrapped.get()
+ CDurationType* dtype = <CDurationType*> sp.type.get()
+ TimeUnit unit = dtype.unit()
+
+ if not sp.is_valid:
+ return None
- cdef int64_t value = ap.Value(self.index)
- converter = _timedelta_conversion_functions()[dtype.unit()]
- return converter(value)
+ if unit == TimeUnit_SECOND:
+ return datetime.timedelta(seconds=sp.value)
+ elif unit == TimeUnit_MILLI:
+ return datetime.timedelta(milliseconds=sp.value)
+ elif unit == TimeUnit_MICRO:
+ return datetime.timedelta(microseconds=sp.value)
+ else:
+ # TimeUnit_NANO: prefer pandas timestamps if available
+ if _pandas_api.have_pandas:
+ return _pandas_api.pd.Timedelta(sp.value, unit='ns')
+ # otherwise safely truncate to microsecond resolution timedelta
+ if sp.value % 1000 != 0:
+ raise ValueError(
+ "Nanosecond duration {} is not safely convertible to "
+ "microseconds to convert to datetime.timedelta. Install "
+ "pandas to return as Timedelta with nanosecond support or "
+ "access the .value attribute.".format(sp.value)
+ )
+ return datetime.timedelta(microseconds=sp.value // 1000)
-cdef class HalfFloatValue(ArrayValue):
+cdef class BinaryScalar(Scalar):
"""
- Concrete class for float16 array elements.
+ Concrete class for binary-like scalars.
"""
- def as_py(self):
+ def as_buffer(self):
"""
- Return this value as a Python float.
+ Return a view over this value as a Buffer object.
"""
- cdef CHalfFloatArray* ap = <CHalfFloatArray*> self.sp_array.get()
- return PyHalf_FromHalf(ap.Value(self.index))
-
-
-cdef class FloatValue(ArrayValue):
- """
- Concrete class for float32 array elements.
- """
+ cdef CBinaryScalar* sp = <CBinaryScalar*> self.wrapped.get()
+ return pyarrow_wrap_buffer(sp.value) if sp.is_valid else None
def as_py(self):
"""
- Return this value as a Python float.
+ Return this value as a Python bytes.
"""
- cdef CFloatArray* ap = <CFloatArray*> self.sp_array.get()
- return ap.Value(self.index)
+ buffer = self.as_buffer()
+ if buffer is not None:
+ return self.as_buffer().to_pybytes()
+ else:
+ return None
-cdef class DoubleValue(ArrayValue):
- """
- Concrete class for float64 array elements.
- """
+cdef class LargeBinaryScalar(BinaryScalar):
+ pass
- def as_py(self):
- """
- Return this value as a Python float.
- """
- cdef CDoubleArray* ap = <CDoubleArray*> self.sp_array.get()
- return ap.Value(self.index)
+
+cdef class FixedSizeBinaryScalar(BinaryScalar):
+ pass
-cdef class DecimalValue(ArrayValue):
+cdef class StringScalar(BinaryScalar):
"""
- Concrete class for decimal128 array elements.
+ Concrete class for string-like (utf8) scalars.
"""
def as_py(self):
"""
- Return this value as a Python Decimal.
+ Return this value as a Python string.
"""
- cdef:
- CDecimal128Array* ap = <CDecimal128Array*> self.sp_array.get()
- c_string s = ap.FormatValue(self.index)
- return _pydecimal.Decimal(s.decode('utf8'))
+ buffer = self.as_buffer()
+ if buffer is not None:
+ return frombytes(self.as_buffer().to_pybytes())
+ else:
+ return None
+
+cdef class LargeStringScalar(StringScalar):
+ pass
-cdef class StringValue(ArrayValue):
+
+cdef class ListScalar(Scalar):
"""
- Concrete class for string (utf8) array elements.
+ Concrete class for list-like scalars.
"""
- def as_py(self):
+ @property
+ def values(self):
+ cdef CListScalar* sp = <CListScalar*> self.wrapped.get()
+ if sp.is_valid:
+ return pyarrow_wrap_array(sp.value)
+ else:
+ return None
+
+ def __len__(self):
"""
- Return this value as a Python unicode string.
+ Return the number of values.
"""
- cdef CStringArray* ap = <CStringArray*> self.sp_array.get()
- return ap.GetString(self.index).decode('utf-8')
+ return len(self.values)
- def as_buffer(self):
+ def __getitem__(self, i):
"""
- Return a view over this value as a Buffer object.
+ Return the value at the given index.
"""
- cdef:
- CStringArray* ap = <CStringArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
-
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
-
+ return self.values[_normalize_index(i, len(self))]
-cdef class LargeStringValue(ArrayValue):
- """
- Concrete class for large string (utf8) array elements.
- """
-
- def as_py(self):
+ def __iter__(self):
"""
- Return this value as a Python unicode string.
+ Iterate over this element's values.
"""
- cdef CLargeStringArray* ap = <CLargeStringArray*> self.sp_array.get()
- return ap.GetString(self.index).decode('utf-8')
+ return iter(self.values)
- def as_buffer(self):
+ def as_py(self):
"""
- Return a view over this value as a Buffer object.
+ Return this value as a Python list.
"""
- cdef:
- CLargeStringArray* ap = <CLargeStringArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
+ arr = self.values
+ return None if arr is None else arr.to_pylist()
+
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
+cdef class FixedSizeListScalar(ListScalar):
+ pass
-cdef class BinaryValue(ArrayValue):
+cdef class LargeListScalar(ListScalar):
Review comment:
Same here.
##########
File path: python/pyarrow/scalar.pxi
##########
@@ -16,1198 +16,745 @@
# under the License.
-_NULL = NA = None
+import collections
cdef class Scalar:
"""
- The base class for all array elements.
+ The base class for scalars.
"""
+ def __init__(self):
+ raise TypeError("Do not call {}'s constructor directly, use "
+ "pa.scalar() instead.".format(self.__class__.__name__))
-cdef class NullType(Scalar):
- """
- Singleton for null array elements.
- """
- # TODO rename this NullValue?
+ cdef void init(self, const shared_ptr[CScalar]& wrapped):
+ self.wrapped = wrapped
- def __cinit__(self):
- global NA
- if NA is not None:
- raise Exception('Cannot create multiple NAType instances')
+ @staticmethod
+ cdef wrap(const shared_ptr[CScalar]& wrapped):
+ cdef:
+ Scalar self
+ Type type_id = wrapped.get().type.get().id()
+
+ if type_id == _Type_NA:
+ return _NULL
+
+ typ = _scalar_classes[type_id]
+ self = typ.__new__(typ)
+ self.init(wrapped)
+
+ return self
+
+ cdef inline shared_ptr[CScalar] unwrap(self) nogil:
+ return self.wrapped
+
+ @property
+ def type(self):
+ return pyarrow_wrap_data_type(self.wrapped.get().type)
- self.type = null()
+ @property
+ def is_valid(self):
+ return self.wrapped.get().is_valid
def __repr__(self):
- return 'NULL'
+ return '<pyarrow.{}: {!r}>'.format(
+ self.__class__.__name__, self.as_py()
+ )
- def as_py(self):
- """
- Return None
- """
- return None
+ def __str__(self):
+ return str(self.as_py())
+
+ def equals(self, Scalar other):
+ return self.wrapped.get().Equals(other.unwrap().get()[0])
def __eq__(self, other):
- return NA
+ try:
+ if not isinstance(other, Scalar):
+ other = scalar(other, type=self.type)
+ return self.equals(other)
+ except (TypeError, ValueError, ArrowInvalid):
+ return NotImplemented
+
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+
+ def as_py(self):
+ raise NotImplementedError()
-_NULL = NA = NullType()
+_NULL = NA = None
-cdef class ArrayValue(Scalar):
+cdef class NullScalar(Scalar):
"""
- The base class for non-null array elements.
+ Concrete class for null scalars.
"""
- def __init__(self):
- raise TypeError("Do not call {}'s constructor directly, use array "
- "subscription instead."
- .format(self.__class__.__name__))
+ def __cinit__(self):
+ global NA
+ if NA is not None:
+ raise Exception('Cannot create multiple NAType instances')
+ self.init(shared_ptr[CScalar](new CNullScalar()))
- cdef void init(self, DataType type, const shared_ptr[CArray]& sp_array,
- int64_t index):
- self.type = type
- self.index = index
- self._set_array(sp_array)
+ def __init__(self):
+ pass
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
+ def __eq__(self, other):
+ return NA
- def __repr__(self):
- if hasattr(self, 'as_py'):
- return repr(self.as_py())
- else:
- return super(Scalar, self).__repr__()
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
- def __str__(self):
- if hasattr(self, 'as_py'):
- return str(self.as_py())
- else:
- return super(Scalar, self).__str__()
+ def as_py(self):
+ """
+ Return this value as a Python None.
+ """
+ return None
- def __eq__(self, other):
- if hasattr(self, 'as_py'):
- if isinstance(other, ArrayValue):
- other = other.as_py()
- return self.as_py() == other
- else:
- raise NotImplementedError(
- "Cannot compare Arrow values that don't support as_py()")
- def __hash__(self):
- return hash(self.as_py())
+_NULL = NA = NullScalar()
-cdef class BooleanValue(ArrayValue):
+cdef class BooleanScalar(Scalar):
"""
- Concrete class for boolean array elements.
+ Concrete class for boolean scalars.
"""
def as_py(self):
"""
Return this value as a Python bool.
"""
- cdef CBooleanArray* ap = <CBooleanArray*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CBooleanScalar* sp = <CBooleanScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int8Value(ArrayValue):
+cdef class UInt8Scalar(Scalar):
"""
- Concrete class for int8 array elements.
+ Concrete class for uint8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt8Array* ap = <CInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt8Scalar* sp = <CUInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt8Value(ArrayValue):
+cdef class Int8Scalar(Scalar):
"""
- Concrete class for uint8 array elements.
+ Concrete class for int8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt8Array* ap = <CUInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt8Scalar* sp = <CInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int16Value(ArrayValue):
+cdef class UInt16Scalar(Scalar):
"""
- Concrete class for int16 array elements.
+ Concrete class for uint16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt16Array* ap = <CInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt16Scalar* sp = <CUInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt16Value(ArrayValue):
+cdef class Int16Scalar(Scalar):
"""
- Concrete class for uint16 array elements.
+ Concrete class for int16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt16Array* ap = <CUInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt16Scalar* sp = <CInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int32Value(ArrayValue):
+cdef class UInt32Scalar(Scalar):
"""
- Concrete class for int32 array elements.
+ Concrete class for uint32 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt32Array* ap = <CInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt32Scalar* sp = <CUInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt32Value(ArrayValue):
+cdef class Int32Scalar(Scalar):
"""
- Concrete class for uint32 array elements.
+ Concrete class for int32 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt32Array* ap = <CUInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt32Scalar* sp = <CInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int64Value(ArrayValue):
+cdef class UInt64Scalar(Scalar):
"""
- Concrete class for int64 array elements.
+ Concrete class for uint64 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt64Array* ap = <CInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt64Scalar* sp = <CUInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt64Value(ArrayValue):
+cdef class Int64Scalar(Scalar):
"""
- Concrete class for uint64 array elements.
+ Concrete class for int64 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt64Array* ap = <CUInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt64Scalar* sp = <CInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date32Value(ArrayValue):
+cdef class HalfFloatScalar(Scalar):
"""
- Concrete class for date32 array elements.
+ Concrete class for float scalars.
"""
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+
+ def __eq__(self, other):
+ if hasattr(self, 'as_py'):
+ if isinstance(other, Scalar):
+ other = other.as_py()
+ return self.as_py() == other
+ else:
+ raise NotImplementedError
+
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python float.
"""
- cdef CDate32Array* ap = <CDate32Array*> self.sp_array.get()
+ cdef CHalfFloatScalar* sp = <CHalfFloatScalar*> self.wrapped.get()
+ return PyHalf_FromHalf(sp.value) if sp.is_valid else None
- # Shift to seconds since epoch
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(days=ap.Value(self.index)))
+
+cdef class FloatScalar(Scalar):
+ """
+ Concrete class for float scalars.
+ """
+
+ def as_py(self):
+ """
+ Return this value as a Python float.
+ """
+ cdef CFloatScalar* sp = <CFloatScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date64Value(ArrayValue):
+cdef class DoubleScalar(Scalar):
"""
- Concrete class for date64 array elements.
+ Concrete class for double scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python float.
"""
- cdef CDate64Array* ap = <CDate64Array*> self.sp_array.get()
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(
- days=ap.Value(self.index) / 86400000))
+ cdef CDoubleScalar* sp = <CDoubleScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Time32Value(ArrayValue):
+cdef class Decimal128Scalar(Scalar):
"""
- Concrete class for time32 array elements.
+ Concrete class for decimal128 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.timedelta instance.
+ Return this value as a Python Decimal.
"""
cdef:
- CTime32Array* ap = <CTime32Array*> self.sp_array.get()
- CTime32Type* dtype = <CTime32Type*> ap.type().get()
-
- if dtype.unit() == TimeUnit_SECOND:
- delta = datetime.timedelta(seconds=ap.Value(self.index))
- return (datetime.datetime(1970, 1, 1) + delta).time()
+ CDecimal128Scalar* sp = <CDecimal128Scalar*> self.wrapped.get()
+ CDecimal128Type* dtype = <CDecimal128Type*> sp.type.get()
+ if sp.is_valid:
+ return _pydecimal.Decimal(
+ frombytes(sp.value.ToString(dtype.scale()))
+ )
else:
- return _box_time_milli(ap.Value(self.index))
+ return None
-cdef class Time64Value(ArrayValue):
+cdef class Date32Scalar(Scalar):
"""
- Concrete class for time64 array elements.
+ Concrete class for date32 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.timedelta instance.
+ Return this value as a Python datetime.datetime instance.
"""
- cdef:
- CTime64Array* ap = <CTime64Array*> self.sp_array.get()
- CTime64Type* dtype = <CTime64Type*> ap.type().get()
+ cdef CDate32Scalar* sp = <CDate32Scalar*> self.wrapped.get()
- cdef int64_t val = ap.Value(self.index)
- if dtype.unit() == TimeUnit_MICRO:
- return _box_time_micro(val)
+ if sp.is_valid:
+ # shift to seconds since epoch
+ return (
+ datetime.date(1970, 1, 1) + datetime.timedelta(days=sp.value)
+ )
else:
- return (datetime.datetime(1970, 1, 1) +
- datetime.timedelta(microseconds=val / 1000)).time()
-
+ return None
-cpdef _box_time_milli(int64_t val):
- delta = datetime.timedelta(milliseconds=val)
- return (datetime.datetime(1970, 1, 1) + delta).time()
+cdef class Date64Scalar(Scalar):
+ """
+ Concrete class for date64 scalars.
+ """
-cpdef _box_time_micro(int64_t val):
- return (datetime.datetime(1970, 1, 1) +
- datetime.timedelta(microseconds=val)).time()
+ def as_py(self):
+ """
+ Return this value as a Python datetime.datetime instance.
+ """
+ cdef CDate64Scalar* sp = <CDate64Scalar*> self.wrapped.get()
+ if sp.is_valid:
+ return (
+ datetime.date(1970, 1, 1) +
+ datetime.timedelta(days=sp.value / 86400000)
+ )
+ else:
+ return None
-cdef dict _DATETIME_CONVERSION_FUNCTIONS = {}
-cdef c_bool _datetime_conversion_initialized = False
+def _datetime_from_int(int64_t value, TimeUnit unit, tzinfo=None):
+ if unit == TimeUnit_SECOND:
+ delta = datetime.timedelta(seconds=value)
+ elif unit == TimeUnit_MILLI:
+ delta = datetime.timedelta(milliseconds=value)
+ elif unit == TimeUnit_MICRO:
+ delta = datetime.timedelta(microseconds=value)
+ else:
+ # TimeUnit_NANO: prefer pandas timestamps if available
+ if _pandas_api.have_pandas:
+ return _pandas_api.pd.Timestamp(value, tz=tzinfo, unit='ns')
+ # otherwise safely truncate to microsecond resolution datetime
+ if value % 1000 != 0:
+ raise ValueError(
+ "Nanosecond resolution temporal type {} is not safely "
+ "convertible to microseconds to convert to datetime.datetime. "
+ "Install pandas to return as Timestamp with nanosecond "
+ "support or access the .value attribute.".format(value)
+ )
+ delta = datetime.timedelta(microseconds=value // 1000)
-cdef _add_micros_maybe_localize(dt, micros, tzinfo):
- import pytz
- dt = dt.replace(microsecond=micros)
+ dt = datetime.datetime(1970, 1, 1) + delta
+ # adjust timezone if set to the datatype
if tzinfo is not None:
- if not isinstance(tzinfo, datetime.tzinfo):
- tzinfo = string_to_tzinfo(tzinfo)
dt = tzinfo.fromutc(dt)
- return dt
-
-
-cdef _datetime_from_seconds(int64_t v):
- return datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=v)
-
-
-def _nanoseconds_to_datetime_safe(v, tzinfo):
- if v % 1000 != 0:
- raise ValueError("Nanosecond timestamp {} is not safely convertible "
- " to microseconds to convert to datetime.datetime."
- " Install pandas to return as Timestamp with "
- " nanosecond support or access the .value attribute.")
- v = v // 1000
- micros = v % 1_000_000
-
- dt = _datetime_from_seconds(v // 1_000_000)
- return _add_micros_maybe_localize(dt, micros, tzinfo)
-
-
-def _microseconds_to_datetime(v, tzinfo):
- micros = v % 1_000_000
- dt = _datetime_from_seconds(v // 1_000_000)
- return _add_micros_maybe_localize(dt, micros, tzinfo)
+ return dt
-def _millis_to_datetime(v, tzinfo):
- millis = v % 1_000
- dt = _datetime_from_seconds(v // 1000)
- return _add_micros_maybe_localize(dt, millis * 1000, tzinfo)
+cdef class Time32Scalar(Scalar):
+ """
+ Concrete class for time32 scalars.
+ """
-def _seconds_to_datetime(v, tzinfo):
- dt = _datetime_from_seconds(v)
- return _add_micros_maybe_localize(dt, 0, tzinfo)
+ def as_py(self):
+ """
+ Return this value as a Python datetime.timedelta instance.
+ """
+ cdef:
+ CTime32Scalar* sp = <CTime32Scalar*> self.wrapped.get()
+ CTime32Type* dtype = <CTime32Type*> sp.type.get()
+ if sp.is_valid:
+ return _datetime_from_int(sp.value, unit=dtype.unit()).time()
+ else:
+ return None
-def _datetime_conversion_functions():
- global _datetime_conversion_initialized
- if _datetime_conversion_initialized:
- return _DATETIME_CONVERSION_FUNCTIONS
- _DATETIME_CONVERSION_FUNCTIONS.update({
- TimeUnit_SECOND: _seconds_to_datetime,
- TimeUnit_MILLI: _millis_to_datetime,
- TimeUnit_MICRO: _microseconds_to_datetime,
- TimeUnit_NANO: _nanoseconds_to_datetime_safe
- })
+cdef class Time64Scalar(Scalar):
+ """
+ Concrete class for time64 scalars.
+ """
- try:
- import pandas as pd
- _DATETIME_CONVERSION_FUNCTIONS[TimeUnit_NANO] = (
- lambda x, tzinfo: pd.Timestamp(
- x, tz=tzinfo, unit='ns',
- )
- )
- except ImportError:
- pass
+ def as_py(self):
+ """
+ Return this value as a Python datetime.timedelta instance.
+ """
+ cdef:
+ CTime64Scalar* sp = <CTime64Scalar*> self.wrapped.get()
+ CTime64Type* dtype = <CTime64Type*> sp.type.get()
- _datetime_conversion_initialized = True
- return _DATETIME_CONVERSION_FUNCTIONS
+ if sp.is_valid:
+ return _datetime_from_int(sp.value, unit=dtype.unit()).time()
+ else:
+ return None
-cdef class TimestampValue(ArrayValue):
+cdef class TimestampScalar(Scalar):
"""
- Concrete class for timestamp array elements.
+ Concrete class for timestamp scalars.
"""
@property
def value(self):
- cdef CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
- cdef CTimestampType* dtype = <CTimestampType*> ap.type().get()
- return ap.Value(self.index)
+ cdef CTimestampScalar* sp = <CTimestampScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
def as_py(self):
"""
Return this value as a Pandas Timestamp instance (if available),
otherwise as a Python datetime.timedelta instance.
"""
- cdef CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
- cdef CTimestampType* dtype = <CTimestampType*> ap.type().get()
+ cdef:
+ CTimestampScalar* sp = <CTimestampScalar*> self.wrapped.get()
+ CTimestampType* dtype = <CTimestampType*> sp.type.get()
- value = self.value
+ if not sp.is_valid:
+ return None
if not dtype.timezone().empty():
tzinfo = string_to_tzinfo(frombytes(dtype.timezone()))
+ if not isinstance(tzinfo, datetime.tzinfo):
+ tzinfo = string_to_tzinfo(tzinfo)
else:
tzinfo = None
- try:
- converter = _datetime_conversion_functions()[dtype.unit()]
- except KeyError:
- raise ValueError(
- 'Cannot convert nanosecond timestamps without pandas'
- )
- return converter(value, tzinfo=tzinfo)
-
-
-cdef dict _TIMEDELTA_CONVERSION_FUNCTIONS = {}
-
-
-def _nanoseconds_to_timedelta_safe(v):
- if v % 1000 != 0:
- raise ValueError(
- "Nanosecond duration {} is not safely convertible to microseconds "
- "to convert to datetime.timedelta. Install pandas to return as "
- "Timedelta with nanosecond support or access the .value "
- "attribute.".format(v))
- micros = v // 1000
-
- return datetime.timedelta(microseconds=micros)
-
-
-def _timedelta_conversion_functions():
- if _TIMEDELTA_CONVERSION_FUNCTIONS:
- return _TIMEDELTA_CONVERSION_FUNCTIONS
-
- _TIMEDELTA_CONVERSION_FUNCTIONS.update({
- TimeUnit_SECOND: lambda v: datetime.timedelta(seconds=v),
- TimeUnit_MILLI: lambda v: datetime.timedelta(milliseconds=v),
- TimeUnit_MICRO: lambda v: datetime.timedelta(microseconds=v),
- TimeUnit_NANO: _nanoseconds_to_timedelta_safe
- })
-
- try:
- import pandas as pd
- _TIMEDELTA_CONVERSION_FUNCTIONS[TimeUnit_NANO] = (
- lambda v: pd.Timedelta(v, unit='ns')
- )
- except ImportError:
- pass
+ return _datetime_from_int(sp.value, unit=dtype.unit(), tzinfo=tzinfo)
- return _TIMEDELTA_CONVERSION_FUNCTIONS
-
-cdef class DurationValue(ArrayValue):
+cdef class DurationScalar(Scalar):
"""
- Concrete class for duration array elements.
+ Concrete class for duration scalars.
"""
@property
def value(self):
- cdef CDurationArray* ap = <CDurationArray*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CDurationScalar* sp = <CDurationScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
def as_py(self):
"""
Return this value as a Pandas Timestamp instance (if available),
otherwise as a Python datetime.timedelta instance.
"""
- cdef CDurationArray* ap = <CDurationArray*> self.sp_array.get()
- cdef CDurationType* dtype = <CDurationType*> ap.type().get()
+ cdef:
+ CDurationScalar* sp = <CDurationScalar*> self.wrapped.get()
+ CDurationType* dtype = <CDurationType*> sp.type.get()
+ TimeUnit unit = dtype.unit()
+
+ if not sp.is_valid:
+ return None
- cdef int64_t value = ap.Value(self.index)
- converter = _timedelta_conversion_functions()[dtype.unit()]
- return converter(value)
+ if unit == TimeUnit_SECOND:
+ return datetime.timedelta(seconds=sp.value)
+ elif unit == TimeUnit_MILLI:
+ return datetime.timedelta(milliseconds=sp.value)
+ elif unit == TimeUnit_MICRO:
+ return datetime.timedelta(microseconds=sp.value)
+ else:
+ # TimeUnit_NANO: prefer pandas timestamps if available
+ if _pandas_api.have_pandas:
+ return _pandas_api.pd.Timedelta(sp.value, unit='ns')
+ # otherwise safely truncate to microsecond resolution timedelta
+ if sp.value % 1000 != 0:
+ raise ValueError(
+ "Nanosecond duration {} is not safely convertible to "
+ "microseconds to convert to datetime.timedelta. Install "
+ "pandas to return as Timedelta with nanosecond support or "
+ "access the .value attribute.".format(sp.value)
+ )
+ return datetime.timedelta(microseconds=sp.value // 1000)
-cdef class HalfFloatValue(ArrayValue):
+cdef class BinaryScalar(Scalar):
"""
- Concrete class for float16 array elements.
+ Concrete class for binary-like scalars.
"""
- def as_py(self):
+ def as_buffer(self):
"""
- Return this value as a Python float.
+ Return a view over this value as a Buffer object.
"""
- cdef CHalfFloatArray* ap = <CHalfFloatArray*> self.sp_array.get()
- return PyHalf_FromHalf(ap.Value(self.index))
-
-
-cdef class FloatValue(ArrayValue):
- """
- Concrete class for float32 array elements.
- """
+ cdef CBinaryScalar* sp = <CBinaryScalar*> self.wrapped.get()
+ return pyarrow_wrap_buffer(sp.value) if sp.is_valid else None
def as_py(self):
"""
- Return this value as a Python float.
+ Return this value as a Python bytes.
"""
- cdef CFloatArray* ap = <CFloatArray*> self.sp_array.get()
- return ap.Value(self.index)
+ buffer = self.as_buffer()
+ if buffer is not None:
+ return self.as_buffer().to_pybytes()
+ else:
+ return None
-cdef class DoubleValue(ArrayValue):
- """
- Concrete class for float64 array elements.
- """
+cdef class LargeBinaryScalar(BinaryScalar):
+ pass
- def as_py(self):
- """
- Return this value as a Python float.
- """
- cdef CDoubleArray* ap = <CDoubleArray*> self.sp_array.get()
- return ap.Value(self.index)
+
+cdef class FixedSizeBinaryScalar(BinaryScalar):
+ pass
-cdef class DecimalValue(ArrayValue):
+cdef class StringScalar(BinaryScalar):
"""
- Concrete class for decimal128 array elements.
+ Concrete class for string-like (utf8) scalars.
"""
def as_py(self):
"""
- Return this value as a Python Decimal.
+ Return this value as a Python string.
"""
- cdef:
- CDecimal128Array* ap = <CDecimal128Array*> self.sp_array.get()
- c_string s = ap.FormatValue(self.index)
- return _pydecimal.Decimal(s.decode('utf8'))
+ buffer = self.as_buffer()
+ if buffer is not None:
+ return frombytes(self.as_buffer().to_pybytes())
+ else:
+ return None
+
+cdef class LargeStringScalar(StringScalar):
+ pass
-cdef class StringValue(ArrayValue):
+
+cdef class ListScalar(Scalar):
"""
- Concrete class for string (utf8) array elements.
+ Concrete class for list-like scalars.
"""
- def as_py(self):
+ @property
+ def values(self):
+ cdef CListScalar* sp = <CListScalar*> self.wrapped.get()
+ if sp.is_valid:
+ return pyarrow_wrap_array(sp.value)
+ else:
+ return None
+
+ def __len__(self):
"""
- Return this value as a Python unicode string.
+ Return the number of values.
"""
- cdef CStringArray* ap = <CStringArray*> self.sp_array.get()
- return ap.GetString(self.index).decode('utf-8')
+ return len(self.values)
- def as_buffer(self):
+ def __getitem__(self, i):
"""
- Return a view over this value as a Buffer object.
+ Return the value at the given index.
"""
- cdef:
- CStringArray* ap = <CStringArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
-
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
-
+ return self.values[_normalize_index(i, len(self))]
-cdef class LargeStringValue(ArrayValue):
- """
- Concrete class for large string (utf8) array elements.
- """
-
- def as_py(self):
+ def __iter__(self):
"""
- Return this value as a Python unicode string.
+ Iterate over this element's values.
"""
- cdef CLargeStringArray* ap = <CLargeStringArray*> self.sp_array.get()
- return ap.GetString(self.index).decode('utf-8')
+ return iter(self.values)
- def as_buffer(self):
+ def as_py(self):
"""
- Return a view over this value as a Buffer object.
+ Return this value as a Python list.
"""
- cdef:
- CLargeStringArray* ap = <CLargeStringArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
+ arr = self.values
+ return None if arr is None else arr.to_pylist()
+
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
+cdef class FixedSizeListScalar(ListScalar):
Review comment:
Like above, you cannot inherit `ListScalar.values` safely.
##########
File path: python/pyarrow/scalar.pxi
##########
@@ -16,1198 +16,745 @@
# under the License.
-_NULL = NA = None
+import collections
cdef class Scalar:
"""
- The base class for all array elements.
+ The base class for scalars.
"""
+ def __init__(self):
+ raise TypeError("Do not call {}'s constructor directly, use "
+ "pa.scalar() instead.".format(self.__class__.__name__))
-cdef class NullType(Scalar):
- """
- Singleton for null array elements.
- """
- # TODO rename this NullValue?
+ cdef void init(self, const shared_ptr[CScalar]& wrapped):
+ self.wrapped = wrapped
- def __cinit__(self):
- global NA
- if NA is not None:
- raise Exception('Cannot create multiple NAType instances')
+ @staticmethod
+ cdef wrap(const shared_ptr[CScalar]& wrapped):
+ cdef:
+ Scalar self
+ Type type_id = wrapped.get().type.get().id()
+
+ if type_id == _Type_NA:
+ return _NULL
+
+ typ = _scalar_classes[type_id]
+ self = typ.__new__(typ)
+ self.init(wrapped)
+
+ return self
+
+ cdef inline shared_ptr[CScalar] unwrap(self) nogil:
+ return self.wrapped
+
+ @property
+ def type(self):
+ return pyarrow_wrap_data_type(self.wrapped.get().type)
- self.type = null()
+ @property
+ def is_valid(self):
+ return self.wrapped.get().is_valid
def __repr__(self):
- return 'NULL'
+ return '<pyarrow.{}: {!r}>'.format(
+ self.__class__.__name__, self.as_py()
+ )
- def as_py(self):
- """
- Return None
- """
- return None
+ def __str__(self):
+ return str(self.as_py())
+
+ def equals(self, Scalar other):
+ return self.wrapped.get().Equals(other.unwrap().get()[0])
def __eq__(self, other):
- return NA
+ try:
+ if not isinstance(other, Scalar):
+ other = scalar(other, type=self.type)
+ return self.equals(other)
+ except (TypeError, ValueError, ArrowInvalid):
+ return NotImplemented
+
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+
+ def as_py(self):
+ raise NotImplementedError()
-_NULL = NA = NullType()
+_NULL = NA = None
-cdef class ArrayValue(Scalar):
+cdef class NullScalar(Scalar):
"""
- The base class for non-null array elements.
+ Concrete class for null scalars.
"""
- def __init__(self):
- raise TypeError("Do not call {}'s constructor directly, use array "
- "subscription instead."
- .format(self.__class__.__name__))
+ def __cinit__(self):
+ global NA
+ if NA is not None:
+ raise Exception('Cannot create multiple NAType instances')
+ self.init(shared_ptr[CScalar](new CNullScalar()))
- cdef void init(self, DataType type, const shared_ptr[CArray]& sp_array,
- int64_t index):
- self.type = type
- self.index = index
- self._set_array(sp_array)
+ def __init__(self):
+ pass
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
+ def __eq__(self, other):
+ return NA
- def __repr__(self):
- if hasattr(self, 'as_py'):
- return repr(self.as_py())
- else:
- return super(Scalar, self).__repr__()
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
- def __str__(self):
- if hasattr(self, 'as_py'):
- return str(self.as_py())
- else:
- return super(Scalar, self).__str__()
+ def as_py(self):
+ """
+ Return this value as a Python None.
+ """
+ return None
- def __eq__(self, other):
- if hasattr(self, 'as_py'):
- if isinstance(other, ArrayValue):
- other = other.as_py()
- return self.as_py() == other
- else:
- raise NotImplementedError(
- "Cannot compare Arrow values that don't support as_py()")
- def __hash__(self):
- return hash(self.as_py())
+_NULL = NA = NullScalar()
-cdef class BooleanValue(ArrayValue):
+cdef class BooleanScalar(Scalar):
"""
- Concrete class for boolean array elements.
+ Concrete class for boolean scalars.
"""
def as_py(self):
"""
Return this value as a Python bool.
"""
- cdef CBooleanArray* ap = <CBooleanArray*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CBooleanScalar* sp = <CBooleanScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int8Value(ArrayValue):
+cdef class UInt8Scalar(Scalar):
"""
- Concrete class for int8 array elements.
+ Concrete class for uint8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt8Array* ap = <CInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt8Scalar* sp = <CUInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt8Value(ArrayValue):
+cdef class Int8Scalar(Scalar):
"""
- Concrete class for uint8 array elements.
+ Concrete class for int8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt8Array* ap = <CUInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt8Scalar* sp = <CInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int16Value(ArrayValue):
+cdef class UInt16Scalar(Scalar):
"""
- Concrete class for int16 array elements.
+ Concrete class for uint16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt16Array* ap = <CInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt16Scalar* sp = <CUInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt16Value(ArrayValue):
+cdef class Int16Scalar(Scalar):
"""
- Concrete class for uint16 array elements.
+ Concrete class for int16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt16Array* ap = <CUInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt16Scalar* sp = <CInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int32Value(ArrayValue):
+cdef class UInt32Scalar(Scalar):
"""
- Concrete class for int32 array elements.
+ Concrete class for uint32 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt32Array* ap = <CInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt32Scalar* sp = <CUInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt32Value(ArrayValue):
+cdef class Int32Scalar(Scalar):
"""
- Concrete class for uint32 array elements.
+ Concrete class for int32 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt32Array* ap = <CUInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt32Scalar* sp = <CInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int64Value(ArrayValue):
+cdef class UInt64Scalar(Scalar):
"""
- Concrete class for int64 array elements.
+ Concrete class for uint64 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt64Array* ap = <CInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt64Scalar* sp = <CUInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt64Value(ArrayValue):
+cdef class Int64Scalar(Scalar):
"""
- Concrete class for uint64 array elements.
+ Concrete class for int64 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt64Array* ap = <CUInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt64Scalar* sp = <CInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date32Value(ArrayValue):
+cdef class HalfFloatScalar(Scalar):
"""
- Concrete class for date32 array elements.
+ Concrete class for float scalars.
"""
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+
+ def __eq__(self, other):
+ if hasattr(self, 'as_py'):
+ if isinstance(other, Scalar):
+ other = other.as_py()
+ return self.as_py() == other
+ else:
+ raise NotImplementedError
+
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python float.
"""
- cdef CDate32Array* ap = <CDate32Array*> self.sp_array.get()
+ cdef CHalfFloatScalar* sp = <CHalfFloatScalar*> self.wrapped.get()
+ return PyHalf_FromHalf(sp.value) if sp.is_valid else None
- # Shift to seconds since epoch
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(days=ap.Value(self.index)))
+
+cdef class FloatScalar(Scalar):
+ """
+ Concrete class for float scalars.
+ """
+
+ def as_py(self):
+ """
+ Return this value as a Python float.
+ """
+ cdef CFloatScalar* sp = <CFloatScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date64Value(ArrayValue):
+cdef class DoubleScalar(Scalar):
"""
- Concrete class for date64 array elements.
+ Concrete class for double scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python float.
"""
- cdef CDate64Array* ap = <CDate64Array*> self.sp_array.get()
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(
- days=ap.Value(self.index) / 86400000))
+ cdef CDoubleScalar* sp = <CDoubleScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Time32Value(ArrayValue):
+cdef class Decimal128Scalar(Scalar):
"""
- Concrete class for time32 array elements.
+ Concrete class for decimal128 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.timedelta instance.
+ Return this value as a Python Decimal.
"""
cdef:
- CTime32Array* ap = <CTime32Array*> self.sp_array.get()
- CTime32Type* dtype = <CTime32Type*> ap.type().get()
-
- if dtype.unit() == TimeUnit_SECOND:
- delta = datetime.timedelta(seconds=ap.Value(self.index))
- return (datetime.datetime(1970, 1, 1) + delta).time()
+ CDecimal128Scalar* sp = <CDecimal128Scalar*> self.wrapped.get()
+ CDecimal128Type* dtype = <CDecimal128Type*> sp.type.get()
+ if sp.is_valid:
+ return _pydecimal.Decimal(
+ frombytes(sp.value.ToString(dtype.scale()))
+ )
else:
- return _box_time_milli(ap.Value(self.index))
+ return None
-cdef class Time64Value(ArrayValue):
+cdef class Date32Scalar(Scalar):
"""
- Concrete class for time64 array elements.
+ Concrete class for date32 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.timedelta instance.
+ Return this value as a Python datetime.datetime instance.
"""
- cdef:
- CTime64Array* ap = <CTime64Array*> self.sp_array.get()
- CTime64Type* dtype = <CTime64Type*> ap.type().get()
+ cdef CDate32Scalar* sp = <CDate32Scalar*> self.wrapped.get()
- cdef int64_t val = ap.Value(self.index)
- if dtype.unit() == TimeUnit_MICRO:
- return _box_time_micro(val)
+ if sp.is_valid:
+ # shift to seconds since epoch
+ return (
+ datetime.date(1970, 1, 1) + datetime.timedelta(days=sp.value)
+ )
else:
- return (datetime.datetime(1970, 1, 1) +
- datetime.timedelta(microseconds=val / 1000)).time()
-
+ return None
-cpdef _box_time_milli(int64_t val):
- delta = datetime.timedelta(milliseconds=val)
- return (datetime.datetime(1970, 1, 1) + delta).time()
+cdef class Date64Scalar(Scalar):
+ """
+ Concrete class for date64 scalars.
+ """
-cpdef _box_time_micro(int64_t val):
- return (datetime.datetime(1970, 1, 1) +
- datetime.timedelta(microseconds=val)).time()
+ def as_py(self):
+ """
+ Return this value as a Python datetime.datetime instance.
+ """
+ cdef CDate64Scalar* sp = <CDate64Scalar*> self.wrapped.get()
+ if sp.is_valid:
+ return (
+ datetime.date(1970, 1, 1) +
+ datetime.timedelta(days=sp.value / 86400000)
+ )
+ else:
+ return None
-cdef dict _DATETIME_CONVERSION_FUNCTIONS = {}
-cdef c_bool _datetime_conversion_initialized = False
+def _datetime_from_int(int64_t value, TimeUnit unit, tzinfo=None):
+ if unit == TimeUnit_SECOND:
+ delta = datetime.timedelta(seconds=value)
+ elif unit == TimeUnit_MILLI:
+ delta = datetime.timedelta(milliseconds=value)
+ elif unit == TimeUnit_MICRO:
+ delta = datetime.timedelta(microseconds=value)
+ else:
+ # TimeUnit_NANO: prefer pandas timestamps if available
+ if _pandas_api.have_pandas:
+ return _pandas_api.pd.Timestamp(value, tz=tzinfo, unit='ns')
+ # otherwise safely truncate to microsecond resolution datetime
+ if value % 1000 != 0:
+ raise ValueError(
+ "Nanosecond resolution temporal type {} is not safely "
+ "convertible to microseconds to convert to datetime.datetime. "
+ "Install pandas to return as Timestamp with nanosecond "
+ "support or access the .value attribute.".format(value)
+ )
+ delta = datetime.timedelta(microseconds=value // 1000)
-cdef _add_micros_maybe_localize(dt, micros, tzinfo):
- import pytz
- dt = dt.replace(microsecond=micros)
+ dt = datetime.datetime(1970, 1, 1) + delta
+ # adjust timezone if set to the datatype
if tzinfo is not None:
- if not isinstance(tzinfo, datetime.tzinfo):
- tzinfo = string_to_tzinfo(tzinfo)
dt = tzinfo.fromutc(dt)
- return dt
-
-
-cdef _datetime_from_seconds(int64_t v):
- return datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=v)
-
-
-def _nanoseconds_to_datetime_safe(v, tzinfo):
- if v % 1000 != 0:
- raise ValueError("Nanosecond timestamp {} is not safely convertible "
- " to microseconds to convert to datetime.datetime."
- " Install pandas to return as Timestamp with "
- " nanosecond support or access the .value attribute.")
- v = v // 1000
- micros = v % 1_000_000
-
- dt = _datetime_from_seconds(v // 1_000_000)
- return _add_micros_maybe_localize(dt, micros, tzinfo)
-
-
-def _microseconds_to_datetime(v, tzinfo):
- micros = v % 1_000_000
- dt = _datetime_from_seconds(v // 1_000_000)
- return _add_micros_maybe_localize(dt, micros, tzinfo)
+ return dt
-def _millis_to_datetime(v, tzinfo):
- millis = v % 1_000
- dt = _datetime_from_seconds(v // 1000)
- return _add_micros_maybe_localize(dt, millis * 1000, tzinfo)
+cdef class Time32Scalar(Scalar):
+ """
+ Concrete class for time32 scalars.
+ """
-def _seconds_to_datetime(v, tzinfo):
- dt = _datetime_from_seconds(v)
- return _add_micros_maybe_localize(dt, 0, tzinfo)
+ def as_py(self):
+ """
+ Return this value as a Python datetime.timedelta instance.
+ """
+ cdef:
+ CTime32Scalar* sp = <CTime32Scalar*> self.wrapped.get()
+ CTime32Type* dtype = <CTime32Type*> sp.type.get()
+ if sp.is_valid:
+ return _datetime_from_int(sp.value, unit=dtype.unit()).time()
+ else:
+ return None
-def _datetime_conversion_functions():
- global _datetime_conversion_initialized
- if _datetime_conversion_initialized:
- return _DATETIME_CONVERSION_FUNCTIONS
- _DATETIME_CONVERSION_FUNCTIONS.update({
- TimeUnit_SECOND: _seconds_to_datetime,
- TimeUnit_MILLI: _millis_to_datetime,
- TimeUnit_MICRO: _microseconds_to_datetime,
- TimeUnit_NANO: _nanoseconds_to_datetime_safe
- })
+cdef class Time64Scalar(Scalar):
+ """
+ Concrete class for time64 scalars.
+ """
- try:
- import pandas as pd
- _DATETIME_CONVERSION_FUNCTIONS[TimeUnit_NANO] = (
- lambda x, tzinfo: pd.Timestamp(
- x, tz=tzinfo, unit='ns',
- )
- )
- except ImportError:
- pass
+ def as_py(self):
+ """
+ Return this value as a Python datetime.timedelta instance.
+ """
+ cdef:
+ CTime64Scalar* sp = <CTime64Scalar*> self.wrapped.get()
+ CTime64Type* dtype = <CTime64Type*> sp.type.get()
- _datetime_conversion_initialized = True
- return _DATETIME_CONVERSION_FUNCTIONS
+ if sp.is_valid:
+ return _datetime_from_int(sp.value, unit=dtype.unit()).time()
+ else:
+ return None
-cdef class TimestampValue(ArrayValue):
+cdef class TimestampScalar(Scalar):
"""
- Concrete class for timestamp array elements.
+ Concrete class for timestamp scalars.
"""
@property
def value(self):
- cdef CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
- cdef CTimestampType* dtype = <CTimestampType*> ap.type().get()
- return ap.Value(self.index)
+ cdef CTimestampScalar* sp = <CTimestampScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
def as_py(self):
"""
Return this value as a Pandas Timestamp instance (if available),
otherwise as a Python datetime.timedelta instance.
"""
- cdef CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
- cdef CTimestampType* dtype = <CTimestampType*> ap.type().get()
+ cdef:
+ CTimestampScalar* sp = <CTimestampScalar*> self.wrapped.get()
+ CTimestampType* dtype = <CTimestampType*> sp.type.get()
- value = self.value
+ if not sp.is_valid:
+ return None
if not dtype.timezone().empty():
tzinfo = string_to_tzinfo(frombytes(dtype.timezone()))
+ if not isinstance(tzinfo, datetime.tzinfo):
+ tzinfo = string_to_tzinfo(tzinfo)
else:
tzinfo = None
- try:
- converter = _datetime_conversion_functions()[dtype.unit()]
- except KeyError:
- raise ValueError(
- 'Cannot convert nanosecond timestamps without pandas'
- )
- return converter(value, tzinfo=tzinfo)
-
-
-cdef dict _TIMEDELTA_CONVERSION_FUNCTIONS = {}
-
-
-def _nanoseconds_to_timedelta_safe(v):
- if v % 1000 != 0:
- raise ValueError(
- "Nanosecond duration {} is not safely convertible to microseconds "
- "to convert to datetime.timedelta. Install pandas to return as "
- "Timedelta with nanosecond support or access the .value "
- "attribute.".format(v))
- micros = v // 1000
-
- return datetime.timedelta(microseconds=micros)
-
-
-def _timedelta_conversion_functions():
- if _TIMEDELTA_CONVERSION_FUNCTIONS:
- return _TIMEDELTA_CONVERSION_FUNCTIONS
-
- _TIMEDELTA_CONVERSION_FUNCTIONS.update({
- TimeUnit_SECOND: lambda v: datetime.timedelta(seconds=v),
- TimeUnit_MILLI: lambda v: datetime.timedelta(milliseconds=v),
- TimeUnit_MICRO: lambda v: datetime.timedelta(microseconds=v),
- TimeUnit_NANO: _nanoseconds_to_timedelta_safe
- })
-
- try:
- import pandas as pd
- _TIMEDELTA_CONVERSION_FUNCTIONS[TimeUnit_NANO] = (
- lambda v: pd.Timedelta(v, unit='ns')
- )
- except ImportError:
- pass
+ return _datetime_from_int(sp.value, unit=dtype.unit(), tzinfo=tzinfo)
- return _TIMEDELTA_CONVERSION_FUNCTIONS
-
-cdef class DurationValue(ArrayValue):
+cdef class DurationScalar(Scalar):
"""
- Concrete class for duration array elements.
+ Concrete class for duration scalars.
"""
@property
def value(self):
- cdef CDurationArray* ap = <CDurationArray*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CDurationScalar* sp = <CDurationScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
def as_py(self):
"""
Return this value as a Pandas Timestamp instance (if available),
otherwise as a Python datetime.timedelta instance.
"""
- cdef CDurationArray* ap = <CDurationArray*> self.sp_array.get()
- cdef CDurationType* dtype = <CDurationType*> ap.type().get()
+ cdef:
+ CDurationScalar* sp = <CDurationScalar*> self.wrapped.get()
+ CDurationType* dtype = <CDurationType*> sp.type.get()
+ TimeUnit unit = dtype.unit()
+
+ if not sp.is_valid:
+ return None
- cdef int64_t value = ap.Value(self.index)
- converter = _timedelta_conversion_functions()[dtype.unit()]
- return converter(value)
+ if unit == TimeUnit_SECOND:
+ return datetime.timedelta(seconds=sp.value)
+ elif unit == TimeUnit_MILLI:
+ return datetime.timedelta(milliseconds=sp.value)
+ elif unit == TimeUnit_MICRO:
+ return datetime.timedelta(microseconds=sp.value)
+ else:
+ # TimeUnit_NANO: prefer pandas timestamps if available
+ if _pandas_api.have_pandas:
+ return _pandas_api.pd.Timedelta(sp.value, unit='ns')
+ # otherwise safely truncate to microsecond resolution timedelta
+ if sp.value % 1000 != 0:
+ raise ValueError(
+ "Nanosecond duration {} is not safely convertible to "
+ "microseconds to convert to datetime.timedelta. Install "
+ "pandas to return as Timedelta with nanosecond support or "
+ "access the .value attribute.".format(sp.value)
+ )
+ return datetime.timedelta(microseconds=sp.value // 1000)
-cdef class HalfFloatValue(ArrayValue):
+cdef class BinaryScalar(Scalar):
"""
- Concrete class for float16 array elements.
+ Concrete class for binary-like scalars.
"""
- def as_py(self):
+ def as_buffer(self):
"""
- Return this value as a Python float.
+ Return a view over this value as a Buffer object.
"""
- cdef CHalfFloatArray* ap = <CHalfFloatArray*> self.sp_array.get()
- return PyHalf_FromHalf(ap.Value(self.index))
-
-
-cdef class FloatValue(ArrayValue):
- """
- Concrete class for float32 array elements.
- """
+ cdef CBinaryScalar* sp = <CBinaryScalar*> self.wrapped.get()
+ return pyarrow_wrap_buffer(sp.value) if sp.is_valid else None
def as_py(self):
"""
- Return this value as a Python float.
+ Return this value as a Python bytes.
"""
- cdef CFloatArray* ap = <CFloatArray*> self.sp_array.get()
- return ap.Value(self.index)
+ buffer = self.as_buffer()
+ if buffer is not None:
+ return self.as_buffer().to_pybytes()
+ else:
+ return None
-cdef class DoubleValue(ArrayValue):
- """
- Concrete class for float64 array elements.
- """
+cdef class LargeBinaryScalar(BinaryScalar):
+ pass
- def as_py(self):
- """
- Return this value as a Python float.
- """
- cdef CDoubleArray* ap = <CDoubleArray*> self.sp_array.get()
- return ap.Value(self.index)
+
+cdef class FixedSizeBinaryScalar(BinaryScalar):
+ pass
-cdef class DecimalValue(ArrayValue):
+cdef class StringScalar(BinaryScalar):
"""
- Concrete class for decimal128 array elements.
+ Concrete class for string-like (utf8) scalars.
"""
def as_py(self):
"""
- Return this value as a Python Decimal.
+ Return this value as a Python string.
"""
- cdef:
- CDecimal128Array* ap = <CDecimal128Array*> self.sp_array.get()
- c_string s = ap.FormatValue(self.index)
- return _pydecimal.Decimal(s.decode('utf8'))
+ buffer = self.as_buffer()
+ if buffer is not None:
+ return frombytes(self.as_buffer().to_pybytes())
+ else:
+ return None
+
+cdef class LargeStringScalar(StringScalar):
+ pass
-cdef class StringValue(ArrayValue):
+
+cdef class ListScalar(Scalar):
"""
- Concrete class for string (utf8) array elements.
+ Concrete class for list-like scalars.
"""
- def as_py(self):
+ @property
+ def values(self):
+ cdef CListScalar* sp = <CListScalar*> self.wrapped.get()
+ if sp.is_valid:
+ return pyarrow_wrap_array(sp.value)
+ else:
+ return None
+
+ def __len__(self):
"""
- Return this value as a Python unicode string.
+ Return the number of values.
"""
- cdef CStringArray* ap = <CStringArray*> self.sp_array.get()
- return ap.GetString(self.index).decode('utf-8')
+ return len(self.values)
- def as_buffer(self):
+ def __getitem__(self, i):
"""
- Return a view over this value as a Buffer object.
+ Return the value at the given index.
"""
- cdef:
- CStringArray* ap = <CStringArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
-
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
-
+ return self.values[_normalize_index(i, len(self))]
-cdef class LargeStringValue(ArrayValue):
- """
- Concrete class for large string (utf8) array elements.
- """
-
- def as_py(self):
+ def __iter__(self):
"""
- Return this value as a Python unicode string.
+ Iterate over this element's values.
"""
- cdef CLargeStringArray* ap = <CLargeStringArray*> self.sp_array.get()
- return ap.GetString(self.index).decode('utf-8')
+ return iter(self.values)
- def as_buffer(self):
+ def as_py(self):
"""
- Return a view over this value as a Buffer object.
+ Return this value as a Python list.
"""
- cdef:
- CLargeStringArray* ap = <CLargeStringArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
+ arr = self.values
+ return None if arr is None else arr.to_pylist()
+
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
+cdef class FixedSizeListScalar(ListScalar):
+ pass
-cdef class BinaryValue(ArrayValue):
+cdef class LargeListScalar(ListScalar):
+ pass
+
+
+cdef class StructScalar(Scalar, collections.abc.Mapping):
"""
- Concrete class for variable-sized binary array elements.
+ Concrete class for struct scalars.
"""
- def as_py(self):
- """
- Return this value as a Python bytes object.
- """
+ def __len__(self):
+ cdef CStructScalar* sp = <CStructScalar*> self.wrapped.get()
+ return sp.value.size()
+
+ def __iter__(self):
cdef:
- const uint8_t* ptr
- int32_t length
- CBinaryArray* ap = <CBinaryArray*> self.sp_array.get()
+ CStructScalar* sp = <CStructScalar*> self.wrapped.get()
+ CStructType* dtype = <CStructType*> sp.type.get()
+ vector[shared_ptr[CField]] fields = dtype.fields()
- ptr = ap.GetValue(self.index, &length)
- return cp.PyBytes_FromStringAndSize(<const char*>(ptr), length)
+ if sp.is_valid:
+ for i in range(dtype.num_fields()):
+ yield frombytes(fields[i].get().name())
- def as_buffer(self):
+ def __contains__(self, key):
+ try:
+ self[key]
+ except IndexError:
+ return False
+ else:
+ return True
+
+ def __getitem__(self, key):
"""
- Return a view over this value as a Buffer object.
+ Return the child value for the given field.
+
+ Parameters
+ ----------
+ index : Union[int, str]
+ Index / position or name of the field.
+
+ Returns
+ -------
+ result : Scalar
"""
cdef:
- CBinaryArray* ap = <CBinaryArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
+ CFieldRef ref
+ CStructScalar* sp = <CStructScalar*> self.wrapped.get()
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
+ if isinstance(key, (bytes, str)):
+ ref = CFieldRef(<c_string> tobytes(key))
+ elif isinstance(key, int):
+ ref = CFieldRef(<int> key)
+ else:
+ raise TypeError('Expected integer or string index')
-
-cdef class LargeBinaryValue(ArrayValue):
- """
- Concrete class for large variable-sized binary array elements.
- """
-
- def as_py(self):
- """
- Return this value as a Python bytes object.
- """
- cdef:
- const uint8_t* ptr
- int64_t length
- CLargeBinaryArray* ap = <CLargeBinaryArray*> self.sp_array.get()
-
- ptr = ap.GetValue(self.index, &length)
- return cp.PyBytes_FromStringAndSize(<const char*>(ptr), length)
-
- def as_buffer(self):
- """
- Return a view over this value as a Buffer object.
- """
- cdef:
- CLargeBinaryArray* ap = <CLargeBinaryArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
-
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
-
-
-cdef class ListValue(ArrayValue):
- """
- Concrete class for list array elements.
- """
-
- def __len__(self):
- """
- Return the number of values.
- """
- return self.length()
-
- def __getitem__(self, i):
- """
- Return the value at the given index.
- """
- return self.getitem(_normalize_index(i, self.length()))
-
- def __iter__(self):
- """
- Iterate over this element's values.
- """
- for i in range(len(self)):
- yield self.getitem(i)
- raise StopIteration
-
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CListArray*> sp_array.get()
- self.value_type = pyarrow_wrap_data_type(self.ap.value_type())
-
- cdef getitem(self, int64_t i):
- cdef int64_t j = self.ap.value_offset(self.index) + i
- return box_scalar(self.value_type, self.ap.values(), j)
-
- cdef int64_t length(self):
- return self.ap.value_length(self.index)
-
- def as_py(self):
- """
- Return this value as a Python list.
- """
- cdef:
- int64_t j
- list result = []
-
- for j in range(len(self)):
- result.append(self.getitem(j).as_py())
-
- return result
-
-
-cdef class LargeListValue(ArrayValue):
- """
- Concrete class for large list array elements.
- """
-
- def __len__(self):
- """
- Return the number of values.
- """
- return self.length()
-
- def __getitem__(self, i):
- """
- Return the value at the given index.
- """
- return self.getitem(_normalize_index(i, self.length()))
-
- def __iter__(self):
- """
- Iterate over this element's values.
- """
- for i in range(len(self)):
- yield self.getitem(i)
- raise StopIteration
-
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CLargeListArray*> sp_array.get()
- self.value_type = pyarrow_wrap_data_type(self.ap.value_type())
-
- cdef getitem(self, int64_t i):
- cdef int64_t j = self.ap.value_offset(self.index) + i
- return box_scalar(self.value_type, self.ap.values(), j)
-
- cdef int64_t length(self):
- return self.ap.value_length(self.index)
-
- def as_py(self):
- """
- Return this value as a Python list.
- """
- cdef:
- int64_t j
- list result = []
-
- for j in range(len(self)):
- result.append(self.getitem(j).as_py())
-
- return result
-
-
-cdef class MapValue(ArrayValue):
- """
- Concrete class for map array elements.
- """
-
- def __len__(self):
- """
- Return the number of values.
- """
- return self.length()
-
- def __getitem__(self, i):
- """
- Return the value at the given index.
- """
- return self.getitem(_normalize_index(i, self.length()))
-
- def __iter__(self):
- """
- Iterate over this element's values.
- """
- for i in range(len(self)):
- yield self.getitem(i)
- raise StopIteration
-
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CMapArray*> sp_array.get()
- self.key_type = pyarrow_wrap_data_type(self.ap.map_type().key_type())
- self.item_type = pyarrow_wrap_data_type(self.ap.map_type().item_type())
-
- cdef getitem(self, int64_t i):
- cdef int64_t j = self.ap.value_offset(self.index) + i
- return (box_scalar(self.key_type, self.ap.keys(), j),
- box_scalar(self.item_type, self.ap.items(), j))
-
- cdef int64_t length(self):
- return self.ap.value_length(self.index)
+ try:
+ return Scalar.wrap(GetResultValue(sp.field(ref)))
+ except ArrowInvalid:
+ raise IndexError(key)
def as_py(self):
"""
- Return this value as a Python list of tuples, each containing a
- key and item.
+ Return this value as a Python dict.
"""
- cdef:
- int64_t j
- list result = []
-
- for j in range(len(self)):
- key, item = self.getitem(j)
- result.append((key.as_py(), item.as_py()))
-
- return result
+ if self.is_valid:
+ return {k: v.as_py() for k, v in self.items()}
+ else:
+ return None
-cdef class FixedSizeListValue(ArrayValue):
+cdef class MapScalar(ListScalar):
"""
- Concrete class for fixed size list array elements.
+ Concrete class for map scalars.
"""
- def __len__(self):
- """
- Return the number of values.
- """
- return self.length()
-
def __getitem__(self, i):
"""
Return the value at the given index.
"""
- return self.getitem(_normalize_index(i, self.length()))
+ arr = self.values
+ if arr is None:
+ raise IndexError(i)
+ dct = arr[_normalize_index(i, len(arr))]
+ return (dct['key'], dct['value'])
def __iter__(self):
"""
Iterate over this element's values.
"""
- for i in range(len(self)):
- yield self.getitem(i)
- raise StopIteration
-
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CFixedSizeListArray*> sp_array.get()
- self.value_type = pyarrow_wrap_data_type(self.ap.value_type())
-
- cdef getitem(self, int64_t i):
- cdef int64_t j = self.ap.value_offset(self.index) + i
- return box_scalar(self.value_type, self.ap.values(), j)
-
- cdef int64_t length(self):
- return self.ap.value_length(self.index)
+ arr = self.values
+ if arr is None:
+ return iter(zip(arr.field('key'), arr.field('value')))
+ else:
+ raise StopIteration
def as_py(self):
"""
Return this value as a Python list.
"""
- cdef:
- int64_t j
- list result = []
-
- for j in range(len(self)):
- result.append(self.getitem(j).as_py())
-
- return result
-
-
-cdef class UnionValue(ArrayValue):
- """
- Concrete class for union array elements.
- """
-
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CUnionArray*> sp_array.get()
-
- cdef getitem(self, int64_t i):
- cdef int child_id = self.ap.child_id(i)
- cdef shared_ptr[CArray] child = self.ap.field(child_id)
- cdef CDenseUnionArray* dense
- if self.ap.mode() == _UnionMode_SPARSE:
- return box_scalar(self.type[child_id].type, child, i)
+ arr = self.values
+ if arr is not None:
+ return list(zip(arr.field('key'), arr.field('value')))
else:
- dense = <CDenseUnionArray*> self.ap
- return box_scalar(self.type[child_id].type, child,
- dense.value_offset(i))
-
- def as_py(self):
- """
- Return this value as a Python object.
-
- The exact type depends on the underlying union member.
- """
- return self.getitem(self.index).as_py()
+ return None
-cdef class FixedSizeBinaryValue(ArrayValue):
- """
- Concrete class for fixed-size binary array elements.
- """
-
- def as_py(self):
- """
- Return this value as a Python bytes object.
- """
- cdef:
- CFixedSizeBinaryArray* ap
- CFixedSizeBinaryType* ap_type
- int32_t length
- const char* data
- ap = <CFixedSizeBinaryArray*> self.sp_array.get()
- ap_type = <CFixedSizeBinaryType*> ap.type().get()
- length = ap_type.byte_width()
- data = <const char*> ap.GetValue(self.index)
- return cp.PyBytes_FromStringAndSize(data, length)
-
-
-cdef class StructValue(ArrayValue):
+cdef class DictionaryScalar(Scalar):
"""
- Concrete class for struct array elements.
+ Concrete class for dictionary-encoded scalars.
"""
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CStructArray*> sp_array.get()
-
- def __getitem__(self, key):
- """
- Return the child value for the given field name.
- """
- cdef:
- CStructType* type
- int index
-
- type = <CStructType*> self.type.type
- index = type.GetFieldIndex(tobytes(key))
-
- if index < 0:
- raise KeyError(key)
+ # @property
+ # def index(self):
+ # """
+ # Return this value's underlying index as a scalar.
+ # """
+ # cdef CDictionaryScalar* sp = <CDictionaryScalar*> self.wrapped.get()
+ # return Scalar.wrap(sp.index)
- return pyarrow_wrap_array(self.ap.field(index))[self.index]
-
- def as_py(self):
+ @property
+ def value(self):
"""
- Return this value as a Python dict.
+ Return this value's underlying dictionary value as a scalar.
"""
- cdef:
- vector[shared_ptr[CField]] child_fields = self.type.type.fields()
-
- wrapped_arrays = [pyarrow_wrap_array(self.ap.field(i))
- for i in range(self.ap.num_fields())]
- child_names = [child.get().name() for child in child_fields]
- # Return the struct as a dict
- return {
- frombytes(name): child_array[self.index].as_py()
- for name, child_array in zip(child_names, wrapped_arrays)
- }
-
-
-cdef class DictionaryValue(ArrayValue):
- """
- Concrete class for dictionary-encoded array elements.
- """
+ cdef CDictionaryScalar* sp = <CDictionaryScalar*> self.wrapped.get()
+ return Scalar.wrap(sp.value)
def as_py(self):
"""
Return this value as a Python object.
The exact type depends on the dictionary value type.
"""
- return self.dictionary_value.as_py()
+ value = self.value
+ return None if value is None else value.as_py()
- @property
- def index_value(self):
- """
- Return this value's underlying index as a ArrayValue of the right
- signed integer type.
- """
- cdef CDictionaryArray* darr = <CDictionaryArray*>(self.sp_array.get())
- indices = pyarrow_wrap_array(darr.indices())
- return indices[self.index]
+ # TODO(kszucs): deprecate these
+ # @property
+ # def index_value(self):
+ # index = self.index
+ # return None if index is None else self.index
@property
def dictionary_value(self):
- """
- Return this value's underlying dictionary value as a ArrayValue.
- """
- cdef CDictionaryArray* darr = <CDictionaryArray*>(self.sp_array.get())
- dictionary = pyarrow_wrap_array(darr.dictionary())
- return dictionary[self.index_value.as_py()]
-
-
-cdef dict _array_value_classes = {
- _Type_BOOL: BooleanValue,
- _Type_UINT8: UInt8Value,
- _Type_UINT16: UInt16Value,
- _Type_UINT32: UInt32Value,
- _Type_UINT64: UInt64Value,
- _Type_INT8: Int8Value,
- _Type_INT16: Int16Value,
- _Type_INT32: Int32Value,
- _Type_INT64: Int64Value,
- _Type_DATE32: Date32Value,
- _Type_DATE64: Date64Value,
- _Type_TIME32: Time32Value,
- _Type_TIME64: Time64Value,
- _Type_TIMESTAMP: TimestampValue,
- _Type_DURATION: DurationValue,
- _Type_HALF_FLOAT: HalfFloatValue,
- _Type_FLOAT: FloatValue,
- _Type_DOUBLE: DoubleValue,
- _Type_LIST: ListValue,
- _Type_LARGE_LIST: LargeListValue,
- _Type_MAP: MapValue,
- _Type_FIXED_SIZE_LIST: FixedSizeListValue,
- _Type_SPARSE_UNION: UnionValue,
- _Type_DENSE_UNION: UnionValue,
- _Type_BINARY: BinaryValue,
- _Type_STRING: StringValue,
- _Type_LARGE_BINARY: LargeBinaryValue,
- _Type_LARGE_STRING: LargeStringValue,
- _Type_FIXED_SIZE_BINARY: FixedSizeBinaryValue,
- _Type_DECIMAL: DecimalValue,
- _Type_STRUCT: StructValue,
- _Type_DICTIONARY: DictionaryValue,
-}
-
-cdef class ScalarValue(Scalar):
- """
- The base class for scalars.
- """
-
- def __init__(self):
- raise TypeError("Do not call {}'s constructor directly."
- .format(self.__class__.__name__))
-
- cdef void init(self, const shared_ptr[CScalar]& sp_scalar):
- self.sp_scalar = sp_scalar
-
- def __repr__(self):
- if hasattr(self, 'as_py'):
- return repr(self.as_py())
- else:
- return super(Scalar, self).__repr__()
-
- def __str__(self):
- if hasattr(self, 'as_py'):
- return str(self.as_py())
- else:
- return super(Scalar, self).__str__()
-
- def __eq__(self, other):
- if hasattr(self, 'as_py'):
- if isinstance(other, ScalarValue):
- other = other.as_py()
- return self.as_py() == other
- else:
- raise NotImplemented(
- "Cannot compare scalars that don't support as_py()")
-
- def __hash__(self):
- return hash(self.as_py())
-
-
-cdef class NullScalar(ScalarValue):
- """
- Concrete class for null scalars.
- """
-
- def as_py(self):
- """
- Return this value as a Python None.
- """
- return None
-
-
-cdef class BooleanScalar(ScalarValue):
- """
- Concrete class for boolean scalars.
- """
-
- def as_py(self):
- """
- Return this value as a Python bool.
- """
- cdef CBooleanScalar* sp = <CBooleanScalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
-
-
-cdef class UInt8Scalar(ScalarValue):
- """
- Concrete class for uint8 scalars.
- """
-
- def as_py(self):
- """
- Return this value as a Python int.
- """
- cdef CUInt8Scalar* sp = <CUInt8Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
-
-
-cdef class Int8Scalar(ScalarValue):
- """
- Concrete class for int8 scalars.
- """
-
- def as_py(self):
- """
- Return this value as a Python int.
- """
- cdef CInt8Scalar* sp = <CInt8Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
-
-
-cdef class UInt16Scalar(ScalarValue):
- """
- Concrete class for uint16 scalars.
- """
-
- def as_py(self):
- """
- Return this value as a Python int.
- """
- cdef CUInt16Scalar* sp = <CUInt16Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
-
-
-cdef class Int16Scalar(ScalarValue):
- """
- Concrete class for int16 scalars.
- """
-
- def as_py(self):
- """
- Return this value as a Python int.
- """
- cdef CInt16Scalar* sp = <CInt16Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
-
-
-cdef class UInt32Scalar(ScalarValue):
- """
- Concrete class for uint32 scalars.
- """
-
- def as_py(self):
- """
- Return this value as a Python int.
- """
- cdef CUInt32Scalar* sp = <CUInt32Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
-
-
-cdef class Int32Scalar(ScalarValue):
- """
- Concrete class for int32 scalars.
- """
-
- def as_py(self):
- """
- Return this value as a Python int.
- """
- cdef CInt32Scalar* sp = <CInt32Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
-
-
-cdef class UInt64Scalar(ScalarValue):
- """
- Concrete class for uint64 scalars.
- """
-
- def as_py(self):
- """
- Return this value as a Python int.
- """
- cdef CUInt64Scalar* sp = <CUInt64Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
-
-
-cdef class Int64Scalar(ScalarValue):
- """
- Concrete class for int64 scalars.
- """
-
- def as_py(self):
- """
- Return this value as a Python int.
- """
- cdef CInt64Scalar* sp = <CInt64Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
+ value = self.value
+ return None if value is None else self.value
-cdef class FloatScalar(ScalarValue):
+cdef class UnionScalar(Scalar):
"""
- Concrete class for float scalars.
+ Concrete class for Union scalars.
"""
- def as_py(self):
+ @property
+ def value(self):
"""
- Return this value as a Python float.
+ Return this value's underlying dictionary value as a scalar.
Review comment:
There's no dictionary here.
##########
File path: python/pyarrow/scalar.pxi
##########
@@ -16,1198 +16,745 @@
# under the License.
-_NULL = NA = None
+import collections
cdef class Scalar:
"""
- The base class for all array elements.
+ The base class for scalars.
"""
+ def __init__(self):
+ raise TypeError("Do not call {}'s constructor directly, use "
+ "pa.scalar() instead.".format(self.__class__.__name__))
-cdef class NullType(Scalar):
- """
- Singleton for null array elements.
- """
- # TODO rename this NullValue?
+ cdef void init(self, const shared_ptr[CScalar]& wrapped):
+ self.wrapped = wrapped
- def __cinit__(self):
- global NA
- if NA is not None:
- raise Exception('Cannot create multiple NAType instances')
+ @staticmethod
+ cdef wrap(const shared_ptr[CScalar]& wrapped):
+ cdef:
+ Scalar self
+ Type type_id = wrapped.get().type.get().id()
+
+ if type_id == _Type_NA:
+ return _NULL
+
+ typ = _scalar_classes[type_id]
+ self = typ.__new__(typ)
+ self.init(wrapped)
+
+ return self
+
+ cdef inline shared_ptr[CScalar] unwrap(self) nogil:
+ return self.wrapped
+
+ @property
+ def type(self):
+ return pyarrow_wrap_data_type(self.wrapped.get().type)
- self.type = null()
+ @property
+ def is_valid(self):
+ return self.wrapped.get().is_valid
def __repr__(self):
- return 'NULL'
+ return '<pyarrow.{}: {!r}>'.format(
+ self.__class__.__name__, self.as_py()
+ )
- def as_py(self):
- """
- Return None
- """
- return None
+ def __str__(self):
+ return str(self.as_py())
+
+ def equals(self, Scalar other):
+ return self.wrapped.get().Equals(other.unwrap().get()[0])
def __eq__(self, other):
- return NA
+ try:
+ if not isinstance(other, Scalar):
+ other = scalar(other, type=self.type)
+ return self.equals(other)
+ except (TypeError, ValueError, ArrowInvalid):
+ return NotImplemented
+
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+
+ def as_py(self):
+ raise NotImplementedError()
-_NULL = NA = NullType()
+_NULL = NA = None
-cdef class ArrayValue(Scalar):
+cdef class NullScalar(Scalar):
"""
- The base class for non-null array elements.
+ Concrete class for null scalars.
"""
- def __init__(self):
- raise TypeError("Do not call {}'s constructor directly, use array "
- "subscription instead."
- .format(self.__class__.__name__))
+ def __cinit__(self):
+ global NA
+ if NA is not None:
+ raise Exception('Cannot create multiple NAType instances')
+ self.init(shared_ptr[CScalar](new CNullScalar()))
- cdef void init(self, DataType type, const shared_ptr[CArray]& sp_array,
- int64_t index):
- self.type = type
- self.index = index
- self._set_array(sp_array)
+ def __init__(self):
+ pass
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
+ def __eq__(self, other):
+ return NA
- def __repr__(self):
- if hasattr(self, 'as_py'):
- return repr(self.as_py())
- else:
- return super(Scalar, self).__repr__()
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
- def __str__(self):
- if hasattr(self, 'as_py'):
- return str(self.as_py())
- else:
- return super(Scalar, self).__str__()
+ def as_py(self):
+ """
+ Return this value as a Python None.
+ """
+ return None
- def __eq__(self, other):
- if hasattr(self, 'as_py'):
- if isinstance(other, ArrayValue):
- other = other.as_py()
- return self.as_py() == other
- else:
- raise NotImplementedError(
- "Cannot compare Arrow values that don't support as_py()")
- def __hash__(self):
- return hash(self.as_py())
+_NULL = NA = NullScalar()
-cdef class BooleanValue(ArrayValue):
+cdef class BooleanScalar(Scalar):
"""
- Concrete class for boolean array elements.
+ Concrete class for boolean scalars.
"""
def as_py(self):
"""
Return this value as a Python bool.
"""
- cdef CBooleanArray* ap = <CBooleanArray*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CBooleanScalar* sp = <CBooleanScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int8Value(ArrayValue):
+cdef class UInt8Scalar(Scalar):
"""
- Concrete class for int8 array elements.
+ Concrete class for uint8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt8Array* ap = <CInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt8Scalar* sp = <CUInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt8Value(ArrayValue):
+cdef class Int8Scalar(Scalar):
"""
- Concrete class for uint8 array elements.
+ Concrete class for int8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt8Array* ap = <CUInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt8Scalar* sp = <CInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int16Value(ArrayValue):
+cdef class UInt16Scalar(Scalar):
"""
- Concrete class for int16 array elements.
+ Concrete class for uint16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt16Array* ap = <CInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt16Scalar* sp = <CUInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt16Value(ArrayValue):
+cdef class Int16Scalar(Scalar):
"""
- Concrete class for uint16 array elements.
+ Concrete class for int16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt16Array* ap = <CUInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt16Scalar* sp = <CInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int32Value(ArrayValue):
+cdef class UInt32Scalar(Scalar):
"""
- Concrete class for int32 array elements.
+ Concrete class for uint32 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt32Array* ap = <CInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt32Scalar* sp = <CUInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt32Value(ArrayValue):
+cdef class Int32Scalar(Scalar):
"""
- Concrete class for uint32 array elements.
+ Concrete class for int32 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt32Array* ap = <CUInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt32Scalar* sp = <CInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int64Value(ArrayValue):
+cdef class UInt64Scalar(Scalar):
"""
- Concrete class for int64 array elements.
+ Concrete class for uint64 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt64Array* ap = <CInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt64Scalar* sp = <CUInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt64Value(ArrayValue):
+cdef class Int64Scalar(Scalar):
"""
- Concrete class for uint64 array elements.
+ Concrete class for int64 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt64Array* ap = <CUInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt64Scalar* sp = <CInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date32Value(ArrayValue):
+cdef class HalfFloatScalar(Scalar):
"""
- Concrete class for date32 array elements.
+ Concrete class for float scalars.
"""
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+
+ def __eq__(self, other):
+ if hasattr(self, 'as_py'):
+ if isinstance(other, Scalar):
+ other = other.as_py()
+ return self.as_py() == other
+ else:
+ raise NotImplementedError
+
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python float.
"""
- cdef CDate32Array* ap = <CDate32Array*> self.sp_array.get()
+ cdef CHalfFloatScalar* sp = <CHalfFloatScalar*> self.wrapped.get()
+ return PyHalf_FromHalf(sp.value) if sp.is_valid else None
- # Shift to seconds since epoch
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(days=ap.Value(self.index)))
+
+cdef class FloatScalar(Scalar):
+ """
+ Concrete class for float scalars.
+ """
+
+ def as_py(self):
+ """
+ Return this value as a Python float.
+ """
+ cdef CFloatScalar* sp = <CFloatScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date64Value(ArrayValue):
+cdef class DoubleScalar(Scalar):
"""
- Concrete class for date64 array elements.
+ Concrete class for double scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python float.
"""
- cdef CDate64Array* ap = <CDate64Array*> self.sp_array.get()
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(
- days=ap.Value(self.index) / 86400000))
+ cdef CDoubleScalar* sp = <CDoubleScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Time32Value(ArrayValue):
+cdef class Decimal128Scalar(Scalar):
"""
- Concrete class for time32 array elements.
+ Concrete class for decimal128 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.timedelta instance.
+ Return this value as a Python Decimal.
"""
cdef:
- CTime32Array* ap = <CTime32Array*> self.sp_array.get()
- CTime32Type* dtype = <CTime32Type*> ap.type().get()
-
- if dtype.unit() == TimeUnit_SECOND:
- delta = datetime.timedelta(seconds=ap.Value(self.index))
- return (datetime.datetime(1970, 1, 1) + delta).time()
+ CDecimal128Scalar* sp = <CDecimal128Scalar*> self.wrapped.get()
+ CDecimal128Type* dtype = <CDecimal128Type*> sp.type.get()
+ if sp.is_valid:
+ return _pydecimal.Decimal(
+ frombytes(sp.value.ToString(dtype.scale()))
+ )
else:
- return _box_time_milli(ap.Value(self.index))
+ return None
-cdef class Time64Value(ArrayValue):
+cdef class Date32Scalar(Scalar):
"""
- Concrete class for time64 array elements.
+ Concrete class for date32 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.timedelta instance.
+ Return this value as a Python datetime.datetime instance.
"""
- cdef:
- CTime64Array* ap = <CTime64Array*> self.sp_array.get()
- CTime64Type* dtype = <CTime64Type*> ap.type().get()
+ cdef CDate32Scalar* sp = <CDate32Scalar*> self.wrapped.get()
- cdef int64_t val = ap.Value(self.index)
- if dtype.unit() == TimeUnit_MICRO:
- return _box_time_micro(val)
+ if sp.is_valid:
+ # shift to seconds since epoch
+ return (
+ datetime.date(1970, 1, 1) + datetime.timedelta(days=sp.value)
+ )
else:
- return (datetime.datetime(1970, 1, 1) +
- datetime.timedelta(microseconds=val / 1000)).time()
-
+ return None
-cpdef _box_time_milli(int64_t val):
- delta = datetime.timedelta(milliseconds=val)
- return (datetime.datetime(1970, 1, 1) + delta).time()
+cdef class Date64Scalar(Scalar):
+ """
+ Concrete class for date64 scalars.
+ """
-cpdef _box_time_micro(int64_t val):
- return (datetime.datetime(1970, 1, 1) +
- datetime.timedelta(microseconds=val)).time()
+ def as_py(self):
+ """
+ Return this value as a Python datetime.datetime instance.
+ """
+ cdef CDate64Scalar* sp = <CDate64Scalar*> self.wrapped.get()
+ if sp.is_valid:
+ return (
+ datetime.date(1970, 1, 1) +
+ datetime.timedelta(days=sp.value / 86400000)
+ )
+ else:
+ return None
-cdef dict _DATETIME_CONVERSION_FUNCTIONS = {}
-cdef c_bool _datetime_conversion_initialized = False
+def _datetime_from_int(int64_t value, TimeUnit unit, tzinfo=None):
+ if unit == TimeUnit_SECOND:
+ delta = datetime.timedelta(seconds=value)
+ elif unit == TimeUnit_MILLI:
+ delta = datetime.timedelta(milliseconds=value)
+ elif unit == TimeUnit_MICRO:
+ delta = datetime.timedelta(microseconds=value)
+ else:
+ # TimeUnit_NANO: prefer pandas timestamps if available
+ if _pandas_api.have_pandas:
+ return _pandas_api.pd.Timestamp(value, tz=tzinfo, unit='ns')
+ # otherwise safely truncate to microsecond resolution datetime
+ if value % 1000 != 0:
+ raise ValueError(
+ "Nanosecond resolution temporal type {} is not safely "
+ "convertible to microseconds to convert to datetime.datetime. "
+ "Install pandas to return as Timestamp with nanosecond "
+ "support or access the .value attribute.".format(value)
+ )
+ delta = datetime.timedelta(microseconds=value // 1000)
-cdef _add_micros_maybe_localize(dt, micros, tzinfo):
- import pytz
- dt = dt.replace(microsecond=micros)
+ dt = datetime.datetime(1970, 1, 1) + delta
+ # adjust timezone if set to the datatype
if tzinfo is not None:
- if not isinstance(tzinfo, datetime.tzinfo):
- tzinfo = string_to_tzinfo(tzinfo)
dt = tzinfo.fromutc(dt)
- return dt
-
-
-cdef _datetime_from_seconds(int64_t v):
- return datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=v)
-
-
-def _nanoseconds_to_datetime_safe(v, tzinfo):
- if v % 1000 != 0:
- raise ValueError("Nanosecond timestamp {} is not safely convertible "
- " to microseconds to convert to datetime.datetime."
- " Install pandas to return as Timestamp with "
- " nanosecond support or access the .value attribute.")
- v = v // 1000
- micros = v % 1_000_000
-
- dt = _datetime_from_seconds(v // 1_000_000)
- return _add_micros_maybe_localize(dt, micros, tzinfo)
-
-
-def _microseconds_to_datetime(v, tzinfo):
- micros = v % 1_000_000
- dt = _datetime_from_seconds(v // 1_000_000)
- return _add_micros_maybe_localize(dt, micros, tzinfo)
+ return dt
-def _millis_to_datetime(v, tzinfo):
- millis = v % 1_000
- dt = _datetime_from_seconds(v // 1000)
- return _add_micros_maybe_localize(dt, millis * 1000, tzinfo)
+cdef class Time32Scalar(Scalar):
+ """
+ Concrete class for time32 scalars.
+ """
-def _seconds_to_datetime(v, tzinfo):
- dt = _datetime_from_seconds(v)
- return _add_micros_maybe_localize(dt, 0, tzinfo)
+ def as_py(self):
+ """
+ Return this value as a Python datetime.timedelta instance.
+ """
+ cdef:
+ CTime32Scalar* sp = <CTime32Scalar*> self.wrapped.get()
+ CTime32Type* dtype = <CTime32Type*> sp.type.get()
+ if sp.is_valid:
+ return _datetime_from_int(sp.value, unit=dtype.unit()).time()
+ else:
+ return None
-def _datetime_conversion_functions():
- global _datetime_conversion_initialized
- if _datetime_conversion_initialized:
- return _DATETIME_CONVERSION_FUNCTIONS
- _DATETIME_CONVERSION_FUNCTIONS.update({
- TimeUnit_SECOND: _seconds_to_datetime,
- TimeUnit_MILLI: _millis_to_datetime,
- TimeUnit_MICRO: _microseconds_to_datetime,
- TimeUnit_NANO: _nanoseconds_to_datetime_safe
- })
+cdef class Time64Scalar(Scalar):
+ """
+ Concrete class for time64 scalars.
+ """
- try:
- import pandas as pd
- _DATETIME_CONVERSION_FUNCTIONS[TimeUnit_NANO] = (
- lambda x, tzinfo: pd.Timestamp(
- x, tz=tzinfo, unit='ns',
- )
- )
- except ImportError:
- pass
+ def as_py(self):
+ """
+ Return this value as a Python datetime.timedelta instance.
+ """
+ cdef:
+ CTime64Scalar* sp = <CTime64Scalar*> self.wrapped.get()
+ CTime64Type* dtype = <CTime64Type*> sp.type.get()
- _datetime_conversion_initialized = True
- return _DATETIME_CONVERSION_FUNCTIONS
+ if sp.is_valid:
+ return _datetime_from_int(sp.value, unit=dtype.unit()).time()
+ else:
+ return None
-cdef class TimestampValue(ArrayValue):
+cdef class TimestampScalar(Scalar):
"""
- Concrete class for timestamp array elements.
+ Concrete class for timestamp scalars.
"""
@property
def value(self):
- cdef CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
- cdef CTimestampType* dtype = <CTimestampType*> ap.type().get()
- return ap.Value(self.index)
+ cdef CTimestampScalar* sp = <CTimestampScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
def as_py(self):
"""
Return this value as a Pandas Timestamp instance (if available),
otherwise as a Python datetime.timedelta instance.
"""
- cdef CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
- cdef CTimestampType* dtype = <CTimestampType*> ap.type().get()
+ cdef:
+ CTimestampScalar* sp = <CTimestampScalar*> self.wrapped.get()
+ CTimestampType* dtype = <CTimestampType*> sp.type.get()
- value = self.value
+ if not sp.is_valid:
+ return None
if not dtype.timezone().empty():
tzinfo = string_to_tzinfo(frombytes(dtype.timezone()))
+ if not isinstance(tzinfo, datetime.tzinfo):
+ tzinfo = string_to_tzinfo(tzinfo)
else:
tzinfo = None
- try:
- converter = _datetime_conversion_functions()[dtype.unit()]
- except KeyError:
- raise ValueError(
- 'Cannot convert nanosecond timestamps without pandas'
- )
- return converter(value, tzinfo=tzinfo)
-
-
-cdef dict _TIMEDELTA_CONVERSION_FUNCTIONS = {}
-
-
-def _nanoseconds_to_timedelta_safe(v):
- if v % 1000 != 0:
- raise ValueError(
- "Nanosecond duration {} is not safely convertible to microseconds "
- "to convert to datetime.timedelta. Install pandas to return as "
- "Timedelta with nanosecond support or access the .value "
- "attribute.".format(v))
- micros = v // 1000
-
- return datetime.timedelta(microseconds=micros)
-
-
-def _timedelta_conversion_functions():
- if _TIMEDELTA_CONVERSION_FUNCTIONS:
- return _TIMEDELTA_CONVERSION_FUNCTIONS
-
- _TIMEDELTA_CONVERSION_FUNCTIONS.update({
- TimeUnit_SECOND: lambda v: datetime.timedelta(seconds=v),
- TimeUnit_MILLI: lambda v: datetime.timedelta(milliseconds=v),
- TimeUnit_MICRO: lambda v: datetime.timedelta(microseconds=v),
- TimeUnit_NANO: _nanoseconds_to_timedelta_safe
- })
-
- try:
- import pandas as pd
- _TIMEDELTA_CONVERSION_FUNCTIONS[TimeUnit_NANO] = (
- lambda v: pd.Timedelta(v, unit='ns')
- )
- except ImportError:
- pass
+ return _datetime_from_int(sp.value, unit=dtype.unit(), tzinfo=tzinfo)
- return _TIMEDELTA_CONVERSION_FUNCTIONS
-
-cdef class DurationValue(ArrayValue):
+cdef class DurationScalar(Scalar):
"""
- Concrete class for duration array elements.
+ Concrete class for duration scalars.
"""
@property
def value(self):
- cdef CDurationArray* ap = <CDurationArray*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CDurationScalar* sp = <CDurationScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
def as_py(self):
"""
Return this value as a Pandas Timestamp instance (if available),
otherwise as a Python datetime.timedelta instance.
"""
- cdef CDurationArray* ap = <CDurationArray*> self.sp_array.get()
- cdef CDurationType* dtype = <CDurationType*> ap.type().get()
+ cdef:
+ CDurationScalar* sp = <CDurationScalar*> self.wrapped.get()
+ CDurationType* dtype = <CDurationType*> sp.type.get()
+ TimeUnit unit = dtype.unit()
+
+ if not sp.is_valid:
+ return None
- cdef int64_t value = ap.Value(self.index)
- converter = _timedelta_conversion_functions()[dtype.unit()]
- return converter(value)
+ if unit == TimeUnit_SECOND:
+ return datetime.timedelta(seconds=sp.value)
+ elif unit == TimeUnit_MILLI:
+ return datetime.timedelta(milliseconds=sp.value)
+ elif unit == TimeUnit_MICRO:
+ return datetime.timedelta(microseconds=sp.value)
+ else:
+ # TimeUnit_NANO: prefer pandas timestamps if available
+ if _pandas_api.have_pandas:
+ return _pandas_api.pd.Timedelta(sp.value, unit='ns')
+ # otherwise safely truncate to microsecond resolution timedelta
+ if sp.value % 1000 != 0:
+ raise ValueError(
+ "Nanosecond duration {} is not safely convertible to "
+ "microseconds to convert to datetime.timedelta. Install "
+ "pandas to return as Timedelta with nanosecond support or "
+ "access the .value attribute.".format(sp.value)
+ )
+ return datetime.timedelta(microseconds=sp.value // 1000)
-cdef class HalfFloatValue(ArrayValue):
+cdef class BinaryScalar(Scalar):
"""
- Concrete class for float16 array elements.
+ Concrete class for binary-like scalars.
"""
- def as_py(self):
+ def as_buffer(self):
"""
- Return this value as a Python float.
+ Return a view over this value as a Buffer object.
"""
- cdef CHalfFloatArray* ap = <CHalfFloatArray*> self.sp_array.get()
- return PyHalf_FromHalf(ap.Value(self.index))
-
-
-cdef class FloatValue(ArrayValue):
- """
- Concrete class for float32 array elements.
- """
+ cdef CBinaryScalar* sp = <CBinaryScalar*> self.wrapped.get()
+ return pyarrow_wrap_buffer(sp.value) if sp.is_valid else None
def as_py(self):
"""
- Return this value as a Python float.
+ Return this value as a Python bytes.
"""
- cdef CFloatArray* ap = <CFloatArray*> self.sp_array.get()
- return ap.Value(self.index)
+ buffer = self.as_buffer()
+ if buffer is not None:
+ return self.as_buffer().to_pybytes()
+ else:
+ return None
-cdef class DoubleValue(ArrayValue):
- """
- Concrete class for float64 array elements.
- """
+cdef class LargeBinaryScalar(BinaryScalar):
+ pass
- def as_py(self):
- """
- Return this value as a Python float.
- """
- cdef CDoubleArray* ap = <CDoubleArray*> self.sp_array.get()
- return ap.Value(self.index)
+
+cdef class FixedSizeBinaryScalar(BinaryScalar):
+ pass
-cdef class DecimalValue(ArrayValue):
+cdef class StringScalar(BinaryScalar):
"""
- Concrete class for decimal128 array elements.
+ Concrete class for string-like (utf8) scalars.
"""
def as_py(self):
"""
- Return this value as a Python Decimal.
+ Return this value as a Python string.
"""
- cdef:
- CDecimal128Array* ap = <CDecimal128Array*> self.sp_array.get()
- c_string s = ap.FormatValue(self.index)
- return _pydecimal.Decimal(s.decode('utf8'))
+ buffer = self.as_buffer()
+ if buffer is not None:
+ return frombytes(self.as_buffer().to_pybytes())
+ else:
+ return None
+
+cdef class LargeStringScalar(StringScalar):
+ pass
-cdef class StringValue(ArrayValue):
+
+cdef class ListScalar(Scalar):
"""
- Concrete class for string (utf8) array elements.
+ Concrete class for list-like scalars.
"""
- def as_py(self):
+ @property
+ def values(self):
+ cdef CListScalar* sp = <CListScalar*> self.wrapped.get()
+ if sp.is_valid:
+ return pyarrow_wrap_array(sp.value)
+ else:
+ return None
+
+ def __len__(self):
"""
- Return this value as a Python unicode string.
+ Return the number of values.
"""
- cdef CStringArray* ap = <CStringArray*> self.sp_array.get()
- return ap.GetString(self.index).decode('utf-8')
+ return len(self.values)
- def as_buffer(self):
+ def __getitem__(self, i):
"""
- Return a view over this value as a Buffer object.
+ Return the value at the given index.
"""
- cdef:
- CStringArray* ap = <CStringArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
-
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
-
+ return self.values[_normalize_index(i, len(self))]
-cdef class LargeStringValue(ArrayValue):
- """
- Concrete class for large string (utf8) array elements.
- """
-
- def as_py(self):
+ def __iter__(self):
"""
- Return this value as a Python unicode string.
+ Iterate over this element's values.
"""
- cdef CLargeStringArray* ap = <CLargeStringArray*> self.sp_array.get()
- return ap.GetString(self.index).decode('utf-8')
+ return iter(self.values)
- def as_buffer(self):
+ def as_py(self):
"""
- Return a view over this value as a Buffer object.
+ Return this value as a Python list.
"""
- cdef:
- CLargeStringArray* ap = <CLargeStringArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
+ arr = self.values
+ return None if arr is None else arr.to_pylist()
+
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
+cdef class FixedSizeListScalar(ListScalar):
+ pass
-cdef class BinaryValue(ArrayValue):
+cdef class LargeListScalar(ListScalar):
+ pass
+
+
+cdef class StructScalar(Scalar, collections.abc.Mapping):
"""
- Concrete class for variable-sized binary array elements.
+ Concrete class for struct scalars.
"""
- def as_py(self):
- """
- Return this value as a Python bytes object.
- """
+ def __len__(self):
+ cdef CStructScalar* sp = <CStructScalar*> self.wrapped.get()
+ return sp.value.size()
+
+ def __iter__(self):
cdef:
- const uint8_t* ptr
- int32_t length
- CBinaryArray* ap = <CBinaryArray*> self.sp_array.get()
+ CStructScalar* sp = <CStructScalar*> self.wrapped.get()
+ CStructType* dtype = <CStructType*> sp.type.get()
+ vector[shared_ptr[CField]] fields = dtype.fields()
- ptr = ap.GetValue(self.index, &length)
- return cp.PyBytes_FromStringAndSize(<const char*>(ptr), length)
+ if sp.is_valid:
+ for i in range(dtype.num_fields()):
+ yield frombytes(fields[i].get().name())
- def as_buffer(self):
+ def __contains__(self, key):
+ try:
+ self[key]
+ except IndexError:
+ return False
+ else:
+ return True
+
+ def __getitem__(self, key):
"""
- Return a view over this value as a Buffer object.
+ Return the child value for the given field.
+
+ Parameters
+ ----------
+ index : Union[int, str]
+ Index / position or name of the field.
+
+ Returns
+ -------
+ result : Scalar
"""
cdef:
- CBinaryArray* ap = <CBinaryArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
+ CFieldRef ref
+ CStructScalar* sp = <CStructScalar*> self.wrapped.get()
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
+ if isinstance(key, (bytes, str)):
+ ref = CFieldRef(<c_string> tobytes(key))
+ elif isinstance(key, int):
+ ref = CFieldRef(<int> key)
+ else:
+ raise TypeError('Expected integer or string index')
-
-cdef class LargeBinaryValue(ArrayValue):
- """
- Concrete class for large variable-sized binary array elements.
- """
-
- def as_py(self):
- """
- Return this value as a Python bytes object.
- """
- cdef:
- const uint8_t* ptr
- int64_t length
- CLargeBinaryArray* ap = <CLargeBinaryArray*> self.sp_array.get()
-
- ptr = ap.GetValue(self.index, &length)
- return cp.PyBytes_FromStringAndSize(<const char*>(ptr), length)
-
- def as_buffer(self):
- """
- Return a view over this value as a Buffer object.
- """
- cdef:
- CLargeBinaryArray* ap = <CLargeBinaryArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
-
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
-
-
-cdef class ListValue(ArrayValue):
- """
- Concrete class for list array elements.
- """
-
- def __len__(self):
- """
- Return the number of values.
- """
- return self.length()
-
- def __getitem__(self, i):
- """
- Return the value at the given index.
- """
- return self.getitem(_normalize_index(i, self.length()))
-
- def __iter__(self):
- """
- Iterate over this element's values.
- """
- for i in range(len(self)):
- yield self.getitem(i)
- raise StopIteration
-
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CListArray*> sp_array.get()
- self.value_type = pyarrow_wrap_data_type(self.ap.value_type())
-
- cdef getitem(self, int64_t i):
- cdef int64_t j = self.ap.value_offset(self.index) + i
- return box_scalar(self.value_type, self.ap.values(), j)
-
- cdef int64_t length(self):
- return self.ap.value_length(self.index)
-
- def as_py(self):
- """
- Return this value as a Python list.
- """
- cdef:
- int64_t j
- list result = []
-
- for j in range(len(self)):
- result.append(self.getitem(j).as_py())
-
- return result
-
-
-cdef class LargeListValue(ArrayValue):
- """
- Concrete class for large list array elements.
- """
-
- def __len__(self):
- """
- Return the number of values.
- """
- return self.length()
-
- def __getitem__(self, i):
- """
- Return the value at the given index.
- """
- return self.getitem(_normalize_index(i, self.length()))
-
- def __iter__(self):
- """
- Iterate over this element's values.
- """
- for i in range(len(self)):
- yield self.getitem(i)
- raise StopIteration
-
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CLargeListArray*> sp_array.get()
- self.value_type = pyarrow_wrap_data_type(self.ap.value_type())
-
- cdef getitem(self, int64_t i):
- cdef int64_t j = self.ap.value_offset(self.index) + i
- return box_scalar(self.value_type, self.ap.values(), j)
-
- cdef int64_t length(self):
- return self.ap.value_length(self.index)
-
- def as_py(self):
- """
- Return this value as a Python list.
- """
- cdef:
- int64_t j
- list result = []
-
- for j in range(len(self)):
- result.append(self.getitem(j).as_py())
-
- return result
-
-
-cdef class MapValue(ArrayValue):
- """
- Concrete class for map array elements.
- """
-
- def __len__(self):
- """
- Return the number of values.
- """
- return self.length()
-
- def __getitem__(self, i):
- """
- Return the value at the given index.
- """
- return self.getitem(_normalize_index(i, self.length()))
-
- def __iter__(self):
- """
- Iterate over this element's values.
- """
- for i in range(len(self)):
- yield self.getitem(i)
- raise StopIteration
-
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CMapArray*> sp_array.get()
- self.key_type = pyarrow_wrap_data_type(self.ap.map_type().key_type())
- self.item_type = pyarrow_wrap_data_type(self.ap.map_type().item_type())
-
- cdef getitem(self, int64_t i):
- cdef int64_t j = self.ap.value_offset(self.index) + i
- return (box_scalar(self.key_type, self.ap.keys(), j),
- box_scalar(self.item_type, self.ap.items(), j))
-
- cdef int64_t length(self):
- return self.ap.value_length(self.index)
+ try:
+ return Scalar.wrap(GetResultValue(sp.field(ref)))
+ except ArrowInvalid:
+ raise IndexError(key)
def as_py(self):
"""
- Return this value as a Python list of tuples, each containing a
- key and item.
+ Return this value as a Python dict.
"""
- cdef:
- int64_t j
- list result = []
-
- for j in range(len(self)):
- key, item = self.getitem(j)
- result.append((key.as_py(), item.as_py()))
-
- return result
+ if self.is_valid:
+ return {k: v.as_py() for k, v in self.items()}
+ else:
+ return None
-cdef class FixedSizeListValue(ArrayValue):
+cdef class MapScalar(ListScalar):
"""
- Concrete class for fixed size list array elements.
+ Concrete class for map scalars.
"""
- def __len__(self):
- """
- Return the number of values.
- """
- return self.length()
-
def __getitem__(self, i):
"""
Return the value at the given index.
"""
- return self.getitem(_normalize_index(i, self.length()))
+ arr = self.values
+ if arr is None:
+ raise IndexError(i)
+ dct = arr[_normalize_index(i, len(arr))]
+ return (dct['key'], dct['value'])
def __iter__(self):
"""
Iterate over this element's values.
"""
- for i in range(len(self)):
- yield self.getitem(i)
- raise StopIteration
-
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CFixedSizeListArray*> sp_array.get()
- self.value_type = pyarrow_wrap_data_type(self.ap.value_type())
-
- cdef getitem(self, int64_t i):
- cdef int64_t j = self.ap.value_offset(self.index) + i
- return box_scalar(self.value_type, self.ap.values(), j)
-
- cdef int64_t length(self):
- return self.ap.value_length(self.index)
+ arr = self.values
+ if arr is None:
+ return iter(zip(arr.field('key'), arr.field('value')))
+ else:
+ raise StopIteration
def as_py(self):
"""
Return this value as a Python list.
"""
- cdef:
- int64_t j
- list result = []
-
- for j in range(len(self)):
- result.append(self.getitem(j).as_py())
-
- return result
-
-
-cdef class UnionValue(ArrayValue):
- """
- Concrete class for union array elements.
- """
-
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CUnionArray*> sp_array.get()
-
- cdef getitem(self, int64_t i):
- cdef int child_id = self.ap.child_id(i)
- cdef shared_ptr[CArray] child = self.ap.field(child_id)
- cdef CDenseUnionArray* dense
- if self.ap.mode() == _UnionMode_SPARSE:
- return box_scalar(self.type[child_id].type, child, i)
+ arr = self.values
+ if arr is not None:
+ return list(zip(arr.field('key'), arr.field('value')))
else:
- dense = <CDenseUnionArray*> self.ap
- return box_scalar(self.type[child_id].type, child,
- dense.value_offset(i))
-
- def as_py(self):
- """
- Return this value as a Python object.
-
- The exact type depends on the underlying union member.
- """
- return self.getitem(self.index).as_py()
+ return None
-cdef class FixedSizeBinaryValue(ArrayValue):
- """
- Concrete class for fixed-size binary array elements.
- """
-
- def as_py(self):
- """
- Return this value as a Python bytes object.
- """
- cdef:
- CFixedSizeBinaryArray* ap
- CFixedSizeBinaryType* ap_type
- int32_t length
- const char* data
- ap = <CFixedSizeBinaryArray*> self.sp_array.get()
- ap_type = <CFixedSizeBinaryType*> ap.type().get()
- length = ap_type.byte_width()
- data = <const char*> ap.GetValue(self.index)
- return cp.PyBytes_FromStringAndSize(data, length)
-
-
-cdef class StructValue(ArrayValue):
+cdef class DictionaryScalar(Scalar):
"""
- Concrete class for struct array elements.
+ Concrete class for dictionary-encoded scalars.
"""
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CStructArray*> sp_array.get()
-
- def __getitem__(self, key):
- """
- Return the child value for the given field name.
- """
- cdef:
- CStructType* type
- int index
-
- type = <CStructType*> self.type.type
- index = type.GetFieldIndex(tobytes(key))
-
- if index < 0:
- raise KeyError(key)
+ # @property
+ # def index(self):
+ # """
+ # Return this value's underlying index as a scalar.
+ # """
+ # cdef CDictionaryScalar* sp = <CDictionaryScalar*> self.wrapped.get()
+ # return Scalar.wrap(sp.index)
- return pyarrow_wrap_array(self.ap.field(index))[self.index]
-
- def as_py(self):
+ @property
+ def value(self):
"""
- Return this value as a Python dict.
+ Return this value's underlying dictionary value as a scalar.
"""
- cdef:
- vector[shared_ptr[CField]] child_fields = self.type.type.fields()
-
- wrapped_arrays = [pyarrow_wrap_array(self.ap.field(i))
- for i in range(self.ap.num_fields())]
- child_names = [child.get().name() for child in child_fields]
- # Return the struct as a dict
- return {
- frombytes(name): child_array[self.index].as_py()
- for name, child_array in zip(child_names, wrapped_arrays)
- }
-
-
-cdef class DictionaryValue(ArrayValue):
- """
- Concrete class for dictionary-encoded array elements.
- """
+ cdef CDictionaryScalar* sp = <CDictionaryScalar*> self.wrapped.get()
+ return Scalar.wrap(sp.value)
def as_py(self):
"""
Return this value as a Python object.
The exact type depends on the dictionary value type.
"""
- return self.dictionary_value.as_py()
+ value = self.value
+ return None if value is None else value.as_py()
- @property
- def index_value(self):
- """
- Return this value's underlying index as a ArrayValue of the right
- signed integer type.
- """
- cdef CDictionaryArray* darr = <CDictionaryArray*>(self.sp_array.get())
- indices = pyarrow_wrap_array(darr.indices())
- return indices[self.index]
+ # TODO(kszucs): deprecate these
+ # @property
+ # def index_value(self):
+ # index = self.index
+ # return None if index is None else self.index
@property
def dictionary_value(self):
- """
- Return this value's underlying dictionary value as a ArrayValue.
- """
- cdef CDictionaryArray* darr = <CDictionaryArray*>(self.sp_array.get())
- dictionary = pyarrow_wrap_array(darr.dictionary())
- return dictionary[self.index_value.as_py()]
-
-
-cdef dict _array_value_classes = {
- _Type_BOOL: BooleanValue,
- _Type_UINT8: UInt8Value,
- _Type_UINT16: UInt16Value,
- _Type_UINT32: UInt32Value,
- _Type_UINT64: UInt64Value,
- _Type_INT8: Int8Value,
- _Type_INT16: Int16Value,
- _Type_INT32: Int32Value,
- _Type_INT64: Int64Value,
- _Type_DATE32: Date32Value,
- _Type_DATE64: Date64Value,
- _Type_TIME32: Time32Value,
- _Type_TIME64: Time64Value,
- _Type_TIMESTAMP: TimestampValue,
- _Type_DURATION: DurationValue,
- _Type_HALF_FLOAT: HalfFloatValue,
- _Type_FLOAT: FloatValue,
- _Type_DOUBLE: DoubleValue,
- _Type_LIST: ListValue,
- _Type_LARGE_LIST: LargeListValue,
- _Type_MAP: MapValue,
- _Type_FIXED_SIZE_LIST: FixedSizeListValue,
- _Type_SPARSE_UNION: UnionValue,
- _Type_DENSE_UNION: UnionValue,
- _Type_BINARY: BinaryValue,
- _Type_STRING: StringValue,
- _Type_LARGE_BINARY: LargeBinaryValue,
- _Type_LARGE_STRING: LargeStringValue,
- _Type_FIXED_SIZE_BINARY: FixedSizeBinaryValue,
- _Type_DECIMAL: DecimalValue,
- _Type_STRUCT: StructValue,
- _Type_DICTIONARY: DictionaryValue,
-}
-
-cdef class ScalarValue(Scalar):
- """
- The base class for scalars.
- """
-
- def __init__(self):
- raise TypeError("Do not call {}'s constructor directly."
- .format(self.__class__.__name__))
-
- cdef void init(self, const shared_ptr[CScalar]& sp_scalar):
- self.sp_scalar = sp_scalar
-
- def __repr__(self):
- if hasattr(self, 'as_py'):
- return repr(self.as_py())
- else:
- return super(Scalar, self).__repr__()
-
- def __str__(self):
- if hasattr(self, 'as_py'):
- return str(self.as_py())
- else:
- return super(Scalar, self).__str__()
-
- def __eq__(self, other):
- if hasattr(self, 'as_py'):
- if isinstance(other, ScalarValue):
- other = other.as_py()
- return self.as_py() == other
- else:
- raise NotImplemented(
- "Cannot compare scalars that don't support as_py()")
-
- def __hash__(self):
- return hash(self.as_py())
-
-
-cdef class NullScalar(ScalarValue):
- """
- Concrete class for null scalars.
- """
-
- def as_py(self):
- """
- Return this value as a Python None.
- """
- return None
-
-
-cdef class BooleanScalar(ScalarValue):
- """
- Concrete class for boolean scalars.
- """
-
- def as_py(self):
- """
- Return this value as a Python bool.
- """
- cdef CBooleanScalar* sp = <CBooleanScalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
-
-
-cdef class UInt8Scalar(ScalarValue):
- """
- Concrete class for uint8 scalars.
- """
-
- def as_py(self):
- """
- Return this value as a Python int.
- """
- cdef CUInt8Scalar* sp = <CUInt8Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
-
-
-cdef class Int8Scalar(ScalarValue):
- """
- Concrete class for int8 scalars.
- """
-
- def as_py(self):
- """
- Return this value as a Python int.
- """
- cdef CInt8Scalar* sp = <CInt8Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
-
-
-cdef class UInt16Scalar(ScalarValue):
- """
- Concrete class for uint16 scalars.
- """
-
- def as_py(self):
- """
- Return this value as a Python int.
- """
- cdef CUInt16Scalar* sp = <CUInt16Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
-
-
-cdef class Int16Scalar(ScalarValue):
- """
- Concrete class for int16 scalars.
- """
-
- def as_py(self):
- """
- Return this value as a Python int.
- """
- cdef CInt16Scalar* sp = <CInt16Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
-
-
-cdef class UInt32Scalar(ScalarValue):
- """
- Concrete class for uint32 scalars.
- """
-
- def as_py(self):
- """
- Return this value as a Python int.
- """
- cdef CUInt32Scalar* sp = <CUInt32Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
-
-
-cdef class Int32Scalar(ScalarValue):
- """
- Concrete class for int32 scalars.
- """
-
- def as_py(self):
- """
- Return this value as a Python int.
- """
- cdef CInt32Scalar* sp = <CInt32Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
-
-
-cdef class UInt64Scalar(ScalarValue):
- """
- Concrete class for uint64 scalars.
- """
-
- def as_py(self):
- """
- Return this value as a Python int.
- """
- cdef CUInt64Scalar* sp = <CUInt64Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
-
-
-cdef class Int64Scalar(ScalarValue):
- """
- Concrete class for int64 scalars.
- """
-
- def as_py(self):
- """
- Return this value as a Python int.
- """
- cdef CInt64Scalar* sp = <CInt64Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
+ value = self.value
+ return None if value is None else self.value
-cdef class FloatScalar(ScalarValue):
+cdef class UnionScalar(Scalar):
"""
- Concrete class for float scalars.
+ Concrete class for Union scalars.
"""
- def as_py(self):
+ @property
+ def value(self):
"""
- Return this value as a Python float.
+ Return this value's underlying dictionary value as a scalar.
"""
- cdef CFloatScalar* sp = <CFloatScalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
-
-
-cdef class DoubleScalar(ScalarValue):
- """
- Concrete class for double scalars.
- """
+ cdef CDictionaryScalar* sp = <CDictionaryScalar*> self.wrapped.get()
+ return Scalar.wrap(sp.value)
def as_py(self):
"""
- Return this value as a Python float.
- """
- cdef CDoubleScalar* sp = <CDoubleScalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
-
-
-cdef class StringScalar(ScalarValue):
- """
- Concrete class for string scalars.
- """
+ Return this value as a Python object.
- def as_py(self):
- """
- Return this value as a Python string.
+ The exact type depends on the dictionary value type.
Review comment:
Not "dictionary"
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] pitrou commented on pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
pitrou commented on pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#issuecomment-652256345
Yes, we could. That may have other annoying implications, though (such as `__contains__` not working anymore). I've started a ML discussion.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] jorisvandenbossche commented on a change in pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
jorisvandenbossche commented on a change in pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#discussion_r445569265
##########
File path: python/pyarrow/scalar.pxi
##########
@@ -16,1198 +16,704 @@
# under the License.
-_NULL = NA = None
-
-
cdef class Scalar:
"""
- The base class for all array elements.
+ The base class for scalars.
"""
+ def __init__(self):
+ raise TypeError("Do not call {}'s constructor directly, use "
+ "pa.scalar() instead.".format(self.__class__.__name__))
-cdef class NullType(Scalar):
- """
- Singleton for null array elements.
- """
- # TODO rename this NullValue?
+ cdef void init(self, const shared_ptr[CScalar]& wrapped):
+ self.wrapped = wrapped
- def __cinit__(self):
- global NA
- if NA is not None:
- raise Exception('Cannot create multiple NAType instances')
+ @staticmethod
+ cdef wrap(const shared_ptr[CScalar]& wrapped):
+ cdef:
+ Scalar self
+ Type type_id = wrapped.get().type.get().id()
+
+ if type_id == _Type_NA:
+ return _NULL
+
+ typ = _scalar_classes[type_id]
+ self = typ.__new__(typ)
+ self.init(wrapped)
+
+ return self
+
+ cdef inline shared_ptr[CScalar] unwrap(self) nogil:
+ return self.wrapped
- self.type = null()
+ @property
+ def type(self):
+ return pyarrow_wrap_data_type(self.wrapped.get().type)
def __repr__(self):
- return 'NULL'
+ return '<pyarrow.{}: {!r}>'.format(
+ self.__class__.__name__, self.as_py()
+ )
- def as_py(self):
- """
- Return None
- """
- return None
+ def __str__(self):
+ return str(self.as_py())
def __eq__(self, other):
- return NA
+ # TODO(kszucs): use c++ Equals
+ if isinstance(other, Scalar):
+ other = other.as_py()
+ return self.as_py() == other
+ def __hash__(self):
+ # TODO(kszucs): use C++ hash if implemented for the type
+ return hash(self.as_py())
+
+ def as_py(self):
+ raise NotImplementedError()
-_NULL = NA = NullType()
+
+_NULL = NA = None
-cdef class ArrayValue(Scalar):
+cdef class NullScalar(Scalar):
"""
- The base class for non-null array elements.
+ Concrete class for null scalars.
"""
- def __init__(self):
- raise TypeError("Do not call {}'s constructor directly, use array "
- "subscription instead."
- .format(self.__class__.__name__))
-
- cdef void init(self, DataType type, const shared_ptr[CArray]& sp_array,
- int64_t index):
- self.type = type
- self.index = index
- self._set_array(sp_array)
+ def __cinit__(self):
+ global NA
+ if NA is not None:
+ raise Exception('Cannot create multiple NAType instances')
+ self.init(shared_ptr[CScalar](new CNullScalar()))
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
+ def __init__(self):
+ pass
- def __repr__(self):
- if hasattr(self, 'as_py'):
- return repr(self.as_py())
- else:
- return super(Scalar, self).__repr__()
+ def __eq__(self, other):
+ return NA
- def __str__(self):
- if hasattr(self, 'as_py'):
- return str(self.as_py())
- else:
- return super(Scalar, self).__str__()
+ def as_py(self):
+ """
+ Return this value as a Python None.
+ """
+ return None
- def __eq__(self, other):
- if hasattr(self, 'as_py'):
- if isinstance(other, ArrayValue):
- other = other.as_py()
- return self.as_py() == other
- else:
- raise NotImplementedError(
- "Cannot compare Arrow values that don't support as_py()")
- def __hash__(self):
- return hash(self.as_py())
+_NULL = NA = NullScalar()
-cdef class BooleanValue(ArrayValue):
+cdef class BooleanScalar(Scalar):
"""
- Concrete class for boolean array elements.
+ Concrete class for boolean scalars.
"""
def as_py(self):
"""
Return this value as a Python bool.
"""
- cdef CBooleanArray* ap = <CBooleanArray*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CBooleanScalar* sp = <CBooleanScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int8Value(ArrayValue):
+cdef class UInt8Scalar(Scalar):
"""
- Concrete class for int8 array elements.
+ Concrete class for uint8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt8Array* ap = <CInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt8Scalar* sp = <CUInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt8Value(ArrayValue):
+cdef class Int8Scalar(Scalar):
"""
- Concrete class for uint8 array elements.
+ Concrete class for int8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt8Array* ap = <CUInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt8Scalar* sp = <CInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int16Value(ArrayValue):
+cdef class UInt16Scalar(Scalar):
"""
- Concrete class for int16 array elements.
+ Concrete class for uint16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt16Array* ap = <CInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt16Scalar* sp = <CUInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt16Value(ArrayValue):
+cdef class Int16Scalar(Scalar):
"""
- Concrete class for uint16 array elements.
+ Concrete class for int16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt16Array* ap = <CUInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt16Scalar* sp = <CInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int32Value(ArrayValue):
+cdef class UInt32Scalar(Scalar):
"""
- Concrete class for int32 array elements.
+ Concrete class for uint32 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt32Array* ap = <CInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt32Scalar* sp = <CUInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt32Value(ArrayValue):
+cdef class Int32Scalar(Scalar):
"""
- Concrete class for uint32 array elements.
+ Concrete class for int32 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt32Array* ap = <CUInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt32Scalar* sp = <CInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int64Value(ArrayValue):
+cdef class UInt64Scalar(Scalar):
"""
- Concrete class for int64 array elements.
+ Concrete class for uint64 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt64Array* ap = <CInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt64Scalar* sp = <CUInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt64Value(ArrayValue):
+cdef class Int64Scalar(Scalar):
"""
- Concrete class for uint64 array elements.
+ Concrete class for int64 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt64Array* ap = <CUInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt64Scalar* sp = <CInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date32Value(ArrayValue):
+cdef class HalfFloatScalar(Scalar):
"""
- Concrete class for date32 array elements.
+ Concrete class for float scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python float.
"""
- cdef CDate32Array* ap = <CDate32Array*> self.sp_array.get()
+ cdef CHalfFloatScalar* sp = <CHalfFloatScalar*> self.wrapped.get()
+ return PyHalf_FromHalf(sp.value) if sp.is_valid else None
+
+
+cdef class FloatScalar(Scalar):
+ """
+ Concrete class for float scalars.
+ """
- # Shift to seconds since epoch
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(days=ap.Value(self.index)))
+ def as_py(self):
+ """
+ Return this value as a Python float.
+ """
+ cdef CFloatScalar* sp = <CFloatScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date64Value(ArrayValue):
+cdef class DoubleScalar(Scalar):
"""
- Concrete class for date64 array elements.
+ Concrete class for double scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python float.
"""
- cdef CDate64Array* ap = <CDate64Array*> self.sp_array.get()
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(
- days=ap.Value(self.index) / 86400000))
+ cdef CDoubleScalar* sp = <CDoubleScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Time32Value(ArrayValue):
+cdef class DecimalScalar(Scalar):
"""
- Concrete class for time32 array elements.
+ Concrete class for decimal128 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.timedelta instance.
+ Return this value as a Python Decimal.
"""
cdef:
- CTime32Array* ap = <CTime32Array*> self.sp_array.get()
- CTime32Type* dtype = <CTime32Type*> ap.type().get()
-
- if dtype.unit() == TimeUnit_SECOND:
- delta = datetime.timedelta(seconds=ap.Value(self.index))
- return (datetime.datetime(1970, 1, 1) + delta).time()
+ CDecimal128Scalar* sp = <CDecimal128Scalar*> self.wrapped.get()
+ CDecimal128Type* dtype = <CDecimal128Type*> sp.type.get()
+ if sp.is_valid:
+ return _pydecimal.Decimal(
+ frombytes(sp.value.ToString(dtype.scale()))
+ )
else:
- return _box_time_milli(ap.Value(self.index))
+ return None
-cdef class Time64Value(ArrayValue):
+cdef class Date32Scalar(Scalar):
"""
- Concrete class for time64 array elements.
+ Concrete class for date32 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.timedelta instance.
+ Return this value as a Python datetime.datetime instance.
"""
- cdef:
- CTime64Array* ap = <CTime64Array*> self.sp_array.get()
- CTime64Type* dtype = <CTime64Type*> ap.type().get()
+ cdef CDate32Scalar* sp = <CDate32Scalar*> self.wrapped.get()
- cdef int64_t val = ap.Value(self.index)
- if dtype.unit() == TimeUnit_MICRO:
- return _box_time_micro(val)
+ if sp.is_valid:
+ # shift to seconds since epoch
+ return (
+ datetime.date(1970, 1, 1) + datetime.timedelta(days=sp.value)
+ )
else:
- return (datetime.datetime(1970, 1, 1) +
- datetime.timedelta(microseconds=val / 1000)).time()
-
+ return None
-cpdef _box_time_milli(int64_t val):
- delta = datetime.timedelta(milliseconds=val)
- return (datetime.datetime(1970, 1, 1) + delta).time()
+cdef class Date64Scalar(Scalar):
+ """
+ Concrete class for date64 scalars.
+ """
-cpdef _box_time_micro(int64_t val):
- return (datetime.datetime(1970, 1, 1) +
- datetime.timedelta(microseconds=val)).time()
+ def as_py(self):
+ """
+ Return this value as a Python datetime.datetime instance.
+ """
+ cdef CDate64Scalar* sp = <CDate64Scalar*> self.wrapped.get()
+ if sp.is_valid:
+ return (
+ datetime.date(1970, 1, 1) +
+ datetime.timedelta(days=sp.value / 86400000)
+ )
+ else:
+ return None
-cdef dict _DATETIME_CONVERSION_FUNCTIONS = {}
-cdef c_bool _datetime_conversion_initialized = False
+def _datetime_from_int(int64_t value, TimeUnit unit, tzinfo=None):
+ if unit == TimeUnit_SECOND:
+ delta = datetime.timedelta(seconds=value)
+ elif unit == TimeUnit_MILLI:
+ delta = datetime.timedelta(milliseconds=value)
+ elif unit == TimeUnit_MICRO:
+ delta = datetime.timedelta(microseconds=value)
+ else:
+ # TimeUnit_NANO: prefer pandas timestamps if available
+ if _pandas_api.have_pandas:
+ return _pandas_api.pd.Timestamp(value, tz=tzinfo, unit='ns')
+ # otherwise safely truncate to microsecond resolution datetime
+ if value % 1000 != 0:
+ raise ValueError(
+ "Nanosecond resolution temporal type {} is not safely "
+ "convertible to microseconds to convert to datetime.datetime. "
+ "Install pandas to return as Timestamp with nanosecond "
+ "support or access the .value attribute.".format(value)
+ )
+ delta = datetime.timedelta(microseconds=value)
-cdef _add_micros_maybe_localize(dt, micros, tzinfo):
- import pytz
- dt = dt.replace(microsecond=micros)
+ dt = datetime.datetime(1970, 1, 1) + delta
+ # adjust timezone if set to the datatype
if tzinfo is not None:
- if not isinstance(tzinfo, datetime.tzinfo):
- tzinfo = string_to_tzinfo(tzinfo)
dt = tzinfo.fromutc(dt)
- return dt
-
-
-cdef _datetime_from_seconds(int64_t v):
- return datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=v)
-
-
-def _nanoseconds_to_datetime_safe(v, tzinfo):
- if v % 1000 != 0:
- raise ValueError("Nanosecond timestamp {} is not safely convertible "
- " to microseconds to convert to datetime.datetime."
- " Install pandas to return as Timestamp with "
- " nanosecond support or access the .value attribute.")
- v = v // 1000
- micros = v % 1_000_000
-
- dt = _datetime_from_seconds(v // 1_000_000)
- return _add_micros_maybe_localize(dt, micros, tzinfo)
-
-
-def _microseconds_to_datetime(v, tzinfo):
- micros = v % 1_000_000
- dt = _datetime_from_seconds(v // 1_000_000)
- return _add_micros_maybe_localize(dt, micros, tzinfo)
+ return dt
-def _millis_to_datetime(v, tzinfo):
- millis = v % 1_000
- dt = _datetime_from_seconds(v // 1000)
- return _add_micros_maybe_localize(dt, millis * 1000, tzinfo)
+cdef class Time32Scalar(Scalar):
+ """
+ Concrete class for time32 scalars.
+ """
-def _seconds_to_datetime(v, tzinfo):
- dt = _datetime_from_seconds(v)
- return _add_micros_maybe_localize(dt, 0, tzinfo)
+ def as_py(self):
+ """
+ Return this value as a Python datetime.timedelta instance.
+ """
+ cdef:
+ CTime32Scalar* sp = <CTime32Scalar*> self.wrapped.get()
+ CTime32Type* dtype = <CTime32Type*> sp.type.get()
+ if sp.is_valid:
+ return _datetime_from_int(sp.value, unit=dtype.unit()).time()
+ else:
+ return None
-def _datetime_conversion_functions():
- global _datetime_conversion_initialized
- if _datetime_conversion_initialized:
- return _DATETIME_CONVERSION_FUNCTIONS
- _DATETIME_CONVERSION_FUNCTIONS.update({
- TimeUnit_SECOND: _seconds_to_datetime,
- TimeUnit_MILLI: _millis_to_datetime,
- TimeUnit_MICRO: _microseconds_to_datetime,
- TimeUnit_NANO: _nanoseconds_to_datetime_safe
- })
+cdef class Time64Scalar(Scalar):
+ """
+ Concrete class for time64 scalars.
+ """
- try:
- import pandas as pd
- _DATETIME_CONVERSION_FUNCTIONS[TimeUnit_NANO] = (
- lambda x, tzinfo: pd.Timestamp(
- x, tz=tzinfo, unit='ns',
- )
- )
- except ImportError:
- pass
+ def as_py(self):
+ """
+ Return this value as a Python datetime.timedelta instance.
+ """
+ cdef:
+ CTime64Scalar* sp = <CTime64Scalar*> self.wrapped.get()
+ CTime64Type* dtype = <CTime64Type*> sp.type.get()
- _datetime_conversion_initialized = True
- return _DATETIME_CONVERSION_FUNCTIONS
+ if sp.is_valid:
+ return _datetime_from_int(sp.value, unit=dtype.unit()).time()
+ else:
+ return None
-cdef class TimestampValue(ArrayValue):
+cdef class TimestampScalar(Scalar):
"""
- Concrete class for timestamp array elements.
+ Concrete class for timestamp scalars.
"""
@property
def value(self):
- cdef CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
- cdef CTimestampType* dtype = <CTimestampType*> ap.type().get()
- return ap.Value(self.index)
+ cdef CTimestampScalar* sp = <CTimestampScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
def as_py(self):
"""
Return this value as a Pandas Timestamp instance (if available),
otherwise as a Python datetime.timedelta instance.
"""
- cdef CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
- cdef CTimestampType* dtype = <CTimestampType*> ap.type().get()
+ cdef:
+ CTimestampScalar* sp = <CTimestampScalar*> self.wrapped.get()
+ CTimestampType* dtype = <CTimestampType*> sp.type.get()
- value = self.value
+ if not sp.is_valid:
+ return None
if not dtype.timezone().empty():
tzinfo = string_to_tzinfo(frombytes(dtype.timezone()))
+ if not isinstance(tzinfo, datetime.tzinfo):
+ tzinfo = string_to_tzinfo(tzinfo)
else:
tzinfo = None
- try:
- converter = _datetime_conversion_functions()[dtype.unit()]
- except KeyError:
- raise ValueError(
- 'Cannot convert nanosecond timestamps without pandas'
- )
- return converter(value, tzinfo=tzinfo)
-
-
-cdef dict _TIMEDELTA_CONVERSION_FUNCTIONS = {}
-
-
-def _nanoseconds_to_timedelta_safe(v):
- if v % 1000 != 0:
- raise ValueError(
- "Nanosecond duration {} is not safely convertible to microseconds "
- "to convert to datetime.timedelta. Install pandas to return as "
- "Timedelta with nanosecond support or access the .value "
- "attribute.".format(v))
- micros = v // 1000
-
- return datetime.timedelta(microseconds=micros)
-
+ return _datetime_from_int(sp.value, unit=dtype.unit(), tzinfo=tzinfo)
-def _timedelta_conversion_functions():
- if _TIMEDELTA_CONVERSION_FUNCTIONS:
- return _TIMEDELTA_CONVERSION_FUNCTIONS
- _TIMEDELTA_CONVERSION_FUNCTIONS.update({
- TimeUnit_SECOND: lambda v: datetime.timedelta(seconds=v),
- TimeUnit_MILLI: lambda v: datetime.timedelta(milliseconds=v),
- TimeUnit_MICRO: lambda v: datetime.timedelta(microseconds=v),
- TimeUnit_NANO: _nanoseconds_to_timedelta_safe
- })
-
- try:
- import pandas as pd
- _TIMEDELTA_CONVERSION_FUNCTIONS[TimeUnit_NANO] = (
- lambda v: pd.Timedelta(v, unit='ns')
- )
- except ImportError:
- pass
-
- return _TIMEDELTA_CONVERSION_FUNCTIONS
-
-
-cdef class DurationValue(ArrayValue):
+cdef class DurationScalar(Scalar):
"""
- Concrete class for duration array elements.
+ Concrete class for duration scalars.
"""
@property
def value(self):
- cdef CDurationArray* ap = <CDurationArray*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CDurationScalar* sp = <CDurationScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
def as_py(self):
"""
Return this value as a Pandas Timestamp instance (if available),
otherwise as a Python datetime.timedelta instance.
"""
- cdef CDurationArray* ap = <CDurationArray*> self.sp_array.get()
- cdef CDurationType* dtype = <CDurationType*> ap.type().get()
-
- cdef int64_t value = ap.Value(self.index)
- converter = _timedelta_conversion_functions()[dtype.unit()]
- return converter(value)
-
+ cdef:
+ CDurationScalar* sp = <CDurationScalar*> self.wrapped.get()
+ CDurationType* dtype = <CDurationType*> sp.type.get()
+ TimeUnit unit = dtype.unit()
-cdef class HalfFloatValue(ArrayValue):
- """
- Concrete class for float16 array elements.
- """
+ if not sp.is_valid:
+ return None
- def as_py(self):
- """
- Return this value as a Python float.
- """
- cdef CHalfFloatArray* ap = <CHalfFloatArray*> self.sp_array.get()
- return PyHalf_FromHalf(ap.Value(self.index))
+ if unit == TimeUnit_SECOND:
+ return datetime.timedelta(seconds=sp.value)
+ elif unit == TimeUnit_MILLI:
+ return datetime.timedelta(milliseconds=sp.value)
+ elif unit == TimeUnit_MICRO:
+ return datetime.timedelta(microseconds=sp.value)
+ else:
+ # TimeUnit_NANO: prefer pandas timestamps if available
+ if _pandas_api.have_pandas:
+ return _pandas_api.pd.Timedelta(sp.value, unit='ns')
+ # otherwise safely truncate to microsecond resolution timedelta
+ if sp.value % 1000 != 0:
+ raise ValueError(
+ "Nanosecond duration {} is not safely convertible to "
+ "microseconds to convert to datetime.timedelta. Install "
+ "pandas to return as Timedelta with nanosecond support or "
+ "access the .value attribute.".format(sp.value)
+ )
+ return datetime.timedelta(microseconds=sp.value // 1000)
-cdef class FloatValue(ArrayValue):
+cdef class BinaryScalar(Scalar):
"""
- Concrete class for float32 array elements.
+ Concrete class for binary-like scalars.
"""
- def as_py(self):
+ def as_buffer(self):
"""
- Return this value as a Python float.
+ Return a view over this value as a Buffer object.
"""
- cdef CFloatArray* ap = <CFloatArray*> self.sp_array.get()
- return ap.Value(self.index)
-
-
-cdef class DoubleValue(ArrayValue):
- """
- Concrete class for float64 array elements.
- """
+ cdef CBinaryScalar* sp = <CBinaryScalar*> self.wrapped.get()
+ return pyarrow_wrap_buffer(sp.value) if sp.is_valid else None
def as_py(self):
"""
- Return this value as a Python float.
+ Return this value as a Python bytes.
"""
- cdef CDoubleArray* ap = <CDoubleArray*> self.sp_array.get()
- return ap.Value(self.index)
+ buffer = self.as_buffer()
+ if buffer is not None:
+ return self.as_buffer().to_pybytes()
+ else:
+ return None
-cdef class DecimalValue(ArrayValue):
- """
- Concrete class for decimal128 array elements.
- """
+cdef class LargeBinaryScalar(BinaryScalar):
+ pass
- def as_py(self):
- """
- Return this value as a Python Decimal.
- """
- cdef:
- CDecimal128Array* ap = <CDecimal128Array*> self.sp_array.get()
- c_string s = ap.FormatValue(self.index)
- return _pydecimal.Decimal(s.decode('utf8'))
+cdef class FixedSizeBinaryScalar(BinaryScalar):
+ pass
-cdef class StringValue(ArrayValue):
+
+cdef class StringScalar(BinaryScalar):
"""
- Concrete class for string (utf8) array elements.
+ Concrete class for string-like (utf8) scalars.
"""
def as_py(self):
"""
- Return this value as a Python unicode string.
+ Return this value as a Python string.
"""
- cdef CStringArray* ap = <CStringArray*> self.sp_array.get()
- return ap.GetString(self.index).decode('utf-8')
+ buffer = self.as_buffer()
+ if buffer is not None:
+ return frombytes(self.as_buffer().to_pybytes())
+ else:
+ return None
- def as_buffer(self):
- """
- Return a view over this value as a Buffer object.
- """
- cdef:
- CStringArray* ap = <CStringArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
+cdef class LargeStringScalar(StringScalar):
+ pass
-cdef class LargeStringValue(ArrayValue):
+cdef class ListScalar(Scalar):
"""
- Concrete class for large string (utf8) array elements.
+ Concrete class for list-like scalars.
"""
- def as_py(self):
+ cdef array(self):
+ cdef CListScalar* sp = <CListScalar*> self.wrapped.get()
+ if sp.is_valid:
+ return pyarrow_wrap_array(sp.value)
+ else:
+ return None
+
+ def __len__(self):
"""
- Return this value as a Python unicode string.
+ Return the number of values.
"""
- cdef CLargeStringArray* ap = <CLargeStringArray*> self.sp_array.get()
- return ap.GetString(self.index).decode('utf-8')
+ return len(self.array())
- def as_buffer(self):
+ def __getitem__(self, i):
"""
- Return a view over this value as a Buffer object.
+ Return the value at the given index.
"""
- cdef:
- CLargeStringArray* ap = <CLargeStringArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
-
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
+ return self.array()[_normalize_index(i, len(self))]
-
-cdef class BinaryValue(ArrayValue):
- """
- Concrete class for variable-sized binary array elements.
- """
+ def __iter__(self):
+ """
+ Iterate over this element's values.
+ """
+ return iter(self.array())
def as_py(self):
"""
- Return this value as a Python bytes object.
+ Return this value as a Python list.
"""
- cdef:
- const uint8_t* ptr
- int32_t length
- CBinaryArray* ap = <CBinaryArray*> self.sp_array.get()
+ arr = self.array()
+ return None if arr is None else arr.to_pylist()
- ptr = ap.GetValue(self.index, &length)
- return cp.PyBytes_FromStringAndSize(<const char*>(ptr), length)
- def as_buffer(self):
- """
- Return a view over this value as a Buffer object.
- """
- cdef:
- CBinaryArray* ap = <CBinaryArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
+cdef class FixedSizeListScalar(ListScalar):
+ pass
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
+cdef class LargeListScalar(ListScalar):
+ pass
-cdef class LargeBinaryValue(ArrayValue):
+
+cdef class StructScalar(Scalar):
"""
- Concrete class for large variable-sized binary array elements.
+ Concrete class for struct scalars.
"""
- def as_py(self):
+ def __len__(self):
+ cdef CStructScalar* sp = <CStructScalar*> self.wrapped.get()
+ return sp.value.size()
+
+ def __getitem__(self, key):
"""
- Return this value as a Python bytes object.
+ Return the child value for the given field.
+
+ Parameters
+ ----------
+ index : Union[int, str]
+ Index / position or name of the field.
+
+ Returns
+ -------
+ result : Scalar
"""
cdef:
- const uint8_t* ptr
- int64_t length
- CLargeBinaryArray* ap = <CLargeBinaryArray*> self.sp_array.get()
+ CFieldRef ref
+ CStructScalar* sp = <CStructScalar*> self.wrapped.get()
+
+ if isinstance(key, (bytes, str)):
+ ref = CFieldRef(<c_string> tobytes(key))
+ elif isinstance(key, int):
+ ref = CFieldRef(<int> key)
+ else:
+ raise TypeError('Expected integer or string index')
- ptr = ap.GetValue(self.index, &length)
- return cp.PyBytes_FromStringAndSize(<const char*>(ptr), length)
+ try:
+ return Scalar.wrap(GetResultValue(sp.field(ref)))
+ except ArrowInvalid:
+ raise IndexError(key)
- def as_buffer(self):
+ def as_py(self):
"""
- Return a view over this value as a Buffer object.
+ Return this value as a Python dict.
"""
cdef:
- CLargeBinaryArray* ap = <CLargeBinaryArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
+ CStructScalar* sp = <CStructScalar*> self.wrapped.get()
+ CStructType* dtype = <CStructType*> sp.type.get()
+ vector[shared_ptr[CField]] fields = dtype.fields()
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
+ if sp.is_valid:
+ return {frombytes(fields[i].get().name()): Scalar.wrap(sp.value[i])
Review comment:
This is also what we seem to do on master
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] kszucs commented on a change in pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
kszucs commented on a change in pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#discussion_r448589973
##########
File path: python/pyarrow/scalar.pxi
##########
@@ -1217,21 +764,50 @@ cdef dict _scalar_classes = {
_Type_INT16: Int16Scalar,
_Type_INT32: Int32Scalar,
_Type_INT64: Int64Scalar,
+ _Type_HALF_FLOAT: HalfFloatScalar,
_Type_FLOAT: FloatScalar,
_Type_DOUBLE: DoubleScalar,
+ _Type_DECIMAL: Decimal128Scalar,
+ _Type_DATE32: Date32Scalar,
+ _Type_DATE64: Date64Scalar,
+ _Type_TIME32: Time32Scalar,
+ _Type_TIME64: Time64Scalar,
+ _Type_TIMESTAMP: TimestampScalar,
+ _Type_DURATION: DurationScalar,
+ _Type_BINARY: BinaryScalar,
+ _Type_LARGE_BINARY: LargeBinaryScalar,
+ _Type_FIXED_SIZE_BINARY: FixedSizeBinaryScalar,
_Type_STRING: StringScalar,
+ _Type_LARGE_STRING: LargeStringScalar,
+ _Type_LIST: ListScalar,
+ _Type_LARGE_LIST: LargeListScalar,
+ _Type_FIXED_SIZE_LIST: FixedSizeListScalar,
+ _Type_STRUCT: StructScalar,
+ _Type_MAP: MapScalar,
+ _Type_DICTIONARY: DictionaryScalar,
+ _Type_SPARSE_UNION: UnionScalar,
+ _Type_DENSE_UNION: UnionScalar,
}
-cdef object box_scalar(DataType type, const shared_ptr[CArray]& sp_array,
- int64_t index):
- cdef ArrayValue value
- if type.type.id() == _Type_NA:
- return _NULL
- elif sp_array.get().IsNull(index):
- return _NULL
- else:
- klass = _array_value_classes[type.type.id()]
- value = klass.__new__(klass)
- value.init(type, sp_array, index)
- return value
+def scalar(value, DataType type=None, bint safe=True,
Review comment:
Removed.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] kszucs commented on a change in pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
kszucs commented on a change in pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#discussion_r446148105
##########
File path: python/pyarrow/tests/test_parquet.py
##########
@@ -2028,7 +2028,7 @@ def test_filters_invalid_pred_op(tempdir, use_legacy_dataset):
use_legacy_dataset=use_legacy_dataset)
assert dataset.read().num_rows == 0
- with pytest.raises(ValueError if use_legacy_dataset else TypeError):
+ with pytest.raises(ValueError if use_legacy_dataset else pa.ArrowInvalid):
# dataset API returns TypeError when trying create invalid comparison
Review comment:
Nice, updated.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] kszucs commented on pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
kszucs commented on pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#issuecomment-653044818
Addressed the review comments and changed the equality checks to be strict, only allowing to compare scalars with scalars of the same type. One exception *could* be to compare invalid scalar values to `pa.NA`, but even if we consider to choose that behavior, it should be implemented on the C++ side.
@pitrou @jorisvandenbossche it should be ready for another review.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] kszucs commented on a change in pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
kszucs commented on a change in pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#discussion_r448589857
##########
File path: python/pyarrow/scalar.pxi
##########
@@ -1217,21 +764,50 @@ cdef dict _scalar_classes = {
_Type_INT16: Int16Scalar,
_Type_INT32: Int32Scalar,
_Type_INT64: Int64Scalar,
+ _Type_HALF_FLOAT: HalfFloatScalar,
_Type_FLOAT: FloatScalar,
_Type_DOUBLE: DoubleScalar,
+ _Type_DECIMAL: Decimal128Scalar,
+ _Type_DATE32: Date32Scalar,
+ _Type_DATE64: Date64Scalar,
+ _Type_TIME32: Time32Scalar,
+ _Type_TIME64: Time64Scalar,
+ _Type_TIMESTAMP: TimestampScalar,
+ _Type_DURATION: DurationScalar,
+ _Type_BINARY: BinaryScalar,
+ _Type_LARGE_BINARY: LargeBinaryScalar,
+ _Type_FIXED_SIZE_BINARY: FixedSizeBinaryScalar,
_Type_STRING: StringScalar,
+ _Type_LARGE_STRING: LargeStringScalar,
+ _Type_LIST: ListScalar,
+ _Type_LARGE_LIST: LargeListScalar,
+ _Type_FIXED_SIZE_LIST: FixedSizeListScalar,
+ _Type_STRUCT: StructScalar,
+ _Type_MAP: MapScalar,
+ _Type_DICTIONARY: DictionaryScalar,
+ _Type_SPARSE_UNION: UnionScalar,
+ _Type_DENSE_UNION: UnionScalar,
}
-cdef object box_scalar(DataType type, const shared_ptr[CArray]& sp_array,
- int64_t index):
- cdef ArrayValue value
- if type.type.id() == _Type_NA:
- return _NULL
- elif sp_array.get().IsNull(index):
- return _NULL
- else:
- klass = _array_value_classes[type.type.id()]
- value = klass.__new__(klass)
- value.init(type, sp_array, index)
- return value
+def scalar(value, DataType type=None, bint safe=True,
+ MemoryPool memory_pool=None):
+ cdef:
+ PyConversionOptions options
+ shared_ptr[CScalar] scalar
+ shared_ptr[CArray] array
+ shared_ptr[CChunkedArray] chunked
+
+ options.size = 1
+ options.pool = maybe_unbox_memory_pool(memory_pool)
+ # options.from_pandas = from_pandas
Review comment:
Implemented it.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] kszucs commented on a change in pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
kszucs commented on a change in pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#discussion_r450344074
##########
File path: cpp/src/arrow/scalar_test.cc
##########
@@ -451,6 +565,127 @@ TEST(TestStructScalar, FieldAccess) {
ASSERT_RAISES(Invalid, abc.field(5).status());
ASSERT_RAISES(Invalid, abc.field("c").status());
+
+ ASSERT_OK_AND_ASSIGN(auto d, abc.field("d"));
+ ASSERT_TRUE(d->Equals(MakeNullScalar(int64())));
+}
+
+TEST(TestDictionaryScalar, Basics) {
+ auto ty = dictionary(int8(), utf8());
+ auto dict = ArrayFromJSON(utf8(), "[\"alpha\", \"beta\", \"gamma\"]");
+
+ DictionaryScalar::ValueType alpha;
+ ASSERT_OK_AND_ASSIGN(alpha.index, MakeScalar(int8(), 0));
+ alpha.dictionary = dict;
+
+ DictionaryScalar::ValueType gamma;
+ ASSERT_OK_AND_ASSIGN(gamma.index, MakeScalar(int8(), 2));
+ gamma.dictionary = dict;
+
+ auto scalar_null = MakeNullScalar(ty);
+ auto scalar_alpha = DictionaryScalar(alpha, ty);
+ auto scalar_gamma = DictionaryScalar(gamma, ty);
+
+ ASSERT_OK_AND_ASSIGN(
+ auto encoded_null,
+ checked_cast<const DictionaryScalar&>(*scalar_null).GetEncodedValue());
+ ASSERT_TRUE(encoded_null->Equals(MakeNullScalar(utf8())));
+
+ ASSERT_OK_AND_ASSIGN(
+ auto encoded_alpha,
+ checked_cast<const DictionaryScalar&>(scalar_alpha).GetEncodedValue());
+ ASSERT_TRUE(encoded_alpha->Equals(MakeScalar("alpha")));
+
+ ASSERT_OK_AND_ASSIGN(
+ auto encoded_gamma,
+ checked_cast<const DictionaryScalar&>(scalar_gamma).GetEncodedValue());
+ ASSERT_TRUE(encoded_gamma->Equals(MakeScalar("gamma")));
+
+ // test Array.GetScalar
+ DictionaryArray arr(ty, ArrayFromJSON(int8(), "[2, 0, 1, null]"), dict);
+ ASSERT_OK_AND_ASSIGN(auto first, arr.GetScalar(0));
+ ASSERT_OK_AND_ASSIGN(auto second, arr.GetScalar(1));
+ ASSERT_OK_AND_ASSIGN(auto last, arr.GetScalar(3));
+
+ ASSERT_TRUE(first->Equals(scalar_gamma));
+ ASSERT_TRUE(second->Equals(scalar_alpha));
+ ASSERT_TRUE(last->Equals(scalar_null));
+}
+
+TEST(TestSparseUnionScalar, Basics) {
+ auto ty = sparse_union({field("string", utf8()), field("number", uint64())});
+
+ auto alpha = MakeScalar("alpha");
+ auto beta = MakeScalar("beta");
+ ASSERT_OK_AND_ASSIGN(auto two, MakeScalar(uint64(), 2));
+
+ auto scalar_alpha = SparseUnionScalar(alpha, ty);
+ auto scalar_beta = SparseUnionScalar(beta, ty);
+ auto scalar_two = SparseUnionScalar(two, ty);
+
+ // test Array.GetScalar
+ std::vector<std::shared_ptr<Array>> children{
+ ArrayFromJSON(utf8(), "[\"alpha\", \"\", \"beta\", null, \"gamma\"]"),
+ ArrayFromJSON(uint64(), "[1, 2, 11, 22, null]")};
+
+ auto type_ids = ArrayFromJSON(int8(), "[0, 1, 0, 0, 1]");
+ SparseUnionArray arr(ty, 5, children, type_ids->data()->buffers[1]);
+ ASSERT_OK(arr.ValidateFull());
+
+ ASSERT_OK_AND_ASSIGN(auto first, arr.GetScalar(0));
+ ASSERT_TRUE(first->Equals(scalar_alpha));
+
+ ASSERT_OK_AND_ASSIGN(auto second, arr.GetScalar(1));
+ ASSERT_TRUE(second->Equals(scalar_two));
+
+ ASSERT_OK_AND_ASSIGN(auto third, arr.GetScalar(2));
+ ASSERT_TRUE(third->Equals(scalar_beta));
+
+ ASSERT_OK_AND_ASSIGN(auto fourth, arr.GetScalar(3));
+ ASSERT_TRUE(fourth->Equals(MakeNullScalar(ty)));
Review comment:
So for UnionScalar we have two `is_valid` flags, une for the union scalar and one for the underlying value scalar. Currently `MakeNullScalar()` sets the validity flag for the union scalar and leaves the value scalar uninitialized.
What do you think, shall we initialize the value scalar as well?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] jorisvandenbossche commented on a change in pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
jorisvandenbossche commented on a change in pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#discussion_r449498839
##########
File path: python/pyarrow/tests/test_array.py
##########
@@ -583,12 +584,14 @@ def test_dictionary_from_numpy():
assert d2.dictionary.to_pylist() == dictionary.tolist()
for i in range(len(indices)):
+ assert d1[i].dictionary == dictionary
assert d1[i].as_py() == dictionary[indices[i]]
- if mask[i]:
- assert d2[i] is pa.NULL
- else:
- assert d2[i].as_py() == dictionary[indices[i]]
+ print(i, d2[i])
+ # if mask[i]:
+ # assert d2[i].as_py() is None
+ # else:
+ # assert d2[i].as_py() == dictionary[indices[i]]
Review comment:
left-over ?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] jorisvandenbossche commented on pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
jorisvandenbossche commented on pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#issuecomment-652249133
We could "fix" that one by raising in `__bool__` (meaning: it will at least give an error instead of silently returning a wrong answer)
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] kszucs commented on a change in pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
kszucs commented on a change in pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#discussion_r450317608
##########
File path: cpp/src/arrow/scalar_test.cc
##########
@@ -451,6 +565,127 @@ TEST(TestStructScalar, FieldAccess) {
ASSERT_RAISES(Invalid, abc.field(5).status());
ASSERT_RAISES(Invalid, abc.field("c").status());
+
+ ASSERT_OK_AND_ASSIGN(auto d, abc.field("d"));
+ ASSERT_TRUE(d->Equals(MakeNullScalar(int64())));
+}
+
+TEST(TestDictionaryScalar, Basics) {
+ auto ty = dictionary(int8(), utf8());
+ auto dict = ArrayFromJSON(utf8(), "[\"alpha\", \"beta\", \"gamma\"]");
+
+ DictionaryScalar::ValueType alpha;
+ ASSERT_OK_AND_ASSIGN(alpha.index, MakeScalar(int8(), 0));
+ alpha.dictionary = dict;
+
+ DictionaryScalar::ValueType gamma;
+ ASSERT_OK_AND_ASSIGN(gamma.index, MakeScalar(int8(), 2));
+ gamma.dictionary = dict;
+
+ auto scalar_null = MakeNullScalar(ty);
+ auto scalar_alpha = DictionaryScalar(alpha, ty);
+ auto scalar_gamma = DictionaryScalar(gamma, ty);
+
+ ASSERT_OK_AND_ASSIGN(
+ auto encoded_null,
+ checked_cast<const DictionaryScalar&>(*scalar_null).GetEncodedValue());
+ ASSERT_TRUE(encoded_null->Equals(MakeNullScalar(utf8())));
+
+ ASSERT_OK_AND_ASSIGN(
+ auto encoded_alpha,
+ checked_cast<const DictionaryScalar&>(scalar_alpha).GetEncodedValue());
+ ASSERT_TRUE(encoded_alpha->Equals(MakeScalar("alpha")));
+
+ ASSERT_OK_AND_ASSIGN(
+ auto encoded_gamma,
+ checked_cast<const DictionaryScalar&>(scalar_gamma).GetEncodedValue());
+ ASSERT_TRUE(encoded_gamma->Equals(MakeScalar("gamma")));
+
+ // test Array.GetScalar
+ DictionaryArray arr(ty, ArrayFromJSON(int8(), "[2, 0, 1, null]"), dict);
+ ASSERT_OK_AND_ASSIGN(auto first, arr.GetScalar(0));
+ ASSERT_OK_AND_ASSIGN(auto second, arr.GetScalar(1));
+ ASSERT_OK_AND_ASSIGN(auto last, arr.GetScalar(3));
+
+ ASSERT_TRUE(first->Equals(scalar_gamma));
+ ASSERT_TRUE(second->Equals(scalar_alpha));
+ ASSERT_TRUE(last->Equals(scalar_null));
+}
+
+TEST(TestSparseUnionScalar, Basics) {
+ auto ty = sparse_union({field("string", utf8()), field("number", uint64())});
+
+ auto alpha = MakeScalar("alpha");
+ auto beta = MakeScalar("beta");
+ ASSERT_OK_AND_ASSIGN(auto two, MakeScalar(uint64(), 2));
+
+ auto scalar_alpha = SparseUnionScalar(alpha, ty);
+ auto scalar_beta = SparseUnionScalar(beta, ty);
+ auto scalar_two = SparseUnionScalar(two, ty);
+
+ // test Array.GetScalar
+ std::vector<std::shared_ptr<Array>> children{
+ ArrayFromJSON(utf8(), "[\"alpha\", \"\", \"beta\", null, \"gamma\"]"),
+ ArrayFromJSON(uint64(), "[1, 2, 11, 22, null]")};
+
+ auto type_ids = ArrayFromJSON(int8(), "[0, 1, 0, 0, 1]");
+ SparseUnionArray arr(ty, 5, children, type_ids->data()->buffers[1]);
+ ASSERT_OK(arr.ValidateFull());
+
+ ASSERT_OK_AND_ASSIGN(auto first, arr.GetScalar(0));
+ ASSERT_TRUE(first->Equals(scalar_alpha));
+
+ ASSERT_OK_AND_ASSIGN(auto second, arr.GetScalar(1));
+ ASSERT_TRUE(second->Equals(scalar_two));
+
+ ASSERT_OK_AND_ASSIGN(auto third, arr.GetScalar(2));
+ ASSERT_TRUE(third->Equals(scalar_beta));
+
+ ASSERT_OK_AND_ASSIGN(auto fourth, arr.GetScalar(3));
+ ASSERT_TRUE(fourth->Equals(MakeNullScalar(ty)));
Review comment:
I think `GetScalar()` should return a Scalar with the same type as the Array.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] kszucs commented on a change in pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
kszucs commented on a change in pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#discussion_r447738979
##########
File path: python/pyarrow/scalar.pxi
##########
@@ -16,1198 +16,745 @@
# under the License.
-_NULL = NA = None
+import collections
cdef class Scalar:
"""
- The base class for all array elements.
+ The base class for scalars.
"""
+ def __init__(self):
+ raise TypeError("Do not call {}'s constructor directly, use "
+ "pa.scalar() instead.".format(self.__class__.__name__))
-cdef class NullType(Scalar):
- """
- Singleton for null array elements.
- """
- # TODO rename this NullValue?
+ cdef void init(self, const shared_ptr[CScalar]& wrapped):
+ self.wrapped = wrapped
- def __cinit__(self):
- global NA
- if NA is not None:
- raise Exception('Cannot create multiple NAType instances')
+ @staticmethod
+ cdef wrap(const shared_ptr[CScalar]& wrapped):
+ cdef:
+ Scalar self
+ Type type_id = wrapped.get().type.get().id()
+
+ if type_id == _Type_NA:
+ return _NULL
+
+ typ = _scalar_classes[type_id]
+ self = typ.__new__(typ)
+ self.init(wrapped)
+
+ return self
+
+ cdef inline shared_ptr[CScalar] unwrap(self) nogil:
+ return self.wrapped
+
+ @property
+ def type(self):
+ return pyarrow_wrap_data_type(self.wrapped.get().type)
- self.type = null()
+ @property
+ def is_valid(self):
+ return self.wrapped.get().is_valid
def __repr__(self):
- return 'NULL'
+ return '<pyarrow.{}: {!r}>'.format(
+ self.__class__.__name__, self.as_py()
+ )
- def as_py(self):
- """
- Return None
- """
- return None
+ def __str__(self):
+ return str(self.as_py())
+
+ def equals(self, Scalar other):
+ return self.wrapped.get().Equals(other.unwrap().get()[0])
def __eq__(self, other):
- return NA
+ try:
+ if not isinstance(other, Scalar):
+ other = scalar(other, type=self.type)
+ return self.equals(other)
+ except (TypeError, ValueError, ArrowInvalid):
+ return NotImplemented
+
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+
+ def as_py(self):
+ raise NotImplementedError()
-_NULL = NA = NullType()
+_NULL = NA = None
-cdef class ArrayValue(Scalar):
+cdef class NullScalar(Scalar):
"""
- The base class for non-null array elements.
+ Concrete class for null scalars.
"""
- def __init__(self):
- raise TypeError("Do not call {}'s constructor directly, use array "
- "subscription instead."
- .format(self.__class__.__name__))
+ def __cinit__(self):
+ global NA
+ if NA is not None:
+ raise Exception('Cannot create multiple NAType instances')
+ self.init(shared_ptr[CScalar](new CNullScalar()))
- cdef void init(self, DataType type, const shared_ptr[CArray]& sp_array,
- int64_t index):
- self.type = type
- self.index = index
- self._set_array(sp_array)
+ def __init__(self):
+ pass
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
+ def __eq__(self, other):
+ return NA
- def __repr__(self):
- if hasattr(self, 'as_py'):
- return repr(self.as_py())
- else:
- return super(Scalar, self).__repr__()
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
- def __str__(self):
- if hasattr(self, 'as_py'):
- return str(self.as_py())
- else:
- return super(Scalar, self).__str__()
+ def as_py(self):
+ """
+ Return this value as a Python None.
+ """
+ return None
- def __eq__(self, other):
- if hasattr(self, 'as_py'):
- if isinstance(other, ArrayValue):
- other = other.as_py()
- return self.as_py() == other
- else:
- raise NotImplementedError(
- "Cannot compare Arrow values that don't support as_py()")
- def __hash__(self):
- return hash(self.as_py())
+_NULL = NA = NullScalar()
-cdef class BooleanValue(ArrayValue):
+cdef class BooleanScalar(Scalar):
"""
- Concrete class for boolean array elements.
+ Concrete class for boolean scalars.
"""
def as_py(self):
"""
Return this value as a Python bool.
"""
- cdef CBooleanArray* ap = <CBooleanArray*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CBooleanScalar* sp = <CBooleanScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int8Value(ArrayValue):
+cdef class UInt8Scalar(Scalar):
"""
- Concrete class for int8 array elements.
+ Concrete class for uint8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt8Array* ap = <CInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt8Scalar* sp = <CUInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt8Value(ArrayValue):
+cdef class Int8Scalar(Scalar):
"""
- Concrete class for uint8 array elements.
+ Concrete class for int8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt8Array* ap = <CUInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt8Scalar* sp = <CInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int16Value(ArrayValue):
+cdef class UInt16Scalar(Scalar):
"""
- Concrete class for int16 array elements.
+ Concrete class for uint16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt16Array* ap = <CInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt16Scalar* sp = <CUInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt16Value(ArrayValue):
+cdef class Int16Scalar(Scalar):
"""
- Concrete class for uint16 array elements.
+ Concrete class for int16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt16Array* ap = <CUInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt16Scalar* sp = <CInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int32Value(ArrayValue):
+cdef class UInt32Scalar(Scalar):
"""
- Concrete class for int32 array elements.
+ Concrete class for uint32 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt32Array* ap = <CInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt32Scalar* sp = <CUInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt32Value(ArrayValue):
+cdef class Int32Scalar(Scalar):
"""
- Concrete class for uint32 array elements.
+ Concrete class for int32 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt32Array* ap = <CUInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt32Scalar* sp = <CInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int64Value(ArrayValue):
+cdef class UInt64Scalar(Scalar):
"""
- Concrete class for int64 array elements.
+ Concrete class for uint64 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt64Array* ap = <CInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt64Scalar* sp = <CUInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt64Value(ArrayValue):
+cdef class Int64Scalar(Scalar):
"""
- Concrete class for uint64 array elements.
+ Concrete class for int64 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt64Array* ap = <CUInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt64Scalar* sp = <CInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date32Value(ArrayValue):
+cdef class HalfFloatScalar(Scalar):
"""
- Concrete class for date32 array elements.
+ Concrete class for float scalars.
"""
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+
+ def __eq__(self, other):
+ if hasattr(self, 'as_py'):
+ if isinstance(other, Scalar):
+ other = other.as_py()
+ return self.as_py() == other
+ else:
+ raise NotImplementedError
+
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python float.
"""
- cdef CDate32Array* ap = <CDate32Array*> self.sp_array.get()
+ cdef CHalfFloatScalar* sp = <CHalfFloatScalar*> self.wrapped.get()
+ return PyHalf_FromHalf(sp.value) if sp.is_valid else None
- # Shift to seconds since epoch
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(days=ap.Value(self.index)))
+
+cdef class FloatScalar(Scalar):
+ """
+ Concrete class for float scalars.
+ """
+
+ def as_py(self):
+ """
+ Return this value as a Python float.
+ """
+ cdef CFloatScalar* sp = <CFloatScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date64Value(ArrayValue):
+cdef class DoubleScalar(Scalar):
"""
- Concrete class for date64 array elements.
+ Concrete class for double scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python float.
"""
- cdef CDate64Array* ap = <CDate64Array*> self.sp_array.get()
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(
- days=ap.Value(self.index) / 86400000))
+ cdef CDoubleScalar* sp = <CDoubleScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Time32Value(ArrayValue):
+cdef class Decimal128Scalar(Scalar):
"""
- Concrete class for time32 array elements.
+ Concrete class for decimal128 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.timedelta instance.
+ Return this value as a Python Decimal.
"""
cdef:
- CTime32Array* ap = <CTime32Array*> self.sp_array.get()
- CTime32Type* dtype = <CTime32Type*> ap.type().get()
-
- if dtype.unit() == TimeUnit_SECOND:
- delta = datetime.timedelta(seconds=ap.Value(self.index))
- return (datetime.datetime(1970, 1, 1) + delta).time()
+ CDecimal128Scalar* sp = <CDecimal128Scalar*> self.wrapped.get()
+ CDecimal128Type* dtype = <CDecimal128Type*> sp.type.get()
+ if sp.is_valid:
+ return _pydecimal.Decimal(
+ frombytes(sp.value.ToString(dtype.scale()))
+ )
else:
- return _box_time_milli(ap.Value(self.index))
+ return None
-cdef class Time64Value(ArrayValue):
+cdef class Date32Scalar(Scalar):
"""
- Concrete class for time64 array elements.
+ Concrete class for date32 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.timedelta instance.
+ Return this value as a Python datetime.datetime instance.
"""
- cdef:
- CTime64Array* ap = <CTime64Array*> self.sp_array.get()
- CTime64Type* dtype = <CTime64Type*> ap.type().get()
+ cdef CDate32Scalar* sp = <CDate32Scalar*> self.wrapped.get()
- cdef int64_t val = ap.Value(self.index)
- if dtype.unit() == TimeUnit_MICRO:
- return _box_time_micro(val)
+ if sp.is_valid:
+ # shift to seconds since epoch
+ return (
+ datetime.date(1970, 1, 1) + datetime.timedelta(days=sp.value)
+ )
else:
- return (datetime.datetime(1970, 1, 1) +
- datetime.timedelta(microseconds=val / 1000)).time()
-
+ return None
-cpdef _box_time_milli(int64_t val):
- delta = datetime.timedelta(milliseconds=val)
- return (datetime.datetime(1970, 1, 1) + delta).time()
+cdef class Date64Scalar(Scalar):
+ """
+ Concrete class for date64 scalars.
+ """
-cpdef _box_time_micro(int64_t val):
- return (datetime.datetime(1970, 1, 1) +
- datetime.timedelta(microseconds=val)).time()
+ def as_py(self):
+ """
+ Return this value as a Python datetime.datetime instance.
+ """
+ cdef CDate64Scalar* sp = <CDate64Scalar*> self.wrapped.get()
+ if sp.is_valid:
+ return (
+ datetime.date(1970, 1, 1) +
+ datetime.timedelta(days=sp.value / 86400000)
+ )
+ else:
+ return None
-cdef dict _DATETIME_CONVERSION_FUNCTIONS = {}
-cdef c_bool _datetime_conversion_initialized = False
+def _datetime_from_int(int64_t value, TimeUnit unit, tzinfo=None):
+ if unit == TimeUnit_SECOND:
+ delta = datetime.timedelta(seconds=value)
+ elif unit == TimeUnit_MILLI:
+ delta = datetime.timedelta(milliseconds=value)
+ elif unit == TimeUnit_MICRO:
+ delta = datetime.timedelta(microseconds=value)
+ else:
+ # TimeUnit_NANO: prefer pandas timestamps if available
+ if _pandas_api.have_pandas:
+ return _pandas_api.pd.Timestamp(value, tz=tzinfo, unit='ns')
+ # otherwise safely truncate to microsecond resolution datetime
+ if value % 1000 != 0:
+ raise ValueError(
+ "Nanosecond resolution temporal type {} is not safely "
+ "convertible to microseconds to convert to datetime.datetime. "
+ "Install pandas to return as Timestamp with nanosecond "
+ "support or access the .value attribute.".format(value)
+ )
+ delta = datetime.timedelta(microseconds=value // 1000)
-cdef _add_micros_maybe_localize(dt, micros, tzinfo):
- import pytz
- dt = dt.replace(microsecond=micros)
+ dt = datetime.datetime(1970, 1, 1) + delta
+ # adjust timezone if set to the datatype
if tzinfo is not None:
- if not isinstance(tzinfo, datetime.tzinfo):
- tzinfo = string_to_tzinfo(tzinfo)
dt = tzinfo.fromutc(dt)
- return dt
-
-
-cdef _datetime_from_seconds(int64_t v):
- return datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=v)
-
-
-def _nanoseconds_to_datetime_safe(v, tzinfo):
- if v % 1000 != 0:
- raise ValueError("Nanosecond timestamp {} is not safely convertible "
- " to microseconds to convert to datetime.datetime."
- " Install pandas to return as Timestamp with "
- " nanosecond support or access the .value attribute.")
- v = v // 1000
- micros = v % 1_000_000
-
- dt = _datetime_from_seconds(v // 1_000_000)
- return _add_micros_maybe_localize(dt, micros, tzinfo)
-
-
-def _microseconds_to_datetime(v, tzinfo):
- micros = v % 1_000_000
- dt = _datetime_from_seconds(v // 1_000_000)
- return _add_micros_maybe_localize(dt, micros, tzinfo)
+ return dt
-def _millis_to_datetime(v, tzinfo):
- millis = v % 1_000
- dt = _datetime_from_seconds(v // 1000)
- return _add_micros_maybe_localize(dt, millis * 1000, tzinfo)
+cdef class Time32Scalar(Scalar):
+ """
+ Concrete class for time32 scalars.
+ """
-def _seconds_to_datetime(v, tzinfo):
- dt = _datetime_from_seconds(v)
- return _add_micros_maybe_localize(dt, 0, tzinfo)
+ def as_py(self):
+ """
+ Return this value as a Python datetime.timedelta instance.
+ """
+ cdef:
+ CTime32Scalar* sp = <CTime32Scalar*> self.wrapped.get()
+ CTime32Type* dtype = <CTime32Type*> sp.type.get()
+ if sp.is_valid:
+ return _datetime_from_int(sp.value, unit=dtype.unit()).time()
+ else:
+ return None
-def _datetime_conversion_functions():
- global _datetime_conversion_initialized
- if _datetime_conversion_initialized:
- return _DATETIME_CONVERSION_FUNCTIONS
- _DATETIME_CONVERSION_FUNCTIONS.update({
- TimeUnit_SECOND: _seconds_to_datetime,
- TimeUnit_MILLI: _millis_to_datetime,
- TimeUnit_MICRO: _microseconds_to_datetime,
- TimeUnit_NANO: _nanoseconds_to_datetime_safe
- })
+cdef class Time64Scalar(Scalar):
+ """
+ Concrete class for time64 scalars.
+ """
- try:
- import pandas as pd
- _DATETIME_CONVERSION_FUNCTIONS[TimeUnit_NANO] = (
- lambda x, tzinfo: pd.Timestamp(
- x, tz=tzinfo, unit='ns',
- )
- )
- except ImportError:
- pass
+ def as_py(self):
+ """
+ Return this value as a Python datetime.timedelta instance.
+ """
+ cdef:
+ CTime64Scalar* sp = <CTime64Scalar*> self.wrapped.get()
+ CTime64Type* dtype = <CTime64Type*> sp.type.get()
- _datetime_conversion_initialized = True
- return _DATETIME_CONVERSION_FUNCTIONS
+ if sp.is_valid:
+ return _datetime_from_int(sp.value, unit=dtype.unit()).time()
+ else:
+ return None
-cdef class TimestampValue(ArrayValue):
+cdef class TimestampScalar(Scalar):
"""
- Concrete class for timestamp array elements.
+ Concrete class for timestamp scalars.
"""
@property
def value(self):
- cdef CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
- cdef CTimestampType* dtype = <CTimestampType*> ap.type().get()
- return ap.Value(self.index)
+ cdef CTimestampScalar* sp = <CTimestampScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
def as_py(self):
"""
Return this value as a Pandas Timestamp instance (if available),
otherwise as a Python datetime.timedelta instance.
"""
- cdef CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
- cdef CTimestampType* dtype = <CTimestampType*> ap.type().get()
+ cdef:
+ CTimestampScalar* sp = <CTimestampScalar*> self.wrapped.get()
+ CTimestampType* dtype = <CTimestampType*> sp.type.get()
- value = self.value
+ if not sp.is_valid:
+ return None
if not dtype.timezone().empty():
tzinfo = string_to_tzinfo(frombytes(dtype.timezone()))
+ if not isinstance(tzinfo, datetime.tzinfo):
+ tzinfo = string_to_tzinfo(tzinfo)
Review comment:
It's a copy issue, I incrementally ported the previous helper functions. Removing the second call.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] pitrou commented on a change in pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
pitrou commented on a change in pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#discussion_r447732301
##########
File path: python/pyarrow/scalar.pxi
##########
@@ -16,1198 +16,745 @@
# under the License.
-_NULL = NA = None
+import collections
cdef class Scalar:
"""
- The base class for all array elements.
+ The base class for scalars.
"""
+ def __init__(self):
+ raise TypeError("Do not call {}'s constructor directly, use "
+ "pa.scalar() instead.".format(self.__class__.__name__))
-cdef class NullType(Scalar):
- """
- Singleton for null array elements.
- """
- # TODO rename this NullValue?
+ cdef void init(self, const shared_ptr[CScalar]& wrapped):
+ self.wrapped = wrapped
- def __cinit__(self):
- global NA
- if NA is not None:
- raise Exception('Cannot create multiple NAType instances')
+ @staticmethod
+ cdef wrap(const shared_ptr[CScalar]& wrapped):
+ cdef:
+ Scalar self
+ Type type_id = wrapped.get().type.get().id()
+
+ if type_id == _Type_NA:
+ return _NULL
+
+ typ = _scalar_classes[type_id]
+ self = typ.__new__(typ)
+ self.init(wrapped)
+
+ return self
+
+ cdef inline shared_ptr[CScalar] unwrap(self) nogil:
+ return self.wrapped
+
+ @property
+ def type(self):
+ return pyarrow_wrap_data_type(self.wrapped.get().type)
- self.type = null()
+ @property
+ def is_valid(self):
+ return self.wrapped.get().is_valid
def __repr__(self):
- return 'NULL'
+ return '<pyarrow.{}: {!r}>'.format(
+ self.__class__.__name__, self.as_py()
+ )
- def as_py(self):
- """
- Return None
- """
- return None
+ def __str__(self):
+ return str(self.as_py())
+
+ def equals(self, Scalar other):
+ return self.wrapped.get().Equals(other.unwrap().get()[0])
def __eq__(self, other):
- return NA
+ try:
+ if not isinstance(other, Scalar):
+ other = scalar(other, type=self.type)
+ return self.equals(other)
+ except (TypeError, ValueError, ArrowInvalid):
+ return NotImplemented
+
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+
+ def as_py(self):
+ raise NotImplementedError()
-_NULL = NA = NullType()
+_NULL = NA = None
-cdef class ArrayValue(Scalar):
+cdef class NullScalar(Scalar):
"""
- The base class for non-null array elements.
+ Concrete class for null scalars.
"""
- def __init__(self):
- raise TypeError("Do not call {}'s constructor directly, use array "
- "subscription instead."
- .format(self.__class__.__name__))
+ def __cinit__(self):
+ global NA
+ if NA is not None:
+ raise Exception('Cannot create multiple NAType instances')
+ self.init(shared_ptr[CScalar](new CNullScalar()))
- cdef void init(self, DataType type, const shared_ptr[CArray]& sp_array,
- int64_t index):
- self.type = type
- self.index = index
- self._set_array(sp_array)
+ def __init__(self):
+ pass
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
+ def __eq__(self, other):
+ return NA
Review comment:
Well, it should probably return `False` for non-nulls?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] kszucs commented on a change in pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
kszucs commented on a change in pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#discussion_r446147913
##########
File path: python/pyarrow/_dataset.pyx
##########
@@ -216,22 +216,18 @@ cdef class Expression:
@staticmethod
def _scalar(value):
cdef:
- shared_ptr[CScalar] scalar
-
- if value is None:
- scalar.reset(new CNullScalar())
- elif isinstance(value, bool):
- scalar = MakeScalar(<c_bool>value)
- elif isinstance(value, float):
- scalar = MakeScalar(<double>value)
- elif isinstance(value, int):
- scalar = MakeScalar(<int64_t>value)
- elif isinstance(value, (bytes, str)):
- scalar = MakeStringScalar(tobytes(value))
Review comment:
Removed it.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] kszucs commented on a change in pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
kszucs commented on a change in pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#discussion_r450327095
##########
File path: cpp/src/arrow/scalar_test.cc
##########
@@ -451,6 +565,127 @@ TEST(TestStructScalar, FieldAccess) {
ASSERT_RAISES(Invalid, abc.field(5).status());
ASSERT_RAISES(Invalid, abc.field("c").status());
+
+ ASSERT_OK_AND_ASSIGN(auto d, abc.field("d"));
+ ASSERT_TRUE(d->Equals(MakeNullScalar(int64())));
+}
+
+TEST(TestDictionaryScalar, Basics) {
+ auto ty = dictionary(int8(), utf8());
+ auto dict = ArrayFromJSON(utf8(), "[\"alpha\", \"beta\", \"gamma\"]");
+
+ DictionaryScalar::ValueType alpha;
+ ASSERT_OK_AND_ASSIGN(alpha.index, MakeScalar(int8(), 0));
+ alpha.dictionary = dict;
+
+ DictionaryScalar::ValueType gamma;
+ ASSERT_OK_AND_ASSIGN(gamma.index, MakeScalar(int8(), 2));
+ gamma.dictionary = dict;
+
+ auto scalar_null = MakeNullScalar(ty);
+ auto scalar_alpha = DictionaryScalar(alpha, ty);
+ auto scalar_gamma = DictionaryScalar(gamma, ty);
+
+ ASSERT_OK_AND_ASSIGN(
+ auto encoded_null,
+ checked_cast<const DictionaryScalar&>(*scalar_null).GetEncodedValue());
+ ASSERT_TRUE(encoded_null->Equals(MakeNullScalar(utf8())));
+
+ ASSERT_OK_AND_ASSIGN(
+ auto encoded_alpha,
+ checked_cast<const DictionaryScalar&>(scalar_alpha).GetEncodedValue());
+ ASSERT_TRUE(encoded_alpha->Equals(MakeScalar("alpha")));
+
+ ASSERT_OK_AND_ASSIGN(
+ auto encoded_gamma,
+ checked_cast<const DictionaryScalar&>(scalar_gamma).GetEncodedValue());
+ ASSERT_TRUE(encoded_gamma->Equals(MakeScalar("gamma")));
+
+ // test Array.GetScalar
+ DictionaryArray arr(ty, ArrayFromJSON(int8(), "[2, 0, 1, null]"), dict);
+ ASSERT_OK_AND_ASSIGN(auto first, arr.GetScalar(0));
+ ASSERT_OK_AND_ASSIGN(auto second, arr.GetScalar(1));
+ ASSERT_OK_AND_ASSIGN(auto last, arr.GetScalar(3));
+
+ ASSERT_TRUE(first->Equals(scalar_gamma));
+ ASSERT_TRUE(second->Equals(scalar_alpha));
+ ASSERT_TRUE(last->Equals(scalar_null));
+}
+
+TEST(TestSparseUnionScalar, Basics) {
+ auto ty = sparse_union({field("string", utf8()), field("number", uint64())});
+
+ auto alpha = MakeScalar("alpha");
+ auto beta = MakeScalar("beta");
+ ASSERT_OK_AND_ASSIGN(auto two, MakeScalar(uint64(), 2));
+
+ auto scalar_alpha = SparseUnionScalar(alpha, ty);
+ auto scalar_beta = SparseUnionScalar(beta, ty);
+ auto scalar_two = SparseUnionScalar(two, ty);
+
+ // test Array.GetScalar
+ std::vector<std::shared_ptr<Array>> children{
+ ArrayFromJSON(utf8(), "[\"alpha\", \"\", \"beta\", null, \"gamma\"]"),
+ ArrayFromJSON(uint64(), "[1, 2, 11, 22, null]")};
+
+ auto type_ids = ArrayFromJSON(int8(), "[0, 1, 0, 0, 1]");
+ SparseUnionArray arr(ty, 5, children, type_ids->data()->buffers[1]);
+ ASSERT_OK(arr.ValidateFull());
+
+ ASSERT_OK_AND_ASSIGN(auto first, arr.GetScalar(0));
+ ASSERT_TRUE(first->Equals(scalar_alpha));
+
+ ASSERT_OK_AND_ASSIGN(auto second, arr.GetScalar(1));
+ ASSERT_TRUE(second->Equals(scalar_two));
+
+ ASSERT_OK_AND_ASSIGN(auto third, arr.GetScalar(2));
+ ASSERT_TRUE(third->Equals(scalar_beta));
+
+ ASSERT_OK_AND_ASSIGN(auto fourth, arr.GetScalar(3));
+ ASSERT_TRUE(fourth->Equals(MakeNullScalar(ty)));
Review comment:
We have the value type in `UnionScalar.value.type`.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] pitrou commented on a change in pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
pitrou commented on a change in pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#discussion_r447751371
##########
File path: python/pyarrow/scalar.pxi
##########
@@ -16,1198 +16,745 @@
# under the License.
-_NULL = NA = None
+import collections
cdef class Scalar:
"""
- The base class for all array elements.
+ The base class for scalars.
"""
+ def __init__(self):
+ raise TypeError("Do not call {}'s constructor directly, use "
+ "pa.scalar() instead.".format(self.__class__.__name__))
-cdef class NullType(Scalar):
- """
- Singleton for null array elements.
- """
- # TODO rename this NullValue?
+ cdef void init(self, const shared_ptr[CScalar]& wrapped):
+ self.wrapped = wrapped
- def __cinit__(self):
- global NA
- if NA is not None:
- raise Exception('Cannot create multiple NAType instances')
+ @staticmethod
+ cdef wrap(const shared_ptr[CScalar]& wrapped):
+ cdef:
+ Scalar self
+ Type type_id = wrapped.get().type.get().id()
+
+ if type_id == _Type_NA:
+ return _NULL
+
+ typ = _scalar_classes[type_id]
+ self = typ.__new__(typ)
+ self.init(wrapped)
+
+ return self
+
+ cdef inline shared_ptr[CScalar] unwrap(self) nogil:
+ return self.wrapped
+
+ @property
+ def type(self):
+ return pyarrow_wrap_data_type(self.wrapped.get().type)
- self.type = null()
+ @property
+ def is_valid(self):
+ return self.wrapped.get().is_valid
def __repr__(self):
- return 'NULL'
+ return '<pyarrow.{}: {!r}>'.format(
+ self.__class__.__name__, self.as_py()
+ )
- def as_py(self):
- """
- Return None
- """
- return None
+ def __str__(self):
+ return str(self.as_py())
+
+ def equals(self, Scalar other):
+ return self.wrapped.get().Equals(other.unwrap().get()[0])
def __eq__(self, other):
- return NA
+ try:
+ if not isinstance(other, Scalar):
+ other = scalar(other, type=self.type)
+ return self.equals(other)
+ except (TypeError, ValueError, ArrowInvalid):
+ return NotImplemented
+
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+
+ def as_py(self):
+ raise NotImplementedError()
-_NULL = NA = NullType()
+_NULL = NA = None
-cdef class ArrayValue(Scalar):
+cdef class NullScalar(Scalar):
"""
- The base class for non-null array elements.
+ Concrete class for null scalars.
"""
- def __init__(self):
- raise TypeError("Do not call {}'s constructor directly, use array "
- "subscription instead."
- .format(self.__class__.__name__))
+ def __cinit__(self):
+ global NA
+ if NA is not None:
+ raise Exception('Cannot create multiple NAType instances')
+ self.init(shared_ptr[CScalar](new CNullScalar()))
- cdef void init(self, DataType type, const shared_ptr[CArray]& sp_array,
- int64_t index):
- self.type = type
- self.index = index
- self._set_array(sp_array)
+ def __init__(self):
+ pass
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
+ def __eq__(self, other):
+ return NA
- def __repr__(self):
- if hasattr(self, 'as_py'):
- return repr(self.as_py())
- else:
- return super(Scalar, self).__repr__()
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
- def __str__(self):
- if hasattr(self, 'as_py'):
- return str(self.as_py())
- else:
- return super(Scalar, self).__str__()
+ def as_py(self):
+ """
+ Return this value as a Python None.
+ """
+ return None
- def __eq__(self, other):
- if hasattr(self, 'as_py'):
- if isinstance(other, ArrayValue):
- other = other.as_py()
- return self.as_py() == other
- else:
- raise NotImplementedError(
- "Cannot compare Arrow values that don't support as_py()")
- def __hash__(self):
- return hash(self.as_py())
+_NULL = NA = NullScalar()
-cdef class BooleanValue(ArrayValue):
+cdef class BooleanScalar(Scalar):
"""
- Concrete class for boolean array elements.
+ Concrete class for boolean scalars.
"""
def as_py(self):
"""
Return this value as a Python bool.
"""
- cdef CBooleanArray* ap = <CBooleanArray*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CBooleanScalar* sp = <CBooleanScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int8Value(ArrayValue):
+cdef class UInt8Scalar(Scalar):
"""
- Concrete class for int8 array elements.
+ Concrete class for uint8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt8Array* ap = <CInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt8Scalar* sp = <CUInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt8Value(ArrayValue):
+cdef class Int8Scalar(Scalar):
"""
- Concrete class for uint8 array elements.
+ Concrete class for int8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt8Array* ap = <CUInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt8Scalar* sp = <CInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int16Value(ArrayValue):
+cdef class UInt16Scalar(Scalar):
"""
- Concrete class for int16 array elements.
+ Concrete class for uint16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt16Array* ap = <CInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt16Scalar* sp = <CUInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt16Value(ArrayValue):
+cdef class Int16Scalar(Scalar):
"""
- Concrete class for uint16 array elements.
+ Concrete class for int16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt16Array* ap = <CUInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt16Scalar* sp = <CInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int32Value(ArrayValue):
+cdef class UInt32Scalar(Scalar):
"""
- Concrete class for int32 array elements.
+ Concrete class for uint32 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt32Array* ap = <CInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt32Scalar* sp = <CUInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt32Value(ArrayValue):
+cdef class Int32Scalar(Scalar):
"""
- Concrete class for uint32 array elements.
+ Concrete class for int32 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt32Array* ap = <CUInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt32Scalar* sp = <CInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int64Value(ArrayValue):
+cdef class UInt64Scalar(Scalar):
"""
- Concrete class for int64 array elements.
+ Concrete class for uint64 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt64Array* ap = <CInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt64Scalar* sp = <CUInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt64Value(ArrayValue):
+cdef class Int64Scalar(Scalar):
"""
- Concrete class for uint64 array elements.
+ Concrete class for int64 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt64Array* ap = <CUInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt64Scalar* sp = <CInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date32Value(ArrayValue):
+cdef class HalfFloatScalar(Scalar):
"""
- Concrete class for date32 array elements.
+ Concrete class for float scalars.
"""
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+
+ def __eq__(self, other):
+ if hasattr(self, 'as_py'):
+ if isinstance(other, Scalar):
+ other = other.as_py()
+ return self.as_py() == other
+ else:
+ raise NotImplementedError
+
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python float.
"""
- cdef CDate32Array* ap = <CDate32Array*> self.sp_array.get()
+ cdef CHalfFloatScalar* sp = <CHalfFloatScalar*> self.wrapped.get()
+ return PyHalf_FromHalf(sp.value) if sp.is_valid else None
- # Shift to seconds since epoch
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(days=ap.Value(self.index)))
+
+cdef class FloatScalar(Scalar):
+ """
+ Concrete class for float scalars.
+ """
+
+ def as_py(self):
+ """
+ Return this value as a Python float.
+ """
+ cdef CFloatScalar* sp = <CFloatScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date64Value(ArrayValue):
+cdef class DoubleScalar(Scalar):
"""
- Concrete class for date64 array elements.
+ Concrete class for double scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python float.
"""
- cdef CDate64Array* ap = <CDate64Array*> self.sp_array.get()
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(
- days=ap.Value(self.index) / 86400000))
+ cdef CDoubleScalar* sp = <CDoubleScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Time32Value(ArrayValue):
+cdef class Decimal128Scalar(Scalar):
"""
- Concrete class for time32 array elements.
+ Concrete class for decimal128 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.timedelta instance.
+ Return this value as a Python Decimal.
"""
cdef:
- CTime32Array* ap = <CTime32Array*> self.sp_array.get()
- CTime32Type* dtype = <CTime32Type*> ap.type().get()
-
- if dtype.unit() == TimeUnit_SECOND:
- delta = datetime.timedelta(seconds=ap.Value(self.index))
- return (datetime.datetime(1970, 1, 1) + delta).time()
+ CDecimal128Scalar* sp = <CDecimal128Scalar*> self.wrapped.get()
+ CDecimal128Type* dtype = <CDecimal128Type*> sp.type.get()
+ if sp.is_valid:
+ return _pydecimal.Decimal(
Review comment:
Wow, this is misleading, before `_pydecimal` is the name of the pure Python implementation of `decimal`:
```python
>>> import _pydecimal
>>> _pydecimal
<module '_pydecimal' from '/home/antoine/miniconda3/envs/pyarrow/lib/python3.7/_pydecimal.py'>
>>> import decimal
>>> decimal is _pydecimal
False
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] kszucs commented on pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
kszucs commented on pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#issuecomment-654308059
@pitrou updated.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] wesm closed pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
wesm closed pull request #7519:
URL: https://github.com/apache/arrow/pull/7519
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] kszucs commented on a change in pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
kszucs commented on a change in pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#discussion_r450383737
##########
File path: cpp/src/arrow/scalar_test.cc
##########
@@ -451,6 +565,127 @@ TEST(TestStructScalar, FieldAccess) {
ASSERT_RAISES(Invalid, abc.field(5).status());
ASSERT_RAISES(Invalid, abc.field("c").status());
+
+ ASSERT_OK_AND_ASSIGN(auto d, abc.field("d"));
+ ASSERT_TRUE(d->Equals(MakeNullScalar(int64())));
+}
+
+TEST(TestDictionaryScalar, Basics) {
+ auto ty = dictionary(int8(), utf8());
+ auto dict = ArrayFromJSON(utf8(), "[\"alpha\", \"beta\", \"gamma\"]");
+
+ DictionaryScalar::ValueType alpha;
+ ASSERT_OK_AND_ASSIGN(alpha.index, MakeScalar(int8(), 0));
+ alpha.dictionary = dict;
+
+ DictionaryScalar::ValueType gamma;
+ ASSERT_OK_AND_ASSIGN(gamma.index, MakeScalar(int8(), 2));
+ gamma.dictionary = dict;
+
+ auto scalar_null = MakeNullScalar(ty);
+ auto scalar_alpha = DictionaryScalar(alpha, ty);
+ auto scalar_gamma = DictionaryScalar(gamma, ty);
+
+ ASSERT_OK_AND_ASSIGN(
+ auto encoded_null,
+ checked_cast<const DictionaryScalar&>(*scalar_null).GetEncodedValue());
+ ASSERT_TRUE(encoded_null->Equals(MakeNullScalar(utf8())));
+
+ ASSERT_OK_AND_ASSIGN(
+ auto encoded_alpha,
+ checked_cast<const DictionaryScalar&>(scalar_alpha).GetEncodedValue());
+ ASSERT_TRUE(encoded_alpha->Equals(MakeScalar("alpha")));
+
+ ASSERT_OK_AND_ASSIGN(
+ auto encoded_gamma,
+ checked_cast<const DictionaryScalar&>(scalar_gamma).GetEncodedValue());
+ ASSERT_TRUE(encoded_gamma->Equals(MakeScalar("gamma")));
+
+ // test Array.GetScalar
+ DictionaryArray arr(ty, ArrayFromJSON(int8(), "[2, 0, 1, null]"), dict);
+ ASSERT_OK_AND_ASSIGN(auto first, arr.GetScalar(0));
+ ASSERT_OK_AND_ASSIGN(auto second, arr.GetScalar(1));
+ ASSERT_OK_AND_ASSIGN(auto last, arr.GetScalar(3));
+
+ ASSERT_TRUE(first->Equals(scalar_gamma));
+ ASSERT_TRUE(second->Equals(scalar_alpha));
+ ASSERT_TRUE(last->Equals(scalar_null));
+}
+
+TEST(TestSparseUnionScalar, Basics) {
+ auto ty = sparse_union({field("string", utf8()), field("number", uint64())});
+
+ auto alpha = MakeScalar("alpha");
+ auto beta = MakeScalar("beta");
+ ASSERT_OK_AND_ASSIGN(auto two, MakeScalar(uint64(), 2));
+
+ auto scalar_alpha = SparseUnionScalar(alpha, ty);
+ auto scalar_beta = SparseUnionScalar(beta, ty);
+ auto scalar_two = SparseUnionScalar(two, ty);
+
+ // test Array.GetScalar
+ std::vector<std::shared_ptr<Array>> children{
+ ArrayFromJSON(utf8(), "[\"alpha\", \"\", \"beta\", null, \"gamma\"]"),
+ ArrayFromJSON(uint64(), "[1, 2, 11, 22, null]")};
+
+ auto type_ids = ArrayFromJSON(int8(), "[0, 1, 0, 0, 1]");
+ SparseUnionArray arr(ty, 5, children, type_ids->data()->buffers[1]);
+ ASSERT_OK(arr.ValidateFull());
+
+ ASSERT_OK_AND_ASSIGN(auto first, arr.GetScalar(0));
+ ASSERT_TRUE(first->Equals(scalar_alpha));
+
+ ASSERT_OK_AND_ASSIGN(auto second, arr.GetScalar(1));
+ ASSERT_TRUE(second->Equals(scalar_two));
+
+ ASSERT_OK_AND_ASSIGN(auto third, arr.GetScalar(2));
+ ASSERT_TRUE(third->Equals(scalar_beta));
+
+ ASSERT_OK_AND_ASSIGN(auto fourth, arr.GetScalar(3));
+ ASSERT_TRUE(fourth->Equals(MakeNullScalar(ty)));
Review comment:
We can ensure to have an explicit type to the value scalar from `array.GetScalar()` but not when a null union scalar is directly constructed, so `MakeNullScalar(union_type)` won't know which type to choose for the underlying value scalar.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] kszucs commented on pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
kszucs commented on pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#issuecomment-650149745
> Could we add a `is_valid` attribute to the python scalar as well? Now the only way to check for a null value is to do `.as_py() is None` ?
Added.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] github-actions[bot] commented on pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
github-actions[bot] commented on pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#issuecomment-649551591
https://issues.apache.org/jira/browse/ARROW-9017
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] kszucs commented on a change in pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
kszucs commented on a change in pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#discussion_r447725569
##########
File path: python/pyarrow/lib.pxd
##########
@@ -179,101 +179,18 @@ cdef class Schema:
cdef class Scalar:
- cdef readonly:
- DataType type
-
-
-cdef class NAType(Scalar):
- pass
-
-
-cdef class ArrayValue(Scalar):
cdef:
- shared_ptr[CArray] sp_array
- int64_t index
-
- cdef void init(self, DataType type,
- const shared_ptr[CArray]& sp_array, int64_t index)
-
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array)
-
-cdef class ScalarValue(Scalar):
- cdef:
- shared_ptr[CScalar] sp_scalar
-
- cdef void init(self, const shared_ptr[CScalar]& sp_scalar)
-
-cdef class Int8Value(ArrayValue):
- pass
+ shared_ptr[CScalar] wrapped
+ cdef void init(self, const shared_ptr[CScalar]& wrapped)
-cdef class Int64Value(ArrayValue):
- pass
-
-
-cdef class ListValue(ArrayValue):
- cdef readonly:
- DataType value_type
-
- cdef:
- CListArray* ap
-
- cdef getitem(self, int64_t i)
- cdef int64_t length(self)
-
-
-cdef class LargeListValue(ArrayValue):
- cdef readonly:
- DataType value_type
-
- cdef:
- CLargeListArray* ap
-
- cdef getitem(self, int64_t i)
- cdef int64_t length(self)
-
-
-cdef class MapValue(ArrayValue):
- cdef readonly:
- DataType key_type
- DataType item_type
-
- cdef:
- CMapArray* ap
-
- cdef getitem(self, int64_t i)
- cdef int64_t length(self)
-
-
-cdef class FixedSizeListValue(ArrayValue):
- cdef readonly:
- DataType value_type
-
- cdef:
- CFixedSizeListArray* ap
-
- cdef getitem(self, int64_t i)
- cdef int64_t length(self)
-
-
-cdef class StructValue(ArrayValue):
- cdef:
- CStructArray* ap
-
-
-cdef class UnionValue(ArrayValue):
- cdef:
- CUnionArray* ap
- list value_types
-
- cdef getitem(self, int64_t i)
-
+ @staticmethod
+ cdef wrap(const shared_ptr[CScalar]& wrapped)
-cdef class StringValue(ArrayValue):
- pass
+ cdef inline shared_ptr[CScalar] unwrap(self) nogil
-cdef class FixedSizeBinaryValue(ArrayValue):
+cdef class NAType(Scalar):
Review comment:
Forgot to rename it in the `.pxd` file.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] kszucs commented on a change in pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
kszucs commented on a change in pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#discussion_r448551033
##########
File path: python/pyarrow/tests/test_scalars.py
##########
@@ -16,427 +16,443 @@
# under the License.
import datetime
+import decimal
import pytest
-import unittest
import numpy as np
import pyarrow as pa
-class TestScalars(unittest.TestCase):
-
- def test_null_singleton(self):
- with pytest.raises(Exception):
- pa.NAType()
+@pytest.mark.parametrize(['value', 'ty', 'klass', 'deprecated'], [
+ (False, None, pa.BooleanScalar, pa.BooleanValue),
+ (True, None, pa.BooleanScalar, pa.BooleanValue),
+ (1, None, pa.Int64Scalar, pa.Int64Value),
+ (-1, None, pa.Int64Scalar, pa.Int64Value),
+ (1, pa.int8(), pa.Int8Scalar, pa.Int8Value),
+ (1, pa.uint8(), pa.UInt8Scalar, pa.UInt8Value),
+ (1, pa.int16(), pa.Int16Scalar, pa.Int16Value),
+ (1, pa.uint16(), pa.UInt16Scalar, pa.UInt16Value),
+ (1, pa.int32(), pa.Int32Scalar, pa.Int32Value),
+ (1, pa.uint32(), pa.UInt32Scalar, pa.UInt32Value),
+ (1, pa.int64(), pa.Int64Scalar, pa.Int64Value),
+ (1, pa.uint64(), pa.UInt64Scalar, pa.UInt64Value),
+ (1.0, None, pa.DoubleScalar, pa.DoubleValue),
+ (np.float16(1.0), pa.float16(), pa.HalfFloatScalar, pa.HalfFloatValue),
+ (1.0, pa.float32(), pa.FloatScalar, pa.FloatValue),
+ ("string", None, pa.StringScalar, pa.StringValue),
+ (b"bytes", None, pa.BinaryScalar, pa.BinaryValue),
+ ([1, 2, 3], None, pa.ListScalar, pa.ListValue),
+ ([1, 2, 3, 4], pa.large_list(pa.int8()), pa.LargeListScalar,
+ pa.LargeListValue),
+ (datetime.date.today(), None, pa.Date32Scalar, pa.Date64Value),
+ (datetime.datetime.now(), None, pa.TimestampScalar, pa.TimestampValue),
+ ({'a': 1, 'b': [1, 2]}, None, pa.StructScalar, pa.StructValue)
+])
+def test_basics(value, ty, klass, deprecated):
+ s = pa.scalar(value, type=ty)
+ assert isinstance(s, klass)
+ assert s == value
+ assert s == s
+ assert s != "else"
+ assert hash(s) == hash(s)
+ assert s.is_valid is True
+ with pytest.warns(FutureWarning):
+ isinstance(s, deprecated)
+
+ s = pa.scalar(None, type=s.type)
+ assert s.is_valid is False
+ assert s.as_py() is None
+
+
+def test_null_singleton():
+ with pytest.raises(Exception):
+ pa.NullScalar()
+
+
+def test_nulls():
+ null = pa.scalar(None)
+ assert null is pa.NA
+ assert null.as_py() is None
+ assert (null == "something") is pa.NA
Review comment:
Added and updated the `__eq__` behavior to return `False`.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] kszucs commented on a change in pull request #7519: ARROW-9153: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
kszucs commented on a change in pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#discussion_r445194648
##########
File path: python/pyarrow/tests/test_misc.py
##########
@@ -120,7 +120,6 @@ def test_cpu_count():
pa.LargeListValue,
pa.MapValue,
pa.FixedSizeListValue,
- pa.UnionValue,
Review comment:
Restore it
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] kszucs commented on a change in pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
kszucs commented on a change in pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#discussion_r448550822
##########
File path: python/pyarrow/scalar.pxi
##########
@@ -16,1198 +16,748 @@
# under the License.
-_NULL = NA = None
+import collections
cdef class Scalar:
"""
- The base class for all array elements.
- """
-
-
-cdef class NullType(Scalar):
- """
- Singleton for null array elements.
+ The base class for scalars.
"""
- # TODO rename this NullValue?
-
- def __cinit__(self):
- global NA
- if NA is not None:
- raise Exception('Cannot create multiple NAType instances')
-
- self.type = null()
-
- def __repr__(self):
- return 'NULL'
- def as_py(self):
- """
- Return None
- """
- return None
+ def __init__(self):
+ raise TypeError("Do not call {}'s constructor directly, use "
+ "pa.scalar() instead.".format(self.__class__.__name__))
- def __eq__(self, other):
- return NA
+ cdef void init(self, const shared_ptr[CScalar]& wrapped):
+ self.wrapped = wrapped
+ @staticmethod
+ cdef wrap(const shared_ptr[CScalar]& wrapped):
+ cdef:
+ Scalar self
+ Type type_id = wrapped.get().type.get().id()
-_NULL = NA = NullType()
+ if type_id == _Type_NA:
+ return _NULL
+ typ = _scalar_classes[type_id]
+ self = typ.__new__(typ)
+ self.init(wrapped)
-cdef class ArrayValue(Scalar):
- """
- The base class for non-null array elements.
- """
+ return self
- def __init__(self):
- raise TypeError("Do not call {}'s constructor directly, use array "
- "subscription instead."
- .format(self.__class__.__name__))
+ cdef inline shared_ptr[CScalar] unwrap(self) nogil:
+ return self.wrapped
- cdef void init(self, DataType type, const shared_ptr[CArray]& sp_array,
- int64_t index):
- self.type = type
- self.index = index
- self._set_array(sp_array)
+ @property
+ def type(self):
+ """
+ Data type of the Scalar object.
+ """
+ return pyarrow_wrap_data_type(self.wrapped.get().type)
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
+ @property
+ def is_valid(self):
+ """
+ Holds a valid (non-null) value.
+ """
+ return self.wrapped.get().is_valid
def __repr__(self):
- if hasattr(self, 'as_py'):
- return repr(self.as_py())
- else:
- return super(Scalar, self).__repr__()
+ return '<pyarrow.{}: {!r}>'.format(
+ self.__class__.__name__, self.as_py()
+ )
def __str__(self):
- if hasattr(self, 'as_py'):
- return str(self.as_py())
- else:
- return super(Scalar, self).__str__()
+ return str(self.as_py())
+
+ def equals(self, Scalar other):
+ return self.wrapped.get().Equals(other.unwrap().get()[0])
def __eq__(self, other):
- if hasattr(self, 'as_py'):
- if isinstance(other, ArrayValue):
- other = other.as_py()
- return self.as_py() == other
- else:
- raise NotImplementedError(
- "Cannot compare Arrow values that don't support as_py()")
+ try:
+ if not isinstance(other, Scalar):
+ other = scalar(other, type=self.type)
+ return self.equals(other)
+ except (TypeError, ValueError, ArrowInvalid):
+ return NotImplemented
def __hash__(self):
- return hash(self.as_py())
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+ def as_py(self):
+ raise NotImplementedError()
-cdef class BooleanValue(ArrayValue):
- """
- Concrete class for boolean array elements.
- """
- def as_py(self):
- """
- Return this value as a Python bool.
- """
- cdef CBooleanArray* ap = <CBooleanArray*> self.sp_array.get()
- return ap.Value(self.index)
+_NULL = NA = None
-cdef class Int8Value(ArrayValue):
+cdef class NullScalar(Scalar):
"""
- Concrete class for int8 array elements.
+ Concrete class for null scalars.
"""
- def as_py(self):
- """
- Return this value as a Python int.
- """
- cdef CInt8Array* ap = <CInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ def __cinit__(self):
+ global NA
+ if NA is not None:
+ raise RuntimeError('Cannot create multiple NullScalar instances')
+ self.init(shared_ptr[CScalar](new CNullScalar()))
+ def __init__(self):
+ pass
-cdef class UInt8Value(ArrayValue):
- """
- Concrete class for uint8 array elements.
- """
+ def __eq__(self, other):
+ return NA
+
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
def as_py(self):
"""
- Return this value as a Python int.
+ Return this value as a Python None.
"""
- cdef CUInt8Array* ap = <CUInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
-
+ return None
-cdef class Int16Value(ArrayValue):
- """
- Concrete class for int16 array elements.
- """
- def as_py(self):
- """
- Return this value as a Python int.
- """
- cdef CInt16Array* ap = <CInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+_NULL = NA = NullScalar()
-cdef class UInt16Value(ArrayValue):
+cdef class BooleanScalar(Scalar):
"""
- Concrete class for uint16 array elements.
+ Concrete class for boolean scalars.
"""
def as_py(self):
"""
- Return this value as a Python int.
+ Return this value as a Python bool.
"""
- cdef CUInt16Array* ap = <CUInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CBooleanScalar* sp = <CBooleanScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int32Value(ArrayValue):
+cdef class UInt8Scalar(Scalar):
"""
- Concrete class for int32 array elements.
+ Concrete class for uint8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt32Array* ap = <CInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt8Scalar* sp = <CUInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt32Value(ArrayValue):
+cdef class Int8Scalar(Scalar):
"""
- Concrete class for uint32 array elements.
+ Concrete class for int8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt32Array* ap = <CUInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt8Scalar* sp = <CInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int64Value(ArrayValue):
+cdef class UInt16Scalar(Scalar):
"""
- Concrete class for int64 array elements.
+ Concrete class for uint16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt64Array* ap = <CInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt16Scalar* sp = <CUInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt64Value(ArrayValue):
+cdef class Int16Scalar(Scalar):
"""
- Concrete class for uint64 array elements.
+ Concrete class for int16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt64Array* ap = <CUInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt16Scalar* sp = <CInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date32Value(ArrayValue):
+cdef class UInt32Scalar(Scalar):
"""
- Concrete class for date32 array elements.
+ Concrete class for uint32 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python int.
"""
- cdef CDate32Array* ap = <CDate32Array*> self.sp_array.get()
-
- # Shift to seconds since epoch
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(days=ap.Value(self.index)))
+ cdef CUInt32Scalar* sp = <CUInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date64Value(ArrayValue):
+cdef class Int32Scalar(Scalar):
"""
- Concrete class for date64 array elements.
+ Concrete class for int32 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python int.
"""
- cdef CDate64Array* ap = <CDate64Array*> self.sp_array.get()
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(
- days=ap.Value(self.index) / 86400000))
+ cdef CInt32Scalar* sp = <CInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Time32Value(ArrayValue):
+cdef class UInt64Scalar(Scalar):
"""
- Concrete class for time32 array elements.
+ Concrete class for uint64 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.timedelta instance.
+ Return this value as a Python int.
"""
- cdef:
- CTime32Array* ap = <CTime32Array*> self.sp_array.get()
- CTime32Type* dtype = <CTime32Type*> ap.type().get()
-
- if dtype.unit() == TimeUnit_SECOND:
- delta = datetime.timedelta(seconds=ap.Value(self.index))
- return (datetime.datetime(1970, 1, 1) + delta).time()
- else:
- return _box_time_milli(ap.Value(self.index))
+ cdef CUInt64Scalar* sp = <CUInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Time64Value(ArrayValue):
+cdef class Int64Scalar(Scalar):
"""
- Concrete class for time64 array elements.
+ Concrete class for int64 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.timedelta instance.
+ Return this value as a Python int.
"""
- cdef:
- CTime64Array* ap = <CTime64Array*> self.sp_array.get()
- CTime64Type* dtype = <CTime64Type*> ap.type().get()
-
- cdef int64_t val = ap.Value(self.index)
- if dtype.unit() == TimeUnit_MICRO:
- return _box_time_micro(val)
- else:
- return (datetime.datetime(1970, 1, 1) +
- datetime.timedelta(microseconds=val / 1000)).time()
-
-
-cpdef _box_time_milli(int64_t val):
- delta = datetime.timedelta(milliseconds=val)
- return (datetime.datetime(1970, 1, 1) + delta).time()
-
-
-cpdef _box_time_micro(int64_t val):
- return (datetime.datetime(1970, 1, 1) +
- datetime.timedelta(microseconds=val)).time()
-
-
-cdef dict _DATETIME_CONVERSION_FUNCTIONS = {}
-cdef c_bool _datetime_conversion_initialized = False
-
-
-cdef _add_micros_maybe_localize(dt, micros, tzinfo):
- import pytz
- dt = dt.replace(microsecond=micros)
- if tzinfo is not None:
- if not isinstance(tzinfo, datetime.tzinfo):
- tzinfo = string_to_tzinfo(tzinfo)
- dt = tzinfo.fromutc(dt)
- return dt
-
-
-cdef _datetime_from_seconds(int64_t v):
- return datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=v)
-
-
-def _nanoseconds_to_datetime_safe(v, tzinfo):
- if v % 1000 != 0:
- raise ValueError("Nanosecond timestamp {} is not safely convertible "
- " to microseconds to convert to datetime.datetime."
- " Install pandas to return as Timestamp with "
- " nanosecond support or access the .value attribute.")
- v = v // 1000
- micros = v % 1_000_000
-
- dt = _datetime_from_seconds(v // 1_000_000)
- return _add_micros_maybe_localize(dt, micros, tzinfo)
-
-
-def _microseconds_to_datetime(v, tzinfo):
- micros = v % 1_000_000
- dt = _datetime_from_seconds(v // 1_000_000)
- return _add_micros_maybe_localize(dt, micros, tzinfo)
-
-
-def _millis_to_datetime(v, tzinfo):
- millis = v % 1_000
- dt = _datetime_from_seconds(v // 1000)
- return _add_micros_maybe_localize(dt, millis * 1000, tzinfo)
-
-
-def _seconds_to_datetime(v, tzinfo):
- dt = _datetime_from_seconds(v)
- return _add_micros_maybe_localize(dt, 0, tzinfo)
-
-
-def _datetime_conversion_functions():
- global _datetime_conversion_initialized
- if _datetime_conversion_initialized:
- return _DATETIME_CONVERSION_FUNCTIONS
-
- _DATETIME_CONVERSION_FUNCTIONS.update({
- TimeUnit_SECOND: _seconds_to_datetime,
- TimeUnit_MILLI: _millis_to_datetime,
- TimeUnit_MICRO: _microseconds_to_datetime,
- TimeUnit_NANO: _nanoseconds_to_datetime_safe
- })
-
- try:
- import pandas as pd
- _DATETIME_CONVERSION_FUNCTIONS[TimeUnit_NANO] = (
- lambda x, tzinfo: pd.Timestamp(
- x, tz=tzinfo, unit='ns',
- )
- )
- except ImportError:
- pass
-
- _datetime_conversion_initialized = True
- return _DATETIME_CONVERSION_FUNCTIONS
+ cdef CInt64Scalar* sp = <CInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class TimestampValue(ArrayValue):
+cdef class HalfFloatScalar(Scalar):
"""
- Concrete class for timestamp array elements.
+ Concrete class for float scalars.
"""
- @property
- def value(self):
- cdef CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
- cdef CTimestampType* dtype = <CTimestampType*> ap.type().get()
- return ap.Value(self.index)
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+
+ def __eq__(self, other):
+ if isinstance(other, Scalar):
+ other = other.as_py()
+ return self.as_py() == other
def as_py(self):
"""
- Return this value as a Pandas Timestamp instance (if available),
- otherwise as a Python datetime.timedelta instance.
+ Return this value as a Python float.
"""
- cdef CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
- cdef CTimestampType* dtype = <CTimestampType*> ap.type().get()
-
- value = self.value
-
- if not dtype.timezone().empty():
- tzinfo = string_to_tzinfo(frombytes(dtype.timezone()))
- else:
- tzinfo = None
-
- try:
- converter = _datetime_conversion_functions()[dtype.unit()]
- except KeyError:
- raise ValueError(
- 'Cannot convert nanosecond timestamps without pandas'
- )
- return converter(value, tzinfo=tzinfo)
-
-
-cdef dict _TIMEDELTA_CONVERSION_FUNCTIONS = {}
-
-
-def _nanoseconds_to_timedelta_safe(v):
- if v % 1000 != 0:
- raise ValueError(
- "Nanosecond duration {} is not safely convertible to microseconds "
- "to convert to datetime.timedelta. Install pandas to return as "
- "Timedelta with nanosecond support or access the .value "
- "attribute.".format(v))
- micros = v // 1000
-
- return datetime.timedelta(microseconds=micros)
+ cdef CHalfFloatScalar* sp = <CHalfFloatScalar*> self.wrapped.get()
+ return PyHalf_FromHalf(sp.value) if sp.is_valid else None
-def _timedelta_conversion_functions():
- if _TIMEDELTA_CONVERSION_FUNCTIONS:
- return _TIMEDELTA_CONVERSION_FUNCTIONS
-
- _TIMEDELTA_CONVERSION_FUNCTIONS.update({
- TimeUnit_SECOND: lambda v: datetime.timedelta(seconds=v),
- TimeUnit_MILLI: lambda v: datetime.timedelta(milliseconds=v),
- TimeUnit_MICRO: lambda v: datetime.timedelta(microseconds=v),
- TimeUnit_NANO: _nanoseconds_to_timedelta_safe
- })
-
- try:
- import pandas as pd
- _TIMEDELTA_CONVERSION_FUNCTIONS[TimeUnit_NANO] = (
- lambda v: pd.Timedelta(v, unit='ns')
- )
- except ImportError:
- pass
-
- return _TIMEDELTA_CONVERSION_FUNCTIONS
-
-
-cdef class DurationValue(ArrayValue):
+cdef class FloatScalar(Scalar):
"""
- Concrete class for duration array elements.
+ Concrete class for float scalars.
"""
- @property
- def value(self):
- cdef CDurationArray* ap = <CDurationArray*> self.sp_array.get()
- return ap.Value(self.index)
-
def as_py(self):
"""
- Return this value as a Pandas Timestamp instance (if available),
- otherwise as a Python datetime.timedelta instance.
+ Return this value as a Python float.
"""
- cdef CDurationArray* ap = <CDurationArray*> self.sp_array.get()
- cdef CDurationType* dtype = <CDurationType*> ap.type().get()
-
- cdef int64_t value = ap.Value(self.index)
- converter = _timedelta_conversion_functions()[dtype.unit()]
- return converter(value)
+ cdef CFloatScalar* sp = <CFloatScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class HalfFloatValue(ArrayValue):
+cdef class DoubleScalar(Scalar):
"""
- Concrete class for float16 array elements.
+ Concrete class for double scalars.
"""
def as_py(self):
"""
Return this value as a Python float.
"""
- cdef CHalfFloatArray* ap = <CHalfFloatArray*> self.sp_array.get()
- return PyHalf_FromHalf(ap.Value(self.index))
+ cdef CDoubleScalar* sp = <CDoubleScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class FloatValue(ArrayValue):
+cdef class Decimal128Scalar(Scalar):
"""
- Concrete class for float32 array elements.
+ Concrete class for decimal128 scalars.
"""
def as_py(self):
"""
- Return this value as a Python float.
+ Return this value as a Python Decimal.
"""
- cdef CFloatArray* ap = <CFloatArray*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef:
+ CDecimal128Scalar* sp = <CDecimal128Scalar*> self.wrapped.get()
+ CDecimal128Type* dtype = <CDecimal128Type*> sp.type.get()
+ if sp.is_valid:
+ return _pydecimal.Decimal(
+ frombytes(sp.value.ToString(dtype.scale()))
+ )
+ else:
+ return None
-cdef class DoubleValue(ArrayValue):
+cdef class Date32Scalar(Scalar):
"""
- Concrete class for float64 array elements.
+ Concrete class for date32 scalars.
"""
def as_py(self):
"""
- Return this value as a Python float.
+ Return this value as a Python datetime.datetime instance.
"""
- cdef CDoubleArray* ap = <CDoubleArray*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CDate32Scalar* sp = <CDate32Scalar*> self.wrapped.get()
+
+ if sp.is_valid:
+ # shift to seconds since epoch
+ return (
+ datetime.date(1970, 1, 1) + datetime.timedelta(days=sp.value)
+ )
+ else:
+ return None
-cdef class DecimalValue(ArrayValue):
+cdef class Date64Scalar(Scalar):
"""
- Concrete class for decimal128 array elements.
+ Concrete class for date64 scalars.
"""
def as_py(self):
"""
- Return this value as a Python Decimal.
+ Return this value as a Python datetime.datetime instance.
"""
- cdef:
- CDecimal128Array* ap = <CDecimal128Array*> self.sp_array.get()
- c_string s = ap.FormatValue(self.index)
- return _pydecimal.Decimal(s.decode('utf8'))
+ cdef CDate64Scalar* sp = <CDate64Scalar*> self.wrapped.get()
+ if sp.is_valid:
+ return (
+ datetime.date(1970, 1, 1) +
+ datetime.timedelta(days=sp.value / 86400000)
+ )
+ else:
+ return None
-cdef class StringValue(ArrayValue):
- """
- Concrete class for string (utf8) array elements.
- """
- def as_py(self):
- """
- Return this value as a Python unicode string.
- """
- cdef CStringArray* ap = <CStringArray*> self.sp_array.get()
- return ap.GetString(self.index).decode('utf-8')
+def _datetime_from_int(int64_t value, TimeUnit unit, tzinfo=None):
+ if unit == TimeUnit_SECOND:
+ delta = datetime.timedelta(seconds=value)
+ elif unit == TimeUnit_MILLI:
+ delta = datetime.timedelta(milliseconds=value)
+ elif unit == TimeUnit_MICRO:
+ delta = datetime.timedelta(microseconds=value)
+ else:
+ # TimeUnit_NANO: prefer pandas timestamps if available
+ if _pandas_api.have_pandas:
+ return _pandas_api.pd.Timestamp(value, tz=tzinfo, unit='ns')
+ # otherwise safely truncate to microsecond resolution datetime
+ if value % 1000 != 0:
+ raise ValueError(
+ "Nanosecond resolution temporal type {} is not safely "
+ "convertible to microseconds to convert to datetime.datetime. "
+ "Install pandas to return as Timestamp with nanosecond "
+ "support or access the .value attribute.".format(value)
+ )
+ delta = datetime.timedelta(microseconds=value // 1000)
- def as_buffer(self):
- """
- Return a view over this value as a Buffer object.
- """
- cdef:
- CStringArray* ap = <CStringArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
+ dt = datetime.datetime(1970, 1, 1) + delta
+ # adjust timezone if set to the datatype
+ if tzinfo is not None:
+ dt = tzinfo.fromutc(dt)
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
+ return dt
-cdef class LargeStringValue(ArrayValue):
+cdef class Time32Scalar(Scalar):
"""
- Concrete class for large string (utf8) array elements.
+ Concrete class for time32 scalars.
"""
def as_py(self):
"""
- Return this value as a Python unicode string.
- """
- cdef CLargeStringArray* ap = <CLargeStringArray*> self.sp_array.get()
- return ap.GetString(self.index).decode('utf-8')
-
- def as_buffer(self):
- """
- Return a view over this value as a Buffer object.
+ Return this value as a Python datetime.timedelta instance.
"""
cdef:
- CLargeStringArray* ap = <CLargeStringArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
+ CTime32Scalar* sp = <CTime32Scalar*> self.wrapped.get()
+ CTime32Type* dtype = <CTime32Type*> sp.type.get()
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
+ if sp.is_valid:
+ return _datetime_from_int(sp.value, unit=dtype.unit()).time()
+ else:
+ return None
-cdef class BinaryValue(ArrayValue):
+cdef class Time64Scalar(Scalar):
"""
- Concrete class for variable-sized binary array elements.
+ Concrete class for time64 scalars.
"""
def as_py(self):
"""
- Return this value as a Python bytes object.
- """
- cdef:
- const uint8_t* ptr
- int32_t length
- CBinaryArray* ap = <CBinaryArray*> self.sp_array.get()
-
- ptr = ap.GetValue(self.index, &length)
- return cp.PyBytes_FromStringAndSize(<const char*>(ptr), length)
-
- def as_buffer(self):
- """
- Return a view over this value as a Buffer object.
+ Return this value as a Python datetime.timedelta instance.
"""
cdef:
- CBinaryArray* ap = <CBinaryArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
+ CTime64Scalar* sp = <CTime64Scalar*> self.wrapped.get()
+ CTime64Type* dtype = <CTime64Type*> sp.type.get()
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
+ if sp.is_valid:
+ return _datetime_from_int(sp.value, unit=dtype.unit()).time()
+ else:
+ return None
-cdef class LargeBinaryValue(ArrayValue):
+cdef class TimestampScalar(Scalar):
"""
- Concrete class for large variable-sized binary array elements.
+ Concrete class for timestamp scalars.
"""
+ @property
+ def value(self):
+ cdef CTimestampScalar* sp = <CTimestampScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
+
def as_py(self):
"""
- Return this value as a Python bytes object.
+ Return this value as a Pandas Timestamp instance (if available),
+ otherwise as a Python datetime.timedelta instance.
"""
cdef:
- const uint8_t* ptr
- int64_t length
- CLargeBinaryArray* ap = <CLargeBinaryArray*> self.sp_array.get()
+ CTimestampScalar* sp = <CTimestampScalar*> self.wrapped.get()
+ CTimestampType* dtype = <CTimestampType*> sp.type.get()
- ptr = ap.GetValue(self.index, &length)
- return cp.PyBytes_FromStringAndSize(<const char*>(ptr), length)
+ if not sp.is_valid:
+ return None
- def as_buffer(self):
- """
- Return a view over this value as a Buffer object.
- """
- cdef:
- CLargeBinaryArray* ap = <CLargeBinaryArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
+ if not dtype.timezone().empty():
+ tzinfo = string_to_tzinfo(frombytes(dtype.timezone()))
+ else:
+ tzinfo = None
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
+ return _datetime_from_int(sp.value, unit=dtype.unit(), tzinfo=tzinfo)
-cdef class ListValue(ArrayValue):
+cdef class DurationScalar(Scalar):
"""
- Concrete class for list array elements.
+ Concrete class for duration scalars.
"""
- def __len__(self):
- """
- Return the number of values.
- """
- return self.length()
-
- def __getitem__(self, i):
- """
- Return the value at the given index.
- """
- return self.getitem(_normalize_index(i, self.length()))
-
- def __iter__(self):
- """
- Iterate over this element's values.
- """
- for i in range(len(self)):
- yield self.getitem(i)
- raise StopIteration
-
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CListArray*> sp_array.get()
- self.value_type = pyarrow_wrap_data_type(self.ap.value_type())
-
- cdef getitem(self, int64_t i):
- cdef int64_t j = self.ap.value_offset(self.index) + i
- return box_scalar(self.value_type, self.ap.values(), j)
-
- cdef int64_t length(self):
- return self.ap.value_length(self.index)
+ @property
+ def value(self):
+ cdef CDurationScalar* sp = <CDurationScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
def as_py(self):
"""
- Return this value as a Python list.
+ Return this value as a Pandas Timestamp instance (if available),
+ otherwise as a Python datetime.timedelta instance.
"""
cdef:
- int64_t j
- list result = []
+ CDurationScalar* sp = <CDurationScalar*> self.wrapped.get()
+ CDurationType* dtype = <CDurationType*> sp.type.get()
+ TimeUnit unit = dtype.unit()
- for j in range(len(self)):
- result.append(self.getitem(j).as_py())
+ if not sp.is_valid:
+ return None
- return result
+ if unit == TimeUnit_SECOND:
+ return datetime.timedelta(seconds=sp.value)
+ elif unit == TimeUnit_MILLI:
+ return datetime.timedelta(milliseconds=sp.value)
+ elif unit == TimeUnit_MICRO:
+ return datetime.timedelta(microseconds=sp.value)
+ else:
+ # TimeUnit_NANO: prefer pandas timestamps if available
+ if _pandas_api.have_pandas:
+ return _pandas_api.pd.Timedelta(sp.value, unit='ns')
+ # otherwise safely truncate to microsecond resolution timedelta
+ if sp.value % 1000 != 0:
+ raise ValueError(
+ "Nanosecond duration {} is not safely convertible to "
+ "microseconds to convert to datetime.timedelta. Install "
+ "pandas to return as Timedelta with nanosecond support or "
+ "access the .value attribute.".format(sp.value)
+ )
+ return datetime.timedelta(microseconds=sp.value // 1000)
-cdef class LargeListValue(ArrayValue):
+cdef class BinaryScalar(Scalar):
"""
- Concrete class for large list array elements.
+ Concrete class for binary-like scalars.
"""
- def __len__(self):
- """
- Return the number of values.
- """
- return self.length()
-
- def __getitem__(self, i):
+ def as_buffer(self):
"""
- Return the value at the given index.
+ Return a view over this value as a Buffer object.
"""
- return self.getitem(_normalize_index(i, self.length()))
+ cdef CBinaryScalar* sp = <CBinaryScalar*> self.wrapped.get()
+ return pyarrow_wrap_buffer(sp.value) if sp.is_valid else None
- def __iter__(self):
+ def as_py(self):
"""
- Iterate over this element's values.
+ Return this value as a Python bytes.
"""
- for i in range(len(self)):
- yield self.getitem(i)
- raise StopIteration
+ buffer = self.as_buffer()
+ return None if buffer is None else buffer.to_pybytes()
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CLargeListArray*> sp_array.get()
- self.value_type = pyarrow_wrap_data_type(self.ap.value_type())
- cdef getitem(self, int64_t i):
- cdef int64_t j = self.ap.value_offset(self.index) + i
- return box_scalar(self.value_type, self.ap.values(), j)
+cdef class LargeBinaryScalar(BinaryScalar):
+ pass
- cdef int64_t length(self):
- return self.ap.value_length(self.index)
- def as_py(self):
- """
- Return this value as a Python list.
- """
- cdef:
- int64_t j
- list result = []
-
- for j in range(len(self)):
- result.append(self.getitem(j).as_py())
+cdef class FixedSizeBinaryScalar(BinaryScalar):
+ pass
- return result
-
-cdef class MapValue(ArrayValue):
+cdef class StringScalar(BinaryScalar):
"""
- Concrete class for map array elements.
+ Concrete class for string-like (utf8) scalars.
"""
- def __len__(self):
- """
- Return the number of values.
- """
- return self.length()
-
- def __getitem__(self, i):
- """
- Return the value at the given index.
- """
- return self.getitem(_normalize_index(i, self.length()))
-
- def __iter__(self):
- """
- Iterate over this element's values.
- """
- for i in range(len(self)):
- yield self.getitem(i)
- raise StopIteration
-
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CMapArray*> sp_array.get()
- self.key_type = pyarrow_wrap_data_type(self.ap.map_type().key_type())
- self.item_type = pyarrow_wrap_data_type(self.ap.map_type().item_type())
-
- cdef getitem(self, int64_t i):
- cdef int64_t j = self.ap.value_offset(self.index) + i
- return (box_scalar(self.key_type, self.ap.keys(), j),
- box_scalar(self.item_type, self.ap.items(), j))
-
- cdef int64_t length(self):
- return self.ap.value_length(self.index)
-
def as_py(self):
"""
- Return this value as a Python list of tuples, each containing a
- key and item.
+ Return this value as a Python string.
"""
- cdef:
- int64_t j
- list result = []
+ buffer = self.as_buffer()
+ return None if buffer is None else str(buffer, 'utf8')
- for j in range(len(self)):
- key, item = self.getitem(j)
- result.append((key.as_py(), item.as_py()))
- return result
+cdef class LargeStringScalar(StringScalar):
+ pass
-cdef class FixedSizeListValue(ArrayValue):
+cdef class ListScalar(Scalar):
"""
- Concrete class for fixed size list array elements.
+ Concrete class for list-like scalars.
"""
+ @property
+ def values(self):
+ cdef CListScalar* sp = <CListScalar*> self.wrapped.get()
+ if sp.is_valid:
+ return pyarrow_wrap_array(sp.value)
+ else:
+ return None
+
def __len__(self):
"""
Return the number of values.
"""
- return self.length()
+ return len(self.values)
def __getitem__(self, i):
"""
Return the value at the given index.
"""
- return self.getitem(_normalize_index(i, self.length()))
+ return self.values[_normalize_index(i, len(self))]
def __iter__(self):
"""
Iterate over this element's values.
"""
- for i in range(len(self)):
- yield self.getitem(i)
- raise StopIteration
-
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CFixedSizeListArray*> sp_array.get()
- self.value_type = pyarrow_wrap_data_type(self.ap.value_type())
-
- cdef getitem(self, int64_t i):
- cdef int64_t j = self.ap.value_offset(self.index) + i
- return box_scalar(self.value_type, self.ap.values(), j)
-
- cdef int64_t length(self):
- return self.ap.value_length(self.index)
+ return iter(self.values)
def as_py(self):
"""
Return this value as a Python list.
"""
- cdef:
- int64_t j
- list result = []
-
- for j in range(len(self)):
- result.append(self.getitem(j).as_py())
-
- return result
-
-
-cdef class UnionValue(ArrayValue):
- """
- Concrete class for union array elements.
- """
+ arr = self.values
+ return None if arr is None else arr.to_pylist()
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CUnionArray*> sp_array.get()
-
- cdef getitem(self, int64_t i):
- cdef int child_id = self.ap.child_id(i)
- cdef shared_ptr[CArray] child = self.ap.field(child_id)
- cdef CDenseUnionArray* dense
- if self.ap.mode() == _UnionMode_SPARSE:
- return box_scalar(self.type[child_id].type, child, i)
- else:
- dense = <CDenseUnionArray*> self.ap
- return box_scalar(self.type[child_id].type, child,
- dense.value_offset(i))
- def as_py(self):
- """
- Return this value as a Python object.
+cdef class FixedSizeListScalar(ListScalar):
+ pass
- The exact type depends on the underlying union member.
- """
- return self.getitem(self.index).as_py()
+cdef class LargeListScalar(ListScalar):
+ pass
-cdef class FixedSizeBinaryValue(ArrayValue):
- """
- Concrete class for fixed-size binary array elements.
- """
- def as_py(self):
- """
- Return this value as a Python bytes object.
- """
- cdef:
- CFixedSizeBinaryArray* ap
- CFixedSizeBinaryType* ap_type
- int32_t length
- const char* data
- ap = <CFixedSizeBinaryArray*> self.sp_array.get()
- ap_type = <CFixedSizeBinaryType*> ap.type().get()
- length = ap_type.byte_width()
- data = <const char*> ap.GetValue(self.index)
- return cp.PyBytes_FromStringAndSize(data, length)
-
-
-cdef class StructValue(ArrayValue):
+cdef class StructScalar(Scalar, collections.abc.Mapping):
"""
- Concrete class for struct array elements.
+ Concrete class for struct scalars.
"""
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CStructArray*> sp_array.get()
-
- def __getitem__(self, key):
- """
- Return the child value for the given field name.
- """
- cdef:
- CStructType* type
- int index
-
- type = <CStructType*> self.type.type
- index = type.GetFieldIndex(tobytes(key))
-
- if index < 0:
- raise KeyError(key)
-
- return pyarrow_wrap_array(self.ap.field(index))[self.index]
+ def __len__(self):
+ cdef CStructScalar* sp = <CStructScalar*> self.wrapped.get()
+ return sp.value.size()
- def as_py(self):
- """
- Return this value as a Python dict.
- """
+ def __iter__(self):
cdef:
- vector[shared_ptr[CField]] child_fields = self.type.type.fields()
-
- wrapped_arrays = [pyarrow_wrap_array(self.ap.field(i))
- for i in range(self.ap.num_fields())]
- child_names = [child.get().name() for child in child_fields]
- # Return the struct as a dict
- return {
- frombytes(name): child_array[self.index].as_py()
- for name, child_array in zip(child_names, wrapped_arrays)
- }
-
-
-cdef class DictionaryValue(ArrayValue):
- """
- Concrete class for dictionary-encoded array elements.
- """
-
- def as_py(self):
- """
- Return this value as a Python object.
-
- The exact type depends on the dictionary value type.
- """
- return self.dictionary_value.as_py()
-
- @property
- def index_value(self):
- """
- Return this value's underlying index as a ArrayValue of the right
- signed integer type.
- """
- cdef CDictionaryArray* darr = <CDictionaryArray*>(self.sp_array.get())
- indices = pyarrow_wrap_array(darr.indices())
- return indices[self.index]
-
- @property
- def dictionary_value(self):
- """
- Return this value's underlying dictionary value as a ArrayValue.
- """
- cdef CDictionaryArray* darr = <CDictionaryArray*>(self.sp_array.get())
- dictionary = pyarrow_wrap_array(darr.dictionary())
- return dictionary[self.index_value.as_py()]
-
-
-cdef dict _array_value_classes = {
- _Type_BOOL: BooleanValue,
- _Type_UINT8: UInt8Value,
- _Type_UINT16: UInt16Value,
- _Type_UINT32: UInt32Value,
- _Type_UINT64: UInt64Value,
- _Type_INT8: Int8Value,
- _Type_INT16: Int16Value,
- _Type_INT32: Int32Value,
- _Type_INT64: Int64Value,
- _Type_DATE32: Date32Value,
- _Type_DATE64: Date64Value,
- _Type_TIME32: Time32Value,
- _Type_TIME64: Time64Value,
- _Type_TIMESTAMP: TimestampValue,
- _Type_DURATION: DurationValue,
- _Type_HALF_FLOAT: HalfFloatValue,
- _Type_FLOAT: FloatValue,
- _Type_DOUBLE: DoubleValue,
- _Type_LIST: ListValue,
- _Type_LARGE_LIST: LargeListValue,
- _Type_MAP: MapValue,
- _Type_FIXED_SIZE_LIST: FixedSizeListValue,
- _Type_SPARSE_UNION: UnionValue,
- _Type_DENSE_UNION: UnionValue,
- _Type_BINARY: BinaryValue,
- _Type_STRING: StringValue,
- _Type_LARGE_BINARY: LargeBinaryValue,
- _Type_LARGE_STRING: LargeStringValue,
- _Type_FIXED_SIZE_BINARY: FixedSizeBinaryValue,
- _Type_DECIMAL: DecimalValue,
- _Type_STRUCT: StructValue,
- _Type_DICTIONARY: DictionaryValue,
-}
-
-cdef class ScalarValue(Scalar):
- """
- The base class for scalars.
- """
-
- def __init__(self):
- raise TypeError("Do not call {}'s constructor directly."
- .format(self.__class__.__name__))
-
- cdef void init(self, const shared_ptr[CScalar]& sp_scalar):
- self.sp_scalar = sp_scalar
-
- def __repr__(self):
- if hasattr(self, 'as_py'):
- return repr(self.as_py())
- else:
- return super(Scalar, self).__repr__()
+ CStructScalar* sp = <CStructScalar*> self.wrapped.get()
+ CStructType* dtype = <CStructType*> sp.type.get()
+ vector[shared_ptr[CField]] fields = dtype.fields()
- def __str__(self):
- if hasattr(self, 'as_py'):
- return str(self.as_py())
- else:
- return super(Scalar, self).__str__()
+ if sp.is_valid:
+ for i in range(dtype.num_fields()):
+ yield frombytes(fields[i].get().name())
- def __eq__(self, other):
- if hasattr(self, 'as_py'):
- if isinstance(other, ScalarValue):
- other = other.as_py()
- return self.as_py() == other
+ def __contains__(self, key):
+ try:
+ self[key]
+ except (KeyError, IndexError):
+ return False
else:
- raise NotImplemented(
- "Cannot compare scalars that don't support as_py()")
-
- def __hash__(self):
- return hash(self.as_py())
-
-
-cdef class NullScalar(ScalarValue):
- """
- Concrete class for null scalars.
- """
-
- def as_py(self):
- """
- Return this value as a Python None.
- """
- return None
-
-
-cdef class BooleanScalar(ScalarValue):
- """
- Concrete class for boolean scalars.
- """
+ return True
- def as_py(self):
- """
- Return this value as a Python bool.
+ def __getitem__(self, key):
"""
- cdef CBooleanScalar* sp = <CBooleanScalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
-
+ Return the child value for the given field.
-cdef class UInt8Scalar(ScalarValue):
- """
- Concrete class for uint8 scalars.
- """
+ Parameters
+ ----------
+ index : Union[int, str]
+ Index / position or name of the field.
- def as_py(self):
- """
- Return this value as a Python int.
+ Returns
+ -------
+ result : Scalar
"""
- cdef CUInt8Scalar* sp = <CUInt8Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
+ cdef:
+ CFieldRef ref
+ CStructScalar* sp = <CStructScalar*> self.wrapped.get()
+ if isinstance(key, (bytes, str)):
+ ref = CFieldRef(<c_string> tobytes(key))
+ elif isinstance(key, int):
+ ref = CFieldRef(<int> key)
+ else:
+ raise TypeError('Expected integer or string index')
-cdef class Int8Scalar(ScalarValue):
- """
- Concrete class for int8 scalars.
- """
+ try:
+ return Scalar.wrap(GetResultValue(sp.field(ref)))
+ except ArrowInvalid:
+ if isinstance(key, int):
+ raise IndexError(key)
+ else:
+ raise KeyError(key)
def as_py(self):
"""
- Return this value as a Python int.
+ Return this value as a Python dict.
"""
- cdef CInt8Scalar* sp = <CInt8Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
+ if self.is_valid:
+ return {k: v.as_py() for k, v in self.items()}
+ else:
+ return None
-cdef class UInt16Scalar(ScalarValue):
+cdef class MapScalar(ListScalar):
"""
- Concrete class for uint16 scalars.
+ Concrete class for map scalars.
"""
- def as_py(self):
+ def __getitem__(self, i):
"""
- Return this value as a Python int.
+ Return the value at the given index.
"""
- cdef CUInt16Scalar* sp = <CUInt16Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
-
-
-cdef class Int16Scalar(ScalarValue):
- """
- Concrete class for int16 scalars.
- """
+ arr = self.values
+ if arr is None:
+ raise IndexError(i)
+ dct = arr[_normalize_index(i, len(arr))]
+ return (dct['key'], dct['value'])
- def as_py(self):
+ def __iter__(self):
"""
- Return this value as a Python int.
+ Iterate over this element's values.
"""
- cdef CInt16Scalar* sp = <CInt16Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
-
-
-cdef class UInt32Scalar(ScalarValue):
- """
- Concrete class for uint32 scalars.
- """
+ arr = self.values
+ if arr is None:
+ return iter(zip(arr.field('key'), arr.field('value')))
+ else:
+ raise StopIteration
def as_py(self):
"""
- Return this value as a Python int.
+ Return this value as a Python list.
"""
- cdef CUInt32Scalar* sp = <CUInt32Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
+ arr = self.values
+ if arr is not None:
+ return list(zip(arr.field('key'), arr.field('value')))
+ else:
+ return None
-cdef class Int32Scalar(ScalarValue):
+cdef class DictionaryScalar(Scalar):
"""
- Concrete class for int32 scalars.
+ Concrete class for dictionary-encoded scalars.
"""
- def as_py(self):
+ @property
+ def index(self):
"""
- Return this value as a Python int.
+ Return this value's underlying index as a scalar.
"""
- cdef CInt32Scalar* sp = <CInt32Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
+ cdef CDictionaryScalar* sp = <CDictionaryScalar*> self.wrapped.get()
+ return Scalar.wrap(sp.value.index)
-
-cdef class UInt64Scalar(ScalarValue):
- """
- Concrete class for uint64 scalars.
- """
-
- def as_py(self):
+ @property
+ def value(self):
"""
- Return this value as a Python int.
+ Return the encoded value as a scalar.
"""
- cdef CUInt64Scalar* sp = <CUInt64Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
+ # TODO(kszucs): optimize it to spare uneccessary python object boxing
+ return self.dictionary[self.index.as_py()] if self.is_valid else None
-
-cdef class Int64Scalar(ScalarValue):
- """
- Concrete class for int64 scalars.
- """
+ @property
+ def dictionary(self):
+ cdef CDictionaryScalar* sp = <CDictionaryScalar*> self.wrapped.get()
+ return pyarrow_wrap_array(sp.value.dictionary)
def as_py(self):
"""
- Return this value as a Python int.
+ Return this encoded value as a Python object.
"""
- cdef CInt64Scalar* sp = <CInt64Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
-
+ value = self.value
+ return None if value is None else value.as_py()
-cdef class FloatScalar(ScalarValue):
- """
- Concrete class for float scalars.
- """
+ @property
+ def index_value(self):
+ warnings.warn("`dictionary_value` property is deprecated as of 1.0.0"
+ "please use the `value` property instead",
+ FutureWarning)
Review comment:
Updated.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] kszucs commented on pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
kszucs commented on pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#issuecomment-654234467
@pitrou added the C++ tests.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] kszucs commented on a change in pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
kszucs commented on a change in pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#discussion_r448560164
##########
File path: python/pyarrow/scalar.pxi
##########
@@ -16,1198 +16,748 @@
# under the License.
-_NULL = NA = None
+import collections
cdef class Scalar:
"""
- The base class for all array elements.
- """
-
-
-cdef class NullType(Scalar):
- """
- Singleton for null array elements.
+ The base class for scalars.
"""
- # TODO rename this NullValue?
-
- def __cinit__(self):
- global NA
- if NA is not None:
- raise Exception('Cannot create multiple NAType instances')
-
- self.type = null()
-
- def __repr__(self):
- return 'NULL'
- def as_py(self):
- """
- Return None
- """
- return None
+ def __init__(self):
+ raise TypeError("Do not call {}'s constructor directly, use "
+ "pa.scalar() instead.".format(self.__class__.__name__))
- def __eq__(self, other):
- return NA
+ cdef void init(self, const shared_ptr[CScalar]& wrapped):
+ self.wrapped = wrapped
+ @staticmethod
+ cdef wrap(const shared_ptr[CScalar]& wrapped):
+ cdef:
+ Scalar self
+ Type type_id = wrapped.get().type.get().id()
-_NULL = NA = NullType()
+ if type_id == _Type_NA:
+ return _NULL
+ typ = _scalar_classes[type_id]
+ self = typ.__new__(typ)
+ self.init(wrapped)
-cdef class ArrayValue(Scalar):
- """
- The base class for non-null array elements.
- """
+ return self
- def __init__(self):
- raise TypeError("Do not call {}'s constructor directly, use array "
- "subscription instead."
- .format(self.__class__.__name__))
+ cdef inline shared_ptr[CScalar] unwrap(self) nogil:
+ return self.wrapped
- cdef void init(self, DataType type, const shared_ptr[CArray]& sp_array,
- int64_t index):
- self.type = type
- self.index = index
- self._set_array(sp_array)
+ @property
+ def type(self):
+ """
+ Data type of the Scalar object.
+ """
+ return pyarrow_wrap_data_type(self.wrapped.get().type)
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
+ @property
+ def is_valid(self):
+ """
+ Holds a valid (non-null) value.
+ """
+ return self.wrapped.get().is_valid
def __repr__(self):
- if hasattr(self, 'as_py'):
- return repr(self.as_py())
- else:
- return super(Scalar, self).__repr__()
+ return '<pyarrow.{}: {!r}>'.format(
+ self.__class__.__name__, self.as_py()
+ )
def __str__(self):
- if hasattr(self, 'as_py'):
- return str(self.as_py())
- else:
- return super(Scalar, self).__str__()
+ return str(self.as_py())
+
+ def equals(self, Scalar other):
+ return self.wrapped.get().Equals(other.unwrap().get()[0])
Review comment:
Removed the options `.as_py()` conversion from the equality check, so now a scalar can be equal to another scalar only.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] kszucs commented on a change in pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
kszucs commented on a change in pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#discussion_r448560952
##########
File path: python/pyarrow/scalar.pxi
##########
@@ -1217,21 +767,95 @@ cdef dict _scalar_classes = {
_Type_INT16: Int16Scalar,
_Type_INT32: Int32Scalar,
_Type_INT64: Int64Scalar,
+ _Type_HALF_FLOAT: HalfFloatScalar,
_Type_FLOAT: FloatScalar,
_Type_DOUBLE: DoubleScalar,
+ _Type_DECIMAL: Decimal128Scalar,
+ _Type_DATE32: Date32Scalar,
+ _Type_DATE64: Date64Scalar,
+ _Type_TIME32: Time32Scalar,
+ _Type_TIME64: Time64Scalar,
+ _Type_TIMESTAMP: TimestampScalar,
+ _Type_DURATION: DurationScalar,
+ _Type_BINARY: BinaryScalar,
+ _Type_LARGE_BINARY: LargeBinaryScalar,
+ _Type_FIXED_SIZE_BINARY: FixedSizeBinaryScalar,
_Type_STRING: StringScalar,
+ _Type_LARGE_STRING: LargeStringScalar,
+ _Type_LIST: ListScalar,
+ _Type_LARGE_LIST: LargeListScalar,
+ _Type_FIXED_SIZE_LIST: FixedSizeListScalar,
+ _Type_STRUCT: StructScalar,
+ _Type_MAP: MapScalar,
+ _Type_DICTIONARY: DictionaryScalar,
+ _Type_SPARSE_UNION: UnionScalar,
+ _Type_DENSE_UNION: UnionScalar,
}
-cdef object box_scalar(DataType type, const shared_ptr[CArray]& sp_array,
- int64_t index):
- cdef ArrayValue value
- if type.type.id() == _Type_NA:
- return _NULL
- elif sp_array.get().IsNull(index):
- return _NULL
- else:
- klass = _array_value_classes[type.type.id()]
- value = klass.__new__(klass)
- value.init(type, sp_array, index)
- return value
+def scalar(value, DataType type=None, *, from_pandas=None,
+ MemoryPool memory_pool=None):
+ """
+ Create a pyarrow.Scalar instance from a Python object.
+
+ Parameters
+ ----------
+ value : Any
+ Python object coercible to arrow's type system.
+ type : pyarrow.DataType
+ Explicit type to attempt to coerce to, otherwise will be inferred from
+ the value.
+ from_pandas : bool, default None
+ Use pandas's semantics for inferring nulls from values in
+ ndarray-like data. Defaults to False if not passed explicitly by user,
+ or True if a pandas object is passed in.
+ memory_pool : pyarrow.MemoryPool, optional
+ If not passed, will allocate memory from the currently-set default
+ memory pool.
+
+ Returns
+ -------
+ scalar : pyarrow.Scalar
+
+ Notes
+ -----
+ Localized timestamps will currently be returned as UTC (pandas's native
+ representation). Timezone-naive data will be implicitly interpreted as
+ UTC.
Review comment:
Removing. Copied from `pa.array()`'s docstring, so we mey need to update it there as well.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] pitrou commented on a change in pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
pitrou commented on a change in pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#discussion_r450328412
##########
File path: cpp/src/arrow/scalar_test.cc
##########
@@ -451,6 +565,127 @@ TEST(TestStructScalar, FieldAccess) {
ASSERT_RAISES(Invalid, abc.field(5).status());
ASSERT_RAISES(Invalid, abc.field("c").status());
+
+ ASSERT_OK_AND_ASSIGN(auto d, abc.field("d"));
+ ASSERT_TRUE(d->Equals(MakeNullScalar(int64())));
+}
+
+TEST(TestDictionaryScalar, Basics) {
+ auto ty = dictionary(int8(), utf8());
+ auto dict = ArrayFromJSON(utf8(), "[\"alpha\", \"beta\", \"gamma\"]");
+
+ DictionaryScalar::ValueType alpha;
+ ASSERT_OK_AND_ASSIGN(alpha.index, MakeScalar(int8(), 0));
+ alpha.dictionary = dict;
+
+ DictionaryScalar::ValueType gamma;
+ ASSERT_OK_AND_ASSIGN(gamma.index, MakeScalar(int8(), 2));
+ gamma.dictionary = dict;
+
+ auto scalar_null = MakeNullScalar(ty);
+ auto scalar_alpha = DictionaryScalar(alpha, ty);
+ auto scalar_gamma = DictionaryScalar(gamma, ty);
+
+ ASSERT_OK_AND_ASSIGN(
+ auto encoded_null,
+ checked_cast<const DictionaryScalar&>(*scalar_null).GetEncodedValue());
+ ASSERT_TRUE(encoded_null->Equals(MakeNullScalar(utf8())));
+
+ ASSERT_OK_AND_ASSIGN(
+ auto encoded_alpha,
+ checked_cast<const DictionaryScalar&>(scalar_alpha).GetEncodedValue());
+ ASSERT_TRUE(encoded_alpha->Equals(MakeScalar("alpha")));
+
+ ASSERT_OK_AND_ASSIGN(
+ auto encoded_gamma,
+ checked_cast<const DictionaryScalar&>(scalar_gamma).GetEncodedValue());
+ ASSERT_TRUE(encoded_gamma->Equals(MakeScalar("gamma")));
+
+ // test Array.GetScalar
+ DictionaryArray arr(ty, ArrayFromJSON(int8(), "[2, 0, 1, null]"), dict);
+ ASSERT_OK_AND_ASSIGN(auto first, arr.GetScalar(0));
+ ASSERT_OK_AND_ASSIGN(auto second, arr.GetScalar(1));
+ ASSERT_OK_AND_ASSIGN(auto last, arr.GetScalar(3));
+
+ ASSERT_TRUE(first->Equals(scalar_gamma));
+ ASSERT_TRUE(second->Equals(scalar_alpha));
+ ASSERT_TRUE(last->Equals(scalar_null));
+}
+
+TEST(TestSparseUnionScalar, Basics) {
+ auto ty = sparse_union({field("string", utf8()), field("number", uint64())});
+
+ auto alpha = MakeScalar("alpha");
+ auto beta = MakeScalar("beta");
+ ASSERT_OK_AND_ASSIGN(auto two, MakeScalar(uint64(), 2));
+
+ auto scalar_alpha = SparseUnionScalar(alpha, ty);
+ auto scalar_beta = SparseUnionScalar(beta, ty);
+ auto scalar_two = SparseUnionScalar(two, ty);
+
+ // test Array.GetScalar
+ std::vector<std::shared_ptr<Array>> children{
+ ArrayFromJSON(utf8(), "[\"alpha\", \"\", \"beta\", null, \"gamma\"]"),
+ ArrayFromJSON(uint64(), "[1, 2, 11, 22, null]")};
+
+ auto type_ids = ArrayFromJSON(int8(), "[0, 1, 0, 0, 1]");
+ SparseUnionArray arr(ty, 5, children, type_ids->data()->buffers[1]);
+ ASSERT_OK(arr.ValidateFull());
+
+ ASSERT_OK_AND_ASSIGN(auto first, arr.GetScalar(0));
+ ASSERT_TRUE(first->Equals(scalar_alpha));
+
+ ASSERT_OK_AND_ASSIGN(auto second, arr.GetScalar(1));
+ ASSERT_TRUE(second->Equals(scalar_two));
+
+ ASSERT_OK_AND_ASSIGN(auto third, arr.GetScalar(2));
+ ASSERT_TRUE(third->Equals(scalar_beta));
+
+ ASSERT_OK_AND_ASSIGN(auto fourth, arr.GetScalar(3));
+ ASSERT_TRUE(fourth->Equals(MakeNullScalar(ty)));
Review comment:
Ah, perhaps check that in the test?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] kszucs commented on a change in pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
kszucs commented on a change in pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#discussion_r450318972
##########
File path: cpp/src/arrow/scalar.cc
##########
@@ -185,6 +192,35 @@ DictionaryScalar::DictionaryScalar(std::shared_ptr<DataType> type)
0)
.ValueOrDie()} {}
+Result<std::shared_ptr<Scalar>> DictionaryScalar::GetEncodedValue() const {
+ const auto& dict_type = checked_cast<DictionaryType&>(*type);
+
+ if (!is_valid) {
+ return MakeNullScalar(dict_type.value_type());
+ }
+
+ int64_t index_value = 0;
+ switch (dict_type.index_type()->id()) {
+ case Type::INT8:
+ index_value = checked_cast<const Int8Scalar&>(*value.index).value;
+ break;
+ case Type::INT16:
+ index_value = checked_cast<const Int16Scalar&>(*value.index).value;
+ break;
+ case Type::INT32:
+ index_value = checked_cast<const Int32Scalar&>(*value.index).value;
+ break;
+ case Type::INT64:
+ index_value = checked_cast<const Int64Scalar&>(*value.index).value;
+ break;
+ default:
+ return Status::Invalid("Not implemented dictionary index type");
Review comment:
Updating.
##########
File path: cpp/src/arrow/scalar_test.cc
##########
@@ -80,6 +84,15 @@ TYPED_TEST(TestNumericScalar, Basics) {
auto dyn_null_value = MakeNullScalar(expected_type);
ASSERT_EQ(*null_value, *dyn_null_value);
+
+ // test Array.GetScalar
+ auto arr = ArrayFromJSON(expected_type, "[null, 1, 2]");
+ ASSERT_OK_AND_ASSIGN(auto null, arr->GetScalar(0));
+ ASSERT_OK_AND_ASSIGN(auto one, arr->GetScalar(1));
+ ASSERT_OK_AND_ASSIGN(auto two, arr->GetScalar(2));
+ ASSERT_TRUE(null->Equals(*null_value));
+ ASSERT_TRUE(one->Equals(ScalarType(1)));
+ ASSERT_TRUE(two->Equals(ScalarType(2)));
Review comment:
Not yet, adding check for it.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] wesm commented on pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
wesm commented on pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#issuecomment-654492131
+1. Thanks @kszucs for this work!
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] kszucs commented on a change in pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
kszucs commented on a change in pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#discussion_r445604627
##########
File path: python/pyarrow/tests/test_scalars.py
##########
@@ -17,426 +17,395 @@
import datetime
import pytest
-import unittest
import numpy as np
import pyarrow as pa
-class TestScalars(unittest.TestCase):
-
- def test_null_singleton(self):
- with pytest.raises(Exception):
- pa.NAType()
+@pytest.mark.parametrize(['value', 'ty', 'klass', 'deprecated'], [
+ (None, None, pa.NullScalar, pa.NullType),
+ (False, None, pa.BooleanScalar, pa.BooleanValue),
+ (True, None, pa.BooleanScalar, pa.BooleanValue),
+ (1, None, pa.Int64Scalar, pa.Int64Value),
+ (-1, None, pa.Int64Scalar, pa.Int64Value),
+ (1, pa.int8(), pa.Int8Scalar, pa.Int8Value),
+ (1, pa.uint8(), pa.UInt8Scalar, pa.UInt8Value),
+ (1, pa.int16(), pa.Int16Scalar, pa.Int16Value),
+ (1, pa.uint16(), pa.UInt16Scalar, pa.UInt16Value),
+ (1, pa.int32(), pa.Int32Scalar, pa.Int32Value),
+ (1, pa.uint32(), pa.UInt32Scalar, pa.UInt32Value),
+ (1, pa.int64(), pa.Int64Scalar, pa.Int64Value),
+ (1, pa.uint64(), pa.UInt64Scalar, pa.UInt64Value),
+ (1.0, None, pa.DoubleScalar, pa.DoubleValue),
+ (np.float16(1.0), pa.float16(), pa.HalfFloatScalar, pa.HalfFloatValue),
+ (1.0, pa.float32(), pa.FloatScalar, pa.FloatValue),
+ ("string", None, pa.StringScalar, pa.StringValue),
+ (b"bytes", None, pa.BinaryScalar, pa.BinaryValue),
+ ([1, 2, 3], None, pa.ListScalar, pa.ListValue),
+ ([1, 2, 3, 4], pa.large_list(pa.int8()), pa.LargeListScalar,
+ pa.LargeListValue),
+ # date
+ # time
+])
+def test_type_inference(value, ty, klass, deprecated):
+ s = pa.scalar(value, type=ty)
+ assert isinstance(s, klass)
+ assert s == value
+ with pytest.warns(FutureWarning):
+ isinstance(s, deprecated)
+
+
+def test_null_singleton():
+ with pytest.raises(Exception):
+ pa.NullScalar()
+
+
+def test_nulls():
+ arr = pa.array([None, None])
+ for v in arr:
+ assert v is pa.NA
+ assert v.as_py() is None
+
+
+def test_null_equality():
+ assert (pa.NA == pa.NA) is pa.NA
+ assert (pa.NA == 1) is pa.NA
Review comment:
Great question, I assume it should follow the same semantics as `Null` does.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] pitrou commented on a change in pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
pitrou commented on a change in pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#discussion_r450371161
##########
File path: cpp/src/arrow/scalar_test.cc
##########
@@ -451,6 +565,127 @@ TEST(TestStructScalar, FieldAccess) {
ASSERT_RAISES(Invalid, abc.field(5).status());
ASSERT_RAISES(Invalid, abc.field("c").status());
+
+ ASSERT_OK_AND_ASSIGN(auto d, abc.field("d"));
+ ASSERT_TRUE(d->Equals(MakeNullScalar(int64())));
+}
+
+TEST(TestDictionaryScalar, Basics) {
+ auto ty = dictionary(int8(), utf8());
+ auto dict = ArrayFromJSON(utf8(), "[\"alpha\", \"beta\", \"gamma\"]");
+
+ DictionaryScalar::ValueType alpha;
+ ASSERT_OK_AND_ASSIGN(alpha.index, MakeScalar(int8(), 0));
+ alpha.dictionary = dict;
+
+ DictionaryScalar::ValueType gamma;
+ ASSERT_OK_AND_ASSIGN(gamma.index, MakeScalar(int8(), 2));
+ gamma.dictionary = dict;
+
+ auto scalar_null = MakeNullScalar(ty);
+ auto scalar_alpha = DictionaryScalar(alpha, ty);
+ auto scalar_gamma = DictionaryScalar(gamma, ty);
+
+ ASSERT_OK_AND_ASSIGN(
+ auto encoded_null,
+ checked_cast<const DictionaryScalar&>(*scalar_null).GetEncodedValue());
+ ASSERT_TRUE(encoded_null->Equals(MakeNullScalar(utf8())));
+
+ ASSERT_OK_AND_ASSIGN(
+ auto encoded_alpha,
+ checked_cast<const DictionaryScalar&>(scalar_alpha).GetEncodedValue());
+ ASSERT_TRUE(encoded_alpha->Equals(MakeScalar("alpha")));
+
+ ASSERT_OK_AND_ASSIGN(
+ auto encoded_gamma,
+ checked_cast<const DictionaryScalar&>(scalar_gamma).GetEncodedValue());
+ ASSERT_TRUE(encoded_gamma->Equals(MakeScalar("gamma")));
+
+ // test Array.GetScalar
+ DictionaryArray arr(ty, ArrayFromJSON(int8(), "[2, 0, 1, null]"), dict);
+ ASSERT_OK_AND_ASSIGN(auto first, arr.GetScalar(0));
+ ASSERT_OK_AND_ASSIGN(auto second, arr.GetScalar(1));
+ ASSERT_OK_AND_ASSIGN(auto last, arr.GetScalar(3));
+
+ ASSERT_TRUE(first->Equals(scalar_gamma));
+ ASSERT_TRUE(second->Equals(scalar_alpha));
+ ASSERT_TRUE(last->Equals(scalar_null));
+}
+
+TEST(TestSparseUnionScalar, Basics) {
+ auto ty = sparse_union({field("string", utf8()), field("number", uint64())});
+
+ auto alpha = MakeScalar("alpha");
+ auto beta = MakeScalar("beta");
+ ASSERT_OK_AND_ASSIGN(auto two, MakeScalar(uint64(), 2));
+
+ auto scalar_alpha = SparseUnionScalar(alpha, ty);
+ auto scalar_beta = SparseUnionScalar(beta, ty);
+ auto scalar_two = SparseUnionScalar(two, ty);
+
+ // test Array.GetScalar
+ std::vector<std::shared_ptr<Array>> children{
+ ArrayFromJSON(utf8(), "[\"alpha\", \"\", \"beta\", null, \"gamma\"]"),
+ ArrayFromJSON(uint64(), "[1, 2, 11, 22, null]")};
+
+ auto type_ids = ArrayFromJSON(int8(), "[0, 1, 0, 0, 1]");
+ SparseUnionArray arr(ty, 5, children, type_ids->data()->buffers[1]);
+ ASSERT_OK(arr.ValidateFull());
+
+ ASSERT_OK_AND_ASSIGN(auto first, arr.GetScalar(0));
+ ASSERT_TRUE(first->Equals(scalar_alpha));
+
+ ASSERT_OK_AND_ASSIGN(auto second, arr.GetScalar(1));
+ ASSERT_TRUE(second->Equals(scalar_two));
+
+ ASSERT_OK_AND_ASSIGN(auto third, arr.GetScalar(2));
+ ASSERT_TRUE(third->Equals(scalar_beta));
+
+ ASSERT_OK_AND_ASSIGN(auto fourth, arr.GetScalar(3));
+ ASSERT_TRUE(fourth->Equals(MakeNullScalar(ty)));
Review comment:
It the value scalar is not a nullptr, then it would seem better, unless that's complicated.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] jorisvandenbossche commented on a change in pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
jorisvandenbossche commented on a change in pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#discussion_r448619631
##########
File path: python/pyarrow/scalar.pxi
##########
@@ -16,1198 +16,745 @@
# under the License.
-_NULL = NA = None
+import collections
cdef class Scalar:
"""
- The base class for all array elements.
+ The base class for scalars.
"""
+ def __init__(self):
+ raise TypeError("Do not call {}'s constructor directly, use "
+ "pa.scalar() instead.".format(self.__class__.__name__))
-cdef class NullType(Scalar):
- """
- Singleton for null array elements.
- """
- # TODO rename this NullValue?
+ cdef void init(self, const shared_ptr[CScalar]& wrapped):
+ self.wrapped = wrapped
- def __cinit__(self):
- global NA
- if NA is not None:
- raise Exception('Cannot create multiple NAType instances')
+ @staticmethod
+ cdef wrap(const shared_ptr[CScalar]& wrapped):
+ cdef:
+ Scalar self
+ Type type_id = wrapped.get().type.get().id()
+
+ if type_id == _Type_NA:
+ return _NULL
+
+ typ = _scalar_classes[type_id]
+ self = typ.__new__(typ)
+ self.init(wrapped)
+
+ return self
+
+ cdef inline shared_ptr[CScalar] unwrap(self) nogil:
+ return self.wrapped
+
+ @property
+ def type(self):
+ return pyarrow_wrap_data_type(self.wrapped.get().type)
- self.type = null()
+ @property
+ def is_valid(self):
+ return self.wrapped.get().is_valid
def __repr__(self):
- return 'NULL'
+ return '<pyarrow.{}: {!r}>'.format(
+ self.__class__.__name__, self.as_py()
+ )
- def as_py(self):
- """
- Return None
- """
- return None
+ def __str__(self):
+ return str(self.as_py())
+
+ def equals(self, Scalar other):
+ return self.wrapped.get().Equals(other.unwrap().get()[0])
def __eq__(self, other):
- return NA
+ try:
+ if not isinstance(other, Scalar):
+ other = scalar(other, type=self.type)
+ return self.equals(other)
+ except (TypeError, ValueError, ArrowInvalid):
+ return NotImplemented
+
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+
+ def as_py(self):
+ raise NotImplementedError()
-_NULL = NA = NullType()
+_NULL = NA = None
-cdef class ArrayValue(Scalar):
+cdef class NullScalar(Scalar):
"""
- The base class for non-null array elements.
+ Concrete class for null scalars.
"""
- def __init__(self):
- raise TypeError("Do not call {}'s constructor directly, use array "
- "subscription instead."
- .format(self.__class__.__name__))
+ def __cinit__(self):
+ global NA
+ if NA is not None:
+ raise Exception('Cannot create multiple NAType instances')
+ self.init(shared_ptr[CScalar](new CNullScalar()))
- cdef void init(self, DataType type, const shared_ptr[CArray]& sp_array,
- int64_t index):
- self.type = type
- self.index = index
- self._set_array(sp_array)
+ def __init__(self):
+ pass
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
+ def __eq__(self, other):
+ return NA
Review comment:
Ah, looking at the actual diff of your latest changes, you didn't update to return False, but to return True (for comparing with another Null value, False for comparing with any other value of course, but I had the first case in my mind when reacting ;))
So that basically does what I proposed above, but, I think we can still remove the `__eq__` alltogether to use the base class implementation, which should give the same behaviour.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] kszucs commented on a change in pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
kszucs commented on a change in pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#discussion_r447725217
##########
File path: python/pyarrow/includes/libarrow.pxd
##########
@@ -44,6 +44,11 @@ cdef extern from "arrow/util/key_value_metadata.h" namespace "arrow" nogil:
c_bool Contains(const c_string& key) const
+cdef extern from "arrow/util/decimal.h" namespace "arrow" nogil:
+ cdef cppclass CDecimal128" arrow::Decimal128":
+ c_string ToString(int32_t scale) const
Review comment:
Created jira https://issues.apache.org/jira/browse/ARROW-9279
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] github-actions[bot] commented on pull request #7519: ARROW-9153: [C++][Python] Refactor scalar bindings [WIP]
Posted by GitBox <gi...@apache.org>.
github-actions[bot] commented on pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#issuecomment-647754455
https://issues.apache.org/jira/browse/ARROW-9153
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] pitrou commented on pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
pitrou commented on pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#issuecomment-654237098
Looks like the PR needs rebasing and fixing for the latest union changes.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] kszucs commented on a change in pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
kszucs commented on a change in pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#discussion_r447758318
##########
File path: python/pyarrow/tests/test_scalars.py
##########
@@ -16,427 +16,443 @@
# under the License.
import datetime
+import decimal
import pytest
-import unittest
import numpy as np
import pyarrow as pa
-class TestScalars(unittest.TestCase):
-
- def test_null_singleton(self):
- with pytest.raises(Exception):
- pa.NAType()
+@pytest.mark.parametrize(['value', 'ty', 'klass', 'deprecated'], [
+ (False, None, pa.BooleanScalar, pa.BooleanValue),
+ (True, None, pa.BooleanScalar, pa.BooleanValue),
+ (1, None, pa.Int64Scalar, pa.Int64Value),
+ (-1, None, pa.Int64Scalar, pa.Int64Value),
+ (1, pa.int8(), pa.Int8Scalar, pa.Int8Value),
+ (1, pa.uint8(), pa.UInt8Scalar, pa.UInt8Value),
+ (1, pa.int16(), pa.Int16Scalar, pa.Int16Value),
+ (1, pa.uint16(), pa.UInt16Scalar, pa.UInt16Value),
+ (1, pa.int32(), pa.Int32Scalar, pa.Int32Value),
+ (1, pa.uint32(), pa.UInt32Scalar, pa.UInt32Value),
+ (1, pa.int64(), pa.Int64Scalar, pa.Int64Value),
+ (1, pa.uint64(), pa.UInt64Scalar, pa.UInt64Value),
+ (1.0, None, pa.DoubleScalar, pa.DoubleValue),
+ (np.float16(1.0), pa.float16(), pa.HalfFloatScalar, pa.HalfFloatValue),
+ (1.0, pa.float32(), pa.FloatScalar, pa.FloatValue),
+ ("string", None, pa.StringScalar, pa.StringValue),
+ (b"bytes", None, pa.BinaryScalar, pa.BinaryValue),
+ ([1, 2, 3], None, pa.ListScalar, pa.ListValue),
+ ([1, 2, 3, 4], pa.large_list(pa.int8()), pa.LargeListScalar,
+ pa.LargeListValue),
+ (datetime.date.today(), None, pa.Date32Scalar, pa.Date64Value),
+ (datetime.datetime.now(), None, pa.TimestampScalar, pa.TimestampValue),
+ ({'a': 1, 'b': [1, 2]}, None, pa.StructScalar, pa.StructValue)
+])
+def test_basics(value, ty, klass, deprecated):
+ s = pa.scalar(value, type=ty)
+ assert isinstance(s, klass)
+ assert s == value
+ assert s == s
+ assert s != "else"
+ assert hash(s) == hash(s)
Review comment:
It tests that different calls of the c++ hashing mechanism produces the same hash values.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] kszucs commented on pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
kszucs commented on pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#issuecomment-650158531
I'm considering to apply [cython.freelist](https://cython.readthedocs.io/en/latest/src/userguide/extension_types.html#fast-instantiation) on the scalar extension classes.
@wesm @pitrou @jorisvandenbossche do you have any positive experience with it?
I assume we should measure its impact before using it, so I can defer it to a follow-up.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] kszucs commented on a change in pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
kszucs commented on a change in pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#discussion_r447731034
##########
File path: python/pyarrow/scalar.pxi
##########
@@ -16,1198 +16,745 @@
# under the License.
-_NULL = NA = None
+import collections
cdef class Scalar:
"""
- The base class for all array elements.
+ The base class for scalars.
"""
+ def __init__(self):
+ raise TypeError("Do not call {}'s constructor directly, use "
+ "pa.scalar() instead.".format(self.__class__.__name__))
-cdef class NullType(Scalar):
- """
- Singleton for null array elements.
- """
- # TODO rename this NullValue?
+ cdef void init(self, const shared_ptr[CScalar]& wrapped):
+ self.wrapped = wrapped
- def __cinit__(self):
- global NA
- if NA is not None:
- raise Exception('Cannot create multiple NAType instances')
+ @staticmethod
+ cdef wrap(const shared_ptr[CScalar]& wrapped):
+ cdef:
+ Scalar self
+ Type type_id = wrapped.get().type.get().id()
+
+ if type_id == _Type_NA:
+ return _NULL
+
+ typ = _scalar_classes[type_id]
+ self = typ.__new__(typ)
+ self.init(wrapped)
+
+ return self
+
+ cdef inline shared_ptr[CScalar] unwrap(self) nogil:
+ return self.wrapped
+
+ @property
+ def type(self):
+ return pyarrow_wrap_data_type(self.wrapped.get().type)
- self.type = null()
+ @property
+ def is_valid(self):
+ return self.wrapped.get().is_valid
def __repr__(self):
- return 'NULL'
+ return '<pyarrow.{}: {!r}>'.format(
+ self.__class__.__name__, self.as_py()
+ )
- def as_py(self):
- """
- Return None
- """
- return None
+ def __str__(self):
+ return str(self.as_py())
+
+ def equals(self, Scalar other):
+ return self.wrapped.get().Equals(other.unwrap().get()[0])
def __eq__(self, other):
- return NA
+ try:
+ if not isinstance(other, Scalar):
+ other = scalar(other, type=self.type)
+ return self.equals(other)
+ except (TypeError, ValueError, ArrowInvalid):
+ return NotImplemented
+
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+
+ def as_py(self):
+ raise NotImplementedError()
-_NULL = NA = NullType()
+_NULL = NA = None
-cdef class ArrayValue(Scalar):
+cdef class NullScalar(Scalar):
"""
- The base class for non-null array elements.
+ Concrete class for null scalars.
"""
- def __init__(self):
- raise TypeError("Do not call {}'s constructor directly, use array "
- "subscription instead."
- .format(self.__class__.__name__))
+ def __cinit__(self):
+ global NA
+ if NA is not None:
+ raise Exception('Cannot create multiple NAType instances')
+ self.init(shared_ptr[CScalar](new CNullScalar()))
- cdef void init(self, DataType type, const shared_ptr[CArray]& sp_array,
- int64_t index):
- self.type = type
- self.index = index
- self._set_array(sp_array)
+ def __init__(self):
+ pass
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
+ def __eq__(self, other):
+ return NA
Review comment:
Well, it was the behavior before the refactor. Shall we check proper equality with NullScalar instead?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] kszucs commented on a change in pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
kszucs commented on a change in pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#discussion_r445603562
##########
File path: python/pyarrow/util.py
##########
@@ -41,6 +41,24 @@ def wrapper(*args, **kwargs):
return wrapper
+def _deprecate_class(old_name, new_class, next_version,
+ instancecheck=True):
+ """
+ Raise warning if a deprecated class is used in an isinstance check.
Review comment:
> Really nice!
>
> Could we add a `is_valid` attribute to the python scalar as well? Now the only way to check for a null value is to do `.as_py() is None` ?
Definitely!
>
> Might not be related to the changes in this PR, but reviewing this triggered me to test scalar casting:
>
> ```
> In [2]: s = pa.scalar(pd.Timestamp("2012-01-01"))
> ...:
> ...: import pyarrow.compute as pc
> ...: pc.cast(s, pa.timestamp('ns'))
> ../src/arrow/compute/kernels/scalar_cast_temporal.cc:130: Check failed: (batch[0].kind()) == (Datum::ARRAY)
> ...
> Aborted (core dumped)
> ```
I think the cast kernel doesn't support scalars yet, on the other hand the scalars have custom `CastTo` implementation which we may want to remove in favor of `compute.Cast`?
cc @bkietz
##########
File path: python/pyarrow/util.py
##########
@@ -41,6 +41,24 @@ def wrapper(*args, **kwargs):
return wrapper
+def _deprecate_class(old_name, new_class, next_version,
+ instancecheck=True):
+ """
+ Raise warning if a deprecated class is used in an isinstance check.
Review comment:
> Really nice!
>
> Could we add a `is_valid` attribute to the python scalar as well? Now the only way to check for a null value is to do `.as_py() is None` ?
Definitely!
>
> Might not be related to the changes in this PR, but reviewing this triggered me to test scalar casting:
>
> ```
> In [2]: s = pa.scalar(pd.Timestamp("2012-01-01"))
> ...:
> ...: import pyarrow.compute as pc
> ...: pc.cast(s, pa.timestamp('ns'))
> ../src/arrow/compute/kernels/scalar_cast_temporal.cc:130: Check failed: (batch[0].kind()) == (Datum::ARRAY)
> ...
> Aborted (core dumped)
> ```
I think the cast kernel doesn't support scalars yet, on the other hand the scalars have custom `CastTo` implementation which we may want to remove in favor of `compute.Cast`?
cc @bkietz
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] kszucs commented on pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
kszucs commented on pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#issuecomment-649590873
> Really nice!
>
> Could we add a `is_valid` attribute to the python scalar as well? Now the only way to check for a null value is to do `.as_py() is None` ?
Definitely!
>
> Might not be related to the changes in this PR, but reviewing this triggered me to test scalar casting:
>
> ```
> In [2]: s = pa.scalar(pd.Timestamp("2012-01-01"))
> ...:
> ...: import pyarrow.compute as pc
> ...: pc.cast(s, pa.timestamp('ns'))
> ../src/arrow/compute/kernels/scalar_cast_temporal.cc:130: Check failed: (batch[0].kind()) == (Datum::ARRAY)
> ...
> Aborted (core dumped)
> ```
I think the cast kernel doesn't support scalars yet, on the other hand the scalars have custom `CastTo` implementation which we may want to remove in favor of `compute.Cast`?
cc @bkietz
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] kszucs commented on a change in pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
kszucs commented on a change in pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#discussion_r448556316
##########
File path: python/pyarrow/scalar.pxi
##########
@@ -16,1198 +16,745 @@
# under the License.
-_NULL = NA = None
+import collections
cdef class Scalar:
"""
- The base class for all array elements.
+ The base class for scalars.
"""
+ def __init__(self):
+ raise TypeError("Do not call {}'s constructor directly, use "
+ "pa.scalar() instead.".format(self.__class__.__name__))
-cdef class NullType(Scalar):
- """
- Singleton for null array elements.
- """
- # TODO rename this NullValue?
+ cdef void init(self, const shared_ptr[CScalar]& wrapped):
+ self.wrapped = wrapped
- def __cinit__(self):
- global NA
- if NA is not None:
- raise Exception('Cannot create multiple NAType instances')
+ @staticmethod
+ cdef wrap(const shared_ptr[CScalar]& wrapped):
+ cdef:
+ Scalar self
+ Type type_id = wrapped.get().type.get().id()
+
+ if type_id == _Type_NA:
+ return _NULL
+
+ typ = _scalar_classes[type_id]
+ self = typ.__new__(typ)
+ self.init(wrapped)
+
+ return self
+
+ cdef inline shared_ptr[CScalar] unwrap(self) nogil:
+ return self.wrapped
+
+ @property
+ def type(self):
+ return pyarrow_wrap_data_type(self.wrapped.get().type)
- self.type = null()
+ @property
+ def is_valid(self):
+ return self.wrapped.get().is_valid
def __repr__(self):
- return 'NULL'
+ return '<pyarrow.{}: {!r}>'.format(
+ self.__class__.__name__, self.as_py()
+ )
- def as_py(self):
- """
- Return None
- """
- return None
+ def __str__(self):
+ return str(self.as_py())
+
+ def equals(self, Scalar other):
+ return self.wrapped.get().Equals(other.unwrap().get()[0])
def __eq__(self, other):
- return NA
+ try:
+ if not isinstance(other, Scalar):
+ other = scalar(other, type=self.type)
+ return self.equals(other)
+ except (TypeError, ValueError, ArrowInvalid):
+ return NotImplemented
+
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+
+ def as_py(self):
+ raise NotImplementedError()
-_NULL = NA = NullType()
+_NULL = NA = None
-cdef class ArrayValue(Scalar):
+cdef class NullScalar(Scalar):
"""
- The base class for non-null array elements.
+ Concrete class for null scalars.
"""
- def __init__(self):
- raise TypeError("Do not call {}'s constructor directly, use array "
- "subscription instead."
- .format(self.__class__.__name__))
+ def __cinit__(self):
+ global NA
+ if NA is not None:
+ raise Exception('Cannot create multiple NAType instances')
+ self.init(shared_ptr[CScalar](new CNullScalar()))
- cdef void init(self, DataType type, const shared_ptr[CArray]& sp_array,
- int64_t index):
- self.type = type
- self.index = index
- self._set_array(sp_array)
+ def __init__(self):
+ pass
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
+ def __eq__(self, other):
+ return NA
- def __repr__(self):
- if hasattr(self, 'as_py'):
- return repr(self.as_py())
- else:
- return super(Scalar, self).__repr__()
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
- def __str__(self):
- if hasattr(self, 'as_py'):
- return str(self.as_py())
- else:
- return super(Scalar, self).__str__()
+ def as_py(self):
+ """
+ Return this value as a Python None.
+ """
+ return None
- def __eq__(self, other):
- if hasattr(self, 'as_py'):
- if isinstance(other, ArrayValue):
- other = other.as_py()
- return self.as_py() == other
- else:
- raise NotImplementedError(
- "Cannot compare Arrow values that don't support as_py()")
- def __hash__(self):
- return hash(self.as_py())
+_NULL = NA = NullScalar()
-cdef class BooleanValue(ArrayValue):
+cdef class BooleanScalar(Scalar):
"""
- Concrete class for boolean array elements.
+ Concrete class for boolean scalars.
"""
def as_py(self):
"""
Return this value as a Python bool.
"""
- cdef CBooleanArray* ap = <CBooleanArray*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CBooleanScalar* sp = <CBooleanScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int8Value(ArrayValue):
+cdef class UInt8Scalar(Scalar):
"""
- Concrete class for int8 array elements.
+ Concrete class for uint8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt8Array* ap = <CInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt8Scalar* sp = <CUInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt8Value(ArrayValue):
+cdef class Int8Scalar(Scalar):
"""
- Concrete class for uint8 array elements.
+ Concrete class for int8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt8Array* ap = <CUInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt8Scalar* sp = <CInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int16Value(ArrayValue):
+cdef class UInt16Scalar(Scalar):
"""
- Concrete class for int16 array elements.
+ Concrete class for uint16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt16Array* ap = <CInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt16Scalar* sp = <CUInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt16Value(ArrayValue):
+cdef class Int16Scalar(Scalar):
"""
- Concrete class for uint16 array elements.
+ Concrete class for int16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt16Array* ap = <CUInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt16Scalar* sp = <CInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int32Value(ArrayValue):
+cdef class UInt32Scalar(Scalar):
"""
- Concrete class for int32 array elements.
+ Concrete class for uint32 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt32Array* ap = <CInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt32Scalar* sp = <CUInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt32Value(ArrayValue):
+cdef class Int32Scalar(Scalar):
"""
- Concrete class for uint32 array elements.
+ Concrete class for int32 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt32Array* ap = <CUInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt32Scalar* sp = <CInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int64Value(ArrayValue):
+cdef class UInt64Scalar(Scalar):
"""
- Concrete class for int64 array elements.
+ Concrete class for uint64 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt64Array* ap = <CInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt64Scalar* sp = <CUInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt64Value(ArrayValue):
+cdef class Int64Scalar(Scalar):
"""
- Concrete class for uint64 array elements.
+ Concrete class for int64 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt64Array* ap = <CUInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt64Scalar* sp = <CInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date32Value(ArrayValue):
+cdef class HalfFloatScalar(Scalar):
"""
- Concrete class for date32 array elements.
+ Concrete class for float scalars.
"""
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+
+ def __eq__(self, other):
+ if hasattr(self, 'as_py'):
+ if isinstance(other, Scalar):
+ other = other.as_py()
+ return self.as_py() == other
+ else:
+ raise NotImplementedError
+
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python float.
"""
- cdef CDate32Array* ap = <CDate32Array*> self.sp_array.get()
+ cdef CHalfFloatScalar* sp = <CHalfFloatScalar*> self.wrapped.get()
+ return PyHalf_FromHalf(sp.value) if sp.is_valid else None
- # Shift to seconds since epoch
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(days=ap.Value(self.index)))
+
+cdef class FloatScalar(Scalar):
+ """
+ Concrete class for float scalars.
+ """
+
+ def as_py(self):
+ """
+ Return this value as a Python float.
+ """
+ cdef CFloatScalar* sp = <CFloatScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date64Value(ArrayValue):
+cdef class DoubleScalar(Scalar):
"""
- Concrete class for date64 array elements.
+ Concrete class for double scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python float.
"""
- cdef CDate64Array* ap = <CDate64Array*> self.sp_array.get()
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(
- days=ap.Value(self.index) / 86400000))
+ cdef CDoubleScalar* sp = <CDoubleScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Time32Value(ArrayValue):
+cdef class Decimal128Scalar(Scalar):
"""
- Concrete class for time32 array elements.
+ Concrete class for decimal128 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.timedelta instance.
+ Return this value as a Python Decimal.
"""
cdef:
- CTime32Array* ap = <CTime32Array*> self.sp_array.get()
- CTime32Type* dtype = <CTime32Type*> ap.type().get()
-
- if dtype.unit() == TimeUnit_SECOND:
- delta = datetime.timedelta(seconds=ap.Value(self.index))
- return (datetime.datetime(1970, 1, 1) + delta).time()
+ CDecimal128Scalar* sp = <CDecimal128Scalar*> self.wrapped.get()
+ CDecimal128Type* dtype = <CDecimal128Type*> sp.type.get()
+ if sp.is_valid:
+ return _pydecimal.Decimal(
+ frombytes(sp.value.ToString(dtype.scale()))
+ )
else:
- return _box_time_milli(ap.Value(self.index))
+ return None
-cdef class Time64Value(ArrayValue):
+cdef class Date32Scalar(Scalar):
"""
- Concrete class for time64 array elements.
+ Concrete class for date32 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.timedelta instance.
+ Return this value as a Python datetime.datetime instance.
"""
- cdef:
- CTime64Array* ap = <CTime64Array*> self.sp_array.get()
- CTime64Type* dtype = <CTime64Type*> ap.type().get()
+ cdef CDate32Scalar* sp = <CDate32Scalar*> self.wrapped.get()
- cdef int64_t val = ap.Value(self.index)
- if dtype.unit() == TimeUnit_MICRO:
- return _box_time_micro(val)
+ if sp.is_valid:
+ # shift to seconds since epoch
+ return (
+ datetime.date(1970, 1, 1) + datetime.timedelta(days=sp.value)
+ )
else:
- return (datetime.datetime(1970, 1, 1) +
- datetime.timedelta(microseconds=val / 1000)).time()
-
+ return None
-cpdef _box_time_milli(int64_t val):
- delta = datetime.timedelta(milliseconds=val)
- return (datetime.datetime(1970, 1, 1) + delta).time()
+cdef class Date64Scalar(Scalar):
+ """
+ Concrete class for date64 scalars.
+ """
-cpdef _box_time_micro(int64_t val):
- return (datetime.datetime(1970, 1, 1) +
- datetime.timedelta(microseconds=val)).time()
+ def as_py(self):
+ """
+ Return this value as a Python datetime.datetime instance.
+ """
+ cdef CDate64Scalar* sp = <CDate64Scalar*> self.wrapped.get()
+ if sp.is_valid:
+ return (
+ datetime.date(1970, 1, 1) +
+ datetime.timedelta(days=sp.value / 86400000)
+ )
+ else:
+ return None
-cdef dict _DATETIME_CONVERSION_FUNCTIONS = {}
-cdef c_bool _datetime_conversion_initialized = False
+def _datetime_from_int(int64_t value, TimeUnit unit, tzinfo=None):
+ if unit == TimeUnit_SECOND:
+ delta = datetime.timedelta(seconds=value)
+ elif unit == TimeUnit_MILLI:
+ delta = datetime.timedelta(milliseconds=value)
+ elif unit == TimeUnit_MICRO:
+ delta = datetime.timedelta(microseconds=value)
+ else:
+ # TimeUnit_NANO: prefer pandas timestamps if available
+ if _pandas_api.have_pandas:
+ return _pandas_api.pd.Timestamp(value, tz=tzinfo, unit='ns')
+ # otherwise safely truncate to microsecond resolution datetime
+ if value % 1000 != 0:
+ raise ValueError(
+ "Nanosecond resolution temporal type {} is not safely "
+ "convertible to microseconds to convert to datetime.datetime. "
+ "Install pandas to return as Timestamp with nanosecond "
+ "support or access the .value attribute.".format(value)
+ )
+ delta = datetime.timedelta(microseconds=value // 1000)
-cdef _add_micros_maybe_localize(dt, micros, tzinfo):
- import pytz
- dt = dt.replace(microsecond=micros)
+ dt = datetime.datetime(1970, 1, 1) + delta
+ # adjust timezone if set to the datatype
if tzinfo is not None:
- if not isinstance(tzinfo, datetime.tzinfo):
- tzinfo = string_to_tzinfo(tzinfo)
dt = tzinfo.fromutc(dt)
- return dt
-
-
-cdef _datetime_from_seconds(int64_t v):
- return datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=v)
-
-
-def _nanoseconds_to_datetime_safe(v, tzinfo):
- if v % 1000 != 0:
- raise ValueError("Nanosecond timestamp {} is not safely convertible "
- " to microseconds to convert to datetime.datetime."
- " Install pandas to return as Timestamp with "
- " nanosecond support or access the .value attribute.")
- v = v // 1000
- micros = v % 1_000_000
-
- dt = _datetime_from_seconds(v // 1_000_000)
- return _add_micros_maybe_localize(dt, micros, tzinfo)
-
-
-def _microseconds_to_datetime(v, tzinfo):
- micros = v % 1_000_000
- dt = _datetime_from_seconds(v // 1_000_000)
- return _add_micros_maybe_localize(dt, micros, tzinfo)
+ return dt
-def _millis_to_datetime(v, tzinfo):
- millis = v % 1_000
- dt = _datetime_from_seconds(v // 1000)
- return _add_micros_maybe_localize(dt, millis * 1000, tzinfo)
+cdef class Time32Scalar(Scalar):
+ """
+ Concrete class for time32 scalars.
+ """
-def _seconds_to_datetime(v, tzinfo):
- dt = _datetime_from_seconds(v)
- return _add_micros_maybe_localize(dt, 0, tzinfo)
+ def as_py(self):
+ """
+ Return this value as a Python datetime.timedelta instance.
+ """
+ cdef:
+ CTime32Scalar* sp = <CTime32Scalar*> self.wrapped.get()
+ CTime32Type* dtype = <CTime32Type*> sp.type.get()
+ if sp.is_valid:
+ return _datetime_from_int(sp.value, unit=dtype.unit()).time()
+ else:
+ return None
-def _datetime_conversion_functions():
- global _datetime_conversion_initialized
- if _datetime_conversion_initialized:
- return _DATETIME_CONVERSION_FUNCTIONS
- _DATETIME_CONVERSION_FUNCTIONS.update({
- TimeUnit_SECOND: _seconds_to_datetime,
- TimeUnit_MILLI: _millis_to_datetime,
- TimeUnit_MICRO: _microseconds_to_datetime,
- TimeUnit_NANO: _nanoseconds_to_datetime_safe
- })
+cdef class Time64Scalar(Scalar):
+ """
+ Concrete class for time64 scalars.
+ """
- try:
- import pandas as pd
- _DATETIME_CONVERSION_FUNCTIONS[TimeUnit_NANO] = (
- lambda x, tzinfo: pd.Timestamp(
- x, tz=tzinfo, unit='ns',
- )
- )
- except ImportError:
- pass
+ def as_py(self):
+ """
+ Return this value as a Python datetime.timedelta instance.
+ """
+ cdef:
+ CTime64Scalar* sp = <CTime64Scalar*> self.wrapped.get()
+ CTime64Type* dtype = <CTime64Type*> sp.type.get()
- _datetime_conversion_initialized = True
- return _DATETIME_CONVERSION_FUNCTIONS
+ if sp.is_valid:
+ return _datetime_from_int(sp.value, unit=dtype.unit()).time()
+ else:
+ return None
-cdef class TimestampValue(ArrayValue):
+cdef class TimestampScalar(Scalar):
"""
- Concrete class for timestamp array elements.
+ Concrete class for timestamp scalars.
"""
@property
def value(self):
- cdef CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
- cdef CTimestampType* dtype = <CTimestampType*> ap.type().get()
- return ap.Value(self.index)
+ cdef CTimestampScalar* sp = <CTimestampScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
def as_py(self):
"""
Return this value as a Pandas Timestamp instance (if available),
otherwise as a Python datetime.timedelta instance.
"""
- cdef CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
- cdef CTimestampType* dtype = <CTimestampType*> ap.type().get()
+ cdef:
+ CTimestampScalar* sp = <CTimestampScalar*> self.wrapped.get()
+ CTimestampType* dtype = <CTimestampType*> sp.type.get()
- value = self.value
+ if not sp.is_valid:
+ return None
if not dtype.timezone().empty():
tzinfo = string_to_tzinfo(frombytes(dtype.timezone()))
+ if not isinstance(tzinfo, datetime.tzinfo):
+ tzinfo = string_to_tzinfo(tzinfo)
else:
tzinfo = None
- try:
- converter = _datetime_conversion_functions()[dtype.unit()]
- except KeyError:
- raise ValueError(
- 'Cannot convert nanosecond timestamps without pandas'
- )
- return converter(value, tzinfo=tzinfo)
-
-
-cdef dict _TIMEDELTA_CONVERSION_FUNCTIONS = {}
-
-
-def _nanoseconds_to_timedelta_safe(v):
- if v % 1000 != 0:
- raise ValueError(
- "Nanosecond duration {} is not safely convertible to microseconds "
- "to convert to datetime.timedelta. Install pandas to return as "
- "Timedelta with nanosecond support or access the .value "
- "attribute.".format(v))
- micros = v // 1000
-
- return datetime.timedelta(microseconds=micros)
-
-
-def _timedelta_conversion_functions():
- if _TIMEDELTA_CONVERSION_FUNCTIONS:
- return _TIMEDELTA_CONVERSION_FUNCTIONS
-
- _TIMEDELTA_CONVERSION_FUNCTIONS.update({
- TimeUnit_SECOND: lambda v: datetime.timedelta(seconds=v),
- TimeUnit_MILLI: lambda v: datetime.timedelta(milliseconds=v),
- TimeUnit_MICRO: lambda v: datetime.timedelta(microseconds=v),
- TimeUnit_NANO: _nanoseconds_to_timedelta_safe
- })
-
- try:
- import pandas as pd
- _TIMEDELTA_CONVERSION_FUNCTIONS[TimeUnit_NANO] = (
- lambda v: pd.Timedelta(v, unit='ns')
- )
- except ImportError:
- pass
+ return _datetime_from_int(sp.value, unit=dtype.unit(), tzinfo=tzinfo)
- return _TIMEDELTA_CONVERSION_FUNCTIONS
-
-cdef class DurationValue(ArrayValue):
+cdef class DurationScalar(Scalar):
"""
- Concrete class for duration array elements.
+ Concrete class for duration scalars.
"""
@property
def value(self):
- cdef CDurationArray* ap = <CDurationArray*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CDurationScalar* sp = <CDurationScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
def as_py(self):
"""
Return this value as a Pandas Timestamp instance (if available),
otherwise as a Python datetime.timedelta instance.
"""
- cdef CDurationArray* ap = <CDurationArray*> self.sp_array.get()
- cdef CDurationType* dtype = <CDurationType*> ap.type().get()
+ cdef:
+ CDurationScalar* sp = <CDurationScalar*> self.wrapped.get()
+ CDurationType* dtype = <CDurationType*> sp.type.get()
+ TimeUnit unit = dtype.unit()
+
+ if not sp.is_valid:
+ return None
- cdef int64_t value = ap.Value(self.index)
- converter = _timedelta_conversion_functions()[dtype.unit()]
- return converter(value)
+ if unit == TimeUnit_SECOND:
+ return datetime.timedelta(seconds=sp.value)
+ elif unit == TimeUnit_MILLI:
+ return datetime.timedelta(milliseconds=sp.value)
+ elif unit == TimeUnit_MICRO:
+ return datetime.timedelta(microseconds=sp.value)
+ else:
+ # TimeUnit_NANO: prefer pandas timestamps if available
+ if _pandas_api.have_pandas:
+ return _pandas_api.pd.Timedelta(sp.value, unit='ns')
+ # otherwise safely truncate to microsecond resolution timedelta
+ if sp.value % 1000 != 0:
+ raise ValueError(
+ "Nanosecond duration {} is not safely convertible to "
+ "microseconds to convert to datetime.timedelta. Install "
+ "pandas to return as Timedelta with nanosecond support or "
+ "access the .value attribute.".format(sp.value)
+ )
+ return datetime.timedelta(microseconds=sp.value // 1000)
-cdef class HalfFloatValue(ArrayValue):
+cdef class BinaryScalar(Scalar):
"""
- Concrete class for float16 array elements.
+ Concrete class for binary-like scalars.
"""
- def as_py(self):
+ def as_buffer(self):
"""
- Return this value as a Python float.
+ Return a view over this value as a Buffer object.
"""
- cdef CHalfFloatArray* ap = <CHalfFloatArray*> self.sp_array.get()
- return PyHalf_FromHalf(ap.Value(self.index))
-
-
-cdef class FloatValue(ArrayValue):
- """
- Concrete class for float32 array elements.
- """
+ cdef CBinaryScalar* sp = <CBinaryScalar*> self.wrapped.get()
+ return pyarrow_wrap_buffer(sp.value) if sp.is_valid else None
def as_py(self):
"""
- Return this value as a Python float.
+ Return this value as a Python bytes.
"""
- cdef CFloatArray* ap = <CFloatArray*> self.sp_array.get()
- return ap.Value(self.index)
+ buffer = self.as_buffer()
+ if buffer is not None:
+ return self.as_buffer().to_pybytes()
+ else:
+ return None
-cdef class DoubleValue(ArrayValue):
- """
- Concrete class for float64 array elements.
- """
+cdef class LargeBinaryScalar(BinaryScalar):
Review comment:
Updated it to cast to`CBaseBinaryScalar` which is the common root of the binary-like scalar types.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] kszucs commented on a change in pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
kszucs commented on a change in pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#discussion_r448551482
##########
File path: python/pyarrow/scalar.pxi
##########
@@ -16,1198 +16,745 @@
# under the License.
-_NULL = NA = None
+import collections
cdef class Scalar:
"""
- The base class for all array elements.
+ The base class for scalars.
"""
+ def __init__(self):
+ raise TypeError("Do not call {}'s constructor directly, use "
+ "pa.scalar() instead.".format(self.__class__.__name__))
-cdef class NullType(Scalar):
- """
- Singleton for null array elements.
- """
- # TODO rename this NullValue?
+ cdef void init(self, const shared_ptr[CScalar]& wrapped):
+ self.wrapped = wrapped
- def __cinit__(self):
- global NA
- if NA is not None:
- raise Exception('Cannot create multiple NAType instances')
+ @staticmethod
+ cdef wrap(const shared_ptr[CScalar]& wrapped):
+ cdef:
+ Scalar self
+ Type type_id = wrapped.get().type.get().id()
+
+ if type_id == _Type_NA:
+ return _NULL
+
+ typ = _scalar_classes[type_id]
+ self = typ.__new__(typ)
+ self.init(wrapped)
+
+ return self
+
+ cdef inline shared_ptr[CScalar] unwrap(self) nogil:
+ return self.wrapped
+
+ @property
+ def type(self):
+ return pyarrow_wrap_data_type(self.wrapped.get().type)
- self.type = null()
+ @property
+ def is_valid(self):
+ return self.wrapped.get().is_valid
def __repr__(self):
- return 'NULL'
+ return '<pyarrow.{}: {!r}>'.format(
+ self.__class__.__name__, self.as_py()
+ )
- def as_py(self):
- """
- Return None
- """
- return None
+ def __str__(self):
+ return str(self.as_py())
+
+ def equals(self, Scalar other):
+ return self.wrapped.get().Equals(other.unwrap().get()[0])
def __eq__(self, other):
- return NA
+ try:
+ if not isinstance(other, Scalar):
+ other = scalar(other, type=self.type)
+ return self.equals(other)
+ except (TypeError, ValueError, ArrowInvalid):
+ return NotImplemented
+
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+
+ def as_py(self):
+ raise NotImplementedError()
-_NULL = NA = NullType()
+_NULL = NA = None
-cdef class ArrayValue(Scalar):
+cdef class NullScalar(Scalar):
"""
- The base class for non-null array elements.
+ Concrete class for null scalars.
"""
- def __init__(self):
- raise TypeError("Do not call {}'s constructor directly, use array "
- "subscription instead."
- .format(self.__class__.__name__))
+ def __cinit__(self):
+ global NA
+ if NA is not None:
+ raise Exception('Cannot create multiple NAType instances')
+ self.init(shared_ptr[CScalar](new CNullScalar()))
- cdef void init(self, DataType type, const shared_ptr[CArray]& sp_array,
- int64_t index):
- self.type = type
- self.index = index
- self._set_array(sp_array)
+ def __init__(self):
+ pass
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
+ def __eq__(self, other):
+ return NA
Review comment:
Updated to return False.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] kszucs commented on a change in pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
kszucs commented on a change in pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#discussion_r448551033
##########
File path: python/pyarrow/tests/test_scalars.py
##########
@@ -16,427 +16,443 @@
# under the License.
import datetime
+import decimal
import pytest
-import unittest
import numpy as np
import pyarrow as pa
-class TestScalars(unittest.TestCase):
-
- def test_null_singleton(self):
- with pytest.raises(Exception):
- pa.NAType()
+@pytest.mark.parametrize(['value', 'ty', 'klass', 'deprecated'], [
+ (False, None, pa.BooleanScalar, pa.BooleanValue),
+ (True, None, pa.BooleanScalar, pa.BooleanValue),
+ (1, None, pa.Int64Scalar, pa.Int64Value),
+ (-1, None, pa.Int64Scalar, pa.Int64Value),
+ (1, pa.int8(), pa.Int8Scalar, pa.Int8Value),
+ (1, pa.uint8(), pa.UInt8Scalar, pa.UInt8Value),
+ (1, pa.int16(), pa.Int16Scalar, pa.Int16Value),
+ (1, pa.uint16(), pa.UInt16Scalar, pa.UInt16Value),
+ (1, pa.int32(), pa.Int32Scalar, pa.Int32Value),
+ (1, pa.uint32(), pa.UInt32Scalar, pa.UInt32Value),
+ (1, pa.int64(), pa.Int64Scalar, pa.Int64Value),
+ (1, pa.uint64(), pa.UInt64Scalar, pa.UInt64Value),
+ (1.0, None, pa.DoubleScalar, pa.DoubleValue),
+ (np.float16(1.0), pa.float16(), pa.HalfFloatScalar, pa.HalfFloatValue),
+ (1.0, pa.float32(), pa.FloatScalar, pa.FloatValue),
+ ("string", None, pa.StringScalar, pa.StringValue),
+ (b"bytes", None, pa.BinaryScalar, pa.BinaryValue),
+ ([1, 2, 3], None, pa.ListScalar, pa.ListValue),
+ ([1, 2, 3, 4], pa.large_list(pa.int8()), pa.LargeListScalar,
+ pa.LargeListValue),
+ (datetime.date.today(), None, pa.Date32Scalar, pa.Date64Value),
+ (datetime.datetime.now(), None, pa.TimestampScalar, pa.TimestampValue),
+ ({'a': 1, 'b': [1, 2]}, None, pa.StructScalar, pa.StructValue)
+])
+def test_basics(value, ty, klass, deprecated):
+ s = pa.scalar(value, type=ty)
+ assert isinstance(s, klass)
+ assert s == value
+ assert s == s
+ assert s != "else"
+ assert hash(s) == hash(s)
+ assert s.is_valid is True
+ with pytest.warns(FutureWarning):
+ isinstance(s, deprecated)
+
+ s = pa.scalar(None, type=s.type)
+ assert s.is_valid is False
+ assert s.as_py() is None
+
+
+def test_null_singleton():
+ with pytest.raises(Exception):
+ pa.NullScalar()
+
+
+def test_nulls():
+ null = pa.scalar(None)
+ assert null is pa.NA
+ assert null.as_py() is None
+ assert (null == "something") is pa.NA
Review comment:
Added.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] kszucs edited a comment on pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
kszucs edited a comment on pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#issuecomment-654308059
@pitrou updated. I assume now we propagate the null values from the selected child array, please double check the union tests.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] jorisvandenbossche commented on pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
jorisvandenbossche commented on pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#issuecomment-652245303
> the null equality tests look like a nuisance for regular Python usage (`__eq__` should return a boolean)
The reason that those scalars return null on equality check and not True/False, it to ensure consistent behaviour between arrays and scalars (to ensure `(arr1 == arr2)[0]` and `arr1[0] == arr2[0]`gives the same answer).
Our element-wise equality propagates null, and so IMO also the scalar should do.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] kszucs commented on a change in pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
kszucs commented on a change in pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#discussion_r446160460
##########
File path: python/pyarrow/scalar.pxi
##########
@@ -16,1198 +16,704 @@
# under the License.
-_NULL = NA = None
-
-
cdef class Scalar:
"""
- The base class for all array elements.
+ The base class for scalars.
"""
+ def __init__(self):
+ raise TypeError("Do not call {}'s constructor directly, use "
+ "pa.scalar() instead.".format(self.__class__.__name__))
-cdef class NullType(Scalar):
- """
- Singleton for null array elements.
- """
- # TODO rename this NullValue?
+ cdef void init(self, const shared_ptr[CScalar]& wrapped):
+ self.wrapped = wrapped
- def __cinit__(self):
- global NA
- if NA is not None:
- raise Exception('Cannot create multiple NAType instances')
+ @staticmethod
+ cdef wrap(const shared_ptr[CScalar]& wrapped):
+ cdef:
+ Scalar self
+ Type type_id = wrapped.get().type.get().id()
+
+ if type_id == _Type_NA:
+ return _NULL
+
+ typ = _scalar_classes[type_id]
+ self = typ.__new__(typ)
+ self.init(wrapped)
+
+ return self
+
+ cdef inline shared_ptr[CScalar] unwrap(self) nogil:
+ return self.wrapped
- self.type = null()
+ @property
+ def type(self):
+ return pyarrow_wrap_data_type(self.wrapped.get().type)
def __repr__(self):
- return 'NULL'
+ return '<pyarrow.{}: {!r}>'.format(
+ self.__class__.__name__, self.as_py()
+ )
- def as_py(self):
- """
- Return None
- """
- return None
+ def __str__(self):
+ return str(self.as_py())
def __eq__(self, other):
- return NA
+ # TODO(kszucs): use c++ Equals
+ if isinstance(other, Scalar):
+ other = other.as_py()
+ return self.as_py() == other
+ def __hash__(self):
+ # TODO(kszucs): use C++ hash if implemented for the type
+ return hash(self.as_py())
+
+ def as_py(self):
+ raise NotImplementedError()
-_NULL = NA = NullType()
+
+_NULL = NA = None
-cdef class ArrayValue(Scalar):
+cdef class NullScalar(Scalar):
"""
- The base class for non-null array elements.
+ Concrete class for null scalars.
"""
- def __init__(self):
- raise TypeError("Do not call {}'s constructor directly, use array "
- "subscription instead."
- .format(self.__class__.__name__))
-
- cdef void init(self, DataType type, const shared_ptr[CArray]& sp_array,
- int64_t index):
- self.type = type
- self.index = index
- self._set_array(sp_array)
+ def __cinit__(self):
+ global NA
+ if NA is not None:
+ raise Exception('Cannot create multiple NAType instances')
+ self.init(shared_ptr[CScalar](new CNullScalar()))
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
+ def __init__(self):
+ pass
- def __repr__(self):
- if hasattr(self, 'as_py'):
- return repr(self.as_py())
- else:
- return super(Scalar, self).__repr__()
+ def __eq__(self, other):
+ return NA
- def __str__(self):
- if hasattr(self, 'as_py'):
- return str(self.as_py())
- else:
- return super(Scalar, self).__str__()
+ def as_py(self):
+ """
+ Return this value as a Python None.
+ """
+ return None
- def __eq__(self, other):
- if hasattr(self, 'as_py'):
- if isinstance(other, ArrayValue):
- other = other.as_py()
- return self.as_py() == other
- else:
- raise NotImplementedError(
- "Cannot compare Arrow values that don't support as_py()")
- def __hash__(self):
- return hash(self.as_py())
+_NULL = NA = NullScalar()
-cdef class BooleanValue(ArrayValue):
+cdef class BooleanScalar(Scalar):
"""
- Concrete class for boolean array elements.
+ Concrete class for boolean scalars.
"""
def as_py(self):
"""
Return this value as a Python bool.
"""
- cdef CBooleanArray* ap = <CBooleanArray*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CBooleanScalar* sp = <CBooleanScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int8Value(ArrayValue):
+cdef class UInt8Scalar(Scalar):
"""
- Concrete class for int8 array elements.
+ Concrete class for uint8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt8Array* ap = <CInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt8Scalar* sp = <CUInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt8Value(ArrayValue):
+cdef class Int8Scalar(Scalar):
"""
- Concrete class for uint8 array elements.
+ Concrete class for int8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt8Array* ap = <CUInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt8Scalar* sp = <CInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int16Value(ArrayValue):
+cdef class UInt16Scalar(Scalar):
"""
- Concrete class for int16 array elements.
+ Concrete class for uint16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt16Array* ap = <CInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt16Scalar* sp = <CUInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt16Value(ArrayValue):
+cdef class Int16Scalar(Scalar):
"""
- Concrete class for uint16 array elements.
+ Concrete class for int16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt16Array* ap = <CUInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt16Scalar* sp = <CInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int32Value(ArrayValue):
+cdef class UInt32Scalar(Scalar):
"""
- Concrete class for int32 array elements.
+ Concrete class for uint32 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt32Array* ap = <CInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt32Scalar* sp = <CUInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt32Value(ArrayValue):
+cdef class Int32Scalar(Scalar):
"""
- Concrete class for uint32 array elements.
+ Concrete class for int32 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt32Array* ap = <CUInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt32Scalar* sp = <CInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int64Value(ArrayValue):
+cdef class UInt64Scalar(Scalar):
"""
- Concrete class for int64 array elements.
+ Concrete class for uint64 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt64Array* ap = <CInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt64Scalar* sp = <CUInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt64Value(ArrayValue):
+cdef class Int64Scalar(Scalar):
"""
- Concrete class for uint64 array elements.
+ Concrete class for int64 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt64Array* ap = <CUInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt64Scalar* sp = <CInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date32Value(ArrayValue):
+cdef class HalfFloatScalar(Scalar):
"""
- Concrete class for date32 array elements.
+ Concrete class for float scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python float.
"""
- cdef CDate32Array* ap = <CDate32Array*> self.sp_array.get()
+ cdef CHalfFloatScalar* sp = <CHalfFloatScalar*> self.wrapped.get()
+ return PyHalf_FromHalf(sp.value) if sp.is_valid else None
+
+
+cdef class FloatScalar(Scalar):
+ """
+ Concrete class for float scalars.
+ """
- # Shift to seconds since epoch
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(days=ap.Value(self.index)))
+ def as_py(self):
+ """
+ Return this value as a Python float.
+ """
+ cdef CFloatScalar* sp = <CFloatScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date64Value(ArrayValue):
+cdef class DoubleScalar(Scalar):
"""
- Concrete class for date64 array elements.
+ Concrete class for double scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python float.
"""
- cdef CDate64Array* ap = <CDate64Array*> self.sp_array.get()
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(
- days=ap.Value(self.index) / 86400000))
+ cdef CDoubleScalar* sp = <CDoubleScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Time32Value(ArrayValue):
+cdef class DecimalScalar(Scalar):
"""
- Concrete class for time32 array elements.
+ Concrete class for decimal128 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.timedelta instance.
+ Return this value as a Python Decimal.
"""
cdef:
- CTime32Array* ap = <CTime32Array*> self.sp_array.get()
- CTime32Type* dtype = <CTime32Type*> ap.type().get()
-
- if dtype.unit() == TimeUnit_SECOND:
- delta = datetime.timedelta(seconds=ap.Value(self.index))
- return (datetime.datetime(1970, 1, 1) + delta).time()
+ CDecimal128Scalar* sp = <CDecimal128Scalar*> self.wrapped.get()
+ CDecimal128Type* dtype = <CDecimal128Type*> sp.type.get()
+ if sp.is_valid:
+ return _pydecimal.Decimal(
+ frombytes(sp.value.ToString(dtype.scale()))
+ )
else:
- return _box_time_milli(ap.Value(self.index))
+ return None
-cdef class Time64Value(ArrayValue):
+cdef class Date32Scalar(Scalar):
"""
- Concrete class for time64 array elements.
+ Concrete class for date32 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.timedelta instance.
+ Return this value as a Python datetime.datetime instance.
"""
- cdef:
- CTime64Array* ap = <CTime64Array*> self.sp_array.get()
- CTime64Type* dtype = <CTime64Type*> ap.type().get()
+ cdef CDate32Scalar* sp = <CDate32Scalar*> self.wrapped.get()
- cdef int64_t val = ap.Value(self.index)
- if dtype.unit() == TimeUnit_MICRO:
- return _box_time_micro(val)
+ if sp.is_valid:
+ # shift to seconds since epoch
+ return (
+ datetime.date(1970, 1, 1) + datetime.timedelta(days=sp.value)
+ )
else:
- return (datetime.datetime(1970, 1, 1) +
- datetime.timedelta(microseconds=val / 1000)).time()
-
+ return None
-cpdef _box_time_milli(int64_t val):
- delta = datetime.timedelta(milliseconds=val)
- return (datetime.datetime(1970, 1, 1) + delta).time()
+cdef class Date64Scalar(Scalar):
+ """
+ Concrete class for date64 scalars.
+ """
-cpdef _box_time_micro(int64_t val):
- return (datetime.datetime(1970, 1, 1) +
- datetime.timedelta(microseconds=val)).time()
+ def as_py(self):
+ """
+ Return this value as a Python datetime.datetime instance.
+ """
+ cdef CDate64Scalar* sp = <CDate64Scalar*> self.wrapped.get()
+ if sp.is_valid:
+ return (
+ datetime.date(1970, 1, 1) +
+ datetime.timedelta(days=sp.value / 86400000)
+ )
+ else:
+ return None
-cdef dict _DATETIME_CONVERSION_FUNCTIONS = {}
-cdef c_bool _datetime_conversion_initialized = False
+def _datetime_from_int(int64_t value, TimeUnit unit, tzinfo=None):
+ if unit == TimeUnit_SECOND:
+ delta = datetime.timedelta(seconds=value)
+ elif unit == TimeUnit_MILLI:
+ delta = datetime.timedelta(milliseconds=value)
+ elif unit == TimeUnit_MICRO:
+ delta = datetime.timedelta(microseconds=value)
+ else:
+ # TimeUnit_NANO: prefer pandas timestamps if available
+ if _pandas_api.have_pandas:
+ return _pandas_api.pd.Timestamp(value, tz=tzinfo, unit='ns')
+ # otherwise safely truncate to microsecond resolution datetime
+ if value % 1000 != 0:
+ raise ValueError(
+ "Nanosecond resolution temporal type {} is not safely "
+ "convertible to microseconds to convert to datetime.datetime. "
+ "Install pandas to return as Timestamp with nanosecond "
+ "support or access the .value attribute.".format(value)
+ )
+ delta = datetime.timedelta(microseconds=value)
-cdef _add_micros_maybe_localize(dt, micros, tzinfo):
- import pytz
- dt = dt.replace(microsecond=micros)
+ dt = datetime.datetime(1970, 1, 1) + delta
+ # adjust timezone if set to the datatype
if tzinfo is not None:
- if not isinstance(tzinfo, datetime.tzinfo):
- tzinfo = string_to_tzinfo(tzinfo)
dt = tzinfo.fromutc(dt)
- return dt
-
-
-cdef _datetime_from_seconds(int64_t v):
- return datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=v)
-
-
-def _nanoseconds_to_datetime_safe(v, tzinfo):
- if v % 1000 != 0:
- raise ValueError("Nanosecond timestamp {} is not safely convertible "
- " to microseconds to convert to datetime.datetime."
- " Install pandas to return as Timestamp with "
- " nanosecond support or access the .value attribute.")
- v = v // 1000
- micros = v % 1_000_000
-
- dt = _datetime_from_seconds(v // 1_000_000)
- return _add_micros_maybe_localize(dt, micros, tzinfo)
-
-
-def _microseconds_to_datetime(v, tzinfo):
- micros = v % 1_000_000
- dt = _datetime_from_seconds(v // 1_000_000)
- return _add_micros_maybe_localize(dt, micros, tzinfo)
+ return dt
-def _millis_to_datetime(v, tzinfo):
- millis = v % 1_000
- dt = _datetime_from_seconds(v // 1000)
- return _add_micros_maybe_localize(dt, millis * 1000, tzinfo)
+cdef class Time32Scalar(Scalar):
+ """
+ Concrete class for time32 scalars.
+ """
-def _seconds_to_datetime(v, tzinfo):
- dt = _datetime_from_seconds(v)
- return _add_micros_maybe_localize(dt, 0, tzinfo)
+ def as_py(self):
+ """
+ Return this value as a Python datetime.timedelta instance.
+ """
+ cdef:
+ CTime32Scalar* sp = <CTime32Scalar*> self.wrapped.get()
+ CTime32Type* dtype = <CTime32Type*> sp.type.get()
+ if sp.is_valid:
+ return _datetime_from_int(sp.value, unit=dtype.unit()).time()
+ else:
+ return None
-def _datetime_conversion_functions():
- global _datetime_conversion_initialized
- if _datetime_conversion_initialized:
- return _DATETIME_CONVERSION_FUNCTIONS
- _DATETIME_CONVERSION_FUNCTIONS.update({
- TimeUnit_SECOND: _seconds_to_datetime,
- TimeUnit_MILLI: _millis_to_datetime,
- TimeUnit_MICRO: _microseconds_to_datetime,
- TimeUnit_NANO: _nanoseconds_to_datetime_safe
- })
+cdef class Time64Scalar(Scalar):
+ """
+ Concrete class for time64 scalars.
+ """
- try:
- import pandas as pd
- _DATETIME_CONVERSION_FUNCTIONS[TimeUnit_NANO] = (
- lambda x, tzinfo: pd.Timestamp(
- x, tz=tzinfo, unit='ns',
- )
- )
- except ImportError:
- pass
+ def as_py(self):
+ """
+ Return this value as a Python datetime.timedelta instance.
+ """
+ cdef:
+ CTime64Scalar* sp = <CTime64Scalar*> self.wrapped.get()
+ CTime64Type* dtype = <CTime64Type*> sp.type.get()
- _datetime_conversion_initialized = True
- return _DATETIME_CONVERSION_FUNCTIONS
+ if sp.is_valid:
+ return _datetime_from_int(sp.value, unit=dtype.unit()).time()
+ else:
+ return None
-cdef class TimestampValue(ArrayValue):
+cdef class TimestampScalar(Scalar):
"""
- Concrete class for timestamp array elements.
+ Concrete class for timestamp scalars.
"""
@property
def value(self):
- cdef CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
- cdef CTimestampType* dtype = <CTimestampType*> ap.type().get()
- return ap.Value(self.index)
+ cdef CTimestampScalar* sp = <CTimestampScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
def as_py(self):
"""
Return this value as a Pandas Timestamp instance (if available),
otherwise as a Python datetime.timedelta instance.
"""
- cdef CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
- cdef CTimestampType* dtype = <CTimestampType*> ap.type().get()
+ cdef:
+ CTimestampScalar* sp = <CTimestampScalar*> self.wrapped.get()
+ CTimestampType* dtype = <CTimestampType*> sp.type.get()
- value = self.value
+ if not sp.is_valid:
+ return None
if not dtype.timezone().empty():
tzinfo = string_to_tzinfo(frombytes(dtype.timezone()))
+ if not isinstance(tzinfo, datetime.tzinfo):
+ tzinfo = string_to_tzinfo(tzinfo)
else:
tzinfo = None
- try:
- converter = _datetime_conversion_functions()[dtype.unit()]
- except KeyError:
- raise ValueError(
- 'Cannot convert nanosecond timestamps without pandas'
- )
- return converter(value, tzinfo=tzinfo)
-
-
-cdef dict _TIMEDELTA_CONVERSION_FUNCTIONS = {}
-
-
-def _nanoseconds_to_timedelta_safe(v):
- if v % 1000 != 0:
- raise ValueError(
- "Nanosecond duration {} is not safely convertible to microseconds "
- "to convert to datetime.timedelta. Install pandas to return as "
- "Timedelta with nanosecond support or access the .value "
- "attribute.".format(v))
- micros = v // 1000
-
- return datetime.timedelta(microseconds=micros)
-
+ return _datetime_from_int(sp.value, unit=dtype.unit(), tzinfo=tzinfo)
-def _timedelta_conversion_functions():
- if _TIMEDELTA_CONVERSION_FUNCTIONS:
- return _TIMEDELTA_CONVERSION_FUNCTIONS
- _TIMEDELTA_CONVERSION_FUNCTIONS.update({
- TimeUnit_SECOND: lambda v: datetime.timedelta(seconds=v),
- TimeUnit_MILLI: lambda v: datetime.timedelta(milliseconds=v),
- TimeUnit_MICRO: lambda v: datetime.timedelta(microseconds=v),
- TimeUnit_NANO: _nanoseconds_to_timedelta_safe
- })
-
- try:
- import pandas as pd
- _TIMEDELTA_CONVERSION_FUNCTIONS[TimeUnit_NANO] = (
- lambda v: pd.Timedelta(v, unit='ns')
- )
- except ImportError:
- pass
-
- return _TIMEDELTA_CONVERSION_FUNCTIONS
-
-
-cdef class DurationValue(ArrayValue):
+cdef class DurationScalar(Scalar):
"""
- Concrete class for duration array elements.
+ Concrete class for duration scalars.
"""
@property
def value(self):
- cdef CDurationArray* ap = <CDurationArray*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CDurationScalar* sp = <CDurationScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
def as_py(self):
"""
Return this value as a Pandas Timestamp instance (if available),
otherwise as a Python datetime.timedelta instance.
"""
- cdef CDurationArray* ap = <CDurationArray*> self.sp_array.get()
- cdef CDurationType* dtype = <CDurationType*> ap.type().get()
-
- cdef int64_t value = ap.Value(self.index)
- converter = _timedelta_conversion_functions()[dtype.unit()]
- return converter(value)
-
+ cdef:
+ CDurationScalar* sp = <CDurationScalar*> self.wrapped.get()
+ CDurationType* dtype = <CDurationType*> sp.type.get()
+ TimeUnit unit = dtype.unit()
-cdef class HalfFloatValue(ArrayValue):
- """
- Concrete class for float16 array elements.
- """
+ if not sp.is_valid:
+ return None
- def as_py(self):
- """
- Return this value as a Python float.
- """
- cdef CHalfFloatArray* ap = <CHalfFloatArray*> self.sp_array.get()
- return PyHalf_FromHalf(ap.Value(self.index))
+ if unit == TimeUnit_SECOND:
+ return datetime.timedelta(seconds=sp.value)
+ elif unit == TimeUnit_MILLI:
+ return datetime.timedelta(milliseconds=sp.value)
+ elif unit == TimeUnit_MICRO:
+ return datetime.timedelta(microseconds=sp.value)
+ else:
+ # TimeUnit_NANO: prefer pandas timestamps if available
+ if _pandas_api.have_pandas:
+ return _pandas_api.pd.Timedelta(sp.value, unit='ns')
+ # otherwise safely truncate to microsecond resolution timedelta
+ if sp.value % 1000 != 0:
+ raise ValueError(
+ "Nanosecond duration {} is not safely convertible to "
+ "microseconds to convert to datetime.timedelta. Install "
+ "pandas to return as Timedelta with nanosecond support or "
+ "access the .value attribute.".format(sp.value)
+ )
+ return datetime.timedelta(microseconds=sp.value // 1000)
-cdef class FloatValue(ArrayValue):
+cdef class BinaryScalar(Scalar):
"""
- Concrete class for float32 array elements.
+ Concrete class for binary-like scalars.
"""
- def as_py(self):
+ def as_buffer(self):
"""
- Return this value as a Python float.
+ Return a view over this value as a Buffer object.
"""
- cdef CFloatArray* ap = <CFloatArray*> self.sp_array.get()
- return ap.Value(self.index)
-
-
-cdef class DoubleValue(ArrayValue):
- """
- Concrete class for float64 array elements.
- """
+ cdef CBinaryScalar* sp = <CBinaryScalar*> self.wrapped.get()
+ return pyarrow_wrap_buffer(sp.value) if sp.is_valid else None
def as_py(self):
"""
- Return this value as a Python float.
+ Return this value as a Python bytes.
"""
- cdef CDoubleArray* ap = <CDoubleArray*> self.sp_array.get()
- return ap.Value(self.index)
+ buffer = self.as_buffer()
+ if buffer is not None:
+ return self.as_buffer().to_pybytes()
+ else:
+ return None
-cdef class DecimalValue(ArrayValue):
- """
- Concrete class for decimal128 array elements.
- """
+cdef class LargeBinaryScalar(BinaryScalar):
+ pass
- def as_py(self):
- """
- Return this value as a Python Decimal.
- """
- cdef:
- CDecimal128Array* ap = <CDecimal128Array*> self.sp_array.get()
- c_string s = ap.FormatValue(self.index)
- return _pydecimal.Decimal(s.decode('utf8'))
+cdef class FixedSizeBinaryScalar(BinaryScalar):
+ pass
-cdef class StringValue(ArrayValue):
+
+cdef class StringScalar(BinaryScalar):
"""
- Concrete class for string (utf8) array elements.
+ Concrete class for string-like (utf8) scalars.
"""
def as_py(self):
"""
- Return this value as a Python unicode string.
+ Return this value as a Python string.
"""
- cdef CStringArray* ap = <CStringArray*> self.sp_array.get()
- return ap.GetString(self.index).decode('utf-8')
+ buffer = self.as_buffer()
+ if buffer is not None:
+ return frombytes(self.as_buffer().to_pybytes())
+ else:
+ return None
- def as_buffer(self):
- """
- Return a view over this value as a Buffer object.
- """
- cdef:
- CStringArray* ap = <CStringArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
+cdef class LargeStringScalar(StringScalar):
+ pass
-cdef class LargeStringValue(ArrayValue):
+cdef class ListScalar(Scalar):
"""
- Concrete class for large string (utf8) array elements.
+ Concrete class for list-like scalars.
"""
- def as_py(self):
+ cdef array(self):
Review comment:
Exposed as `values`
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] pitrou commented on pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
pitrou commented on pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#issuecomment-652247947
The problem is that it breaks Python semantics in potentially annoying places:
```python
>>> import pyarrow as pa
>>> na = pa.scalar(None)
>>> na in [5]
True
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] kszucs commented on a change in pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
kszucs commented on a change in pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#discussion_r448559330
##########
File path: python/pyarrow/tests/test_convert_builtin.py
##########
@@ -968,25 +968,31 @@ def test_sequence_timestamp_from_int_with_unit():
arr_s = pa.array(data, type=s)
assert len(arr_s) == 1
assert arr_s.type == s
- assert repr(arr_s[0]) == "datetime.datetime(1970, 1, 1, 0, 0, 1)"
+ assert repr(arr_s[0].as_py()) == "datetime.datetime(1970, 1, 1, 0, 0, 1)"
Review comment:
Updated.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] jorisvandenbossche commented on a change in pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
jorisvandenbossche commented on a change in pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#discussion_r448619823
##########
File path: python/pyarrow/scalar.pxi
##########
@@ -16,1198 +16,748 @@
# under the License.
-_NULL = NA = None
+import collections
cdef class Scalar:
"""
- The base class for all array elements.
- """
-
-
-cdef class NullType(Scalar):
- """
- Singleton for null array elements.
+ The base class for scalars.
"""
- # TODO rename this NullValue?
-
- def __cinit__(self):
- global NA
- if NA is not None:
- raise Exception('Cannot create multiple NAType instances')
-
- self.type = null()
-
- def __repr__(self):
- return 'NULL'
- def as_py(self):
- """
- Return None
- """
- return None
+ def __init__(self):
+ raise TypeError("Do not call {}'s constructor directly, use "
+ "pa.scalar() instead.".format(self.__class__.__name__))
- def __eq__(self, other):
- return NA
+ cdef void init(self, const shared_ptr[CScalar]& wrapped):
+ self.wrapped = wrapped
+ @staticmethod
+ cdef wrap(const shared_ptr[CScalar]& wrapped):
+ cdef:
+ Scalar self
+ Type type_id = wrapped.get().type.get().id()
-_NULL = NA = NullType()
+ if type_id == _Type_NA:
+ return _NULL
+ typ = _scalar_classes[type_id]
+ self = typ.__new__(typ)
+ self.init(wrapped)
-cdef class ArrayValue(Scalar):
- """
- The base class for non-null array elements.
- """
+ return self
- def __init__(self):
- raise TypeError("Do not call {}'s constructor directly, use array "
- "subscription instead."
- .format(self.__class__.__name__))
+ cdef inline shared_ptr[CScalar] unwrap(self) nogil:
+ return self.wrapped
- cdef void init(self, DataType type, const shared_ptr[CArray]& sp_array,
- int64_t index):
- self.type = type
- self.index = index
- self._set_array(sp_array)
+ @property
+ def type(self):
+ """
+ Data type of the Scalar object.
+ """
+ return pyarrow_wrap_data_type(self.wrapped.get().type)
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
+ @property
+ def is_valid(self):
+ """
+ Holds a valid (non-null) value.
+ """
+ return self.wrapped.get().is_valid
def __repr__(self):
- if hasattr(self, 'as_py'):
- return repr(self.as_py())
- else:
- return super(Scalar, self).__repr__()
+ return '<pyarrow.{}: {!r}>'.format(
+ self.__class__.__name__, self.as_py()
+ )
def __str__(self):
- if hasattr(self, 'as_py'):
- return str(self.as_py())
- else:
- return super(Scalar, self).__str__()
+ return str(self.as_py())
+
+ def equals(self, Scalar other):
+ return self.wrapped.get().Equals(other.unwrap().get()[0])
Review comment:
> Removed the options .as_py() conversion from the equality check, so now a scalar can be equal to another scalar only.
Sounds good!
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] kszucs commented on a change in pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
kszucs commented on a change in pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#discussion_r450317608
##########
File path: cpp/src/arrow/scalar_test.cc
##########
@@ -451,6 +565,127 @@ TEST(TestStructScalar, FieldAccess) {
ASSERT_RAISES(Invalid, abc.field(5).status());
ASSERT_RAISES(Invalid, abc.field("c").status());
+
+ ASSERT_OK_AND_ASSIGN(auto d, abc.field("d"));
+ ASSERT_TRUE(d->Equals(MakeNullScalar(int64())));
+}
+
+TEST(TestDictionaryScalar, Basics) {
+ auto ty = dictionary(int8(), utf8());
+ auto dict = ArrayFromJSON(utf8(), "[\"alpha\", \"beta\", \"gamma\"]");
+
+ DictionaryScalar::ValueType alpha;
+ ASSERT_OK_AND_ASSIGN(alpha.index, MakeScalar(int8(), 0));
+ alpha.dictionary = dict;
+
+ DictionaryScalar::ValueType gamma;
+ ASSERT_OK_AND_ASSIGN(gamma.index, MakeScalar(int8(), 2));
+ gamma.dictionary = dict;
+
+ auto scalar_null = MakeNullScalar(ty);
+ auto scalar_alpha = DictionaryScalar(alpha, ty);
+ auto scalar_gamma = DictionaryScalar(gamma, ty);
+
+ ASSERT_OK_AND_ASSIGN(
+ auto encoded_null,
+ checked_cast<const DictionaryScalar&>(*scalar_null).GetEncodedValue());
+ ASSERT_TRUE(encoded_null->Equals(MakeNullScalar(utf8())));
+
+ ASSERT_OK_AND_ASSIGN(
+ auto encoded_alpha,
+ checked_cast<const DictionaryScalar&>(scalar_alpha).GetEncodedValue());
+ ASSERT_TRUE(encoded_alpha->Equals(MakeScalar("alpha")));
+
+ ASSERT_OK_AND_ASSIGN(
+ auto encoded_gamma,
+ checked_cast<const DictionaryScalar&>(scalar_gamma).GetEncodedValue());
+ ASSERT_TRUE(encoded_gamma->Equals(MakeScalar("gamma")));
+
+ // test Array.GetScalar
+ DictionaryArray arr(ty, ArrayFromJSON(int8(), "[2, 0, 1, null]"), dict);
+ ASSERT_OK_AND_ASSIGN(auto first, arr.GetScalar(0));
+ ASSERT_OK_AND_ASSIGN(auto second, arr.GetScalar(1));
+ ASSERT_OK_AND_ASSIGN(auto last, arr.GetScalar(3));
+
+ ASSERT_TRUE(first->Equals(scalar_gamma));
+ ASSERT_TRUE(second->Equals(scalar_alpha));
+ ASSERT_TRUE(last->Equals(scalar_null));
+}
+
+TEST(TestSparseUnionScalar, Basics) {
+ auto ty = sparse_union({field("string", utf8()), field("number", uint64())});
+
+ auto alpha = MakeScalar("alpha");
+ auto beta = MakeScalar("beta");
+ ASSERT_OK_AND_ASSIGN(auto two, MakeScalar(uint64(), 2));
+
+ auto scalar_alpha = SparseUnionScalar(alpha, ty);
+ auto scalar_beta = SparseUnionScalar(beta, ty);
+ auto scalar_two = SparseUnionScalar(two, ty);
+
+ // test Array.GetScalar
+ std::vector<std::shared_ptr<Array>> children{
+ ArrayFromJSON(utf8(), "[\"alpha\", \"\", \"beta\", null, \"gamma\"]"),
+ ArrayFromJSON(uint64(), "[1, 2, 11, 22, null]")};
+
+ auto type_ids = ArrayFromJSON(int8(), "[0, 1, 0, 0, 1]");
+ SparseUnionArray arr(ty, 5, children, type_ids->data()->buffers[1]);
+ ASSERT_OK(arr.ValidateFull());
+
+ ASSERT_OK_AND_ASSIGN(auto first, arr.GetScalar(0));
+ ASSERT_TRUE(first->Equals(scalar_alpha));
+
+ ASSERT_OK_AND_ASSIGN(auto second, arr.GetScalar(1));
+ ASSERT_TRUE(second->Equals(scalar_two));
+
+ ASSERT_OK_AND_ASSIGN(auto third, arr.GetScalar(2));
+ ASSERT_TRUE(third->Equals(scalar_beta));
+
+ ASSERT_OK_AND_ASSIGN(auto fourth, arr.GetScalar(3));
+ ASSERT_TRUE(fourth->Equals(MakeNullScalar(ty)));
Review comment:
I think `GetScalar()` should return a Scalar with the same type as the Array, so in case of unions with the union type rather than the type selected by the union type id (the union scalar's value is another scalar with the type id).
I don't have a strong opinion on this though.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] jorisvandenbossche commented on a change in pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
jorisvandenbossche commented on a change in pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#discussion_r448188888
##########
File path: python/pyarrow/scalar.pxi
##########
@@ -16,1198 +16,748 @@
# under the License.
-_NULL = NA = None
+import collections
cdef class Scalar:
"""
- The base class for all array elements.
- """
-
-
-cdef class NullType(Scalar):
- """
- Singleton for null array elements.
+ The base class for scalars.
"""
- # TODO rename this NullValue?
-
- def __cinit__(self):
- global NA
- if NA is not None:
- raise Exception('Cannot create multiple NAType instances')
-
- self.type = null()
-
- def __repr__(self):
- return 'NULL'
- def as_py(self):
- """
- Return None
- """
- return None
+ def __init__(self):
+ raise TypeError("Do not call {}'s constructor directly, use "
+ "pa.scalar() instead.".format(self.__class__.__name__))
- def __eq__(self, other):
- return NA
+ cdef void init(self, const shared_ptr[CScalar]& wrapped):
+ self.wrapped = wrapped
+ @staticmethod
+ cdef wrap(const shared_ptr[CScalar]& wrapped):
+ cdef:
+ Scalar self
+ Type type_id = wrapped.get().type.get().id()
-_NULL = NA = NullType()
+ if type_id == _Type_NA:
+ return _NULL
+ typ = _scalar_classes[type_id]
+ self = typ.__new__(typ)
+ self.init(wrapped)
-cdef class ArrayValue(Scalar):
- """
- The base class for non-null array elements.
- """
+ return self
- def __init__(self):
- raise TypeError("Do not call {}'s constructor directly, use array "
- "subscription instead."
- .format(self.__class__.__name__))
+ cdef inline shared_ptr[CScalar] unwrap(self) nogil:
+ return self.wrapped
- cdef void init(self, DataType type, const shared_ptr[CArray]& sp_array,
- int64_t index):
- self.type = type
- self.index = index
- self._set_array(sp_array)
+ @property
+ def type(self):
+ """
+ Data type of the Scalar object.
+ """
+ return pyarrow_wrap_data_type(self.wrapped.get().type)
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
+ @property
+ def is_valid(self):
+ """
+ Holds a valid (non-null) value.
+ """
+ return self.wrapped.get().is_valid
def __repr__(self):
- if hasattr(self, 'as_py'):
- return repr(self.as_py())
- else:
- return super(Scalar, self).__repr__()
+ return '<pyarrow.{}: {!r}>'.format(
+ self.__class__.__name__, self.as_py()
+ )
def __str__(self):
- if hasattr(self, 'as_py'):
- return str(self.as_py())
- else:
- return super(Scalar, self).__str__()
+ return str(self.as_py())
+
+ def equals(self, Scalar other):
+ return self.wrapped.get().Equals(other.unwrap().get()[0])
Review comment:
Although, also for arrays this currently doesn't work. But for arrays it gives an error about no kernel matching the types (which is more informative than "silently" giving False)
##########
File path: python/pyarrow/scalar.pxi
##########
@@ -16,1198 +16,748 @@
# under the License.
-_NULL = NA = None
+import collections
cdef class Scalar:
"""
- The base class for all array elements.
- """
-
-
-cdef class NullType(Scalar):
- """
- Singleton for null array elements.
+ The base class for scalars.
"""
- # TODO rename this NullValue?
-
- def __cinit__(self):
- global NA
- if NA is not None:
- raise Exception('Cannot create multiple NAType instances')
-
- self.type = null()
-
- def __repr__(self):
- return 'NULL'
- def as_py(self):
- """
- Return None
- """
- return None
+ def __init__(self):
+ raise TypeError("Do not call {}'s constructor directly, use "
+ "pa.scalar() instead.".format(self.__class__.__name__))
- def __eq__(self, other):
- return NA
+ cdef void init(self, const shared_ptr[CScalar]& wrapped):
+ self.wrapped = wrapped
+ @staticmethod
+ cdef wrap(const shared_ptr[CScalar]& wrapped):
+ cdef:
+ Scalar self
+ Type type_id = wrapped.get().type.get().id()
-_NULL = NA = NullType()
+ if type_id == _Type_NA:
+ return _NULL
+ typ = _scalar_classes[type_id]
+ self = typ.__new__(typ)
+ self.init(wrapped)
-cdef class ArrayValue(Scalar):
- """
- The base class for non-null array elements.
- """
+ return self
- def __init__(self):
- raise TypeError("Do not call {}'s constructor directly, use array "
- "subscription instead."
- .format(self.__class__.__name__))
+ cdef inline shared_ptr[CScalar] unwrap(self) nogil:
+ return self.wrapped
- cdef void init(self, DataType type, const shared_ptr[CArray]& sp_array,
- int64_t index):
- self.type = type
- self.index = index
- self._set_array(sp_array)
+ @property
+ def type(self):
+ """
+ Data type of the Scalar object.
+ """
+ return pyarrow_wrap_data_type(self.wrapped.get().type)
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
+ @property
+ def is_valid(self):
+ """
+ Holds a valid (non-null) value.
+ """
+ return self.wrapped.get().is_valid
def __repr__(self):
- if hasattr(self, 'as_py'):
- return repr(self.as_py())
- else:
- return super(Scalar, self).__repr__()
+ return '<pyarrow.{}: {!r}>'.format(
+ self.__class__.__name__, self.as_py()
+ )
def __str__(self):
- if hasattr(self, 'as_py'):
- return str(self.as_py())
- else:
- return super(Scalar, self).__str__()
+ return str(self.as_py())
+
+ def equals(self, Scalar other):
+ return self.wrapped.get().Equals(other.unwrap().get()[0])
def __eq__(self, other):
- if hasattr(self, 'as_py'):
- if isinstance(other, ArrayValue):
- other = other.as_py()
- return self.as_py() == other
- else:
- raise NotImplementedError(
- "Cannot compare Arrow values that don't support as_py()")
+ try:
+ if not isinstance(other, Scalar):
+ other = scalar(other, type=self.type)
+ return self.equals(other)
+ except (TypeError, ValueError, ArrowInvalid):
+ return NotImplemented
def __hash__(self):
- return hash(self.as_py())
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+ def as_py(self):
+ raise NotImplementedError()
-cdef class BooleanValue(ArrayValue):
- """
- Concrete class for boolean array elements.
- """
- def as_py(self):
- """
- Return this value as a Python bool.
- """
- cdef CBooleanArray* ap = <CBooleanArray*> self.sp_array.get()
- return ap.Value(self.index)
+_NULL = NA = None
-cdef class Int8Value(ArrayValue):
+cdef class NullScalar(Scalar):
"""
- Concrete class for int8 array elements.
+ Concrete class for null scalars.
"""
- def as_py(self):
- """
- Return this value as a Python int.
- """
- cdef CInt8Array* ap = <CInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ def __cinit__(self):
+ global NA
+ if NA is not None:
+ raise RuntimeError('Cannot create multiple NullScalar instances')
+ self.init(shared_ptr[CScalar](new CNullScalar()))
+ def __init__(self):
+ pass
-cdef class UInt8Value(ArrayValue):
- """
- Concrete class for uint8 array elements.
- """
+ def __eq__(self, other):
+ return NA
+
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
def as_py(self):
"""
- Return this value as a Python int.
+ Return this value as a Python None.
"""
- cdef CUInt8Array* ap = <CUInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
-
+ return None
-cdef class Int16Value(ArrayValue):
- """
- Concrete class for int16 array elements.
- """
- def as_py(self):
- """
- Return this value as a Python int.
- """
- cdef CInt16Array* ap = <CInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+_NULL = NA = NullScalar()
-cdef class UInt16Value(ArrayValue):
+cdef class BooleanScalar(Scalar):
"""
- Concrete class for uint16 array elements.
+ Concrete class for boolean scalars.
"""
def as_py(self):
"""
- Return this value as a Python int.
+ Return this value as a Python bool.
"""
- cdef CUInt16Array* ap = <CUInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CBooleanScalar* sp = <CBooleanScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int32Value(ArrayValue):
+cdef class UInt8Scalar(Scalar):
"""
- Concrete class for int32 array elements.
+ Concrete class for uint8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt32Array* ap = <CInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt8Scalar* sp = <CUInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt32Value(ArrayValue):
+cdef class Int8Scalar(Scalar):
"""
- Concrete class for uint32 array elements.
+ Concrete class for int8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt32Array* ap = <CUInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt8Scalar* sp = <CInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int64Value(ArrayValue):
+cdef class UInt16Scalar(Scalar):
"""
- Concrete class for int64 array elements.
+ Concrete class for uint16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt64Array* ap = <CInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt16Scalar* sp = <CUInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt64Value(ArrayValue):
+cdef class Int16Scalar(Scalar):
"""
- Concrete class for uint64 array elements.
+ Concrete class for int16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt64Array* ap = <CUInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt16Scalar* sp = <CInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date32Value(ArrayValue):
+cdef class UInt32Scalar(Scalar):
"""
- Concrete class for date32 array elements.
+ Concrete class for uint32 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python int.
"""
- cdef CDate32Array* ap = <CDate32Array*> self.sp_array.get()
-
- # Shift to seconds since epoch
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(days=ap.Value(self.index)))
+ cdef CUInt32Scalar* sp = <CUInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date64Value(ArrayValue):
+cdef class Int32Scalar(Scalar):
"""
- Concrete class for date64 array elements.
+ Concrete class for int32 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python int.
"""
- cdef CDate64Array* ap = <CDate64Array*> self.sp_array.get()
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(
- days=ap.Value(self.index) / 86400000))
+ cdef CInt32Scalar* sp = <CInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Time32Value(ArrayValue):
+cdef class UInt64Scalar(Scalar):
"""
- Concrete class for time32 array elements.
+ Concrete class for uint64 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.timedelta instance.
+ Return this value as a Python int.
"""
- cdef:
- CTime32Array* ap = <CTime32Array*> self.sp_array.get()
- CTime32Type* dtype = <CTime32Type*> ap.type().get()
-
- if dtype.unit() == TimeUnit_SECOND:
- delta = datetime.timedelta(seconds=ap.Value(self.index))
- return (datetime.datetime(1970, 1, 1) + delta).time()
- else:
- return _box_time_milli(ap.Value(self.index))
+ cdef CUInt64Scalar* sp = <CUInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Time64Value(ArrayValue):
+cdef class Int64Scalar(Scalar):
"""
- Concrete class for time64 array elements.
+ Concrete class for int64 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.timedelta instance.
+ Return this value as a Python int.
"""
- cdef:
- CTime64Array* ap = <CTime64Array*> self.sp_array.get()
- CTime64Type* dtype = <CTime64Type*> ap.type().get()
-
- cdef int64_t val = ap.Value(self.index)
- if dtype.unit() == TimeUnit_MICRO:
- return _box_time_micro(val)
- else:
- return (datetime.datetime(1970, 1, 1) +
- datetime.timedelta(microseconds=val / 1000)).time()
-
-
-cpdef _box_time_milli(int64_t val):
- delta = datetime.timedelta(milliseconds=val)
- return (datetime.datetime(1970, 1, 1) + delta).time()
-
-
-cpdef _box_time_micro(int64_t val):
- return (datetime.datetime(1970, 1, 1) +
- datetime.timedelta(microseconds=val)).time()
-
-
-cdef dict _DATETIME_CONVERSION_FUNCTIONS = {}
-cdef c_bool _datetime_conversion_initialized = False
-
-
-cdef _add_micros_maybe_localize(dt, micros, tzinfo):
- import pytz
- dt = dt.replace(microsecond=micros)
- if tzinfo is not None:
- if not isinstance(tzinfo, datetime.tzinfo):
- tzinfo = string_to_tzinfo(tzinfo)
- dt = tzinfo.fromutc(dt)
- return dt
-
-
-cdef _datetime_from_seconds(int64_t v):
- return datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=v)
-
-
-def _nanoseconds_to_datetime_safe(v, tzinfo):
- if v % 1000 != 0:
- raise ValueError("Nanosecond timestamp {} is not safely convertible "
- " to microseconds to convert to datetime.datetime."
- " Install pandas to return as Timestamp with "
- " nanosecond support or access the .value attribute.")
- v = v // 1000
- micros = v % 1_000_000
-
- dt = _datetime_from_seconds(v // 1_000_000)
- return _add_micros_maybe_localize(dt, micros, tzinfo)
-
-
-def _microseconds_to_datetime(v, tzinfo):
- micros = v % 1_000_000
- dt = _datetime_from_seconds(v // 1_000_000)
- return _add_micros_maybe_localize(dt, micros, tzinfo)
-
-
-def _millis_to_datetime(v, tzinfo):
- millis = v % 1_000
- dt = _datetime_from_seconds(v // 1000)
- return _add_micros_maybe_localize(dt, millis * 1000, tzinfo)
-
-
-def _seconds_to_datetime(v, tzinfo):
- dt = _datetime_from_seconds(v)
- return _add_micros_maybe_localize(dt, 0, tzinfo)
-
-
-def _datetime_conversion_functions():
- global _datetime_conversion_initialized
- if _datetime_conversion_initialized:
- return _DATETIME_CONVERSION_FUNCTIONS
-
- _DATETIME_CONVERSION_FUNCTIONS.update({
- TimeUnit_SECOND: _seconds_to_datetime,
- TimeUnit_MILLI: _millis_to_datetime,
- TimeUnit_MICRO: _microseconds_to_datetime,
- TimeUnit_NANO: _nanoseconds_to_datetime_safe
- })
-
- try:
- import pandas as pd
- _DATETIME_CONVERSION_FUNCTIONS[TimeUnit_NANO] = (
- lambda x, tzinfo: pd.Timestamp(
- x, tz=tzinfo, unit='ns',
- )
- )
- except ImportError:
- pass
-
- _datetime_conversion_initialized = True
- return _DATETIME_CONVERSION_FUNCTIONS
+ cdef CInt64Scalar* sp = <CInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class TimestampValue(ArrayValue):
+cdef class HalfFloatScalar(Scalar):
"""
- Concrete class for timestamp array elements.
+ Concrete class for float scalars.
"""
- @property
- def value(self):
- cdef CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
- cdef CTimestampType* dtype = <CTimestampType*> ap.type().get()
- return ap.Value(self.index)
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+
+ def __eq__(self, other):
+ if isinstance(other, Scalar):
+ other = other.as_py()
+ return self.as_py() == other
def as_py(self):
"""
- Return this value as a Pandas Timestamp instance (if available),
- otherwise as a Python datetime.timedelta instance.
+ Return this value as a Python float.
"""
- cdef CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
- cdef CTimestampType* dtype = <CTimestampType*> ap.type().get()
-
- value = self.value
-
- if not dtype.timezone().empty():
- tzinfo = string_to_tzinfo(frombytes(dtype.timezone()))
- else:
- tzinfo = None
-
- try:
- converter = _datetime_conversion_functions()[dtype.unit()]
- except KeyError:
- raise ValueError(
- 'Cannot convert nanosecond timestamps without pandas'
- )
- return converter(value, tzinfo=tzinfo)
-
-
-cdef dict _TIMEDELTA_CONVERSION_FUNCTIONS = {}
-
-
-def _nanoseconds_to_timedelta_safe(v):
- if v % 1000 != 0:
- raise ValueError(
- "Nanosecond duration {} is not safely convertible to microseconds "
- "to convert to datetime.timedelta. Install pandas to return as "
- "Timedelta with nanosecond support or access the .value "
- "attribute.".format(v))
- micros = v // 1000
-
- return datetime.timedelta(microseconds=micros)
+ cdef CHalfFloatScalar* sp = <CHalfFloatScalar*> self.wrapped.get()
+ return PyHalf_FromHalf(sp.value) if sp.is_valid else None
-def _timedelta_conversion_functions():
- if _TIMEDELTA_CONVERSION_FUNCTIONS:
- return _TIMEDELTA_CONVERSION_FUNCTIONS
-
- _TIMEDELTA_CONVERSION_FUNCTIONS.update({
- TimeUnit_SECOND: lambda v: datetime.timedelta(seconds=v),
- TimeUnit_MILLI: lambda v: datetime.timedelta(milliseconds=v),
- TimeUnit_MICRO: lambda v: datetime.timedelta(microseconds=v),
- TimeUnit_NANO: _nanoseconds_to_timedelta_safe
- })
-
- try:
- import pandas as pd
- _TIMEDELTA_CONVERSION_FUNCTIONS[TimeUnit_NANO] = (
- lambda v: pd.Timedelta(v, unit='ns')
- )
- except ImportError:
- pass
-
- return _TIMEDELTA_CONVERSION_FUNCTIONS
-
-
-cdef class DurationValue(ArrayValue):
+cdef class FloatScalar(Scalar):
"""
- Concrete class for duration array elements.
+ Concrete class for float scalars.
"""
- @property
- def value(self):
- cdef CDurationArray* ap = <CDurationArray*> self.sp_array.get()
- return ap.Value(self.index)
-
def as_py(self):
"""
- Return this value as a Pandas Timestamp instance (if available),
- otherwise as a Python datetime.timedelta instance.
+ Return this value as a Python float.
"""
- cdef CDurationArray* ap = <CDurationArray*> self.sp_array.get()
- cdef CDurationType* dtype = <CDurationType*> ap.type().get()
-
- cdef int64_t value = ap.Value(self.index)
- converter = _timedelta_conversion_functions()[dtype.unit()]
- return converter(value)
+ cdef CFloatScalar* sp = <CFloatScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class HalfFloatValue(ArrayValue):
+cdef class DoubleScalar(Scalar):
"""
- Concrete class for float16 array elements.
+ Concrete class for double scalars.
"""
def as_py(self):
"""
Return this value as a Python float.
"""
- cdef CHalfFloatArray* ap = <CHalfFloatArray*> self.sp_array.get()
- return PyHalf_FromHalf(ap.Value(self.index))
+ cdef CDoubleScalar* sp = <CDoubleScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class FloatValue(ArrayValue):
+cdef class Decimal128Scalar(Scalar):
"""
- Concrete class for float32 array elements.
+ Concrete class for decimal128 scalars.
"""
def as_py(self):
"""
- Return this value as a Python float.
+ Return this value as a Python Decimal.
"""
- cdef CFloatArray* ap = <CFloatArray*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef:
+ CDecimal128Scalar* sp = <CDecimal128Scalar*> self.wrapped.get()
+ CDecimal128Type* dtype = <CDecimal128Type*> sp.type.get()
+ if sp.is_valid:
+ return _pydecimal.Decimal(
+ frombytes(sp.value.ToString(dtype.scale()))
+ )
+ else:
+ return None
-cdef class DoubleValue(ArrayValue):
+cdef class Date32Scalar(Scalar):
"""
- Concrete class for float64 array elements.
+ Concrete class for date32 scalars.
"""
def as_py(self):
"""
- Return this value as a Python float.
+ Return this value as a Python datetime.datetime instance.
"""
- cdef CDoubleArray* ap = <CDoubleArray*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CDate32Scalar* sp = <CDate32Scalar*> self.wrapped.get()
+
+ if sp.is_valid:
+ # shift to seconds since epoch
+ return (
+ datetime.date(1970, 1, 1) + datetime.timedelta(days=sp.value)
+ )
+ else:
+ return None
-cdef class DecimalValue(ArrayValue):
+cdef class Date64Scalar(Scalar):
"""
- Concrete class for decimal128 array elements.
+ Concrete class for date64 scalars.
"""
def as_py(self):
"""
- Return this value as a Python Decimal.
+ Return this value as a Python datetime.datetime instance.
"""
- cdef:
- CDecimal128Array* ap = <CDecimal128Array*> self.sp_array.get()
- c_string s = ap.FormatValue(self.index)
- return _pydecimal.Decimal(s.decode('utf8'))
+ cdef CDate64Scalar* sp = <CDate64Scalar*> self.wrapped.get()
+ if sp.is_valid:
+ return (
+ datetime.date(1970, 1, 1) +
+ datetime.timedelta(days=sp.value / 86400000)
+ )
+ else:
+ return None
-cdef class StringValue(ArrayValue):
- """
- Concrete class for string (utf8) array elements.
- """
- def as_py(self):
- """
- Return this value as a Python unicode string.
- """
- cdef CStringArray* ap = <CStringArray*> self.sp_array.get()
- return ap.GetString(self.index).decode('utf-8')
+def _datetime_from_int(int64_t value, TimeUnit unit, tzinfo=None):
+ if unit == TimeUnit_SECOND:
+ delta = datetime.timedelta(seconds=value)
+ elif unit == TimeUnit_MILLI:
+ delta = datetime.timedelta(milliseconds=value)
+ elif unit == TimeUnit_MICRO:
+ delta = datetime.timedelta(microseconds=value)
+ else:
+ # TimeUnit_NANO: prefer pandas timestamps if available
+ if _pandas_api.have_pandas:
+ return _pandas_api.pd.Timestamp(value, tz=tzinfo, unit='ns')
+ # otherwise safely truncate to microsecond resolution datetime
+ if value % 1000 != 0:
+ raise ValueError(
+ "Nanosecond resolution temporal type {} is not safely "
+ "convertible to microseconds to convert to datetime.datetime. "
+ "Install pandas to return as Timestamp with nanosecond "
+ "support or access the .value attribute.".format(value)
+ )
+ delta = datetime.timedelta(microseconds=value // 1000)
- def as_buffer(self):
- """
- Return a view over this value as a Buffer object.
- """
- cdef:
- CStringArray* ap = <CStringArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
+ dt = datetime.datetime(1970, 1, 1) + delta
+ # adjust timezone if set to the datatype
+ if tzinfo is not None:
+ dt = tzinfo.fromutc(dt)
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
+ return dt
-cdef class LargeStringValue(ArrayValue):
+cdef class Time32Scalar(Scalar):
"""
- Concrete class for large string (utf8) array elements.
+ Concrete class for time32 scalars.
"""
def as_py(self):
"""
- Return this value as a Python unicode string.
- """
- cdef CLargeStringArray* ap = <CLargeStringArray*> self.sp_array.get()
- return ap.GetString(self.index).decode('utf-8')
-
- def as_buffer(self):
- """
- Return a view over this value as a Buffer object.
+ Return this value as a Python datetime.timedelta instance.
"""
cdef:
- CLargeStringArray* ap = <CLargeStringArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
+ CTime32Scalar* sp = <CTime32Scalar*> self.wrapped.get()
+ CTime32Type* dtype = <CTime32Type*> sp.type.get()
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
+ if sp.is_valid:
+ return _datetime_from_int(sp.value, unit=dtype.unit()).time()
+ else:
+ return None
-cdef class BinaryValue(ArrayValue):
+cdef class Time64Scalar(Scalar):
"""
- Concrete class for variable-sized binary array elements.
+ Concrete class for time64 scalars.
"""
def as_py(self):
"""
- Return this value as a Python bytes object.
- """
- cdef:
- const uint8_t* ptr
- int32_t length
- CBinaryArray* ap = <CBinaryArray*> self.sp_array.get()
-
- ptr = ap.GetValue(self.index, &length)
- return cp.PyBytes_FromStringAndSize(<const char*>(ptr), length)
-
- def as_buffer(self):
- """
- Return a view over this value as a Buffer object.
+ Return this value as a Python datetime.timedelta instance.
"""
cdef:
- CBinaryArray* ap = <CBinaryArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
+ CTime64Scalar* sp = <CTime64Scalar*> self.wrapped.get()
+ CTime64Type* dtype = <CTime64Type*> sp.type.get()
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
+ if sp.is_valid:
+ return _datetime_from_int(sp.value, unit=dtype.unit()).time()
+ else:
+ return None
-cdef class LargeBinaryValue(ArrayValue):
+cdef class TimestampScalar(Scalar):
"""
- Concrete class for large variable-sized binary array elements.
+ Concrete class for timestamp scalars.
"""
+ @property
+ def value(self):
+ cdef CTimestampScalar* sp = <CTimestampScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
+
def as_py(self):
"""
- Return this value as a Python bytes object.
+ Return this value as a Pandas Timestamp instance (if available),
+ otherwise as a Python datetime.timedelta instance.
"""
cdef:
- const uint8_t* ptr
- int64_t length
- CLargeBinaryArray* ap = <CLargeBinaryArray*> self.sp_array.get()
+ CTimestampScalar* sp = <CTimestampScalar*> self.wrapped.get()
+ CTimestampType* dtype = <CTimestampType*> sp.type.get()
- ptr = ap.GetValue(self.index, &length)
- return cp.PyBytes_FromStringAndSize(<const char*>(ptr), length)
+ if not sp.is_valid:
+ return None
- def as_buffer(self):
- """
- Return a view over this value as a Buffer object.
- """
- cdef:
- CLargeBinaryArray* ap = <CLargeBinaryArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
+ if not dtype.timezone().empty():
+ tzinfo = string_to_tzinfo(frombytes(dtype.timezone()))
+ else:
+ tzinfo = None
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
+ return _datetime_from_int(sp.value, unit=dtype.unit(), tzinfo=tzinfo)
-cdef class ListValue(ArrayValue):
+cdef class DurationScalar(Scalar):
"""
- Concrete class for list array elements.
+ Concrete class for duration scalars.
"""
- def __len__(self):
- """
- Return the number of values.
- """
- return self.length()
-
- def __getitem__(self, i):
- """
- Return the value at the given index.
- """
- return self.getitem(_normalize_index(i, self.length()))
-
- def __iter__(self):
- """
- Iterate over this element's values.
- """
- for i in range(len(self)):
- yield self.getitem(i)
- raise StopIteration
-
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CListArray*> sp_array.get()
- self.value_type = pyarrow_wrap_data_type(self.ap.value_type())
-
- cdef getitem(self, int64_t i):
- cdef int64_t j = self.ap.value_offset(self.index) + i
- return box_scalar(self.value_type, self.ap.values(), j)
-
- cdef int64_t length(self):
- return self.ap.value_length(self.index)
+ @property
+ def value(self):
+ cdef CDurationScalar* sp = <CDurationScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
def as_py(self):
"""
- Return this value as a Python list.
+ Return this value as a Pandas Timestamp instance (if available),
+ otherwise as a Python datetime.timedelta instance.
"""
cdef:
- int64_t j
- list result = []
+ CDurationScalar* sp = <CDurationScalar*> self.wrapped.get()
+ CDurationType* dtype = <CDurationType*> sp.type.get()
+ TimeUnit unit = dtype.unit()
- for j in range(len(self)):
- result.append(self.getitem(j).as_py())
+ if not sp.is_valid:
+ return None
- return result
+ if unit == TimeUnit_SECOND:
+ return datetime.timedelta(seconds=sp.value)
+ elif unit == TimeUnit_MILLI:
+ return datetime.timedelta(milliseconds=sp.value)
+ elif unit == TimeUnit_MICRO:
+ return datetime.timedelta(microseconds=sp.value)
+ else:
+ # TimeUnit_NANO: prefer pandas timestamps if available
+ if _pandas_api.have_pandas:
+ return _pandas_api.pd.Timedelta(sp.value, unit='ns')
+ # otherwise safely truncate to microsecond resolution timedelta
+ if sp.value % 1000 != 0:
+ raise ValueError(
+ "Nanosecond duration {} is not safely convertible to "
+ "microseconds to convert to datetime.timedelta. Install "
+ "pandas to return as Timedelta with nanosecond support or "
+ "access the .value attribute.".format(sp.value)
+ )
+ return datetime.timedelta(microseconds=sp.value // 1000)
-cdef class LargeListValue(ArrayValue):
+cdef class BinaryScalar(Scalar):
"""
- Concrete class for large list array elements.
+ Concrete class for binary-like scalars.
"""
- def __len__(self):
- """
- Return the number of values.
- """
- return self.length()
-
- def __getitem__(self, i):
+ def as_buffer(self):
"""
- Return the value at the given index.
+ Return a view over this value as a Buffer object.
"""
- return self.getitem(_normalize_index(i, self.length()))
+ cdef CBinaryScalar* sp = <CBinaryScalar*> self.wrapped.get()
+ return pyarrow_wrap_buffer(sp.value) if sp.is_valid else None
- def __iter__(self):
+ def as_py(self):
"""
- Iterate over this element's values.
+ Return this value as a Python bytes.
"""
- for i in range(len(self)):
- yield self.getitem(i)
- raise StopIteration
+ buffer = self.as_buffer()
+ return None if buffer is None else buffer.to_pybytes()
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CLargeListArray*> sp_array.get()
- self.value_type = pyarrow_wrap_data_type(self.ap.value_type())
- cdef getitem(self, int64_t i):
- cdef int64_t j = self.ap.value_offset(self.index) + i
- return box_scalar(self.value_type, self.ap.values(), j)
+cdef class LargeBinaryScalar(BinaryScalar):
+ pass
- cdef int64_t length(self):
- return self.ap.value_length(self.index)
- def as_py(self):
- """
- Return this value as a Python list.
- """
- cdef:
- int64_t j
- list result = []
-
- for j in range(len(self)):
- result.append(self.getitem(j).as_py())
+cdef class FixedSizeBinaryScalar(BinaryScalar):
+ pass
- return result
-
-cdef class MapValue(ArrayValue):
+cdef class StringScalar(BinaryScalar):
"""
- Concrete class for map array elements.
+ Concrete class for string-like (utf8) scalars.
"""
- def __len__(self):
- """
- Return the number of values.
- """
- return self.length()
-
- def __getitem__(self, i):
- """
- Return the value at the given index.
- """
- return self.getitem(_normalize_index(i, self.length()))
-
- def __iter__(self):
- """
- Iterate over this element's values.
- """
- for i in range(len(self)):
- yield self.getitem(i)
- raise StopIteration
-
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CMapArray*> sp_array.get()
- self.key_type = pyarrow_wrap_data_type(self.ap.map_type().key_type())
- self.item_type = pyarrow_wrap_data_type(self.ap.map_type().item_type())
-
- cdef getitem(self, int64_t i):
- cdef int64_t j = self.ap.value_offset(self.index) + i
- return (box_scalar(self.key_type, self.ap.keys(), j),
- box_scalar(self.item_type, self.ap.items(), j))
-
- cdef int64_t length(self):
- return self.ap.value_length(self.index)
-
def as_py(self):
"""
- Return this value as a Python list of tuples, each containing a
- key and item.
+ Return this value as a Python string.
"""
- cdef:
- int64_t j
- list result = []
+ buffer = self.as_buffer()
+ return None if buffer is None else str(buffer, 'utf8')
- for j in range(len(self)):
- key, item = self.getitem(j)
- result.append((key.as_py(), item.as_py()))
- return result
+cdef class LargeStringScalar(StringScalar):
+ pass
-cdef class FixedSizeListValue(ArrayValue):
+cdef class ListScalar(Scalar):
"""
- Concrete class for fixed size list array elements.
+ Concrete class for list-like scalars.
"""
+ @property
+ def values(self):
+ cdef CListScalar* sp = <CListScalar*> self.wrapped.get()
+ if sp.is_valid:
+ return pyarrow_wrap_array(sp.value)
+ else:
+ return None
+
def __len__(self):
"""
Return the number of values.
"""
- return self.length()
+ return len(self.values)
def __getitem__(self, i):
"""
Return the value at the given index.
"""
- return self.getitem(_normalize_index(i, self.length()))
+ return self.values[_normalize_index(i, len(self))]
def __iter__(self):
"""
Iterate over this element's values.
"""
- for i in range(len(self)):
- yield self.getitem(i)
- raise StopIteration
-
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CFixedSizeListArray*> sp_array.get()
- self.value_type = pyarrow_wrap_data_type(self.ap.value_type())
-
- cdef getitem(self, int64_t i):
- cdef int64_t j = self.ap.value_offset(self.index) + i
- return box_scalar(self.value_type, self.ap.values(), j)
-
- cdef int64_t length(self):
- return self.ap.value_length(self.index)
+ return iter(self.values)
def as_py(self):
"""
Return this value as a Python list.
"""
- cdef:
- int64_t j
- list result = []
-
- for j in range(len(self)):
- result.append(self.getitem(j).as_py())
-
- return result
-
-
-cdef class UnionValue(ArrayValue):
- """
- Concrete class for union array elements.
- """
+ arr = self.values
+ return None if arr is None else arr.to_pylist()
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CUnionArray*> sp_array.get()
-
- cdef getitem(self, int64_t i):
- cdef int child_id = self.ap.child_id(i)
- cdef shared_ptr[CArray] child = self.ap.field(child_id)
- cdef CDenseUnionArray* dense
- if self.ap.mode() == _UnionMode_SPARSE:
- return box_scalar(self.type[child_id].type, child, i)
- else:
- dense = <CDenseUnionArray*> self.ap
- return box_scalar(self.type[child_id].type, child,
- dense.value_offset(i))
- def as_py(self):
- """
- Return this value as a Python object.
+cdef class FixedSizeListScalar(ListScalar):
+ pass
- The exact type depends on the underlying union member.
- """
- return self.getitem(self.index).as_py()
+cdef class LargeListScalar(ListScalar):
+ pass
-cdef class FixedSizeBinaryValue(ArrayValue):
- """
- Concrete class for fixed-size binary array elements.
- """
- def as_py(self):
- """
- Return this value as a Python bytes object.
- """
- cdef:
- CFixedSizeBinaryArray* ap
- CFixedSizeBinaryType* ap_type
- int32_t length
- const char* data
- ap = <CFixedSizeBinaryArray*> self.sp_array.get()
- ap_type = <CFixedSizeBinaryType*> ap.type().get()
- length = ap_type.byte_width()
- data = <const char*> ap.GetValue(self.index)
- return cp.PyBytes_FromStringAndSize(data, length)
-
-
-cdef class StructValue(ArrayValue):
+cdef class StructScalar(Scalar, collections.abc.Mapping):
"""
- Concrete class for struct array elements.
+ Concrete class for struct scalars.
"""
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CStructArray*> sp_array.get()
-
- def __getitem__(self, key):
- """
- Return the child value for the given field name.
- """
- cdef:
- CStructType* type
- int index
-
- type = <CStructType*> self.type.type
- index = type.GetFieldIndex(tobytes(key))
-
- if index < 0:
- raise KeyError(key)
-
- return pyarrow_wrap_array(self.ap.field(index))[self.index]
+ def __len__(self):
+ cdef CStructScalar* sp = <CStructScalar*> self.wrapped.get()
+ return sp.value.size()
- def as_py(self):
- """
- Return this value as a Python dict.
- """
+ def __iter__(self):
cdef:
- vector[shared_ptr[CField]] child_fields = self.type.type.fields()
-
- wrapped_arrays = [pyarrow_wrap_array(self.ap.field(i))
- for i in range(self.ap.num_fields())]
- child_names = [child.get().name() for child in child_fields]
- # Return the struct as a dict
- return {
- frombytes(name): child_array[self.index].as_py()
- for name, child_array in zip(child_names, wrapped_arrays)
- }
-
-
-cdef class DictionaryValue(ArrayValue):
- """
- Concrete class for dictionary-encoded array elements.
- """
-
- def as_py(self):
- """
- Return this value as a Python object.
-
- The exact type depends on the dictionary value type.
- """
- return self.dictionary_value.as_py()
-
- @property
- def index_value(self):
- """
- Return this value's underlying index as a ArrayValue of the right
- signed integer type.
- """
- cdef CDictionaryArray* darr = <CDictionaryArray*>(self.sp_array.get())
- indices = pyarrow_wrap_array(darr.indices())
- return indices[self.index]
-
- @property
- def dictionary_value(self):
- """
- Return this value's underlying dictionary value as a ArrayValue.
- """
- cdef CDictionaryArray* darr = <CDictionaryArray*>(self.sp_array.get())
- dictionary = pyarrow_wrap_array(darr.dictionary())
- return dictionary[self.index_value.as_py()]
-
-
-cdef dict _array_value_classes = {
- _Type_BOOL: BooleanValue,
- _Type_UINT8: UInt8Value,
- _Type_UINT16: UInt16Value,
- _Type_UINT32: UInt32Value,
- _Type_UINT64: UInt64Value,
- _Type_INT8: Int8Value,
- _Type_INT16: Int16Value,
- _Type_INT32: Int32Value,
- _Type_INT64: Int64Value,
- _Type_DATE32: Date32Value,
- _Type_DATE64: Date64Value,
- _Type_TIME32: Time32Value,
- _Type_TIME64: Time64Value,
- _Type_TIMESTAMP: TimestampValue,
- _Type_DURATION: DurationValue,
- _Type_HALF_FLOAT: HalfFloatValue,
- _Type_FLOAT: FloatValue,
- _Type_DOUBLE: DoubleValue,
- _Type_LIST: ListValue,
- _Type_LARGE_LIST: LargeListValue,
- _Type_MAP: MapValue,
- _Type_FIXED_SIZE_LIST: FixedSizeListValue,
- _Type_SPARSE_UNION: UnionValue,
- _Type_DENSE_UNION: UnionValue,
- _Type_BINARY: BinaryValue,
- _Type_STRING: StringValue,
- _Type_LARGE_BINARY: LargeBinaryValue,
- _Type_LARGE_STRING: LargeStringValue,
- _Type_FIXED_SIZE_BINARY: FixedSizeBinaryValue,
- _Type_DECIMAL: DecimalValue,
- _Type_STRUCT: StructValue,
- _Type_DICTIONARY: DictionaryValue,
-}
-
-cdef class ScalarValue(Scalar):
- """
- The base class for scalars.
- """
-
- def __init__(self):
- raise TypeError("Do not call {}'s constructor directly."
- .format(self.__class__.__name__))
-
- cdef void init(self, const shared_ptr[CScalar]& sp_scalar):
- self.sp_scalar = sp_scalar
-
- def __repr__(self):
- if hasattr(self, 'as_py'):
- return repr(self.as_py())
- else:
- return super(Scalar, self).__repr__()
+ CStructScalar* sp = <CStructScalar*> self.wrapped.get()
+ CStructType* dtype = <CStructType*> sp.type.get()
+ vector[shared_ptr[CField]] fields = dtype.fields()
- def __str__(self):
- if hasattr(self, 'as_py'):
- return str(self.as_py())
- else:
- return super(Scalar, self).__str__()
+ if sp.is_valid:
+ for i in range(dtype.num_fields()):
+ yield frombytes(fields[i].get().name())
- def __eq__(self, other):
- if hasattr(self, 'as_py'):
- if isinstance(other, ScalarValue):
- other = other.as_py()
- return self.as_py() == other
+ def __contains__(self, key):
+ try:
+ self[key]
+ except (KeyError, IndexError):
+ return False
else:
- raise NotImplemented(
- "Cannot compare scalars that don't support as_py()")
-
- def __hash__(self):
- return hash(self.as_py())
-
-
-cdef class NullScalar(ScalarValue):
- """
- Concrete class for null scalars.
- """
-
- def as_py(self):
- """
- Return this value as a Python None.
- """
- return None
-
-
-cdef class BooleanScalar(ScalarValue):
- """
- Concrete class for boolean scalars.
- """
+ return True
- def as_py(self):
- """
- Return this value as a Python bool.
+ def __getitem__(self, key):
"""
- cdef CBooleanScalar* sp = <CBooleanScalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
-
+ Return the child value for the given field.
-cdef class UInt8Scalar(ScalarValue):
- """
- Concrete class for uint8 scalars.
- """
+ Parameters
+ ----------
+ index : Union[int, str]
+ Index / position or name of the field.
- def as_py(self):
- """
- Return this value as a Python int.
+ Returns
+ -------
+ result : Scalar
"""
- cdef CUInt8Scalar* sp = <CUInt8Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
+ cdef:
+ CFieldRef ref
+ CStructScalar* sp = <CStructScalar*> self.wrapped.get()
+ if isinstance(key, (bytes, str)):
+ ref = CFieldRef(<c_string> tobytes(key))
+ elif isinstance(key, int):
+ ref = CFieldRef(<int> key)
+ else:
+ raise TypeError('Expected integer or string index')
-cdef class Int8Scalar(ScalarValue):
- """
- Concrete class for int8 scalars.
- """
+ try:
+ return Scalar.wrap(GetResultValue(sp.field(ref)))
+ except ArrowInvalid:
+ if isinstance(key, int):
+ raise IndexError(key)
+ else:
+ raise KeyError(key)
def as_py(self):
"""
- Return this value as a Python int.
+ Return this value as a Python dict.
"""
- cdef CInt8Scalar* sp = <CInt8Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
+ if self.is_valid:
+ return {k: v.as_py() for k, v in self.items()}
+ else:
+ return None
-cdef class UInt16Scalar(ScalarValue):
+cdef class MapScalar(ListScalar):
"""
- Concrete class for uint16 scalars.
+ Concrete class for map scalars.
"""
- def as_py(self):
+ def __getitem__(self, i):
"""
- Return this value as a Python int.
+ Return the value at the given index.
"""
- cdef CUInt16Scalar* sp = <CUInt16Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
-
-
-cdef class Int16Scalar(ScalarValue):
- """
- Concrete class for int16 scalars.
- """
+ arr = self.values
+ if arr is None:
+ raise IndexError(i)
+ dct = arr[_normalize_index(i, len(arr))]
+ return (dct['key'], dct['value'])
- def as_py(self):
+ def __iter__(self):
"""
- Return this value as a Python int.
+ Iterate over this element's values.
"""
- cdef CInt16Scalar* sp = <CInt16Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
-
-
-cdef class UInt32Scalar(ScalarValue):
- """
- Concrete class for uint32 scalars.
- """
+ arr = self.values
+ if arr is None:
+ return iter(zip(arr.field('key'), arr.field('value')))
+ else:
+ raise StopIteration
def as_py(self):
"""
- Return this value as a Python int.
+ Return this value as a Python list.
"""
- cdef CUInt32Scalar* sp = <CUInt32Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
+ arr = self.values
+ if arr is not None:
+ return list(zip(arr.field('key'), arr.field('value')))
+ else:
+ return None
-cdef class Int32Scalar(ScalarValue):
+cdef class DictionaryScalar(Scalar):
"""
- Concrete class for int32 scalars.
+ Concrete class for dictionary-encoded scalars.
"""
- def as_py(self):
+ @property
+ def index(self):
"""
- Return this value as a Python int.
+ Return this value's underlying index as a scalar.
"""
- cdef CInt32Scalar* sp = <CInt32Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
+ cdef CDictionaryScalar* sp = <CDictionaryScalar*> self.wrapped.get()
+ return Scalar.wrap(sp.value.index)
-
-cdef class UInt64Scalar(ScalarValue):
- """
- Concrete class for uint64 scalars.
- """
-
- def as_py(self):
+ @property
+ def value(self):
"""
- Return this value as a Python int.
+ Return the encoded value as a scalar.
"""
- cdef CUInt64Scalar* sp = <CUInt64Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
+ # TODO(kszucs): optimize it to spare uneccessary python object boxing
+ return self.dictionary[self.index.as_py()] if self.is_valid else None
-
-cdef class Int64Scalar(ScalarValue):
- """
- Concrete class for int64 scalars.
- """
+ @property
+ def dictionary(self):
+ cdef CDictionaryScalar* sp = <CDictionaryScalar*> self.wrapped.get()
+ return pyarrow_wrap_array(sp.value.dictionary)
def as_py(self):
"""
- Return this value as a Python int.
+ Return this encoded value as a Python object.
"""
- cdef CInt64Scalar* sp = <CInt64Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
-
+ value = self.value
+ return None if value is None else value.as_py()
-cdef class FloatScalar(ScalarValue):
- """
- Concrete class for float scalars.
- """
+ @property
+ def index_value(self):
+ warnings.warn("`dictionary_value` property is deprecated as of 1.0.0"
+ "please use the `value` property instead",
+ FutureWarning)
+ index = self.index
+ return None if index is None else self.index
Review comment:
Can't this be simplified to `return self.index` ?
##########
File path: python/pyarrow/scalar.pxi
##########
@@ -1217,21 +767,95 @@ cdef dict _scalar_classes = {
_Type_INT16: Int16Scalar,
_Type_INT32: Int32Scalar,
_Type_INT64: Int64Scalar,
+ _Type_HALF_FLOAT: HalfFloatScalar,
_Type_FLOAT: FloatScalar,
_Type_DOUBLE: DoubleScalar,
+ _Type_DECIMAL: Decimal128Scalar,
+ _Type_DATE32: Date32Scalar,
+ _Type_DATE64: Date64Scalar,
+ _Type_TIME32: Time32Scalar,
+ _Type_TIME64: Time64Scalar,
+ _Type_TIMESTAMP: TimestampScalar,
+ _Type_DURATION: DurationScalar,
+ _Type_BINARY: BinaryScalar,
+ _Type_LARGE_BINARY: LargeBinaryScalar,
+ _Type_FIXED_SIZE_BINARY: FixedSizeBinaryScalar,
_Type_STRING: StringScalar,
+ _Type_LARGE_STRING: LargeStringScalar,
+ _Type_LIST: ListScalar,
+ _Type_LARGE_LIST: LargeListScalar,
+ _Type_FIXED_SIZE_LIST: FixedSizeListScalar,
+ _Type_STRUCT: StructScalar,
+ _Type_MAP: MapScalar,
+ _Type_DICTIONARY: DictionaryScalar,
+ _Type_SPARSE_UNION: UnionScalar,
+ _Type_DENSE_UNION: UnionScalar,
}
-cdef object box_scalar(DataType type, const shared_ptr[CArray]& sp_array,
- int64_t index):
- cdef ArrayValue value
- if type.type.id() == _Type_NA:
- return _NULL
- elif sp_array.get().IsNull(index):
- return _NULL
- else:
- klass = _array_value_classes[type.type.id()]
- value = klass.__new__(klass)
- value.init(type, sp_array, index)
- return value
+def scalar(value, DataType type=None, *, from_pandas=None,
+ MemoryPool memory_pool=None):
+ """
+ Create a pyarrow.Scalar instance from a Python object.
+
+ Parameters
+ ----------
+ value : Any
+ Python object coercible to arrow's type system.
+ type : pyarrow.DataType
+ Explicit type to attempt to coerce to, otherwise will be inferred from
+ the value.
+ from_pandas : bool, default None
+ Use pandas's semantics for inferring nulls from values in
+ ndarray-like data. Defaults to False if not passed explicitly by user,
+ or True if a pandas object is passed in.
+ memory_pool : pyarrow.MemoryPool, optional
+ If not passed, will allocate memory from the currently-set default
+ memory pool.
+
+ Returns
+ -------
+ scalar : pyarrow.Scalar
+
+ Notes
+ -----
+ Localized timestamps will currently be returned as UTC (pandas's native
+ representation). Timezone-naive data will be implicitly interpreted as
+ UTC.
Review comment:
I don't think this is fully correct? (or at least a bit misleading/confusing)
Timezone-naive data simply have no timezone, I am not aware of places in arrow where we implicitly assume UTC?
##########
File path: python/pyarrow/scalar.pxi
##########
@@ -16,1198 +16,748 @@
# under the License.
-_NULL = NA = None
+import collections
cdef class Scalar:
"""
- The base class for all array elements.
- """
-
-
-cdef class NullType(Scalar):
- """
- Singleton for null array elements.
+ The base class for scalars.
"""
- # TODO rename this NullValue?
-
- def __cinit__(self):
- global NA
- if NA is not None:
- raise Exception('Cannot create multiple NAType instances')
-
- self.type = null()
-
- def __repr__(self):
- return 'NULL'
- def as_py(self):
- """
- Return None
- """
- return None
+ def __init__(self):
+ raise TypeError("Do not call {}'s constructor directly, use "
+ "pa.scalar() instead.".format(self.__class__.__name__))
- def __eq__(self, other):
- return NA
+ cdef void init(self, const shared_ptr[CScalar]& wrapped):
+ self.wrapped = wrapped
+ @staticmethod
+ cdef wrap(const shared_ptr[CScalar]& wrapped):
+ cdef:
+ Scalar self
+ Type type_id = wrapped.get().type.get().id()
-_NULL = NA = NullType()
+ if type_id == _Type_NA:
+ return _NULL
+ typ = _scalar_classes[type_id]
+ self = typ.__new__(typ)
+ self.init(wrapped)
-cdef class ArrayValue(Scalar):
- """
- The base class for non-null array elements.
- """
+ return self
- def __init__(self):
- raise TypeError("Do not call {}'s constructor directly, use array "
- "subscription instead."
- .format(self.__class__.__name__))
+ cdef inline shared_ptr[CScalar] unwrap(self) nogil:
+ return self.wrapped
- cdef void init(self, DataType type, const shared_ptr[CArray]& sp_array,
- int64_t index):
- self.type = type
- self.index = index
- self._set_array(sp_array)
+ @property
+ def type(self):
+ """
+ Data type of the Scalar object.
+ """
+ return pyarrow_wrap_data_type(self.wrapped.get().type)
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
+ @property
+ def is_valid(self):
+ """
+ Holds a valid (non-null) value.
+ """
+ return self.wrapped.get().is_valid
def __repr__(self):
- if hasattr(self, 'as_py'):
- return repr(self.as_py())
- else:
- return super(Scalar, self).__repr__()
+ return '<pyarrow.{}: {!r}>'.format(
+ self.__class__.__name__, self.as_py()
+ )
def __str__(self):
- if hasattr(self, 'as_py'):
- return str(self.as_py())
- else:
- return super(Scalar, self).__str__()
+ return str(self.as_py())
+
+ def equals(self, Scalar other):
+ return self.wrapped.get().Equals(other.unwrap().get()[0])
def __eq__(self, other):
- if hasattr(self, 'as_py'):
- if isinstance(other, ArrayValue):
- other = other.as_py()
- return self.as_py() == other
- else:
- raise NotImplementedError(
- "Cannot compare Arrow values that don't support as_py()")
+ try:
+ if not isinstance(other, Scalar):
+ other = scalar(other, type=self.type)
+ return self.equals(other)
+ except (TypeError, ValueError, ArrowInvalid):
+ return NotImplemented
def __hash__(self):
- return hash(self.as_py())
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+ def as_py(self):
+ raise NotImplementedError()
-cdef class BooleanValue(ArrayValue):
- """
- Concrete class for boolean array elements.
- """
- def as_py(self):
- """
- Return this value as a Python bool.
- """
- cdef CBooleanArray* ap = <CBooleanArray*> self.sp_array.get()
- return ap.Value(self.index)
+_NULL = NA = None
-cdef class Int8Value(ArrayValue):
+cdef class NullScalar(Scalar):
"""
- Concrete class for int8 array elements.
+ Concrete class for null scalars.
"""
- def as_py(self):
- """
- Return this value as a Python int.
- """
- cdef CInt8Array* ap = <CInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ def __cinit__(self):
+ global NA
+ if NA is not None:
+ raise RuntimeError('Cannot create multiple NullScalar instances')
+ self.init(shared_ptr[CScalar](new CNullScalar()))
+ def __init__(self):
+ pass
-cdef class UInt8Value(ArrayValue):
- """
- Concrete class for uint8 array elements.
- """
+ def __eq__(self, other):
+ return NA
+
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
def as_py(self):
"""
- Return this value as a Python int.
+ Return this value as a Python None.
"""
- cdef CUInt8Array* ap = <CUInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
-
+ return None
-cdef class Int16Value(ArrayValue):
- """
- Concrete class for int16 array elements.
- """
- def as_py(self):
- """
- Return this value as a Python int.
- """
- cdef CInt16Array* ap = <CInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+_NULL = NA = NullScalar()
-cdef class UInt16Value(ArrayValue):
+cdef class BooleanScalar(Scalar):
"""
- Concrete class for uint16 array elements.
+ Concrete class for boolean scalars.
"""
def as_py(self):
"""
- Return this value as a Python int.
+ Return this value as a Python bool.
"""
- cdef CUInt16Array* ap = <CUInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CBooleanScalar* sp = <CBooleanScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int32Value(ArrayValue):
+cdef class UInt8Scalar(Scalar):
"""
- Concrete class for int32 array elements.
+ Concrete class for uint8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt32Array* ap = <CInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt8Scalar* sp = <CUInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt32Value(ArrayValue):
+cdef class Int8Scalar(Scalar):
"""
- Concrete class for uint32 array elements.
+ Concrete class for int8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt32Array* ap = <CUInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt8Scalar* sp = <CInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int64Value(ArrayValue):
+cdef class UInt16Scalar(Scalar):
"""
- Concrete class for int64 array elements.
+ Concrete class for uint16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt64Array* ap = <CInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt16Scalar* sp = <CUInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt64Value(ArrayValue):
+cdef class Int16Scalar(Scalar):
"""
- Concrete class for uint64 array elements.
+ Concrete class for int16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt64Array* ap = <CUInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt16Scalar* sp = <CInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date32Value(ArrayValue):
+cdef class UInt32Scalar(Scalar):
"""
- Concrete class for date32 array elements.
+ Concrete class for uint32 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python int.
"""
- cdef CDate32Array* ap = <CDate32Array*> self.sp_array.get()
-
- # Shift to seconds since epoch
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(days=ap.Value(self.index)))
+ cdef CUInt32Scalar* sp = <CUInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date64Value(ArrayValue):
+cdef class Int32Scalar(Scalar):
"""
- Concrete class for date64 array elements.
+ Concrete class for int32 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python int.
"""
- cdef CDate64Array* ap = <CDate64Array*> self.sp_array.get()
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(
- days=ap.Value(self.index) / 86400000))
+ cdef CInt32Scalar* sp = <CInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Time32Value(ArrayValue):
+cdef class UInt64Scalar(Scalar):
"""
- Concrete class for time32 array elements.
+ Concrete class for uint64 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.timedelta instance.
+ Return this value as a Python int.
"""
- cdef:
- CTime32Array* ap = <CTime32Array*> self.sp_array.get()
- CTime32Type* dtype = <CTime32Type*> ap.type().get()
-
- if dtype.unit() == TimeUnit_SECOND:
- delta = datetime.timedelta(seconds=ap.Value(self.index))
- return (datetime.datetime(1970, 1, 1) + delta).time()
- else:
- return _box_time_milli(ap.Value(self.index))
+ cdef CUInt64Scalar* sp = <CUInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Time64Value(ArrayValue):
+cdef class Int64Scalar(Scalar):
"""
- Concrete class for time64 array elements.
+ Concrete class for int64 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.timedelta instance.
+ Return this value as a Python int.
"""
- cdef:
- CTime64Array* ap = <CTime64Array*> self.sp_array.get()
- CTime64Type* dtype = <CTime64Type*> ap.type().get()
-
- cdef int64_t val = ap.Value(self.index)
- if dtype.unit() == TimeUnit_MICRO:
- return _box_time_micro(val)
- else:
- return (datetime.datetime(1970, 1, 1) +
- datetime.timedelta(microseconds=val / 1000)).time()
-
-
-cpdef _box_time_milli(int64_t val):
- delta = datetime.timedelta(milliseconds=val)
- return (datetime.datetime(1970, 1, 1) + delta).time()
-
-
-cpdef _box_time_micro(int64_t val):
- return (datetime.datetime(1970, 1, 1) +
- datetime.timedelta(microseconds=val)).time()
-
-
-cdef dict _DATETIME_CONVERSION_FUNCTIONS = {}
-cdef c_bool _datetime_conversion_initialized = False
-
-
-cdef _add_micros_maybe_localize(dt, micros, tzinfo):
- import pytz
- dt = dt.replace(microsecond=micros)
- if tzinfo is not None:
- if not isinstance(tzinfo, datetime.tzinfo):
- tzinfo = string_to_tzinfo(tzinfo)
- dt = tzinfo.fromutc(dt)
- return dt
-
-
-cdef _datetime_from_seconds(int64_t v):
- return datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=v)
-
-
-def _nanoseconds_to_datetime_safe(v, tzinfo):
- if v % 1000 != 0:
- raise ValueError("Nanosecond timestamp {} is not safely convertible "
- " to microseconds to convert to datetime.datetime."
- " Install pandas to return as Timestamp with "
- " nanosecond support or access the .value attribute.")
- v = v // 1000
- micros = v % 1_000_000
-
- dt = _datetime_from_seconds(v // 1_000_000)
- return _add_micros_maybe_localize(dt, micros, tzinfo)
-
-
-def _microseconds_to_datetime(v, tzinfo):
- micros = v % 1_000_000
- dt = _datetime_from_seconds(v // 1_000_000)
- return _add_micros_maybe_localize(dt, micros, tzinfo)
-
-
-def _millis_to_datetime(v, tzinfo):
- millis = v % 1_000
- dt = _datetime_from_seconds(v // 1000)
- return _add_micros_maybe_localize(dt, millis * 1000, tzinfo)
-
-
-def _seconds_to_datetime(v, tzinfo):
- dt = _datetime_from_seconds(v)
- return _add_micros_maybe_localize(dt, 0, tzinfo)
-
-
-def _datetime_conversion_functions():
- global _datetime_conversion_initialized
- if _datetime_conversion_initialized:
- return _DATETIME_CONVERSION_FUNCTIONS
-
- _DATETIME_CONVERSION_FUNCTIONS.update({
- TimeUnit_SECOND: _seconds_to_datetime,
- TimeUnit_MILLI: _millis_to_datetime,
- TimeUnit_MICRO: _microseconds_to_datetime,
- TimeUnit_NANO: _nanoseconds_to_datetime_safe
- })
-
- try:
- import pandas as pd
- _DATETIME_CONVERSION_FUNCTIONS[TimeUnit_NANO] = (
- lambda x, tzinfo: pd.Timestamp(
- x, tz=tzinfo, unit='ns',
- )
- )
- except ImportError:
- pass
-
- _datetime_conversion_initialized = True
- return _DATETIME_CONVERSION_FUNCTIONS
+ cdef CInt64Scalar* sp = <CInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class TimestampValue(ArrayValue):
+cdef class HalfFloatScalar(Scalar):
"""
- Concrete class for timestamp array elements.
+ Concrete class for float scalars.
"""
- @property
- def value(self):
- cdef CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
- cdef CTimestampType* dtype = <CTimestampType*> ap.type().get()
- return ap.Value(self.index)
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+
+ def __eq__(self, other):
+ if isinstance(other, Scalar):
+ other = other.as_py()
+ return self.as_py() == other
def as_py(self):
"""
- Return this value as a Pandas Timestamp instance (if available),
- otherwise as a Python datetime.timedelta instance.
+ Return this value as a Python float.
"""
- cdef CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
- cdef CTimestampType* dtype = <CTimestampType*> ap.type().get()
-
- value = self.value
-
- if not dtype.timezone().empty():
- tzinfo = string_to_tzinfo(frombytes(dtype.timezone()))
- else:
- tzinfo = None
-
- try:
- converter = _datetime_conversion_functions()[dtype.unit()]
- except KeyError:
- raise ValueError(
- 'Cannot convert nanosecond timestamps without pandas'
- )
- return converter(value, tzinfo=tzinfo)
-
-
-cdef dict _TIMEDELTA_CONVERSION_FUNCTIONS = {}
-
-
-def _nanoseconds_to_timedelta_safe(v):
- if v % 1000 != 0:
- raise ValueError(
- "Nanosecond duration {} is not safely convertible to microseconds "
- "to convert to datetime.timedelta. Install pandas to return as "
- "Timedelta with nanosecond support or access the .value "
- "attribute.".format(v))
- micros = v // 1000
-
- return datetime.timedelta(microseconds=micros)
+ cdef CHalfFloatScalar* sp = <CHalfFloatScalar*> self.wrapped.get()
+ return PyHalf_FromHalf(sp.value) if sp.is_valid else None
-def _timedelta_conversion_functions():
- if _TIMEDELTA_CONVERSION_FUNCTIONS:
- return _TIMEDELTA_CONVERSION_FUNCTIONS
-
- _TIMEDELTA_CONVERSION_FUNCTIONS.update({
- TimeUnit_SECOND: lambda v: datetime.timedelta(seconds=v),
- TimeUnit_MILLI: lambda v: datetime.timedelta(milliseconds=v),
- TimeUnit_MICRO: lambda v: datetime.timedelta(microseconds=v),
- TimeUnit_NANO: _nanoseconds_to_timedelta_safe
- })
-
- try:
- import pandas as pd
- _TIMEDELTA_CONVERSION_FUNCTIONS[TimeUnit_NANO] = (
- lambda v: pd.Timedelta(v, unit='ns')
- )
- except ImportError:
- pass
-
- return _TIMEDELTA_CONVERSION_FUNCTIONS
-
-
-cdef class DurationValue(ArrayValue):
+cdef class FloatScalar(Scalar):
"""
- Concrete class for duration array elements.
+ Concrete class for float scalars.
"""
- @property
- def value(self):
- cdef CDurationArray* ap = <CDurationArray*> self.sp_array.get()
- return ap.Value(self.index)
-
def as_py(self):
"""
- Return this value as a Pandas Timestamp instance (if available),
- otherwise as a Python datetime.timedelta instance.
+ Return this value as a Python float.
"""
- cdef CDurationArray* ap = <CDurationArray*> self.sp_array.get()
- cdef CDurationType* dtype = <CDurationType*> ap.type().get()
-
- cdef int64_t value = ap.Value(self.index)
- converter = _timedelta_conversion_functions()[dtype.unit()]
- return converter(value)
+ cdef CFloatScalar* sp = <CFloatScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class HalfFloatValue(ArrayValue):
+cdef class DoubleScalar(Scalar):
"""
- Concrete class for float16 array elements.
+ Concrete class for double scalars.
"""
def as_py(self):
"""
Return this value as a Python float.
"""
- cdef CHalfFloatArray* ap = <CHalfFloatArray*> self.sp_array.get()
- return PyHalf_FromHalf(ap.Value(self.index))
+ cdef CDoubleScalar* sp = <CDoubleScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class FloatValue(ArrayValue):
+cdef class Decimal128Scalar(Scalar):
"""
- Concrete class for float32 array elements.
+ Concrete class for decimal128 scalars.
"""
def as_py(self):
"""
- Return this value as a Python float.
+ Return this value as a Python Decimal.
"""
- cdef CFloatArray* ap = <CFloatArray*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef:
+ CDecimal128Scalar* sp = <CDecimal128Scalar*> self.wrapped.get()
+ CDecimal128Type* dtype = <CDecimal128Type*> sp.type.get()
+ if sp.is_valid:
+ return _pydecimal.Decimal(
+ frombytes(sp.value.ToString(dtype.scale()))
+ )
+ else:
+ return None
-cdef class DoubleValue(ArrayValue):
+cdef class Date32Scalar(Scalar):
"""
- Concrete class for float64 array elements.
+ Concrete class for date32 scalars.
"""
def as_py(self):
"""
- Return this value as a Python float.
+ Return this value as a Python datetime.datetime instance.
"""
- cdef CDoubleArray* ap = <CDoubleArray*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CDate32Scalar* sp = <CDate32Scalar*> self.wrapped.get()
+
+ if sp.is_valid:
+ # shift to seconds since epoch
+ return (
+ datetime.date(1970, 1, 1) + datetime.timedelta(days=sp.value)
+ )
+ else:
+ return None
-cdef class DecimalValue(ArrayValue):
+cdef class Date64Scalar(Scalar):
"""
- Concrete class for decimal128 array elements.
+ Concrete class for date64 scalars.
"""
def as_py(self):
"""
- Return this value as a Python Decimal.
+ Return this value as a Python datetime.datetime instance.
"""
- cdef:
- CDecimal128Array* ap = <CDecimal128Array*> self.sp_array.get()
- c_string s = ap.FormatValue(self.index)
- return _pydecimal.Decimal(s.decode('utf8'))
+ cdef CDate64Scalar* sp = <CDate64Scalar*> self.wrapped.get()
+ if sp.is_valid:
+ return (
+ datetime.date(1970, 1, 1) +
+ datetime.timedelta(days=sp.value / 86400000)
+ )
+ else:
+ return None
-cdef class StringValue(ArrayValue):
- """
- Concrete class for string (utf8) array elements.
- """
- def as_py(self):
- """
- Return this value as a Python unicode string.
- """
- cdef CStringArray* ap = <CStringArray*> self.sp_array.get()
- return ap.GetString(self.index).decode('utf-8')
+def _datetime_from_int(int64_t value, TimeUnit unit, tzinfo=None):
+ if unit == TimeUnit_SECOND:
+ delta = datetime.timedelta(seconds=value)
+ elif unit == TimeUnit_MILLI:
+ delta = datetime.timedelta(milliseconds=value)
+ elif unit == TimeUnit_MICRO:
+ delta = datetime.timedelta(microseconds=value)
+ else:
+ # TimeUnit_NANO: prefer pandas timestamps if available
+ if _pandas_api.have_pandas:
+ return _pandas_api.pd.Timestamp(value, tz=tzinfo, unit='ns')
+ # otherwise safely truncate to microsecond resolution datetime
+ if value % 1000 != 0:
+ raise ValueError(
+ "Nanosecond resolution temporal type {} is not safely "
+ "convertible to microseconds to convert to datetime.datetime. "
+ "Install pandas to return as Timestamp with nanosecond "
+ "support or access the .value attribute.".format(value)
+ )
+ delta = datetime.timedelta(microseconds=value // 1000)
- def as_buffer(self):
- """
- Return a view over this value as a Buffer object.
- """
- cdef:
- CStringArray* ap = <CStringArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
+ dt = datetime.datetime(1970, 1, 1) + delta
+ # adjust timezone if set to the datatype
+ if tzinfo is not None:
+ dt = tzinfo.fromutc(dt)
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
+ return dt
-cdef class LargeStringValue(ArrayValue):
+cdef class Time32Scalar(Scalar):
"""
- Concrete class for large string (utf8) array elements.
+ Concrete class for time32 scalars.
"""
def as_py(self):
"""
- Return this value as a Python unicode string.
- """
- cdef CLargeStringArray* ap = <CLargeStringArray*> self.sp_array.get()
- return ap.GetString(self.index).decode('utf-8')
-
- def as_buffer(self):
- """
- Return a view over this value as a Buffer object.
+ Return this value as a Python datetime.timedelta instance.
"""
cdef:
- CLargeStringArray* ap = <CLargeStringArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
+ CTime32Scalar* sp = <CTime32Scalar*> self.wrapped.get()
+ CTime32Type* dtype = <CTime32Type*> sp.type.get()
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
+ if sp.is_valid:
+ return _datetime_from_int(sp.value, unit=dtype.unit()).time()
+ else:
+ return None
-cdef class BinaryValue(ArrayValue):
+cdef class Time64Scalar(Scalar):
"""
- Concrete class for variable-sized binary array elements.
+ Concrete class for time64 scalars.
"""
def as_py(self):
"""
- Return this value as a Python bytes object.
- """
- cdef:
- const uint8_t* ptr
- int32_t length
- CBinaryArray* ap = <CBinaryArray*> self.sp_array.get()
-
- ptr = ap.GetValue(self.index, &length)
- return cp.PyBytes_FromStringAndSize(<const char*>(ptr), length)
-
- def as_buffer(self):
- """
- Return a view over this value as a Buffer object.
+ Return this value as a Python datetime.timedelta instance.
"""
cdef:
- CBinaryArray* ap = <CBinaryArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
+ CTime64Scalar* sp = <CTime64Scalar*> self.wrapped.get()
+ CTime64Type* dtype = <CTime64Type*> sp.type.get()
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
+ if sp.is_valid:
+ return _datetime_from_int(sp.value, unit=dtype.unit()).time()
+ else:
+ return None
-cdef class LargeBinaryValue(ArrayValue):
+cdef class TimestampScalar(Scalar):
"""
- Concrete class for large variable-sized binary array elements.
+ Concrete class for timestamp scalars.
"""
+ @property
+ def value(self):
+ cdef CTimestampScalar* sp = <CTimestampScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
+
def as_py(self):
"""
- Return this value as a Python bytes object.
+ Return this value as a Pandas Timestamp instance (if available),
+ otherwise as a Python datetime.timedelta instance.
"""
cdef:
- const uint8_t* ptr
- int64_t length
- CLargeBinaryArray* ap = <CLargeBinaryArray*> self.sp_array.get()
+ CTimestampScalar* sp = <CTimestampScalar*> self.wrapped.get()
+ CTimestampType* dtype = <CTimestampType*> sp.type.get()
- ptr = ap.GetValue(self.index, &length)
- return cp.PyBytes_FromStringAndSize(<const char*>(ptr), length)
+ if not sp.is_valid:
+ return None
- def as_buffer(self):
- """
- Return a view over this value as a Buffer object.
- """
- cdef:
- CLargeBinaryArray* ap = <CLargeBinaryArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
+ if not dtype.timezone().empty():
+ tzinfo = string_to_tzinfo(frombytes(dtype.timezone()))
+ else:
+ tzinfo = None
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
+ return _datetime_from_int(sp.value, unit=dtype.unit(), tzinfo=tzinfo)
-cdef class ListValue(ArrayValue):
+cdef class DurationScalar(Scalar):
"""
- Concrete class for list array elements.
+ Concrete class for duration scalars.
"""
- def __len__(self):
- """
- Return the number of values.
- """
- return self.length()
-
- def __getitem__(self, i):
- """
- Return the value at the given index.
- """
- return self.getitem(_normalize_index(i, self.length()))
-
- def __iter__(self):
- """
- Iterate over this element's values.
- """
- for i in range(len(self)):
- yield self.getitem(i)
- raise StopIteration
-
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CListArray*> sp_array.get()
- self.value_type = pyarrow_wrap_data_type(self.ap.value_type())
-
- cdef getitem(self, int64_t i):
- cdef int64_t j = self.ap.value_offset(self.index) + i
- return box_scalar(self.value_type, self.ap.values(), j)
-
- cdef int64_t length(self):
- return self.ap.value_length(self.index)
+ @property
+ def value(self):
+ cdef CDurationScalar* sp = <CDurationScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
def as_py(self):
"""
- Return this value as a Python list.
+ Return this value as a Pandas Timestamp instance (if available),
+ otherwise as a Python datetime.timedelta instance.
"""
cdef:
- int64_t j
- list result = []
+ CDurationScalar* sp = <CDurationScalar*> self.wrapped.get()
+ CDurationType* dtype = <CDurationType*> sp.type.get()
+ TimeUnit unit = dtype.unit()
- for j in range(len(self)):
- result.append(self.getitem(j).as_py())
+ if not sp.is_valid:
+ return None
- return result
+ if unit == TimeUnit_SECOND:
+ return datetime.timedelta(seconds=sp.value)
+ elif unit == TimeUnit_MILLI:
+ return datetime.timedelta(milliseconds=sp.value)
+ elif unit == TimeUnit_MICRO:
+ return datetime.timedelta(microseconds=sp.value)
+ else:
+ # TimeUnit_NANO: prefer pandas timestamps if available
+ if _pandas_api.have_pandas:
+ return _pandas_api.pd.Timedelta(sp.value, unit='ns')
+ # otherwise safely truncate to microsecond resolution timedelta
+ if sp.value % 1000 != 0:
+ raise ValueError(
+ "Nanosecond duration {} is not safely convertible to "
+ "microseconds to convert to datetime.timedelta. Install "
+ "pandas to return as Timedelta with nanosecond support or "
+ "access the .value attribute.".format(sp.value)
+ )
+ return datetime.timedelta(microseconds=sp.value // 1000)
-cdef class LargeListValue(ArrayValue):
+cdef class BinaryScalar(Scalar):
"""
- Concrete class for large list array elements.
+ Concrete class for binary-like scalars.
"""
- def __len__(self):
- """
- Return the number of values.
- """
- return self.length()
-
- def __getitem__(self, i):
+ def as_buffer(self):
"""
- Return the value at the given index.
+ Return a view over this value as a Buffer object.
"""
- return self.getitem(_normalize_index(i, self.length()))
+ cdef CBinaryScalar* sp = <CBinaryScalar*> self.wrapped.get()
+ return pyarrow_wrap_buffer(sp.value) if sp.is_valid else None
- def __iter__(self):
+ def as_py(self):
"""
- Iterate over this element's values.
+ Return this value as a Python bytes.
"""
- for i in range(len(self)):
- yield self.getitem(i)
- raise StopIteration
+ buffer = self.as_buffer()
+ return None if buffer is None else buffer.to_pybytes()
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CLargeListArray*> sp_array.get()
- self.value_type = pyarrow_wrap_data_type(self.ap.value_type())
- cdef getitem(self, int64_t i):
- cdef int64_t j = self.ap.value_offset(self.index) + i
- return box_scalar(self.value_type, self.ap.values(), j)
+cdef class LargeBinaryScalar(BinaryScalar):
+ pass
- cdef int64_t length(self):
- return self.ap.value_length(self.index)
- def as_py(self):
- """
- Return this value as a Python list.
- """
- cdef:
- int64_t j
- list result = []
-
- for j in range(len(self)):
- result.append(self.getitem(j).as_py())
+cdef class FixedSizeBinaryScalar(BinaryScalar):
+ pass
- return result
-
-cdef class MapValue(ArrayValue):
+cdef class StringScalar(BinaryScalar):
"""
- Concrete class for map array elements.
+ Concrete class for string-like (utf8) scalars.
"""
- def __len__(self):
- """
- Return the number of values.
- """
- return self.length()
-
- def __getitem__(self, i):
- """
- Return the value at the given index.
- """
- return self.getitem(_normalize_index(i, self.length()))
-
- def __iter__(self):
- """
- Iterate over this element's values.
- """
- for i in range(len(self)):
- yield self.getitem(i)
- raise StopIteration
-
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CMapArray*> sp_array.get()
- self.key_type = pyarrow_wrap_data_type(self.ap.map_type().key_type())
- self.item_type = pyarrow_wrap_data_type(self.ap.map_type().item_type())
-
- cdef getitem(self, int64_t i):
- cdef int64_t j = self.ap.value_offset(self.index) + i
- return (box_scalar(self.key_type, self.ap.keys(), j),
- box_scalar(self.item_type, self.ap.items(), j))
-
- cdef int64_t length(self):
- return self.ap.value_length(self.index)
-
def as_py(self):
"""
- Return this value as a Python list of tuples, each containing a
- key and item.
+ Return this value as a Python string.
"""
- cdef:
- int64_t j
- list result = []
+ buffer = self.as_buffer()
+ return None if buffer is None else str(buffer, 'utf8')
- for j in range(len(self)):
- key, item = self.getitem(j)
- result.append((key.as_py(), item.as_py()))
- return result
+cdef class LargeStringScalar(StringScalar):
+ pass
-cdef class FixedSizeListValue(ArrayValue):
+cdef class ListScalar(Scalar):
"""
- Concrete class for fixed size list array elements.
+ Concrete class for list-like scalars.
"""
+ @property
+ def values(self):
+ cdef CListScalar* sp = <CListScalar*> self.wrapped.get()
+ if sp.is_valid:
+ return pyarrow_wrap_array(sp.value)
+ else:
+ return None
+
def __len__(self):
"""
Return the number of values.
"""
- return self.length()
+ return len(self.values)
def __getitem__(self, i):
"""
Return the value at the given index.
"""
- return self.getitem(_normalize_index(i, self.length()))
+ return self.values[_normalize_index(i, len(self))]
def __iter__(self):
"""
Iterate over this element's values.
"""
- for i in range(len(self)):
- yield self.getitem(i)
- raise StopIteration
-
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CFixedSizeListArray*> sp_array.get()
- self.value_type = pyarrow_wrap_data_type(self.ap.value_type())
-
- cdef getitem(self, int64_t i):
- cdef int64_t j = self.ap.value_offset(self.index) + i
- return box_scalar(self.value_type, self.ap.values(), j)
-
- cdef int64_t length(self):
- return self.ap.value_length(self.index)
+ return iter(self.values)
def as_py(self):
"""
Return this value as a Python list.
"""
- cdef:
- int64_t j
- list result = []
-
- for j in range(len(self)):
- result.append(self.getitem(j).as_py())
-
- return result
-
-
-cdef class UnionValue(ArrayValue):
- """
- Concrete class for union array elements.
- """
+ arr = self.values
+ return None if arr is None else arr.to_pylist()
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CUnionArray*> sp_array.get()
-
- cdef getitem(self, int64_t i):
- cdef int child_id = self.ap.child_id(i)
- cdef shared_ptr[CArray] child = self.ap.field(child_id)
- cdef CDenseUnionArray* dense
- if self.ap.mode() == _UnionMode_SPARSE:
- return box_scalar(self.type[child_id].type, child, i)
- else:
- dense = <CDenseUnionArray*> self.ap
- return box_scalar(self.type[child_id].type, child,
- dense.value_offset(i))
- def as_py(self):
- """
- Return this value as a Python object.
+cdef class FixedSizeListScalar(ListScalar):
+ pass
- The exact type depends on the underlying union member.
- """
- return self.getitem(self.index).as_py()
+cdef class LargeListScalar(ListScalar):
+ pass
-cdef class FixedSizeBinaryValue(ArrayValue):
- """
- Concrete class for fixed-size binary array elements.
- """
- def as_py(self):
- """
- Return this value as a Python bytes object.
- """
- cdef:
- CFixedSizeBinaryArray* ap
- CFixedSizeBinaryType* ap_type
- int32_t length
- const char* data
- ap = <CFixedSizeBinaryArray*> self.sp_array.get()
- ap_type = <CFixedSizeBinaryType*> ap.type().get()
- length = ap_type.byte_width()
- data = <const char*> ap.GetValue(self.index)
- return cp.PyBytes_FromStringAndSize(data, length)
-
-
-cdef class StructValue(ArrayValue):
+cdef class StructScalar(Scalar, collections.abc.Mapping):
"""
- Concrete class for struct array elements.
+ Concrete class for struct scalars.
"""
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CStructArray*> sp_array.get()
-
- def __getitem__(self, key):
- """
- Return the child value for the given field name.
- """
- cdef:
- CStructType* type
- int index
-
- type = <CStructType*> self.type.type
- index = type.GetFieldIndex(tobytes(key))
-
- if index < 0:
- raise KeyError(key)
-
- return pyarrow_wrap_array(self.ap.field(index))[self.index]
+ def __len__(self):
+ cdef CStructScalar* sp = <CStructScalar*> self.wrapped.get()
+ return sp.value.size()
- def as_py(self):
- """
- Return this value as a Python dict.
- """
+ def __iter__(self):
cdef:
- vector[shared_ptr[CField]] child_fields = self.type.type.fields()
-
- wrapped_arrays = [pyarrow_wrap_array(self.ap.field(i))
- for i in range(self.ap.num_fields())]
- child_names = [child.get().name() for child in child_fields]
- # Return the struct as a dict
- return {
- frombytes(name): child_array[self.index].as_py()
- for name, child_array in zip(child_names, wrapped_arrays)
- }
-
-
-cdef class DictionaryValue(ArrayValue):
- """
- Concrete class for dictionary-encoded array elements.
- """
-
- def as_py(self):
- """
- Return this value as a Python object.
-
- The exact type depends on the dictionary value type.
- """
- return self.dictionary_value.as_py()
-
- @property
- def index_value(self):
- """
- Return this value's underlying index as a ArrayValue of the right
- signed integer type.
- """
- cdef CDictionaryArray* darr = <CDictionaryArray*>(self.sp_array.get())
- indices = pyarrow_wrap_array(darr.indices())
- return indices[self.index]
-
- @property
- def dictionary_value(self):
- """
- Return this value's underlying dictionary value as a ArrayValue.
- """
- cdef CDictionaryArray* darr = <CDictionaryArray*>(self.sp_array.get())
- dictionary = pyarrow_wrap_array(darr.dictionary())
- return dictionary[self.index_value.as_py()]
-
-
-cdef dict _array_value_classes = {
- _Type_BOOL: BooleanValue,
- _Type_UINT8: UInt8Value,
- _Type_UINT16: UInt16Value,
- _Type_UINT32: UInt32Value,
- _Type_UINT64: UInt64Value,
- _Type_INT8: Int8Value,
- _Type_INT16: Int16Value,
- _Type_INT32: Int32Value,
- _Type_INT64: Int64Value,
- _Type_DATE32: Date32Value,
- _Type_DATE64: Date64Value,
- _Type_TIME32: Time32Value,
- _Type_TIME64: Time64Value,
- _Type_TIMESTAMP: TimestampValue,
- _Type_DURATION: DurationValue,
- _Type_HALF_FLOAT: HalfFloatValue,
- _Type_FLOAT: FloatValue,
- _Type_DOUBLE: DoubleValue,
- _Type_LIST: ListValue,
- _Type_LARGE_LIST: LargeListValue,
- _Type_MAP: MapValue,
- _Type_FIXED_SIZE_LIST: FixedSizeListValue,
- _Type_SPARSE_UNION: UnionValue,
- _Type_DENSE_UNION: UnionValue,
- _Type_BINARY: BinaryValue,
- _Type_STRING: StringValue,
- _Type_LARGE_BINARY: LargeBinaryValue,
- _Type_LARGE_STRING: LargeStringValue,
- _Type_FIXED_SIZE_BINARY: FixedSizeBinaryValue,
- _Type_DECIMAL: DecimalValue,
- _Type_STRUCT: StructValue,
- _Type_DICTIONARY: DictionaryValue,
-}
-
-cdef class ScalarValue(Scalar):
- """
- The base class for scalars.
- """
-
- def __init__(self):
- raise TypeError("Do not call {}'s constructor directly."
- .format(self.__class__.__name__))
-
- cdef void init(self, const shared_ptr[CScalar]& sp_scalar):
- self.sp_scalar = sp_scalar
-
- def __repr__(self):
- if hasattr(self, 'as_py'):
- return repr(self.as_py())
- else:
- return super(Scalar, self).__repr__()
+ CStructScalar* sp = <CStructScalar*> self.wrapped.get()
+ CStructType* dtype = <CStructType*> sp.type.get()
+ vector[shared_ptr[CField]] fields = dtype.fields()
- def __str__(self):
- if hasattr(self, 'as_py'):
- return str(self.as_py())
- else:
- return super(Scalar, self).__str__()
+ if sp.is_valid:
+ for i in range(dtype.num_fields()):
+ yield frombytes(fields[i].get().name())
- def __eq__(self, other):
- if hasattr(self, 'as_py'):
- if isinstance(other, ScalarValue):
- other = other.as_py()
- return self.as_py() == other
+ def __contains__(self, key):
+ try:
+ self[key]
+ except (KeyError, IndexError):
+ return False
else:
- raise NotImplemented(
- "Cannot compare scalars that don't support as_py()")
-
- def __hash__(self):
- return hash(self.as_py())
-
-
-cdef class NullScalar(ScalarValue):
- """
- Concrete class for null scalars.
- """
-
- def as_py(self):
- """
- Return this value as a Python None.
- """
- return None
-
-
-cdef class BooleanScalar(ScalarValue):
- """
- Concrete class for boolean scalars.
- """
+ return True
- def as_py(self):
- """
- Return this value as a Python bool.
+ def __getitem__(self, key):
"""
- cdef CBooleanScalar* sp = <CBooleanScalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
-
+ Return the child value for the given field.
-cdef class UInt8Scalar(ScalarValue):
- """
- Concrete class for uint8 scalars.
- """
+ Parameters
+ ----------
+ index : Union[int, str]
+ Index / position or name of the field.
- def as_py(self):
- """
- Return this value as a Python int.
+ Returns
+ -------
+ result : Scalar
"""
- cdef CUInt8Scalar* sp = <CUInt8Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
+ cdef:
+ CFieldRef ref
+ CStructScalar* sp = <CStructScalar*> self.wrapped.get()
+ if isinstance(key, (bytes, str)):
+ ref = CFieldRef(<c_string> tobytes(key))
+ elif isinstance(key, int):
+ ref = CFieldRef(<int> key)
+ else:
+ raise TypeError('Expected integer or string index')
-cdef class Int8Scalar(ScalarValue):
- """
- Concrete class for int8 scalars.
- """
+ try:
+ return Scalar.wrap(GetResultValue(sp.field(ref)))
+ except ArrowInvalid:
+ if isinstance(key, int):
+ raise IndexError(key)
+ else:
+ raise KeyError(key)
def as_py(self):
"""
- Return this value as a Python int.
+ Return this value as a Python dict.
"""
- cdef CInt8Scalar* sp = <CInt8Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
+ if self.is_valid:
+ return {k: v.as_py() for k, v in self.items()}
+ else:
+ return None
-cdef class UInt16Scalar(ScalarValue):
+cdef class MapScalar(ListScalar):
"""
- Concrete class for uint16 scalars.
+ Concrete class for map scalars.
"""
- def as_py(self):
+ def __getitem__(self, i):
"""
- Return this value as a Python int.
+ Return the value at the given index.
"""
- cdef CUInt16Scalar* sp = <CUInt16Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
-
-
-cdef class Int16Scalar(ScalarValue):
- """
- Concrete class for int16 scalars.
- """
+ arr = self.values
+ if arr is None:
+ raise IndexError(i)
+ dct = arr[_normalize_index(i, len(arr))]
+ return (dct['key'], dct['value'])
- def as_py(self):
+ def __iter__(self):
"""
- Return this value as a Python int.
+ Iterate over this element's values.
"""
- cdef CInt16Scalar* sp = <CInt16Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
-
-
-cdef class UInt32Scalar(ScalarValue):
- """
- Concrete class for uint32 scalars.
- """
+ arr = self.values
+ if arr is None:
+ return iter(zip(arr.field('key'), arr.field('value')))
+ else:
+ raise StopIteration
def as_py(self):
"""
- Return this value as a Python int.
+ Return this value as a Python list.
"""
- cdef CUInt32Scalar* sp = <CUInt32Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
+ arr = self.values
+ if arr is not None:
+ return list(zip(arr.field('key'), arr.field('value')))
+ else:
+ return None
-cdef class Int32Scalar(ScalarValue):
+cdef class DictionaryScalar(Scalar):
"""
- Concrete class for int32 scalars.
+ Concrete class for dictionary-encoded scalars.
"""
- def as_py(self):
+ @property
+ def index(self):
"""
- Return this value as a Python int.
+ Return this value's underlying index as a scalar.
"""
- cdef CInt32Scalar* sp = <CInt32Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
+ cdef CDictionaryScalar* sp = <CDictionaryScalar*> self.wrapped.get()
+ return Scalar.wrap(sp.value.index)
-
-cdef class UInt64Scalar(ScalarValue):
- """
- Concrete class for uint64 scalars.
- """
-
- def as_py(self):
+ @property
+ def value(self):
"""
- Return this value as a Python int.
+ Return the encoded value as a scalar.
"""
- cdef CUInt64Scalar* sp = <CUInt64Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
+ # TODO(kszucs): optimize it to spare uneccessary python object boxing
+ return self.dictionary[self.index.as_py()] if self.is_valid else None
-
-cdef class Int64Scalar(ScalarValue):
- """
- Concrete class for int64 scalars.
- """
+ @property
+ def dictionary(self):
+ cdef CDictionaryScalar* sp = <CDictionaryScalar*> self.wrapped.get()
+ return pyarrow_wrap_array(sp.value.dictionary)
def as_py(self):
"""
- Return this value as a Python int.
+ Return this encoded value as a Python object.
"""
- cdef CInt64Scalar* sp = <CInt64Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
-
+ value = self.value
+ return None if value is None else value.as_py()
-cdef class FloatScalar(ScalarValue):
- """
- Concrete class for float scalars.
- """
+ @property
+ def index_value(self):
+ warnings.warn("`dictionary_value` property is deprecated as of 1.0.0"
+ "please use the `value` property instead",
+ FutureWarning)
Review comment:
```suggestion
warnings.warn("`index_value` property is deprecated as of 1.0.0"
"please use the `index` property instead",
FutureWarning)
```
##########
File path: python/pyarrow/scalar.pxi
##########
@@ -16,1198 +16,748 @@
# under the License.
-_NULL = NA = None
+import collections
cdef class Scalar:
"""
- The base class for all array elements.
- """
-
-
-cdef class NullType(Scalar):
- """
- Singleton for null array elements.
+ The base class for scalars.
"""
- # TODO rename this NullValue?
-
- def __cinit__(self):
- global NA
- if NA is not None:
- raise Exception('Cannot create multiple NAType instances')
-
- self.type = null()
-
- def __repr__(self):
- return 'NULL'
- def as_py(self):
- """
- Return None
- """
- return None
+ def __init__(self):
+ raise TypeError("Do not call {}'s constructor directly, use "
+ "pa.scalar() instead.".format(self.__class__.__name__))
- def __eq__(self, other):
- return NA
+ cdef void init(self, const shared_ptr[CScalar]& wrapped):
+ self.wrapped = wrapped
+ @staticmethod
+ cdef wrap(const shared_ptr[CScalar]& wrapped):
+ cdef:
+ Scalar self
+ Type type_id = wrapped.get().type.get().id()
-_NULL = NA = NullType()
+ if type_id == _Type_NA:
+ return _NULL
+ typ = _scalar_classes[type_id]
+ self = typ.__new__(typ)
+ self.init(wrapped)
-cdef class ArrayValue(Scalar):
- """
- The base class for non-null array elements.
- """
+ return self
- def __init__(self):
- raise TypeError("Do not call {}'s constructor directly, use array "
- "subscription instead."
- .format(self.__class__.__name__))
+ cdef inline shared_ptr[CScalar] unwrap(self) nogil:
+ return self.wrapped
- cdef void init(self, DataType type, const shared_ptr[CArray]& sp_array,
- int64_t index):
- self.type = type
- self.index = index
- self._set_array(sp_array)
+ @property
+ def type(self):
+ """
+ Data type of the Scalar object.
+ """
+ return pyarrow_wrap_data_type(self.wrapped.get().type)
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
+ @property
+ def is_valid(self):
+ """
+ Holds a valid (non-null) value.
+ """
+ return self.wrapped.get().is_valid
def __repr__(self):
- if hasattr(self, 'as_py'):
- return repr(self.as_py())
- else:
- return super(Scalar, self).__repr__()
+ return '<pyarrow.{}: {!r}>'.format(
+ self.__class__.__name__, self.as_py()
+ )
def __str__(self):
- if hasattr(self, 'as_py'):
- return str(self.as_py())
- else:
- return super(Scalar, self).__str__()
+ return str(self.as_py())
+
+ def equals(self, Scalar other):
+ return self.wrapped.get().Equals(other.unwrap().get()[0])
def __eq__(self, other):
- if hasattr(self, 'as_py'):
- if isinstance(other, ArrayValue):
- other = other.as_py()
- return self.as_py() == other
- else:
- raise NotImplementedError(
- "Cannot compare Arrow values that don't support as_py()")
+ try:
+ if not isinstance(other, Scalar):
+ other = scalar(other, type=self.type)
+ return self.equals(other)
+ except (TypeError, ValueError, ArrowInvalid):
+ return NotImplemented
def __hash__(self):
- return hash(self.as_py())
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+ def as_py(self):
+ raise NotImplementedError()
-cdef class BooleanValue(ArrayValue):
- """
- Concrete class for boolean array elements.
- """
- def as_py(self):
- """
- Return this value as a Python bool.
- """
- cdef CBooleanArray* ap = <CBooleanArray*> self.sp_array.get()
- return ap.Value(self.index)
+_NULL = NA = None
-cdef class Int8Value(ArrayValue):
+cdef class NullScalar(Scalar):
"""
- Concrete class for int8 array elements.
+ Concrete class for null scalars.
"""
- def as_py(self):
- """
- Return this value as a Python int.
- """
- cdef CInt8Array* ap = <CInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ def __cinit__(self):
+ global NA
+ if NA is not None:
+ raise RuntimeError('Cannot create multiple NullScalar instances')
+ self.init(shared_ptr[CScalar](new CNullScalar()))
+ def __init__(self):
+ pass
-cdef class UInt8Value(ArrayValue):
- """
- Concrete class for uint8 array elements.
- """
+ def __eq__(self, other):
+ return NA
+
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
def as_py(self):
"""
- Return this value as a Python int.
+ Return this value as a Python None.
"""
- cdef CUInt8Array* ap = <CUInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
-
+ return None
-cdef class Int16Value(ArrayValue):
- """
- Concrete class for int16 array elements.
- """
- def as_py(self):
- """
- Return this value as a Python int.
- """
- cdef CInt16Array* ap = <CInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+_NULL = NA = NullScalar()
-cdef class UInt16Value(ArrayValue):
+cdef class BooleanScalar(Scalar):
"""
- Concrete class for uint16 array elements.
+ Concrete class for boolean scalars.
"""
def as_py(self):
"""
- Return this value as a Python int.
+ Return this value as a Python bool.
"""
- cdef CUInt16Array* ap = <CUInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CBooleanScalar* sp = <CBooleanScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int32Value(ArrayValue):
+cdef class UInt8Scalar(Scalar):
"""
- Concrete class for int32 array elements.
+ Concrete class for uint8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt32Array* ap = <CInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt8Scalar* sp = <CUInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt32Value(ArrayValue):
+cdef class Int8Scalar(Scalar):
"""
- Concrete class for uint32 array elements.
+ Concrete class for int8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt32Array* ap = <CUInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt8Scalar* sp = <CInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int64Value(ArrayValue):
+cdef class UInt16Scalar(Scalar):
"""
- Concrete class for int64 array elements.
+ Concrete class for uint16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt64Array* ap = <CInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt16Scalar* sp = <CUInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt64Value(ArrayValue):
+cdef class Int16Scalar(Scalar):
"""
- Concrete class for uint64 array elements.
+ Concrete class for int16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt64Array* ap = <CUInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt16Scalar* sp = <CInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date32Value(ArrayValue):
+cdef class UInt32Scalar(Scalar):
"""
- Concrete class for date32 array elements.
+ Concrete class for uint32 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python int.
"""
- cdef CDate32Array* ap = <CDate32Array*> self.sp_array.get()
-
- # Shift to seconds since epoch
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(days=ap.Value(self.index)))
+ cdef CUInt32Scalar* sp = <CUInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date64Value(ArrayValue):
+cdef class Int32Scalar(Scalar):
"""
- Concrete class for date64 array elements.
+ Concrete class for int32 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python int.
"""
- cdef CDate64Array* ap = <CDate64Array*> self.sp_array.get()
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(
- days=ap.Value(self.index) / 86400000))
+ cdef CInt32Scalar* sp = <CInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Time32Value(ArrayValue):
+cdef class UInt64Scalar(Scalar):
"""
- Concrete class for time32 array elements.
+ Concrete class for uint64 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.timedelta instance.
+ Return this value as a Python int.
"""
- cdef:
- CTime32Array* ap = <CTime32Array*> self.sp_array.get()
- CTime32Type* dtype = <CTime32Type*> ap.type().get()
-
- if dtype.unit() == TimeUnit_SECOND:
- delta = datetime.timedelta(seconds=ap.Value(self.index))
- return (datetime.datetime(1970, 1, 1) + delta).time()
- else:
- return _box_time_milli(ap.Value(self.index))
+ cdef CUInt64Scalar* sp = <CUInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Time64Value(ArrayValue):
+cdef class Int64Scalar(Scalar):
"""
- Concrete class for time64 array elements.
+ Concrete class for int64 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.timedelta instance.
+ Return this value as a Python int.
"""
- cdef:
- CTime64Array* ap = <CTime64Array*> self.sp_array.get()
- CTime64Type* dtype = <CTime64Type*> ap.type().get()
-
- cdef int64_t val = ap.Value(self.index)
- if dtype.unit() == TimeUnit_MICRO:
- return _box_time_micro(val)
- else:
- return (datetime.datetime(1970, 1, 1) +
- datetime.timedelta(microseconds=val / 1000)).time()
-
-
-cpdef _box_time_milli(int64_t val):
- delta = datetime.timedelta(milliseconds=val)
- return (datetime.datetime(1970, 1, 1) + delta).time()
-
-
-cpdef _box_time_micro(int64_t val):
- return (datetime.datetime(1970, 1, 1) +
- datetime.timedelta(microseconds=val)).time()
-
-
-cdef dict _DATETIME_CONVERSION_FUNCTIONS = {}
-cdef c_bool _datetime_conversion_initialized = False
-
-
-cdef _add_micros_maybe_localize(dt, micros, tzinfo):
- import pytz
- dt = dt.replace(microsecond=micros)
- if tzinfo is not None:
- if not isinstance(tzinfo, datetime.tzinfo):
- tzinfo = string_to_tzinfo(tzinfo)
- dt = tzinfo.fromutc(dt)
- return dt
-
-
-cdef _datetime_from_seconds(int64_t v):
- return datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=v)
-
-
-def _nanoseconds_to_datetime_safe(v, tzinfo):
- if v % 1000 != 0:
- raise ValueError("Nanosecond timestamp {} is not safely convertible "
- " to microseconds to convert to datetime.datetime."
- " Install pandas to return as Timestamp with "
- " nanosecond support or access the .value attribute.")
- v = v // 1000
- micros = v % 1_000_000
-
- dt = _datetime_from_seconds(v // 1_000_000)
- return _add_micros_maybe_localize(dt, micros, tzinfo)
-
-
-def _microseconds_to_datetime(v, tzinfo):
- micros = v % 1_000_000
- dt = _datetime_from_seconds(v // 1_000_000)
- return _add_micros_maybe_localize(dt, micros, tzinfo)
-
-
-def _millis_to_datetime(v, tzinfo):
- millis = v % 1_000
- dt = _datetime_from_seconds(v // 1000)
- return _add_micros_maybe_localize(dt, millis * 1000, tzinfo)
-
-
-def _seconds_to_datetime(v, tzinfo):
- dt = _datetime_from_seconds(v)
- return _add_micros_maybe_localize(dt, 0, tzinfo)
-
-
-def _datetime_conversion_functions():
- global _datetime_conversion_initialized
- if _datetime_conversion_initialized:
- return _DATETIME_CONVERSION_FUNCTIONS
-
- _DATETIME_CONVERSION_FUNCTIONS.update({
- TimeUnit_SECOND: _seconds_to_datetime,
- TimeUnit_MILLI: _millis_to_datetime,
- TimeUnit_MICRO: _microseconds_to_datetime,
- TimeUnit_NANO: _nanoseconds_to_datetime_safe
- })
-
- try:
- import pandas as pd
- _DATETIME_CONVERSION_FUNCTIONS[TimeUnit_NANO] = (
- lambda x, tzinfo: pd.Timestamp(
- x, tz=tzinfo, unit='ns',
- )
- )
- except ImportError:
- pass
-
- _datetime_conversion_initialized = True
- return _DATETIME_CONVERSION_FUNCTIONS
+ cdef CInt64Scalar* sp = <CInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class TimestampValue(ArrayValue):
+cdef class HalfFloatScalar(Scalar):
"""
- Concrete class for timestamp array elements.
+ Concrete class for float scalars.
"""
- @property
- def value(self):
- cdef CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
- cdef CTimestampType* dtype = <CTimestampType*> ap.type().get()
- return ap.Value(self.index)
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+
+ def __eq__(self, other):
+ if isinstance(other, Scalar):
+ other = other.as_py()
+ return self.as_py() == other
def as_py(self):
"""
- Return this value as a Pandas Timestamp instance (if available),
- otherwise as a Python datetime.timedelta instance.
+ Return this value as a Python float.
"""
- cdef CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
- cdef CTimestampType* dtype = <CTimestampType*> ap.type().get()
-
- value = self.value
-
- if not dtype.timezone().empty():
- tzinfo = string_to_tzinfo(frombytes(dtype.timezone()))
- else:
- tzinfo = None
-
- try:
- converter = _datetime_conversion_functions()[dtype.unit()]
- except KeyError:
- raise ValueError(
- 'Cannot convert nanosecond timestamps without pandas'
- )
- return converter(value, tzinfo=tzinfo)
-
-
-cdef dict _TIMEDELTA_CONVERSION_FUNCTIONS = {}
-
-
-def _nanoseconds_to_timedelta_safe(v):
- if v % 1000 != 0:
- raise ValueError(
- "Nanosecond duration {} is not safely convertible to microseconds "
- "to convert to datetime.timedelta. Install pandas to return as "
- "Timedelta with nanosecond support or access the .value "
- "attribute.".format(v))
- micros = v // 1000
-
- return datetime.timedelta(microseconds=micros)
+ cdef CHalfFloatScalar* sp = <CHalfFloatScalar*> self.wrapped.get()
+ return PyHalf_FromHalf(sp.value) if sp.is_valid else None
-def _timedelta_conversion_functions():
- if _TIMEDELTA_CONVERSION_FUNCTIONS:
- return _TIMEDELTA_CONVERSION_FUNCTIONS
-
- _TIMEDELTA_CONVERSION_FUNCTIONS.update({
- TimeUnit_SECOND: lambda v: datetime.timedelta(seconds=v),
- TimeUnit_MILLI: lambda v: datetime.timedelta(milliseconds=v),
- TimeUnit_MICRO: lambda v: datetime.timedelta(microseconds=v),
- TimeUnit_NANO: _nanoseconds_to_timedelta_safe
- })
-
- try:
- import pandas as pd
- _TIMEDELTA_CONVERSION_FUNCTIONS[TimeUnit_NANO] = (
- lambda v: pd.Timedelta(v, unit='ns')
- )
- except ImportError:
- pass
-
- return _TIMEDELTA_CONVERSION_FUNCTIONS
-
-
-cdef class DurationValue(ArrayValue):
+cdef class FloatScalar(Scalar):
"""
- Concrete class for duration array elements.
+ Concrete class for float scalars.
"""
- @property
- def value(self):
- cdef CDurationArray* ap = <CDurationArray*> self.sp_array.get()
- return ap.Value(self.index)
-
def as_py(self):
"""
- Return this value as a Pandas Timestamp instance (if available),
- otherwise as a Python datetime.timedelta instance.
+ Return this value as a Python float.
"""
- cdef CDurationArray* ap = <CDurationArray*> self.sp_array.get()
- cdef CDurationType* dtype = <CDurationType*> ap.type().get()
-
- cdef int64_t value = ap.Value(self.index)
- converter = _timedelta_conversion_functions()[dtype.unit()]
- return converter(value)
+ cdef CFloatScalar* sp = <CFloatScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class HalfFloatValue(ArrayValue):
+cdef class DoubleScalar(Scalar):
"""
- Concrete class for float16 array elements.
+ Concrete class for double scalars.
"""
def as_py(self):
"""
Return this value as a Python float.
"""
- cdef CHalfFloatArray* ap = <CHalfFloatArray*> self.sp_array.get()
- return PyHalf_FromHalf(ap.Value(self.index))
+ cdef CDoubleScalar* sp = <CDoubleScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class FloatValue(ArrayValue):
+cdef class Decimal128Scalar(Scalar):
"""
- Concrete class for float32 array elements.
+ Concrete class for decimal128 scalars.
"""
def as_py(self):
"""
- Return this value as a Python float.
+ Return this value as a Python Decimal.
"""
- cdef CFloatArray* ap = <CFloatArray*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef:
+ CDecimal128Scalar* sp = <CDecimal128Scalar*> self.wrapped.get()
+ CDecimal128Type* dtype = <CDecimal128Type*> sp.type.get()
+ if sp.is_valid:
+ return _pydecimal.Decimal(
+ frombytes(sp.value.ToString(dtype.scale()))
+ )
+ else:
+ return None
-cdef class DoubleValue(ArrayValue):
+cdef class Date32Scalar(Scalar):
"""
- Concrete class for float64 array elements.
+ Concrete class for date32 scalars.
"""
def as_py(self):
"""
- Return this value as a Python float.
+ Return this value as a Python datetime.datetime instance.
"""
- cdef CDoubleArray* ap = <CDoubleArray*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CDate32Scalar* sp = <CDate32Scalar*> self.wrapped.get()
+
+ if sp.is_valid:
+ # shift to seconds since epoch
+ return (
+ datetime.date(1970, 1, 1) + datetime.timedelta(days=sp.value)
+ )
+ else:
+ return None
-cdef class DecimalValue(ArrayValue):
+cdef class Date64Scalar(Scalar):
"""
- Concrete class for decimal128 array elements.
+ Concrete class for date64 scalars.
"""
def as_py(self):
"""
- Return this value as a Python Decimal.
+ Return this value as a Python datetime.datetime instance.
"""
- cdef:
- CDecimal128Array* ap = <CDecimal128Array*> self.sp_array.get()
- c_string s = ap.FormatValue(self.index)
- return _pydecimal.Decimal(s.decode('utf8'))
+ cdef CDate64Scalar* sp = <CDate64Scalar*> self.wrapped.get()
+ if sp.is_valid:
+ return (
+ datetime.date(1970, 1, 1) +
+ datetime.timedelta(days=sp.value / 86400000)
+ )
+ else:
+ return None
-cdef class StringValue(ArrayValue):
- """
- Concrete class for string (utf8) array elements.
- """
- def as_py(self):
- """
- Return this value as a Python unicode string.
- """
- cdef CStringArray* ap = <CStringArray*> self.sp_array.get()
- return ap.GetString(self.index).decode('utf-8')
+def _datetime_from_int(int64_t value, TimeUnit unit, tzinfo=None):
+ if unit == TimeUnit_SECOND:
+ delta = datetime.timedelta(seconds=value)
+ elif unit == TimeUnit_MILLI:
+ delta = datetime.timedelta(milliseconds=value)
+ elif unit == TimeUnit_MICRO:
+ delta = datetime.timedelta(microseconds=value)
+ else:
+ # TimeUnit_NANO: prefer pandas timestamps if available
+ if _pandas_api.have_pandas:
+ return _pandas_api.pd.Timestamp(value, tz=tzinfo, unit='ns')
+ # otherwise safely truncate to microsecond resolution datetime
+ if value % 1000 != 0:
+ raise ValueError(
+ "Nanosecond resolution temporal type {} is not safely "
+ "convertible to microseconds to convert to datetime.datetime. "
+ "Install pandas to return as Timestamp with nanosecond "
+ "support or access the .value attribute.".format(value)
+ )
+ delta = datetime.timedelta(microseconds=value // 1000)
- def as_buffer(self):
- """
- Return a view over this value as a Buffer object.
- """
- cdef:
- CStringArray* ap = <CStringArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
+ dt = datetime.datetime(1970, 1, 1) + delta
+ # adjust timezone if set to the datatype
+ if tzinfo is not None:
+ dt = tzinfo.fromutc(dt)
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
+ return dt
-cdef class LargeStringValue(ArrayValue):
+cdef class Time32Scalar(Scalar):
"""
- Concrete class for large string (utf8) array elements.
+ Concrete class for time32 scalars.
"""
def as_py(self):
"""
- Return this value as a Python unicode string.
- """
- cdef CLargeStringArray* ap = <CLargeStringArray*> self.sp_array.get()
- return ap.GetString(self.index).decode('utf-8')
-
- def as_buffer(self):
- """
- Return a view over this value as a Buffer object.
+ Return this value as a Python datetime.timedelta instance.
"""
cdef:
- CLargeStringArray* ap = <CLargeStringArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
+ CTime32Scalar* sp = <CTime32Scalar*> self.wrapped.get()
+ CTime32Type* dtype = <CTime32Type*> sp.type.get()
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
+ if sp.is_valid:
+ return _datetime_from_int(sp.value, unit=dtype.unit()).time()
+ else:
+ return None
-cdef class BinaryValue(ArrayValue):
+cdef class Time64Scalar(Scalar):
"""
- Concrete class for variable-sized binary array elements.
+ Concrete class for time64 scalars.
"""
def as_py(self):
"""
- Return this value as a Python bytes object.
- """
- cdef:
- const uint8_t* ptr
- int32_t length
- CBinaryArray* ap = <CBinaryArray*> self.sp_array.get()
-
- ptr = ap.GetValue(self.index, &length)
- return cp.PyBytes_FromStringAndSize(<const char*>(ptr), length)
-
- def as_buffer(self):
- """
- Return a view over this value as a Buffer object.
+ Return this value as a Python datetime.timedelta instance.
"""
cdef:
- CBinaryArray* ap = <CBinaryArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
+ CTime64Scalar* sp = <CTime64Scalar*> self.wrapped.get()
+ CTime64Type* dtype = <CTime64Type*> sp.type.get()
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
+ if sp.is_valid:
+ return _datetime_from_int(sp.value, unit=dtype.unit()).time()
+ else:
+ return None
-cdef class LargeBinaryValue(ArrayValue):
+cdef class TimestampScalar(Scalar):
"""
- Concrete class for large variable-sized binary array elements.
+ Concrete class for timestamp scalars.
"""
+ @property
+ def value(self):
+ cdef CTimestampScalar* sp = <CTimestampScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
+
def as_py(self):
"""
- Return this value as a Python bytes object.
+ Return this value as a Pandas Timestamp instance (if available),
+ otherwise as a Python datetime.timedelta instance.
"""
cdef:
- const uint8_t* ptr
- int64_t length
- CLargeBinaryArray* ap = <CLargeBinaryArray*> self.sp_array.get()
+ CTimestampScalar* sp = <CTimestampScalar*> self.wrapped.get()
+ CTimestampType* dtype = <CTimestampType*> sp.type.get()
- ptr = ap.GetValue(self.index, &length)
- return cp.PyBytes_FromStringAndSize(<const char*>(ptr), length)
+ if not sp.is_valid:
+ return None
- def as_buffer(self):
- """
- Return a view over this value as a Buffer object.
- """
- cdef:
- CLargeBinaryArray* ap = <CLargeBinaryArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
+ if not dtype.timezone().empty():
+ tzinfo = string_to_tzinfo(frombytes(dtype.timezone()))
+ else:
+ tzinfo = None
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
+ return _datetime_from_int(sp.value, unit=dtype.unit(), tzinfo=tzinfo)
-cdef class ListValue(ArrayValue):
+cdef class DurationScalar(Scalar):
"""
- Concrete class for list array elements.
+ Concrete class for duration scalars.
"""
- def __len__(self):
- """
- Return the number of values.
- """
- return self.length()
-
- def __getitem__(self, i):
- """
- Return the value at the given index.
- """
- return self.getitem(_normalize_index(i, self.length()))
-
- def __iter__(self):
- """
- Iterate over this element's values.
- """
- for i in range(len(self)):
- yield self.getitem(i)
- raise StopIteration
-
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CListArray*> sp_array.get()
- self.value_type = pyarrow_wrap_data_type(self.ap.value_type())
-
- cdef getitem(self, int64_t i):
- cdef int64_t j = self.ap.value_offset(self.index) + i
- return box_scalar(self.value_type, self.ap.values(), j)
-
- cdef int64_t length(self):
- return self.ap.value_length(self.index)
+ @property
+ def value(self):
+ cdef CDurationScalar* sp = <CDurationScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
def as_py(self):
"""
- Return this value as a Python list.
+ Return this value as a Pandas Timestamp instance (if available),
+ otherwise as a Python datetime.timedelta instance.
"""
cdef:
- int64_t j
- list result = []
+ CDurationScalar* sp = <CDurationScalar*> self.wrapped.get()
+ CDurationType* dtype = <CDurationType*> sp.type.get()
+ TimeUnit unit = dtype.unit()
- for j in range(len(self)):
- result.append(self.getitem(j).as_py())
+ if not sp.is_valid:
+ return None
- return result
+ if unit == TimeUnit_SECOND:
+ return datetime.timedelta(seconds=sp.value)
+ elif unit == TimeUnit_MILLI:
+ return datetime.timedelta(milliseconds=sp.value)
+ elif unit == TimeUnit_MICRO:
+ return datetime.timedelta(microseconds=sp.value)
+ else:
+ # TimeUnit_NANO: prefer pandas timestamps if available
+ if _pandas_api.have_pandas:
+ return _pandas_api.pd.Timedelta(sp.value, unit='ns')
+ # otherwise safely truncate to microsecond resolution timedelta
+ if sp.value % 1000 != 0:
+ raise ValueError(
+ "Nanosecond duration {} is not safely convertible to "
+ "microseconds to convert to datetime.timedelta. Install "
+ "pandas to return as Timedelta with nanosecond support or "
+ "access the .value attribute.".format(sp.value)
+ )
+ return datetime.timedelta(microseconds=sp.value // 1000)
-cdef class LargeListValue(ArrayValue):
+cdef class BinaryScalar(Scalar):
"""
- Concrete class for large list array elements.
+ Concrete class for binary-like scalars.
"""
- def __len__(self):
- """
- Return the number of values.
- """
- return self.length()
-
- def __getitem__(self, i):
+ def as_buffer(self):
"""
- Return the value at the given index.
+ Return a view over this value as a Buffer object.
"""
- return self.getitem(_normalize_index(i, self.length()))
+ cdef CBinaryScalar* sp = <CBinaryScalar*> self.wrapped.get()
+ return pyarrow_wrap_buffer(sp.value) if sp.is_valid else None
- def __iter__(self):
+ def as_py(self):
"""
- Iterate over this element's values.
+ Return this value as a Python bytes.
"""
- for i in range(len(self)):
- yield self.getitem(i)
- raise StopIteration
+ buffer = self.as_buffer()
+ return None if buffer is None else buffer.to_pybytes()
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CLargeListArray*> sp_array.get()
- self.value_type = pyarrow_wrap_data_type(self.ap.value_type())
- cdef getitem(self, int64_t i):
- cdef int64_t j = self.ap.value_offset(self.index) + i
- return box_scalar(self.value_type, self.ap.values(), j)
+cdef class LargeBinaryScalar(BinaryScalar):
+ pass
- cdef int64_t length(self):
- return self.ap.value_length(self.index)
- def as_py(self):
- """
- Return this value as a Python list.
- """
- cdef:
- int64_t j
- list result = []
-
- for j in range(len(self)):
- result.append(self.getitem(j).as_py())
+cdef class FixedSizeBinaryScalar(BinaryScalar):
+ pass
- return result
-
-cdef class MapValue(ArrayValue):
+cdef class StringScalar(BinaryScalar):
"""
- Concrete class for map array elements.
+ Concrete class for string-like (utf8) scalars.
"""
- def __len__(self):
- """
- Return the number of values.
- """
- return self.length()
-
- def __getitem__(self, i):
- """
- Return the value at the given index.
- """
- return self.getitem(_normalize_index(i, self.length()))
-
- def __iter__(self):
- """
- Iterate over this element's values.
- """
- for i in range(len(self)):
- yield self.getitem(i)
- raise StopIteration
-
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CMapArray*> sp_array.get()
- self.key_type = pyarrow_wrap_data_type(self.ap.map_type().key_type())
- self.item_type = pyarrow_wrap_data_type(self.ap.map_type().item_type())
-
- cdef getitem(self, int64_t i):
- cdef int64_t j = self.ap.value_offset(self.index) + i
- return (box_scalar(self.key_type, self.ap.keys(), j),
- box_scalar(self.item_type, self.ap.items(), j))
-
- cdef int64_t length(self):
- return self.ap.value_length(self.index)
-
def as_py(self):
"""
- Return this value as a Python list of tuples, each containing a
- key and item.
+ Return this value as a Python string.
"""
- cdef:
- int64_t j
- list result = []
+ buffer = self.as_buffer()
+ return None if buffer is None else str(buffer, 'utf8')
- for j in range(len(self)):
- key, item = self.getitem(j)
- result.append((key.as_py(), item.as_py()))
- return result
+cdef class LargeStringScalar(StringScalar):
+ pass
-cdef class FixedSizeListValue(ArrayValue):
+cdef class ListScalar(Scalar):
"""
- Concrete class for fixed size list array elements.
+ Concrete class for list-like scalars.
"""
+ @property
+ def values(self):
+ cdef CListScalar* sp = <CListScalar*> self.wrapped.get()
+ if sp.is_valid:
+ return pyarrow_wrap_array(sp.value)
+ else:
+ return None
+
def __len__(self):
"""
Return the number of values.
"""
- return self.length()
+ return len(self.values)
def __getitem__(self, i):
"""
Return the value at the given index.
"""
- return self.getitem(_normalize_index(i, self.length()))
+ return self.values[_normalize_index(i, len(self))]
def __iter__(self):
"""
Iterate over this element's values.
"""
- for i in range(len(self)):
- yield self.getitem(i)
- raise StopIteration
-
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CFixedSizeListArray*> sp_array.get()
- self.value_type = pyarrow_wrap_data_type(self.ap.value_type())
-
- cdef getitem(self, int64_t i):
- cdef int64_t j = self.ap.value_offset(self.index) + i
- return box_scalar(self.value_type, self.ap.values(), j)
-
- cdef int64_t length(self):
- return self.ap.value_length(self.index)
+ return iter(self.values)
def as_py(self):
"""
Return this value as a Python list.
"""
- cdef:
- int64_t j
- list result = []
-
- for j in range(len(self)):
- result.append(self.getitem(j).as_py())
-
- return result
-
-
-cdef class UnionValue(ArrayValue):
- """
- Concrete class for union array elements.
- """
+ arr = self.values
+ return None if arr is None else arr.to_pylist()
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CUnionArray*> sp_array.get()
-
- cdef getitem(self, int64_t i):
- cdef int child_id = self.ap.child_id(i)
- cdef shared_ptr[CArray] child = self.ap.field(child_id)
- cdef CDenseUnionArray* dense
- if self.ap.mode() == _UnionMode_SPARSE:
- return box_scalar(self.type[child_id].type, child, i)
- else:
- dense = <CDenseUnionArray*> self.ap
- return box_scalar(self.type[child_id].type, child,
- dense.value_offset(i))
- def as_py(self):
- """
- Return this value as a Python object.
+cdef class FixedSizeListScalar(ListScalar):
+ pass
- The exact type depends on the underlying union member.
- """
- return self.getitem(self.index).as_py()
+cdef class LargeListScalar(ListScalar):
+ pass
-cdef class FixedSizeBinaryValue(ArrayValue):
- """
- Concrete class for fixed-size binary array elements.
- """
- def as_py(self):
- """
- Return this value as a Python bytes object.
- """
- cdef:
- CFixedSizeBinaryArray* ap
- CFixedSizeBinaryType* ap_type
- int32_t length
- const char* data
- ap = <CFixedSizeBinaryArray*> self.sp_array.get()
- ap_type = <CFixedSizeBinaryType*> ap.type().get()
- length = ap_type.byte_width()
- data = <const char*> ap.GetValue(self.index)
- return cp.PyBytes_FromStringAndSize(data, length)
-
-
-cdef class StructValue(ArrayValue):
+cdef class StructScalar(Scalar, collections.abc.Mapping):
"""
- Concrete class for struct array elements.
+ Concrete class for struct scalars.
"""
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CStructArray*> sp_array.get()
-
- def __getitem__(self, key):
- """
- Return the child value for the given field name.
- """
- cdef:
- CStructType* type
- int index
-
- type = <CStructType*> self.type.type
- index = type.GetFieldIndex(tobytes(key))
-
- if index < 0:
- raise KeyError(key)
-
- return pyarrow_wrap_array(self.ap.field(index))[self.index]
+ def __len__(self):
+ cdef CStructScalar* sp = <CStructScalar*> self.wrapped.get()
+ return sp.value.size()
- def as_py(self):
- """
- Return this value as a Python dict.
- """
+ def __iter__(self):
cdef:
- vector[shared_ptr[CField]] child_fields = self.type.type.fields()
-
- wrapped_arrays = [pyarrow_wrap_array(self.ap.field(i))
- for i in range(self.ap.num_fields())]
- child_names = [child.get().name() for child in child_fields]
- # Return the struct as a dict
- return {
- frombytes(name): child_array[self.index].as_py()
- for name, child_array in zip(child_names, wrapped_arrays)
- }
-
-
-cdef class DictionaryValue(ArrayValue):
- """
- Concrete class for dictionary-encoded array elements.
- """
-
- def as_py(self):
- """
- Return this value as a Python object.
-
- The exact type depends on the dictionary value type.
- """
- return self.dictionary_value.as_py()
-
- @property
- def index_value(self):
- """
- Return this value's underlying index as a ArrayValue of the right
- signed integer type.
- """
- cdef CDictionaryArray* darr = <CDictionaryArray*>(self.sp_array.get())
- indices = pyarrow_wrap_array(darr.indices())
- return indices[self.index]
-
- @property
- def dictionary_value(self):
- """
- Return this value's underlying dictionary value as a ArrayValue.
- """
- cdef CDictionaryArray* darr = <CDictionaryArray*>(self.sp_array.get())
- dictionary = pyarrow_wrap_array(darr.dictionary())
- return dictionary[self.index_value.as_py()]
-
-
-cdef dict _array_value_classes = {
- _Type_BOOL: BooleanValue,
- _Type_UINT8: UInt8Value,
- _Type_UINT16: UInt16Value,
- _Type_UINT32: UInt32Value,
- _Type_UINT64: UInt64Value,
- _Type_INT8: Int8Value,
- _Type_INT16: Int16Value,
- _Type_INT32: Int32Value,
- _Type_INT64: Int64Value,
- _Type_DATE32: Date32Value,
- _Type_DATE64: Date64Value,
- _Type_TIME32: Time32Value,
- _Type_TIME64: Time64Value,
- _Type_TIMESTAMP: TimestampValue,
- _Type_DURATION: DurationValue,
- _Type_HALF_FLOAT: HalfFloatValue,
- _Type_FLOAT: FloatValue,
- _Type_DOUBLE: DoubleValue,
- _Type_LIST: ListValue,
- _Type_LARGE_LIST: LargeListValue,
- _Type_MAP: MapValue,
- _Type_FIXED_SIZE_LIST: FixedSizeListValue,
- _Type_SPARSE_UNION: UnionValue,
- _Type_DENSE_UNION: UnionValue,
- _Type_BINARY: BinaryValue,
- _Type_STRING: StringValue,
- _Type_LARGE_BINARY: LargeBinaryValue,
- _Type_LARGE_STRING: LargeStringValue,
- _Type_FIXED_SIZE_BINARY: FixedSizeBinaryValue,
- _Type_DECIMAL: DecimalValue,
- _Type_STRUCT: StructValue,
- _Type_DICTIONARY: DictionaryValue,
-}
-
-cdef class ScalarValue(Scalar):
- """
- The base class for scalars.
- """
-
- def __init__(self):
- raise TypeError("Do not call {}'s constructor directly."
- .format(self.__class__.__name__))
-
- cdef void init(self, const shared_ptr[CScalar]& sp_scalar):
- self.sp_scalar = sp_scalar
-
- def __repr__(self):
- if hasattr(self, 'as_py'):
- return repr(self.as_py())
- else:
- return super(Scalar, self).__repr__()
+ CStructScalar* sp = <CStructScalar*> self.wrapped.get()
+ CStructType* dtype = <CStructType*> sp.type.get()
+ vector[shared_ptr[CField]] fields = dtype.fields()
- def __str__(self):
- if hasattr(self, 'as_py'):
- return str(self.as_py())
- else:
- return super(Scalar, self).__str__()
+ if sp.is_valid:
+ for i in range(dtype.num_fields()):
+ yield frombytes(fields[i].get().name())
- def __eq__(self, other):
- if hasattr(self, 'as_py'):
- if isinstance(other, ScalarValue):
- other = other.as_py()
- return self.as_py() == other
+ def __contains__(self, key):
+ try:
+ self[key]
+ except (KeyError, IndexError):
+ return False
else:
- raise NotImplemented(
- "Cannot compare scalars that don't support as_py()")
-
- def __hash__(self):
- return hash(self.as_py())
-
-
-cdef class NullScalar(ScalarValue):
- """
- Concrete class for null scalars.
- """
-
- def as_py(self):
- """
- Return this value as a Python None.
- """
- return None
-
-
-cdef class BooleanScalar(ScalarValue):
- """
- Concrete class for boolean scalars.
- """
+ return True
- def as_py(self):
- """
- Return this value as a Python bool.
+ def __getitem__(self, key):
"""
- cdef CBooleanScalar* sp = <CBooleanScalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
-
+ Return the child value for the given field.
-cdef class UInt8Scalar(ScalarValue):
- """
- Concrete class for uint8 scalars.
- """
+ Parameters
+ ----------
+ index : Union[int, str]
+ Index / position or name of the field.
- def as_py(self):
- """
- Return this value as a Python int.
+ Returns
+ -------
+ result : Scalar
"""
- cdef CUInt8Scalar* sp = <CUInt8Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
+ cdef:
+ CFieldRef ref
+ CStructScalar* sp = <CStructScalar*> self.wrapped.get()
+ if isinstance(key, (bytes, str)):
+ ref = CFieldRef(<c_string> tobytes(key))
+ elif isinstance(key, int):
+ ref = CFieldRef(<int> key)
+ else:
+ raise TypeError('Expected integer or string index')
-cdef class Int8Scalar(ScalarValue):
- """
- Concrete class for int8 scalars.
- """
+ try:
+ return Scalar.wrap(GetResultValue(sp.field(ref)))
+ except ArrowInvalid:
+ if isinstance(key, int):
+ raise IndexError(key)
+ else:
+ raise KeyError(key)
def as_py(self):
"""
- Return this value as a Python int.
+ Return this value as a Python dict.
"""
- cdef CInt8Scalar* sp = <CInt8Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
+ if self.is_valid:
+ return {k: v.as_py() for k, v in self.items()}
+ else:
+ return None
-cdef class UInt16Scalar(ScalarValue):
+cdef class MapScalar(ListScalar):
"""
- Concrete class for uint16 scalars.
+ Concrete class for map scalars.
"""
- def as_py(self):
+ def __getitem__(self, i):
"""
- Return this value as a Python int.
+ Return the value at the given index.
"""
- cdef CUInt16Scalar* sp = <CUInt16Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
-
-
-cdef class Int16Scalar(ScalarValue):
- """
- Concrete class for int16 scalars.
- """
+ arr = self.values
+ if arr is None:
+ raise IndexError(i)
+ dct = arr[_normalize_index(i, len(arr))]
+ return (dct['key'], dct['value'])
- def as_py(self):
+ def __iter__(self):
"""
- Return this value as a Python int.
+ Iterate over this element's values.
"""
- cdef CInt16Scalar* sp = <CInt16Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
-
-
-cdef class UInt32Scalar(ScalarValue):
- """
- Concrete class for uint32 scalars.
- """
+ arr = self.values
+ if arr is None:
+ return iter(zip(arr.field('key'), arr.field('value')))
+ else:
+ raise StopIteration
def as_py(self):
"""
- Return this value as a Python int.
+ Return this value as a Python list.
"""
- cdef CUInt32Scalar* sp = <CUInt32Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
+ arr = self.values
+ if arr is not None:
+ return list(zip(arr.field('key'), arr.field('value')))
+ else:
+ return None
-cdef class Int32Scalar(ScalarValue):
+cdef class DictionaryScalar(Scalar):
"""
- Concrete class for int32 scalars.
+ Concrete class for dictionary-encoded scalars.
"""
- def as_py(self):
+ @property
+ def index(self):
"""
- Return this value as a Python int.
+ Return this value's underlying index as a scalar.
"""
- cdef CInt32Scalar* sp = <CInt32Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
+ cdef CDictionaryScalar* sp = <CDictionaryScalar*> self.wrapped.get()
+ return Scalar.wrap(sp.value.index)
-
-cdef class UInt64Scalar(ScalarValue):
- """
- Concrete class for uint64 scalars.
- """
-
- def as_py(self):
+ @property
+ def value(self):
"""
- Return this value as a Python int.
+ Return the encoded value as a scalar.
"""
- cdef CUInt64Scalar* sp = <CUInt64Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
+ # TODO(kszucs): optimize it to spare uneccessary python object boxing
+ return self.dictionary[self.index.as_py()] if self.is_valid else None
-
-cdef class Int64Scalar(ScalarValue):
- """
- Concrete class for int64 scalars.
- """
+ @property
+ def dictionary(self):
+ cdef CDictionaryScalar* sp = <CDictionaryScalar*> self.wrapped.get()
+ return pyarrow_wrap_array(sp.value.dictionary)
def as_py(self):
"""
- Return this value as a Python int.
+ Return this encoded value as a Python object.
"""
- cdef CInt64Scalar* sp = <CInt64Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
-
+ value = self.value
+ return None if value is None else value.as_py()
-cdef class FloatScalar(ScalarValue):
- """
- Concrete class for float scalars.
- """
+ @property
+ def index_value(self):
+ warnings.warn("`dictionary_value` property is deprecated as of 1.0.0"
+ "please use the `value` property instead",
+ FutureWarning)
+ index = self.index
+ return None if index is None else self.index
- def as_py(self):
- """
- Return this value as a Python float.
- """
- cdef CFloatScalar* sp = <CFloatScalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
+ @property
+ def dictionary_value(self):
+ warnings.warn("`dictionary_value` property is deprecated as of 1.0.0, "
+ "please use the `value` property instead", FutureWarning)
+ value = self.value
+ return None if value is None else self.value
Review comment:
same here
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] pitrou commented on pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
pitrou commented on pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#issuecomment-651806049
Ok, at a quick glance, it seems that null container tests work properly regardless:
```python
>>> s = set()
>>> s.add(pa.scalar(None))
>>> s
{<pyarrow.NullScalar: None>}
>>> pa.scalar(None) in s
True
>>> s.add(pa.scalar(None, pa.int64()))
>>> s.add(pa.scalar(12, pa.int64()))
>>> s
{<pyarrow.Int64Scalar: 12>,
<pyarrow.NullScalar: None>,
<pyarrow.Int64Scalar: None>}
>>> pa.scalar(None, pa.int64()) in s
True
>>> pa.scalar(None, pa.int32()) in s
False
```
```python
>>> l = [pa.scalar(None)]
>>> pa.scalar(None) in l
True
>>> pa.scalar(None, pa.int64()) in l
False
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] kszucs commented on a change in pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
kszucs commented on a change in pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#discussion_r447733611
##########
File path: python/pyarrow/scalar.pxi
##########
@@ -16,1198 +16,745 @@
# under the License.
-_NULL = NA = None
+import collections
cdef class Scalar:
"""
- The base class for all array elements.
+ The base class for scalars.
"""
+ def __init__(self):
+ raise TypeError("Do not call {}'s constructor directly, use "
+ "pa.scalar() instead.".format(self.__class__.__name__))
-cdef class NullType(Scalar):
- """
- Singleton for null array elements.
- """
- # TODO rename this NullValue?
+ cdef void init(self, const shared_ptr[CScalar]& wrapped):
+ self.wrapped = wrapped
- def __cinit__(self):
- global NA
- if NA is not None:
- raise Exception('Cannot create multiple NAType instances')
+ @staticmethod
+ cdef wrap(const shared_ptr[CScalar]& wrapped):
+ cdef:
+ Scalar self
+ Type type_id = wrapped.get().type.get().id()
+
+ if type_id == _Type_NA:
+ return _NULL
+
+ typ = _scalar_classes[type_id]
+ self = typ.__new__(typ)
+ self.init(wrapped)
+
+ return self
+
+ cdef inline shared_ptr[CScalar] unwrap(self) nogil:
+ return self.wrapped
+
+ @property
+ def type(self):
+ return pyarrow_wrap_data_type(self.wrapped.get().type)
- self.type = null()
+ @property
+ def is_valid(self):
+ return self.wrapped.get().is_valid
def __repr__(self):
- return 'NULL'
+ return '<pyarrow.{}: {!r}>'.format(
+ self.__class__.__name__, self.as_py()
+ )
- def as_py(self):
- """
- Return None
- """
- return None
+ def __str__(self):
+ return str(self.as_py())
+
+ def equals(self, Scalar other):
+ return self.wrapped.get().Equals(other.unwrap().get()[0])
def __eq__(self, other):
- return NA
+ try:
+ if not isinstance(other, Scalar):
+ other = scalar(other, type=self.type)
+ return self.equals(other)
+ except (TypeError, ValueError, ArrowInvalid):
+ return NotImplemented
+
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+
+ def as_py(self):
+ raise NotImplementedError()
-_NULL = NA = NullType()
+_NULL = NA = None
-cdef class ArrayValue(Scalar):
+cdef class NullScalar(Scalar):
"""
- The base class for non-null array elements.
+ Concrete class for null scalars.
"""
- def __init__(self):
- raise TypeError("Do not call {}'s constructor directly, use array "
- "subscription instead."
- .format(self.__class__.__name__))
+ def __cinit__(self):
+ global NA
+ if NA is not None:
+ raise Exception('Cannot create multiple NAType instances')
+ self.init(shared_ptr[CScalar](new CNullScalar()))
- cdef void init(self, DataType type, const shared_ptr[CArray]& sp_array,
- int64_t index):
- self.type = type
- self.index = index
- self._set_array(sp_array)
+ def __init__(self):
+ pass
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
+ def __eq__(self, other):
+ return NA
- def __repr__(self):
- if hasattr(self, 'as_py'):
- return repr(self.as_py())
- else:
- return super(Scalar, self).__repr__()
+ def __hash__(self):
Review comment:
Once I override `__eq__` the inherited `__hash__` gets removed.
##########
File path: python/pyarrow/scalar.pxi
##########
@@ -16,1198 +16,745 @@
# under the License.
-_NULL = NA = None
+import collections
cdef class Scalar:
"""
- The base class for all array elements.
+ The base class for scalars.
"""
+ def __init__(self):
+ raise TypeError("Do not call {}'s constructor directly, use "
+ "pa.scalar() instead.".format(self.__class__.__name__))
-cdef class NullType(Scalar):
- """
- Singleton for null array elements.
- """
- # TODO rename this NullValue?
+ cdef void init(self, const shared_ptr[CScalar]& wrapped):
+ self.wrapped = wrapped
- def __cinit__(self):
- global NA
- if NA is not None:
- raise Exception('Cannot create multiple NAType instances')
+ @staticmethod
+ cdef wrap(const shared_ptr[CScalar]& wrapped):
+ cdef:
+ Scalar self
+ Type type_id = wrapped.get().type.get().id()
+
+ if type_id == _Type_NA:
+ return _NULL
+
+ typ = _scalar_classes[type_id]
+ self = typ.__new__(typ)
+ self.init(wrapped)
+
+ return self
+
+ cdef inline shared_ptr[CScalar] unwrap(self) nogil:
+ return self.wrapped
+
+ @property
+ def type(self):
+ return pyarrow_wrap_data_type(self.wrapped.get().type)
- self.type = null()
+ @property
+ def is_valid(self):
+ return self.wrapped.get().is_valid
def __repr__(self):
- return 'NULL'
+ return '<pyarrow.{}: {!r}>'.format(
+ self.__class__.__name__, self.as_py()
+ )
- def as_py(self):
- """
- Return None
- """
- return None
+ def __str__(self):
+ return str(self.as_py())
+
+ def equals(self, Scalar other):
+ return self.wrapped.get().Equals(other.unwrap().get()[0])
def __eq__(self, other):
- return NA
+ try:
+ if not isinstance(other, Scalar):
+ other = scalar(other, type=self.type)
+ return self.equals(other)
+ except (TypeError, ValueError, ArrowInvalid):
+ return NotImplemented
+
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+
+ def as_py(self):
+ raise NotImplementedError()
-_NULL = NA = NullType()
+_NULL = NA = None
-cdef class ArrayValue(Scalar):
+cdef class NullScalar(Scalar):
"""
- The base class for non-null array elements.
+ Concrete class for null scalars.
"""
- def __init__(self):
- raise TypeError("Do not call {}'s constructor directly, use array "
- "subscription instead."
- .format(self.__class__.__name__))
+ def __cinit__(self):
+ global NA
+ if NA is not None:
+ raise Exception('Cannot create multiple NAType instances')
+ self.init(shared_ptr[CScalar](new CNullScalar()))
- cdef void init(self, DataType type, const shared_ptr[CArray]& sp_array,
- int64_t index):
- self.type = type
- self.index = index
- self._set_array(sp_array)
+ def __init__(self):
+ pass
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
+ def __eq__(self, other):
+ return NA
- def __repr__(self):
- if hasattr(self, 'as_py'):
- return repr(self.as_py())
- else:
- return super(Scalar, self).__repr__()
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
- def __str__(self):
- if hasattr(self, 'as_py'):
- return str(self.as_py())
- else:
- return super(Scalar, self).__str__()
+ def as_py(self):
+ """
+ Return this value as a Python None.
+ """
+ return None
- def __eq__(self, other):
- if hasattr(self, 'as_py'):
- if isinstance(other, ArrayValue):
- other = other.as_py()
- return self.as_py() == other
- else:
- raise NotImplementedError(
- "Cannot compare Arrow values that don't support as_py()")
- def __hash__(self):
- return hash(self.as_py())
+_NULL = NA = NullScalar()
-cdef class BooleanValue(ArrayValue):
+cdef class BooleanScalar(Scalar):
"""
- Concrete class for boolean array elements.
+ Concrete class for boolean scalars.
"""
def as_py(self):
"""
Return this value as a Python bool.
"""
- cdef CBooleanArray* ap = <CBooleanArray*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CBooleanScalar* sp = <CBooleanScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int8Value(ArrayValue):
+cdef class UInt8Scalar(Scalar):
"""
- Concrete class for int8 array elements.
+ Concrete class for uint8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt8Array* ap = <CInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt8Scalar* sp = <CUInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt8Value(ArrayValue):
+cdef class Int8Scalar(Scalar):
"""
- Concrete class for uint8 array elements.
+ Concrete class for int8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt8Array* ap = <CUInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt8Scalar* sp = <CInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int16Value(ArrayValue):
+cdef class UInt16Scalar(Scalar):
"""
- Concrete class for int16 array elements.
+ Concrete class for uint16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt16Array* ap = <CInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt16Scalar* sp = <CUInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt16Value(ArrayValue):
+cdef class Int16Scalar(Scalar):
"""
- Concrete class for uint16 array elements.
+ Concrete class for int16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt16Array* ap = <CUInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt16Scalar* sp = <CInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int32Value(ArrayValue):
+cdef class UInt32Scalar(Scalar):
"""
- Concrete class for int32 array elements.
+ Concrete class for uint32 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt32Array* ap = <CInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt32Scalar* sp = <CUInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt32Value(ArrayValue):
+cdef class Int32Scalar(Scalar):
"""
- Concrete class for uint32 array elements.
+ Concrete class for int32 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt32Array* ap = <CUInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt32Scalar* sp = <CInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int64Value(ArrayValue):
+cdef class UInt64Scalar(Scalar):
"""
- Concrete class for int64 array elements.
+ Concrete class for uint64 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt64Array* ap = <CInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt64Scalar* sp = <CUInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt64Value(ArrayValue):
+cdef class Int64Scalar(Scalar):
"""
- Concrete class for uint64 array elements.
+ Concrete class for int64 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt64Array* ap = <CUInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt64Scalar* sp = <CInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date32Value(ArrayValue):
+cdef class HalfFloatScalar(Scalar):
"""
- Concrete class for date32 array elements.
+ Concrete class for float scalars.
"""
+ def __hash__(self):
Review comment:
Same as above.
##########
File path: python/pyarrow/scalar.pxi
##########
@@ -16,1198 +16,745 @@
# under the License.
-_NULL = NA = None
+import collections
cdef class Scalar:
"""
- The base class for all array elements.
+ The base class for scalars.
"""
+ def __init__(self):
+ raise TypeError("Do not call {}'s constructor directly, use "
+ "pa.scalar() instead.".format(self.__class__.__name__))
-cdef class NullType(Scalar):
- """
- Singleton for null array elements.
- """
- # TODO rename this NullValue?
+ cdef void init(self, const shared_ptr[CScalar]& wrapped):
+ self.wrapped = wrapped
- def __cinit__(self):
- global NA
- if NA is not None:
- raise Exception('Cannot create multiple NAType instances')
+ @staticmethod
+ cdef wrap(const shared_ptr[CScalar]& wrapped):
+ cdef:
+ Scalar self
+ Type type_id = wrapped.get().type.get().id()
+
+ if type_id == _Type_NA:
+ return _NULL
+
+ typ = _scalar_classes[type_id]
+ self = typ.__new__(typ)
+ self.init(wrapped)
+
+ return self
+
+ cdef inline shared_ptr[CScalar] unwrap(self) nogil:
+ return self.wrapped
+
+ @property
+ def type(self):
+ return pyarrow_wrap_data_type(self.wrapped.get().type)
- self.type = null()
+ @property
+ def is_valid(self):
+ return self.wrapped.get().is_valid
def __repr__(self):
- return 'NULL'
+ return '<pyarrow.{}: {!r}>'.format(
+ self.__class__.__name__, self.as_py()
+ )
- def as_py(self):
- """
- Return None
- """
- return None
+ def __str__(self):
+ return str(self.as_py())
+
+ def equals(self, Scalar other):
+ return self.wrapped.get().Equals(other.unwrap().get()[0])
def __eq__(self, other):
- return NA
+ try:
+ if not isinstance(other, Scalar):
+ other = scalar(other, type=self.type)
+ return self.equals(other)
+ except (TypeError, ValueError, ArrowInvalid):
+ return NotImplemented
+
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+
+ def as_py(self):
+ raise NotImplementedError()
-_NULL = NA = NullType()
+_NULL = NA = None
-cdef class ArrayValue(Scalar):
+cdef class NullScalar(Scalar):
"""
- The base class for non-null array elements.
+ Concrete class for null scalars.
"""
- def __init__(self):
- raise TypeError("Do not call {}'s constructor directly, use array "
- "subscription instead."
- .format(self.__class__.__name__))
+ def __cinit__(self):
+ global NA
+ if NA is not None:
+ raise Exception('Cannot create multiple NAType instances')
+ self.init(shared_ptr[CScalar](new CNullScalar()))
- cdef void init(self, DataType type, const shared_ptr[CArray]& sp_array,
- int64_t index):
- self.type = type
- self.index = index
- self._set_array(sp_array)
+ def __init__(self):
+ pass
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
+ def __eq__(self, other):
+ return NA
- def __repr__(self):
- if hasattr(self, 'as_py'):
- return repr(self.as_py())
- else:
- return super(Scalar, self).__repr__()
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
- def __str__(self):
- if hasattr(self, 'as_py'):
- return str(self.as_py())
- else:
- return super(Scalar, self).__str__()
+ def as_py(self):
+ """
+ Return this value as a Python None.
+ """
+ return None
- def __eq__(self, other):
- if hasattr(self, 'as_py'):
- if isinstance(other, ArrayValue):
- other = other.as_py()
- return self.as_py() == other
- else:
- raise NotImplementedError(
- "Cannot compare Arrow values that don't support as_py()")
- def __hash__(self):
- return hash(self.as_py())
+_NULL = NA = NullScalar()
-cdef class BooleanValue(ArrayValue):
+cdef class BooleanScalar(Scalar):
"""
- Concrete class for boolean array elements.
+ Concrete class for boolean scalars.
"""
def as_py(self):
"""
Return this value as a Python bool.
"""
- cdef CBooleanArray* ap = <CBooleanArray*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CBooleanScalar* sp = <CBooleanScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int8Value(ArrayValue):
+cdef class UInt8Scalar(Scalar):
"""
- Concrete class for int8 array elements.
+ Concrete class for uint8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt8Array* ap = <CInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt8Scalar* sp = <CUInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt8Value(ArrayValue):
+cdef class Int8Scalar(Scalar):
"""
- Concrete class for uint8 array elements.
+ Concrete class for int8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt8Array* ap = <CUInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt8Scalar* sp = <CInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int16Value(ArrayValue):
+cdef class UInt16Scalar(Scalar):
"""
- Concrete class for int16 array elements.
+ Concrete class for uint16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt16Array* ap = <CInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt16Scalar* sp = <CUInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt16Value(ArrayValue):
+cdef class Int16Scalar(Scalar):
"""
- Concrete class for uint16 array elements.
+ Concrete class for int16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt16Array* ap = <CUInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt16Scalar* sp = <CInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int32Value(ArrayValue):
+cdef class UInt32Scalar(Scalar):
"""
- Concrete class for int32 array elements.
+ Concrete class for uint32 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt32Array* ap = <CInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt32Scalar* sp = <CUInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt32Value(ArrayValue):
+cdef class Int32Scalar(Scalar):
"""
- Concrete class for uint32 array elements.
+ Concrete class for int32 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt32Array* ap = <CUInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt32Scalar* sp = <CInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int64Value(ArrayValue):
+cdef class UInt64Scalar(Scalar):
"""
- Concrete class for int64 array elements.
+ Concrete class for uint64 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt64Array* ap = <CInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt64Scalar* sp = <CUInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt64Value(ArrayValue):
+cdef class Int64Scalar(Scalar):
"""
- Concrete class for uint64 array elements.
+ Concrete class for int64 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt64Array* ap = <CUInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt64Scalar* sp = <CInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date32Value(ArrayValue):
+cdef class HalfFloatScalar(Scalar):
"""
- Concrete class for date32 array elements.
+ Concrete class for float scalars.
"""
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+
+ def __eq__(self, other):
+ if hasattr(self, 'as_py'):
Review comment:
Right.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] kszucs commented on a change in pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
kszucs commented on a change in pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#discussion_r448550619
##########
File path: python/pyarrow/scalar.pxi
##########
@@ -16,1198 +16,748 @@
# under the License.
-_NULL = NA = None
+import collections
cdef class Scalar:
"""
- The base class for all array elements.
- """
-
-
-cdef class NullType(Scalar):
- """
- Singleton for null array elements.
+ The base class for scalars.
"""
- # TODO rename this NullValue?
-
- def __cinit__(self):
- global NA
- if NA is not None:
- raise Exception('Cannot create multiple NAType instances')
-
- self.type = null()
-
- def __repr__(self):
- return 'NULL'
- def as_py(self):
- """
- Return None
- """
- return None
+ def __init__(self):
+ raise TypeError("Do not call {}'s constructor directly, use "
+ "pa.scalar() instead.".format(self.__class__.__name__))
- def __eq__(self, other):
- return NA
+ cdef void init(self, const shared_ptr[CScalar]& wrapped):
+ self.wrapped = wrapped
+ @staticmethod
+ cdef wrap(const shared_ptr[CScalar]& wrapped):
+ cdef:
+ Scalar self
+ Type type_id = wrapped.get().type.get().id()
-_NULL = NA = NullType()
+ if type_id == _Type_NA:
+ return _NULL
+ typ = _scalar_classes[type_id]
+ self = typ.__new__(typ)
+ self.init(wrapped)
-cdef class ArrayValue(Scalar):
- """
- The base class for non-null array elements.
- """
+ return self
- def __init__(self):
- raise TypeError("Do not call {}'s constructor directly, use array "
- "subscription instead."
- .format(self.__class__.__name__))
+ cdef inline shared_ptr[CScalar] unwrap(self) nogil:
+ return self.wrapped
- cdef void init(self, DataType type, const shared_ptr[CArray]& sp_array,
- int64_t index):
- self.type = type
- self.index = index
- self._set_array(sp_array)
+ @property
+ def type(self):
+ """
+ Data type of the Scalar object.
+ """
+ return pyarrow_wrap_data_type(self.wrapped.get().type)
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
+ @property
+ def is_valid(self):
+ """
+ Holds a valid (non-null) value.
+ """
+ return self.wrapped.get().is_valid
def __repr__(self):
- if hasattr(self, 'as_py'):
- return repr(self.as_py())
- else:
- return super(Scalar, self).__repr__()
+ return '<pyarrow.{}: {!r}>'.format(
+ self.__class__.__name__, self.as_py()
+ )
def __str__(self):
- if hasattr(self, 'as_py'):
- return str(self.as_py())
- else:
- return super(Scalar, self).__str__()
+ return str(self.as_py())
+
+ def equals(self, Scalar other):
+ return self.wrapped.get().Equals(other.unwrap().get()[0])
def __eq__(self, other):
- if hasattr(self, 'as_py'):
- if isinstance(other, ArrayValue):
- other = other.as_py()
- return self.as_py() == other
- else:
- raise NotImplementedError(
- "Cannot compare Arrow values that don't support as_py()")
+ try:
+ if not isinstance(other, Scalar):
+ other = scalar(other, type=self.type)
+ return self.equals(other)
+ except (TypeError, ValueError, ArrowInvalid):
+ return NotImplemented
def __hash__(self):
- return hash(self.as_py())
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+ def as_py(self):
+ raise NotImplementedError()
-cdef class BooleanValue(ArrayValue):
- """
- Concrete class for boolean array elements.
- """
- def as_py(self):
- """
- Return this value as a Python bool.
- """
- cdef CBooleanArray* ap = <CBooleanArray*> self.sp_array.get()
- return ap.Value(self.index)
+_NULL = NA = None
-cdef class Int8Value(ArrayValue):
+cdef class NullScalar(Scalar):
"""
- Concrete class for int8 array elements.
+ Concrete class for null scalars.
"""
- def as_py(self):
- """
- Return this value as a Python int.
- """
- cdef CInt8Array* ap = <CInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ def __cinit__(self):
+ global NA
+ if NA is not None:
+ raise RuntimeError('Cannot create multiple NullScalar instances')
+ self.init(shared_ptr[CScalar](new CNullScalar()))
+ def __init__(self):
+ pass
-cdef class UInt8Value(ArrayValue):
- """
- Concrete class for uint8 array elements.
- """
+ def __eq__(self, other):
+ return NA
+
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
def as_py(self):
"""
- Return this value as a Python int.
+ Return this value as a Python None.
"""
- cdef CUInt8Array* ap = <CUInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
-
+ return None
-cdef class Int16Value(ArrayValue):
- """
- Concrete class for int16 array elements.
- """
- def as_py(self):
- """
- Return this value as a Python int.
- """
- cdef CInt16Array* ap = <CInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+_NULL = NA = NullScalar()
-cdef class UInt16Value(ArrayValue):
+cdef class BooleanScalar(Scalar):
"""
- Concrete class for uint16 array elements.
+ Concrete class for boolean scalars.
"""
def as_py(self):
"""
- Return this value as a Python int.
+ Return this value as a Python bool.
"""
- cdef CUInt16Array* ap = <CUInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CBooleanScalar* sp = <CBooleanScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int32Value(ArrayValue):
+cdef class UInt8Scalar(Scalar):
"""
- Concrete class for int32 array elements.
+ Concrete class for uint8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt32Array* ap = <CInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt8Scalar* sp = <CUInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt32Value(ArrayValue):
+cdef class Int8Scalar(Scalar):
"""
- Concrete class for uint32 array elements.
+ Concrete class for int8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt32Array* ap = <CUInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt8Scalar* sp = <CInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int64Value(ArrayValue):
+cdef class UInt16Scalar(Scalar):
"""
- Concrete class for int64 array elements.
+ Concrete class for uint16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt64Array* ap = <CInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt16Scalar* sp = <CUInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt64Value(ArrayValue):
+cdef class Int16Scalar(Scalar):
"""
- Concrete class for uint64 array elements.
+ Concrete class for int16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt64Array* ap = <CUInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt16Scalar* sp = <CInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date32Value(ArrayValue):
+cdef class UInt32Scalar(Scalar):
"""
- Concrete class for date32 array elements.
+ Concrete class for uint32 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python int.
"""
- cdef CDate32Array* ap = <CDate32Array*> self.sp_array.get()
-
- # Shift to seconds since epoch
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(days=ap.Value(self.index)))
+ cdef CUInt32Scalar* sp = <CUInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date64Value(ArrayValue):
+cdef class Int32Scalar(Scalar):
"""
- Concrete class for date64 array elements.
+ Concrete class for int32 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python int.
"""
- cdef CDate64Array* ap = <CDate64Array*> self.sp_array.get()
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(
- days=ap.Value(self.index) / 86400000))
+ cdef CInt32Scalar* sp = <CInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Time32Value(ArrayValue):
+cdef class UInt64Scalar(Scalar):
"""
- Concrete class for time32 array elements.
+ Concrete class for uint64 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.timedelta instance.
+ Return this value as a Python int.
"""
- cdef:
- CTime32Array* ap = <CTime32Array*> self.sp_array.get()
- CTime32Type* dtype = <CTime32Type*> ap.type().get()
-
- if dtype.unit() == TimeUnit_SECOND:
- delta = datetime.timedelta(seconds=ap.Value(self.index))
- return (datetime.datetime(1970, 1, 1) + delta).time()
- else:
- return _box_time_milli(ap.Value(self.index))
+ cdef CUInt64Scalar* sp = <CUInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Time64Value(ArrayValue):
+cdef class Int64Scalar(Scalar):
"""
- Concrete class for time64 array elements.
+ Concrete class for int64 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.timedelta instance.
+ Return this value as a Python int.
"""
- cdef:
- CTime64Array* ap = <CTime64Array*> self.sp_array.get()
- CTime64Type* dtype = <CTime64Type*> ap.type().get()
-
- cdef int64_t val = ap.Value(self.index)
- if dtype.unit() == TimeUnit_MICRO:
- return _box_time_micro(val)
- else:
- return (datetime.datetime(1970, 1, 1) +
- datetime.timedelta(microseconds=val / 1000)).time()
-
-
-cpdef _box_time_milli(int64_t val):
- delta = datetime.timedelta(milliseconds=val)
- return (datetime.datetime(1970, 1, 1) + delta).time()
-
-
-cpdef _box_time_micro(int64_t val):
- return (datetime.datetime(1970, 1, 1) +
- datetime.timedelta(microseconds=val)).time()
-
-
-cdef dict _DATETIME_CONVERSION_FUNCTIONS = {}
-cdef c_bool _datetime_conversion_initialized = False
-
-
-cdef _add_micros_maybe_localize(dt, micros, tzinfo):
- import pytz
- dt = dt.replace(microsecond=micros)
- if tzinfo is not None:
- if not isinstance(tzinfo, datetime.tzinfo):
- tzinfo = string_to_tzinfo(tzinfo)
- dt = tzinfo.fromutc(dt)
- return dt
-
-
-cdef _datetime_from_seconds(int64_t v):
- return datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=v)
-
-
-def _nanoseconds_to_datetime_safe(v, tzinfo):
- if v % 1000 != 0:
- raise ValueError("Nanosecond timestamp {} is not safely convertible "
- " to microseconds to convert to datetime.datetime."
- " Install pandas to return as Timestamp with "
- " nanosecond support or access the .value attribute.")
- v = v // 1000
- micros = v % 1_000_000
-
- dt = _datetime_from_seconds(v // 1_000_000)
- return _add_micros_maybe_localize(dt, micros, tzinfo)
-
-
-def _microseconds_to_datetime(v, tzinfo):
- micros = v % 1_000_000
- dt = _datetime_from_seconds(v // 1_000_000)
- return _add_micros_maybe_localize(dt, micros, tzinfo)
-
-
-def _millis_to_datetime(v, tzinfo):
- millis = v % 1_000
- dt = _datetime_from_seconds(v // 1000)
- return _add_micros_maybe_localize(dt, millis * 1000, tzinfo)
-
-
-def _seconds_to_datetime(v, tzinfo):
- dt = _datetime_from_seconds(v)
- return _add_micros_maybe_localize(dt, 0, tzinfo)
-
-
-def _datetime_conversion_functions():
- global _datetime_conversion_initialized
- if _datetime_conversion_initialized:
- return _DATETIME_CONVERSION_FUNCTIONS
-
- _DATETIME_CONVERSION_FUNCTIONS.update({
- TimeUnit_SECOND: _seconds_to_datetime,
- TimeUnit_MILLI: _millis_to_datetime,
- TimeUnit_MICRO: _microseconds_to_datetime,
- TimeUnit_NANO: _nanoseconds_to_datetime_safe
- })
-
- try:
- import pandas as pd
- _DATETIME_CONVERSION_FUNCTIONS[TimeUnit_NANO] = (
- lambda x, tzinfo: pd.Timestamp(
- x, tz=tzinfo, unit='ns',
- )
- )
- except ImportError:
- pass
-
- _datetime_conversion_initialized = True
- return _DATETIME_CONVERSION_FUNCTIONS
+ cdef CInt64Scalar* sp = <CInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class TimestampValue(ArrayValue):
+cdef class HalfFloatScalar(Scalar):
"""
- Concrete class for timestamp array elements.
+ Concrete class for float scalars.
"""
- @property
- def value(self):
- cdef CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
- cdef CTimestampType* dtype = <CTimestampType*> ap.type().get()
- return ap.Value(self.index)
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+
+ def __eq__(self, other):
+ if isinstance(other, Scalar):
+ other = other.as_py()
+ return self.as_py() == other
def as_py(self):
"""
- Return this value as a Pandas Timestamp instance (if available),
- otherwise as a Python datetime.timedelta instance.
+ Return this value as a Python float.
"""
- cdef CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
- cdef CTimestampType* dtype = <CTimestampType*> ap.type().get()
-
- value = self.value
-
- if not dtype.timezone().empty():
- tzinfo = string_to_tzinfo(frombytes(dtype.timezone()))
- else:
- tzinfo = None
-
- try:
- converter = _datetime_conversion_functions()[dtype.unit()]
- except KeyError:
- raise ValueError(
- 'Cannot convert nanosecond timestamps without pandas'
- )
- return converter(value, tzinfo=tzinfo)
-
-
-cdef dict _TIMEDELTA_CONVERSION_FUNCTIONS = {}
-
-
-def _nanoseconds_to_timedelta_safe(v):
- if v % 1000 != 0:
- raise ValueError(
- "Nanosecond duration {} is not safely convertible to microseconds "
- "to convert to datetime.timedelta. Install pandas to return as "
- "Timedelta with nanosecond support or access the .value "
- "attribute.".format(v))
- micros = v // 1000
-
- return datetime.timedelta(microseconds=micros)
+ cdef CHalfFloatScalar* sp = <CHalfFloatScalar*> self.wrapped.get()
+ return PyHalf_FromHalf(sp.value) if sp.is_valid else None
-def _timedelta_conversion_functions():
- if _TIMEDELTA_CONVERSION_FUNCTIONS:
- return _TIMEDELTA_CONVERSION_FUNCTIONS
-
- _TIMEDELTA_CONVERSION_FUNCTIONS.update({
- TimeUnit_SECOND: lambda v: datetime.timedelta(seconds=v),
- TimeUnit_MILLI: lambda v: datetime.timedelta(milliseconds=v),
- TimeUnit_MICRO: lambda v: datetime.timedelta(microseconds=v),
- TimeUnit_NANO: _nanoseconds_to_timedelta_safe
- })
-
- try:
- import pandas as pd
- _TIMEDELTA_CONVERSION_FUNCTIONS[TimeUnit_NANO] = (
- lambda v: pd.Timedelta(v, unit='ns')
- )
- except ImportError:
- pass
-
- return _TIMEDELTA_CONVERSION_FUNCTIONS
-
-
-cdef class DurationValue(ArrayValue):
+cdef class FloatScalar(Scalar):
"""
- Concrete class for duration array elements.
+ Concrete class for float scalars.
"""
- @property
- def value(self):
- cdef CDurationArray* ap = <CDurationArray*> self.sp_array.get()
- return ap.Value(self.index)
-
def as_py(self):
"""
- Return this value as a Pandas Timestamp instance (if available),
- otherwise as a Python datetime.timedelta instance.
+ Return this value as a Python float.
"""
- cdef CDurationArray* ap = <CDurationArray*> self.sp_array.get()
- cdef CDurationType* dtype = <CDurationType*> ap.type().get()
-
- cdef int64_t value = ap.Value(self.index)
- converter = _timedelta_conversion_functions()[dtype.unit()]
- return converter(value)
+ cdef CFloatScalar* sp = <CFloatScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class HalfFloatValue(ArrayValue):
+cdef class DoubleScalar(Scalar):
"""
- Concrete class for float16 array elements.
+ Concrete class for double scalars.
"""
def as_py(self):
"""
Return this value as a Python float.
"""
- cdef CHalfFloatArray* ap = <CHalfFloatArray*> self.sp_array.get()
- return PyHalf_FromHalf(ap.Value(self.index))
+ cdef CDoubleScalar* sp = <CDoubleScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class FloatValue(ArrayValue):
+cdef class Decimal128Scalar(Scalar):
"""
- Concrete class for float32 array elements.
+ Concrete class for decimal128 scalars.
"""
def as_py(self):
"""
- Return this value as a Python float.
+ Return this value as a Python Decimal.
"""
- cdef CFloatArray* ap = <CFloatArray*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef:
+ CDecimal128Scalar* sp = <CDecimal128Scalar*> self.wrapped.get()
+ CDecimal128Type* dtype = <CDecimal128Type*> sp.type.get()
+ if sp.is_valid:
+ return _pydecimal.Decimal(
+ frombytes(sp.value.ToString(dtype.scale()))
+ )
+ else:
+ return None
-cdef class DoubleValue(ArrayValue):
+cdef class Date32Scalar(Scalar):
"""
- Concrete class for float64 array elements.
+ Concrete class for date32 scalars.
"""
def as_py(self):
"""
- Return this value as a Python float.
+ Return this value as a Python datetime.datetime instance.
"""
- cdef CDoubleArray* ap = <CDoubleArray*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CDate32Scalar* sp = <CDate32Scalar*> self.wrapped.get()
+
+ if sp.is_valid:
+ # shift to seconds since epoch
+ return (
+ datetime.date(1970, 1, 1) + datetime.timedelta(days=sp.value)
+ )
+ else:
+ return None
-cdef class DecimalValue(ArrayValue):
+cdef class Date64Scalar(Scalar):
"""
- Concrete class for decimal128 array elements.
+ Concrete class for date64 scalars.
"""
def as_py(self):
"""
- Return this value as a Python Decimal.
+ Return this value as a Python datetime.datetime instance.
"""
- cdef:
- CDecimal128Array* ap = <CDecimal128Array*> self.sp_array.get()
- c_string s = ap.FormatValue(self.index)
- return _pydecimal.Decimal(s.decode('utf8'))
+ cdef CDate64Scalar* sp = <CDate64Scalar*> self.wrapped.get()
+ if sp.is_valid:
+ return (
+ datetime.date(1970, 1, 1) +
+ datetime.timedelta(days=sp.value / 86400000)
+ )
+ else:
+ return None
-cdef class StringValue(ArrayValue):
- """
- Concrete class for string (utf8) array elements.
- """
- def as_py(self):
- """
- Return this value as a Python unicode string.
- """
- cdef CStringArray* ap = <CStringArray*> self.sp_array.get()
- return ap.GetString(self.index).decode('utf-8')
+def _datetime_from_int(int64_t value, TimeUnit unit, tzinfo=None):
+ if unit == TimeUnit_SECOND:
+ delta = datetime.timedelta(seconds=value)
+ elif unit == TimeUnit_MILLI:
+ delta = datetime.timedelta(milliseconds=value)
+ elif unit == TimeUnit_MICRO:
+ delta = datetime.timedelta(microseconds=value)
+ else:
+ # TimeUnit_NANO: prefer pandas timestamps if available
+ if _pandas_api.have_pandas:
+ return _pandas_api.pd.Timestamp(value, tz=tzinfo, unit='ns')
+ # otherwise safely truncate to microsecond resolution datetime
+ if value % 1000 != 0:
+ raise ValueError(
+ "Nanosecond resolution temporal type {} is not safely "
+ "convertible to microseconds to convert to datetime.datetime. "
+ "Install pandas to return as Timestamp with nanosecond "
+ "support or access the .value attribute.".format(value)
+ )
+ delta = datetime.timedelta(microseconds=value // 1000)
- def as_buffer(self):
- """
- Return a view over this value as a Buffer object.
- """
- cdef:
- CStringArray* ap = <CStringArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
+ dt = datetime.datetime(1970, 1, 1) + delta
+ # adjust timezone if set to the datatype
+ if tzinfo is not None:
+ dt = tzinfo.fromutc(dt)
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
+ return dt
-cdef class LargeStringValue(ArrayValue):
+cdef class Time32Scalar(Scalar):
"""
- Concrete class for large string (utf8) array elements.
+ Concrete class for time32 scalars.
"""
def as_py(self):
"""
- Return this value as a Python unicode string.
- """
- cdef CLargeStringArray* ap = <CLargeStringArray*> self.sp_array.get()
- return ap.GetString(self.index).decode('utf-8')
-
- def as_buffer(self):
- """
- Return a view over this value as a Buffer object.
+ Return this value as a Python datetime.timedelta instance.
"""
cdef:
- CLargeStringArray* ap = <CLargeStringArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
+ CTime32Scalar* sp = <CTime32Scalar*> self.wrapped.get()
+ CTime32Type* dtype = <CTime32Type*> sp.type.get()
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
+ if sp.is_valid:
+ return _datetime_from_int(sp.value, unit=dtype.unit()).time()
+ else:
+ return None
-cdef class BinaryValue(ArrayValue):
+cdef class Time64Scalar(Scalar):
"""
- Concrete class for variable-sized binary array elements.
+ Concrete class for time64 scalars.
"""
def as_py(self):
"""
- Return this value as a Python bytes object.
- """
- cdef:
- const uint8_t* ptr
- int32_t length
- CBinaryArray* ap = <CBinaryArray*> self.sp_array.get()
-
- ptr = ap.GetValue(self.index, &length)
- return cp.PyBytes_FromStringAndSize(<const char*>(ptr), length)
-
- def as_buffer(self):
- """
- Return a view over this value as a Buffer object.
+ Return this value as a Python datetime.timedelta instance.
"""
cdef:
- CBinaryArray* ap = <CBinaryArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
+ CTime64Scalar* sp = <CTime64Scalar*> self.wrapped.get()
+ CTime64Type* dtype = <CTime64Type*> sp.type.get()
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
+ if sp.is_valid:
+ return _datetime_from_int(sp.value, unit=dtype.unit()).time()
+ else:
+ return None
-cdef class LargeBinaryValue(ArrayValue):
+cdef class TimestampScalar(Scalar):
"""
- Concrete class for large variable-sized binary array elements.
+ Concrete class for timestamp scalars.
"""
+ @property
+ def value(self):
+ cdef CTimestampScalar* sp = <CTimestampScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
+
def as_py(self):
"""
- Return this value as a Python bytes object.
+ Return this value as a Pandas Timestamp instance (if available),
+ otherwise as a Python datetime.timedelta instance.
"""
cdef:
- const uint8_t* ptr
- int64_t length
- CLargeBinaryArray* ap = <CLargeBinaryArray*> self.sp_array.get()
+ CTimestampScalar* sp = <CTimestampScalar*> self.wrapped.get()
+ CTimestampType* dtype = <CTimestampType*> sp.type.get()
- ptr = ap.GetValue(self.index, &length)
- return cp.PyBytes_FromStringAndSize(<const char*>(ptr), length)
+ if not sp.is_valid:
+ return None
- def as_buffer(self):
- """
- Return a view over this value as a Buffer object.
- """
- cdef:
- CLargeBinaryArray* ap = <CLargeBinaryArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
+ if not dtype.timezone().empty():
+ tzinfo = string_to_tzinfo(frombytes(dtype.timezone()))
+ else:
+ tzinfo = None
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
+ return _datetime_from_int(sp.value, unit=dtype.unit(), tzinfo=tzinfo)
-cdef class ListValue(ArrayValue):
+cdef class DurationScalar(Scalar):
"""
- Concrete class for list array elements.
+ Concrete class for duration scalars.
"""
- def __len__(self):
- """
- Return the number of values.
- """
- return self.length()
-
- def __getitem__(self, i):
- """
- Return the value at the given index.
- """
- return self.getitem(_normalize_index(i, self.length()))
-
- def __iter__(self):
- """
- Iterate over this element's values.
- """
- for i in range(len(self)):
- yield self.getitem(i)
- raise StopIteration
-
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CListArray*> sp_array.get()
- self.value_type = pyarrow_wrap_data_type(self.ap.value_type())
-
- cdef getitem(self, int64_t i):
- cdef int64_t j = self.ap.value_offset(self.index) + i
- return box_scalar(self.value_type, self.ap.values(), j)
-
- cdef int64_t length(self):
- return self.ap.value_length(self.index)
+ @property
+ def value(self):
+ cdef CDurationScalar* sp = <CDurationScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
def as_py(self):
"""
- Return this value as a Python list.
+ Return this value as a Pandas Timestamp instance (if available),
+ otherwise as a Python datetime.timedelta instance.
"""
cdef:
- int64_t j
- list result = []
+ CDurationScalar* sp = <CDurationScalar*> self.wrapped.get()
+ CDurationType* dtype = <CDurationType*> sp.type.get()
+ TimeUnit unit = dtype.unit()
- for j in range(len(self)):
- result.append(self.getitem(j).as_py())
+ if not sp.is_valid:
+ return None
- return result
+ if unit == TimeUnit_SECOND:
+ return datetime.timedelta(seconds=sp.value)
+ elif unit == TimeUnit_MILLI:
+ return datetime.timedelta(milliseconds=sp.value)
+ elif unit == TimeUnit_MICRO:
+ return datetime.timedelta(microseconds=sp.value)
+ else:
+ # TimeUnit_NANO: prefer pandas timestamps if available
+ if _pandas_api.have_pandas:
+ return _pandas_api.pd.Timedelta(sp.value, unit='ns')
+ # otherwise safely truncate to microsecond resolution timedelta
+ if sp.value % 1000 != 0:
+ raise ValueError(
+ "Nanosecond duration {} is not safely convertible to "
+ "microseconds to convert to datetime.timedelta. Install "
+ "pandas to return as Timedelta with nanosecond support or "
+ "access the .value attribute.".format(sp.value)
+ )
+ return datetime.timedelta(microseconds=sp.value // 1000)
-cdef class LargeListValue(ArrayValue):
+cdef class BinaryScalar(Scalar):
"""
- Concrete class for large list array elements.
+ Concrete class for binary-like scalars.
"""
- def __len__(self):
- """
- Return the number of values.
- """
- return self.length()
-
- def __getitem__(self, i):
+ def as_buffer(self):
"""
- Return the value at the given index.
+ Return a view over this value as a Buffer object.
"""
- return self.getitem(_normalize_index(i, self.length()))
+ cdef CBinaryScalar* sp = <CBinaryScalar*> self.wrapped.get()
+ return pyarrow_wrap_buffer(sp.value) if sp.is_valid else None
- def __iter__(self):
+ def as_py(self):
"""
- Iterate over this element's values.
+ Return this value as a Python bytes.
"""
- for i in range(len(self)):
- yield self.getitem(i)
- raise StopIteration
+ buffer = self.as_buffer()
+ return None if buffer is None else buffer.to_pybytes()
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CLargeListArray*> sp_array.get()
- self.value_type = pyarrow_wrap_data_type(self.ap.value_type())
- cdef getitem(self, int64_t i):
- cdef int64_t j = self.ap.value_offset(self.index) + i
- return box_scalar(self.value_type, self.ap.values(), j)
+cdef class LargeBinaryScalar(BinaryScalar):
+ pass
- cdef int64_t length(self):
- return self.ap.value_length(self.index)
- def as_py(self):
- """
- Return this value as a Python list.
- """
- cdef:
- int64_t j
- list result = []
-
- for j in range(len(self)):
- result.append(self.getitem(j).as_py())
+cdef class FixedSizeBinaryScalar(BinaryScalar):
+ pass
- return result
-
-cdef class MapValue(ArrayValue):
+cdef class StringScalar(BinaryScalar):
"""
- Concrete class for map array elements.
+ Concrete class for string-like (utf8) scalars.
"""
- def __len__(self):
- """
- Return the number of values.
- """
- return self.length()
-
- def __getitem__(self, i):
- """
- Return the value at the given index.
- """
- return self.getitem(_normalize_index(i, self.length()))
-
- def __iter__(self):
- """
- Iterate over this element's values.
- """
- for i in range(len(self)):
- yield self.getitem(i)
- raise StopIteration
-
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CMapArray*> sp_array.get()
- self.key_type = pyarrow_wrap_data_type(self.ap.map_type().key_type())
- self.item_type = pyarrow_wrap_data_type(self.ap.map_type().item_type())
-
- cdef getitem(self, int64_t i):
- cdef int64_t j = self.ap.value_offset(self.index) + i
- return (box_scalar(self.key_type, self.ap.keys(), j),
- box_scalar(self.item_type, self.ap.items(), j))
-
- cdef int64_t length(self):
- return self.ap.value_length(self.index)
-
def as_py(self):
"""
- Return this value as a Python list of tuples, each containing a
- key and item.
+ Return this value as a Python string.
"""
- cdef:
- int64_t j
- list result = []
+ buffer = self.as_buffer()
+ return None if buffer is None else str(buffer, 'utf8')
- for j in range(len(self)):
- key, item = self.getitem(j)
- result.append((key.as_py(), item.as_py()))
- return result
+cdef class LargeStringScalar(StringScalar):
+ pass
-cdef class FixedSizeListValue(ArrayValue):
+cdef class ListScalar(Scalar):
"""
- Concrete class for fixed size list array elements.
+ Concrete class for list-like scalars.
"""
+ @property
+ def values(self):
+ cdef CListScalar* sp = <CListScalar*> self.wrapped.get()
+ if sp.is_valid:
+ return pyarrow_wrap_array(sp.value)
+ else:
+ return None
+
def __len__(self):
"""
Return the number of values.
"""
- return self.length()
+ return len(self.values)
def __getitem__(self, i):
"""
Return the value at the given index.
"""
- return self.getitem(_normalize_index(i, self.length()))
+ return self.values[_normalize_index(i, len(self))]
def __iter__(self):
"""
Iterate over this element's values.
"""
- for i in range(len(self)):
- yield self.getitem(i)
- raise StopIteration
-
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CFixedSizeListArray*> sp_array.get()
- self.value_type = pyarrow_wrap_data_type(self.ap.value_type())
-
- cdef getitem(self, int64_t i):
- cdef int64_t j = self.ap.value_offset(self.index) + i
- return box_scalar(self.value_type, self.ap.values(), j)
-
- cdef int64_t length(self):
- return self.ap.value_length(self.index)
+ return iter(self.values)
def as_py(self):
"""
Return this value as a Python list.
"""
- cdef:
- int64_t j
- list result = []
-
- for j in range(len(self)):
- result.append(self.getitem(j).as_py())
-
- return result
-
-
-cdef class UnionValue(ArrayValue):
- """
- Concrete class for union array elements.
- """
+ arr = self.values
+ return None if arr is None else arr.to_pylist()
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CUnionArray*> sp_array.get()
-
- cdef getitem(self, int64_t i):
- cdef int child_id = self.ap.child_id(i)
- cdef shared_ptr[CArray] child = self.ap.field(child_id)
- cdef CDenseUnionArray* dense
- if self.ap.mode() == _UnionMode_SPARSE:
- return box_scalar(self.type[child_id].type, child, i)
- else:
- dense = <CDenseUnionArray*> self.ap
- return box_scalar(self.type[child_id].type, child,
- dense.value_offset(i))
- def as_py(self):
- """
- Return this value as a Python object.
+cdef class FixedSizeListScalar(ListScalar):
+ pass
- The exact type depends on the underlying union member.
- """
- return self.getitem(self.index).as_py()
+cdef class LargeListScalar(ListScalar):
+ pass
-cdef class FixedSizeBinaryValue(ArrayValue):
- """
- Concrete class for fixed-size binary array elements.
- """
- def as_py(self):
- """
- Return this value as a Python bytes object.
- """
- cdef:
- CFixedSizeBinaryArray* ap
- CFixedSizeBinaryType* ap_type
- int32_t length
- const char* data
- ap = <CFixedSizeBinaryArray*> self.sp_array.get()
- ap_type = <CFixedSizeBinaryType*> ap.type().get()
- length = ap_type.byte_width()
- data = <const char*> ap.GetValue(self.index)
- return cp.PyBytes_FromStringAndSize(data, length)
-
-
-cdef class StructValue(ArrayValue):
+cdef class StructScalar(Scalar, collections.abc.Mapping):
"""
- Concrete class for struct array elements.
+ Concrete class for struct scalars.
"""
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CStructArray*> sp_array.get()
-
- def __getitem__(self, key):
- """
- Return the child value for the given field name.
- """
- cdef:
- CStructType* type
- int index
-
- type = <CStructType*> self.type.type
- index = type.GetFieldIndex(tobytes(key))
-
- if index < 0:
- raise KeyError(key)
-
- return pyarrow_wrap_array(self.ap.field(index))[self.index]
+ def __len__(self):
+ cdef CStructScalar* sp = <CStructScalar*> self.wrapped.get()
+ return sp.value.size()
- def as_py(self):
- """
- Return this value as a Python dict.
- """
+ def __iter__(self):
cdef:
- vector[shared_ptr[CField]] child_fields = self.type.type.fields()
-
- wrapped_arrays = [pyarrow_wrap_array(self.ap.field(i))
- for i in range(self.ap.num_fields())]
- child_names = [child.get().name() for child in child_fields]
- # Return the struct as a dict
- return {
- frombytes(name): child_array[self.index].as_py()
- for name, child_array in zip(child_names, wrapped_arrays)
- }
-
-
-cdef class DictionaryValue(ArrayValue):
- """
- Concrete class for dictionary-encoded array elements.
- """
-
- def as_py(self):
- """
- Return this value as a Python object.
-
- The exact type depends on the dictionary value type.
- """
- return self.dictionary_value.as_py()
-
- @property
- def index_value(self):
- """
- Return this value's underlying index as a ArrayValue of the right
- signed integer type.
- """
- cdef CDictionaryArray* darr = <CDictionaryArray*>(self.sp_array.get())
- indices = pyarrow_wrap_array(darr.indices())
- return indices[self.index]
-
- @property
- def dictionary_value(self):
- """
- Return this value's underlying dictionary value as a ArrayValue.
- """
- cdef CDictionaryArray* darr = <CDictionaryArray*>(self.sp_array.get())
- dictionary = pyarrow_wrap_array(darr.dictionary())
- return dictionary[self.index_value.as_py()]
-
-
-cdef dict _array_value_classes = {
- _Type_BOOL: BooleanValue,
- _Type_UINT8: UInt8Value,
- _Type_UINT16: UInt16Value,
- _Type_UINT32: UInt32Value,
- _Type_UINT64: UInt64Value,
- _Type_INT8: Int8Value,
- _Type_INT16: Int16Value,
- _Type_INT32: Int32Value,
- _Type_INT64: Int64Value,
- _Type_DATE32: Date32Value,
- _Type_DATE64: Date64Value,
- _Type_TIME32: Time32Value,
- _Type_TIME64: Time64Value,
- _Type_TIMESTAMP: TimestampValue,
- _Type_DURATION: DurationValue,
- _Type_HALF_FLOAT: HalfFloatValue,
- _Type_FLOAT: FloatValue,
- _Type_DOUBLE: DoubleValue,
- _Type_LIST: ListValue,
- _Type_LARGE_LIST: LargeListValue,
- _Type_MAP: MapValue,
- _Type_FIXED_SIZE_LIST: FixedSizeListValue,
- _Type_SPARSE_UNION: UnionValue,
- _Type_DENSE_UNION: UnionValue,
- _Type_BINARY: BinaryValue,
- _Type_STRING: StringValue,
- _Type_LARGE_BINARY: LargeBinaryValue,
- _Type_LARGE_STRING: LargeStringValue,
- _Type_FIXED_SIZE_BINARY: FixedSizeBinaryValue,
- _Type_DECIMAL: DecimalValue,
- _Type_STRUCT: StructValue,
- _Type_DICTIONARY: DictionaryValue,
-}
-
-cdef class ScalarValue(Scalar):
- """
- The base class for scalars.
- """
-
- def __init__(self):
- raise TypeError("Do not call {}'s constructor directly."
- .format(self.__class__.__name__))
-
- cdef void init(self, const shared_ptr[CScalar]& sp_scalar):
- self.sp_scalar = sp_scalar
-
- def __repr__(self):
- if hasattr(self, 'as_py'):
- return repr(self.as_py())
- else:
- return super(Scalar, self).__repr__()
+ CStructScalar* sp = <CStructScalar*> self.wrapped.get()
+ CStructType* dtype = <CStructType*> sp.type.get()
+ vector[shared_ptr[CField]] fields = dtype.fields()
- def __str__(self):
- if hasattr(self, 'as_py'):
- return str(self.as_py())
- else:
- return super(Scalar, self).__str__()
+ if sp.is_valid:
+ for i in range(dtype.num_fields()):
+ yield frombytes(fields[i].get().name())
- def __eq__(self, other):
- if hasattr(self, 'as_py'):
- if isinstance(other, ScalarValue):
- other = other.as_py()
- return self.as_py() == other
+ def __contains__(self, key):
+ try:
+ self[key]
+ except (KeyError, IndexError):
+ return False
else:
- raise NotImplemented(
- "Cannot compare scalars that don't support as_py()")
-
- def __hash__(self):
- return hash(self.as_py())
-
-
-cdef class NullScalar(ScalarValue):
- """
- Concrete class for null scalars.
- """
-
- def as_py(self):
- """
- Return this value as a Python None.
- """
- return None
-
-
-cdef class BooleanScalar(ScalarValue):
- """
- Concrete class for boolean scalars.
- """
+ return True
- def as_py(self):
- """
- Return this value as a Python bool.
+ def __getitem__(self, key):
"""
- cdef CBooleanScalar* sp = <CBooleanScalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
-
+ Return the child value for the given field.
-cdef class UInt8Scalar(ScalarValue):
- """
- Concrete class for uint8 scalars.
- """
+ Parameters
+ ----------
+ index : Union[int, str]
+ Index / position or name of the field.
- def as_py(self):
- """
- Return this value as a Python int.
+ Returns
+ -------
+ result : Scalar
"""
- cdef CUInt8Scalar* sp = <CUInt8Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
+ cdef:
+ CFieldRef ref
+ CStructScalar* sp = <CStructScalar*> self.wrapped.get()
+ if isinstance(key, (bytes, str)):
+ ref = CFieldRef(<c_string> tobytes(key))
+ elif isinstance(key, int):
+ ref = CFieldRef(<int> key)
+ else:
+ raise TypeError('Expected integer or string index')
-cdef class Int8Scalar(ScalarValue):
- """
- Concrete class for int8 scalars.
- """
+ try:
+ return Scalar.wrap(GetResultValue(sp.field(ref)))
+ except ArrowInvalid:
+ if isinstance(key, int):
+ raise IndexError(key)
+ else:
+ raise KeyError(key)
def as_py(self):
"""
- Return this value as a Python int.
+ Return this value as a Python dict.
"""
- cdef CInt8Scalar* sp = <CInt8Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
+ if self.is_valid:
+ return {k: v.as_py() for k, v in self.items()}
+ else:
+ return None
-cdef class UInt16Scalar(ScalarValue):
+cdef class MapScalar(ListScalar):
"""
- Concrete class for uint16 scalars.
+ Concrete class for map scalars.
"""
- def as_py(self):
+ def __getitem__(self, i):
"""
- Return this value as a Python int.
+ Return the value at the given index.
"""
- cdef CUInt16Scalar* sp = <CUInt16Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
-
-
-cdef class Int16Scalar(ScalarValue):
- """
- Concrete class for int16 scalars.
- """
+ arr = self.values
+ if arr is None:
+ raise IndexError(i)
+ dct = arr[_normalize_index(i, len(arr))]
+ return (dct['key'], dct['value'])
- def as_py(self):
+ def __iter__(self):
"""
- Return this value as a Python int.
+ Iterate over this element's values.
"""
- cdef CInt16Scalar* sp = <CInt16Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
-
-
-cdef class UInt32Scalar(ScalarValue):
- """
- Concrete class for uint32 scalars.
- """
+ arr = self.values
+ if arr is None:
+ return iter(zip(arr.field('key'), arr.field('value')))
+ else:
+ raise StopIteration
def as_py(self):
"""
- Return this value as a Python int.
+ Return this value as a Python list.
"""
- cdef CUInt32Scalar* sp = <CUInt32Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
+ arr = self.values
+ if arr is not None:
+ return list(zip(arr.field('key'), arr.field('value')))
+ else:
+ return None
-cdef class Int32Scalar(ScalarValue):
+cdef class DictionaryScalar(Scalar):
"""
- Concrete class for int32 scalars.
+ Concrete class for dictionary-encoded scalars.
"""
- def as_py(self):
+ @property
+ def index(self):
"""
- Return this value as a Python int.
+ Return this value's underlying index as a scalar.
"""
- cdef CInt32Scalar* sp = <CInt32Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
+ cdef CDictionaryScalar* sp = <CDictionaryScalar*> self.wrapped.get()
+ return Scalar.wrap(sp.value.index)
-
-cdef class UInt64Scalar(ScalarValue):
- """
- Concrete class for uint64 scalars.
- """
-
- def as_py(self):
+ @property
+ def value(self):
"""
- Return this value as a Python int.
+ Return the encoded value as a scalar.
"""
- cdef CUInt64Scalar* sp = <CUInt64Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
+ # TODO(kszucs): optimize it to spare uneccessary python object boxing
+ return self.dictionary[self.index.as_py()] if self.is_valid else None
-
-cdef class Int64Scalar(ScalarValue):
- """
- Concrete class for int64 scalars.
- """
+ @property
+ def dictionary(self):
+ cdef CDictionaryScalar* sp = <CDictionaryScalar*> self.wrapped.get()
+ return pyarrow_wrap_array(sp.value.dictionary)
def as_py(self):
"""
- Return this value as a Python int.
+ Return this encoded value as a Python object.
"""
- cdef CInt64Scalar* sp = <CInt64Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
-
+ value = self.value
+ return None if value is None else value.as_py()
-cdef class FloatScalar(ScalarValue):
- """
- Concrete class for float scalars.
- """
+ @property
+ def index_value(self):
+ warnings.warn("`dictionary_value` property is deprecated as of 1.0.0"
+ "please use the `value` property instead",
+ FutureWarning)
+ index = self.index
+ return None if index is None else self.index
Review comment:
Updated.
##########
File path: python/pyarrow/scalar.pxi
##########
@@ -16,1198 +16,748 @@
# under the License.
-_NULL = NA = None
+import collections
cdef class Scalar:
"""
- The base class for all array elements.
- """
-
-
-cdef class NullType(Scalar):
- """
- Singleton for null array elements.
+ The base class for scalars.
"""
- # TODO rename this NullValue?
-
- def __cinit__(self):
- global NA
- if NA is not None:
- raise Exception('Cannot create multiple NAType instances')
-
- self.type = null()
-
- def __repr__(self):
- return 'NULL'
- def as_py(self):
- """
- Return None
- """
- return None
+ def __init__(self):
+ raise TypeError("Do not call {}'s constructor directly, use "
+ "pa.scalar() instead.".format(self.__class__.__name__))
- def __eq__(self, other):
- return NA
+ cdef void init(self, const shared_ptr[CScalar]& wrapped):
+ self.wrapped = wrapped
+ @staticmethod
+ cdef wrap(const shared_ptr[CScalar]& wrapped):
+ cdef:
+ Scalar self
+ Type type_id = wrapped.get().type.get().id()
-_NULL = NA = NullType()
+ if type_id == _Type_NA:
+ return _NULL
+ typ = _scalar_classes[type_id]
+ self = typ.__new__(typ)
+ self.init(wrapped)
-cdef class ArrayValue(Scalar):
- """
- The base class for non-null array elements.
- """
+ return self
- def __init__(self):
- raise TypeError("Do not call {}'s constructor directly, use array "
- "subscription instead."
- .format(self.__class__.__name__))
+ cdef inline shared_ptr[CScalar] unwrap(self) nogil:
+ return self.wrapped
- cdef void init(self, DataType type, const shared_ptr[CArray]& sp_array,
- int64_t index):
- self.type = type
- self.index = index
- self._set_array(sp_array)
+ @property
+ def type(self):
+ """
+ Data type of the Scalar object.
+ """
+ return pyarrow_wrap_data_type(self.wrapped.get().type)
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
+ @property
+ def is_valid(self):
+ """
+ Holds a valid (non-null) value.
+ """
+ return self.wrapped.get().is_valid
def __repr__(self):
- if hasattr(self, 'as_py'):
- return repr(self.as_py())
- else:
- return super(Scalar, self).__repr__()
+ return '<pyarrow.{}: {!r}>'.format(
+ self.__class__.__name__, self.as_py()
+ )
def __str__(self):
- if hasattr(self, 'as_py'):
- return str(self.as_py())
- else:
- return super(Scalar, self).__str__()
+ return str(self.as_py())
+
+ def equals(self, Scalar other):
+ return self.wrapped.get().Equals(other.unwrap().get()[0])
def __eq__(self, other):
- if hasattr(self, 'as_py'):
- if isinstance(other, ArrayValue):
- other = other.as_py()
- return self.as_py() == other
- else:
- raise NotImplementedError(
- "Cannot compare Arrow values that don't support as_py()")
+ try:
+ if not isinstance(other, Scalar):
+ other = scalar(other, type=self.type)
+ return self.equals(other)
+ except (TypeError, ValueError, ArrowInvalid):
+ return NotImplemented
def __hash__(self):
- return hash(self.as_py())
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+ def as_py(self):
+ raise NotImplementedError()
-cdef class BooleanValue(ArrayValue):
- """
- Concrete class for boolean array elements.
- """
- def as_py(self):
- """
- Return this value as a Python bool.
- """
- cdef CBooleanArray* ap = <CBooleanArray*> self.sp_array.get()
- return ap.Value(self.index)
+_NULL = NA = None
-cdef class Int8Value(ArrayValue):
+cdef class NullScalar(Scalar):
"""
- Concrete class for int8 array elements.
+ Concrete class for null scalars.
"""
- def as_py(self):
- """
- Return this value as a Python int.
- """
- cdef CInt8Array* ap = <CInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ def __cinit__(self):
+ global NA
+ if NA is not None:
+ raise RuntimeError('Cannot create multiple NullScalar instances')
+ self.init(shared_ptr[CScalar](new CNullScalar()))
+ def __init__(self):
+ pass
-cdef class UInt8Value(ArrayValue):
- """
- Concrete class for uint8 array elements.
- """
+ def __eq__(self, other):
+ return NA
+
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
def as_py(self):
"""
- Return this value as a Python int.
+ Return this value as a Python None.
"""
- cdef CUInt8Array* ap = <CUInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
-
+ return None
-cdef class Int16Value(ArrayValue):
- """
- Concrete class for int16 array elements.
- """
- def as_py(self):
- """
- Return this value as a Python int.
- """
- cdef CInt16Array* ap = <CInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+_NULL = NA = NullScalar()
-cdef class UInt16Value(ArrayValue):
+cdef class BooleanScalar(Scalar):
"""
- Concrete class for uint16 array elements.
+ Concrete class for boolean scalars.
"""
def as_py(self):
"""
- Return this value as a Python int.
+ Return this value as a Python bool.
"""
- cdef CUInt16Array* ap = <CUInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CBooleanScalar* sp = <CBooleanScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int32Value(ArrayValue):
+cdef class UInt8Scalar(Scalar):
"""
- Concrete class for int32 array elements.
+ Concrete class for uint8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt32Array* ap = <CInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt8Scalar* sp = <CUInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt32Value(ArrayValue):
+cdef class Int8Scalar(Scalar):
"""
- Concrete class for uint32 array elements.
+ Concrete class for int8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt32Array* ap = <CUInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt8Scalar* sp = <CInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int64Value(ArrayValue):
+cdef class UInt16Scalar(Scalar):
"""
- Concrete class for int64 array elements.
+ Concrete class for uint16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt64Array* ap = <CInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt16Scalar* sp = <CUInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt64Value(ArrayValue):
+cdef class Int16Scalar(Scalar):
"""
- Concrete class for uint64 array elements.
+ Concrete class for int16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt64Array* ap = <CUInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt16Scalar* sp = <CInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date32Value(ArrayValue):
+cdef class UInt32Scalar(Scalar):
"""
- Concrete class for date32 array elements.
+ Concrete class for uint32 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python int.
"""
- cdef CDate32Array* ap = <CDate32Array*> self.sp_array.get()
-
- # Shift to seconds since epoch
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(days=ap.Value(self.index)))
+ cdef CUInt32Scalar* sp = <CUInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date64Value(ArrayValue):
+cdef class Int32Scalar(Scalar):
"""
- Concrete class for date64 array elements.
+ Concrete class for int32 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python int.
"""
- cdef CDate64Array* ap = <CDate64Array*> self.sp_array.get()
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(
- days=ap.Value(self.index) / 86400000))
+ cdef CInt32Scalar* sp = <CInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Time32Value(ArrayValue):
+cdef class UInt64Scalar(Scalar):
"""
- Concrete class for time32 array elements.
+ Concrete class for uint64 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.timedelta instance.
+ Return this value as a Python int.
"""
- cdef:
- CTime32Array* ap = <CTime32Array*> self.sp_array.get()
- CTime32Type* dtype = <CTime32Type*> ap.type().get()
-
- if dtype.unit() == TimeUnit_SECOND:
- delta = datetime.timedelta(seconds=ap.Value(self.index))
- return (datetime.datetime(1970, 1, 1) + delta).time()
- else:
- return _box_time_milli(ap.Value(self.index))
+ cdef CUInt64Scalar* sp = <CUInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Time64Value(ArrayValue):
+cdef class Int64Scalar(Scalar):
"""
- Concrete class for time64 array elements.
+ Concrete class for int64 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.timedelta instance.
+ Return this value as a Python int.
"""
- cdef:
- CTime64Array* ap = <CTime64Array*> self.sp_array.get()
- CTime64Type* dtype = <CTime64Type*> ap.type().get()
-
- cdef int64_t val = ap.Value(self.index)
- if dtype.unit() == TimeUnit_MICRO:
- return _box_time_micro(val)
- else:
- return (datetime.datetime(1970, 1, 1) +
- datetime.timedelta(microseconds=val / 1000)).time()
-
-
-cpdef _box_time_milli(int64_t val):
- delta = datetime.timedelta(milliseconds=val)
- return (datetime.datetime(1970, 1, 1) + delta).time()
-
-
-cpdef _box_time_micro(int64_t val):
- return (datetime.datetime(1970, 1, 1) +
- datetime.timedelta(microseconds=val)).time()
-
-
-cdef dict _DATETIME_CONVERSION_FUNCTIONS = {}
-cdef c_bool _datetime_conversion_initialized = False
-
-
-cdef _add_micros_maybe_localize(dt, micros, tzinfo):
- import pytz
- dt = dt.replace(microsecond=micros)
- if tzinfo is not None:
- if not isinstance(tzinfo, datetime.tzinfo):
- tzinfo = string_to_tzinfo(tzinfo)
- dt = tzinfo.fromutc(dt)
- return dt
-
-
-cdef _datetime_from_seconds(int64_t v):
- return datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=v)
-
-
-def _nanoseconds_to_datetime_safe(v, tzinfo):
- if v % 1000 != 0:
- raise ValueError("Nanosecond timestamp {} is not safely convertible "
- " to microseconds to convert to datetime.datetime."
- " Install pandas to return as Timestamp with "
- " nanosecond support or access the .value attribute.")
- v = v // 1000
- micros = v % 1_000_000
-
- dt = _datetime_from_seconds(v // 1_000_000)
- return _add_micros_maybe_localize(dt, micros, tzinfo)
-
-
-def _microseconds_to_datetime(v, tzinfo):
- micros = v % 1_000_000
- dt = _datetime_from_seconds(v // 1_000_000)
- return _add_micros_maybe_localize(dt, micros, tzinfo)
-
-
-def _millis_to_datetime(v, tzinfo):
- millis = v % 1_000
- dt = _datetime_from_seconds(v // 1000)
- return _add_micros_maybe_localize(dt, millis * 1000, tzinfo)
-
-
-def _seconds_to_datetime(v, tzinfo):
- dt = _datetime_from_seconds(v)
- return _add_micros_maybe_localize(dt, 0, tzinfo)
-
-
-def _datetime_conversion_functions():
- global _datetime_conversion_initialized
- if _datetime_conversion_initialized:
- return _DATETIME_CONVERSION_FUNCTIONS
-
- _DATETIME_CONVERSION_FUNCTIONS.update({
- TimeUnit_SECOND: _seconds_to_datetime,
- TimeUnit_MILLI: _millis_to_datetime,
- TimeUnit_MICRO: _microseconds_to_datetime,
- TimeUnit_NANO: _nanoseconds_to_datetime_safe
- })
-
- try:
- import pandas as pd
- _DATETIME_CONVERSION_FUNCTIONS[TimeUnit_NANO] = (
- lambda x, tzinfo: pd.Timestamp(
- x, tz=tzinfo, unit='ns',
- )
- )
- except ImportError:
- pass
-
- _datetime_conversion_initialized = True
- return _DATETIME_CONVERSION_FUNCTIONS
+ cdef CInt64Scalar* sp = <CInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class TimestampValue(ArrayValue):
+cdef class HalfFloatScalar(Scalar):
"""
- Concrete class for timestamp array elements.
+ Concrete class for float scalars.
"""
- @property
- def value(self):
- cdef CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
- cdef CTimestampType* dtype = <CTimestampType*> ap.type().get()
- return ap.Value(self.index)
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+
+ def __eq__(self, other):
+ if isinstance(other, Scalar):
+ other = other.as_py()
+ return self.as_py() == other
def as_py(self):
"""
- Return this value as a Pandas Timestamp instance (if available),
- otherwise as a Python datetime.timedelta instance.
+ Return this value as a Python float.
"""
- cdef CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
- cdef CTimestampType* dtype = <CTimestampType*> ap.type().get()
-
- value = self.value
-
- if not dtype.timezone().empty():
- tzinfo = string_to_tzinfo(frombytes(dtype.timezone()))
- else:
- tzinfo = None
-
- try:
- converter = _datetime_conversion_functions()[dtype.unit()]
- except KeyError:
- raise ValueError(
- 'Cannot convert nanosecond timestamps without pandas'
- )
- return converter(value, tzinfo=tzinfo)
-
-
-cdef dict _TIMEDELTA_CONVERSION_FUNCTIONS = {}
-
-
-def _nanoseconds_to_timedelta_safe(v):
- if v % 1000 != 0:
- raise ValueError(
- "Nanosecond duration {} is not safely convertible to microseconds "
- "to convert to datetime.timedelta. Install pandas to return as "
- "Timedelta with nanosecond support or access the .value "
- "attribute.".format(v))
- micros = v // 1000
-
- return datetime.timedelta(microseconds=micros)
+ cdef CHalfFloatScalar* sp = <CHalfFloatScalar*> self.wrapped.get()
+ return PyHalf_FromHalf(sp.value) if sp.is_valid else None
-def _timedelta_conversion_functions():
- if _TIMEDELTA_CONVERSION_FUNCTIONS:
- return _TIMEDELTA_CONVERSION_FUNCTIONS
-
- _TIMEDELTA_CONVERSION_FUNCTIONS.update({
- TimeUnit_SECOND: lambda v: datetime.timedelta(seconds=v),
- TimeUnit_MILLI: lambda v: datetime.timedelta(milliseconds=v),
- TimeUnit_MICRO: lambda v: datetime.timedelta(microseconds=v),
- TimeUnit_NANO: _nanoseconds_to_timedelta_safe
- })
-
- try:
- import pandas as pd
- _TIMEDELTA_CONVERSION_FUNCTIONS[TimeUnit_NANO] = (
- lambda v: pd.Timedelta(v, unit='ns')
- )
- except ImportError:
- pass
-
- return _TIMEDELTA_CONVERSION_FUNCTIONS
-
-
-cdef class DurationValue(ArrayValue):
+cdef class FloatScalar(Scalar):
"""
- Concrete class for duration array elements.
+ Concrete class for float scalars.
"""
- @property
- def value(self):
- cdef CDurationArray* ap = <CDurationArray*> self.sp_array.get()
- return ap.Value(self.index)
-
def as_py(self):
"""
- Return this value as a Pandas Timestamp instance (if available),
- otherwise as a Python datetime.timedelta instance.
+ Return this value as a Python float.
"""
- cdef CDurationArray* ap = <CDurationArray*> self.sp_array.get()
- cdef CDurationType* dtype = <CDurationType*> ap.type().get()
-
- cdef int64_t value = ap.Value(self.index)
- converter = _timedelta_conversion_functions()[dtype.unit()]
- return converter(value)
+ cdef CFloatScalar* sp = <CFloatScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class HalfFloatValue(ArrayValue):
+cdef class DoubleScalar(Scalar):
"""
- Concrete class for float16 array elements.
+ Concrete class for double scalars.
"""
def as_py(self):
"""
Return this value as a Python float.
"""
- cdef CHalfFloatArray* ap = <CHalfFloatArray*> self.sp_array.get()
- return PyHalf_FromHalf(ap.Value(self.index))
+ cdef CDoubleScalar* sp = <CDoubleScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class FloatValue(ArrayValue):
+cdef class Decimal128Scalar(Scalar):
"""
- Concrete class for float32 array elements.
+ Concrete class for decimal128 scalars.
"""
def as_py(self):
"""
- Return this value as a Python float.
+ Return this value as a Python Decimal.
"""
- cdef CFloatArray* ap = <CFloatArray*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef:
+ CDecimal128Scalar* sp = <CDecimal128Scalar*> self.wrapped.get()
+ CDecimal128Type* dtype = <CDecimal128Type*> sp.type.get()
+ if sp.is_valid:
+ return _pydecimal.Decimal(
+ frombytes(sp.value.ToString(dtype.scale()))
+ )
+ else:
+ return None
-cdef class DoubleValue(ArrayValue):
+cdef class Date32Scalar(Scalar):
"""
- Concrete class for float64 array elements.
+ Concrete class for date32 scalars.
"""
def as_py(self):
"""
- Return this value as a Python float.
+ Return this value as a Python datetime.datetime instance.
"""
- cdef CDoubleArray* ap = <CDoubleArray*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CDate32Scalar* sp = <CDate32Scalar*> self.wrapped.get()
+
+ if sp.is_valid:
+ # shift to seconds since epoch
+ return (
+ datetime.date(1970, 1, 1) + datetime.timedelta(days=sp.value)
+ )
+ else:
+ return None
-cdef class DecimalValue(ArrayValue):
+cdef class Date64Scalar(Scalar):
"""
- Concrete class for decimal128 array elements.
+ Concrete class for date64 scalars.
"""
def as_py(self):
"""
- Return this value as a Python Decimal.
+ Return this value as a Python datetime.datetime instance.
"""
- cdef:
- CDecimal128Array* ap = <CDecimal128Array*> self.sp_array.get()
- c_string s = ap.FormatValue(self.index)
- return _pydecimal.Decimal(s.decode('utf8'))
+ cdef CDate64Scalar* sp = <CDate64Scalar*> self.wrapped.get()
+ if sp.is_valid:
+ return (
+ datetime.date(1970, 1, 1) +
+ datetime.timedelta(days=sp.value / 86400000)
+ )
+ else:
+ return None
-cdef class StringValue(ArrayValue):
- """
- Concrete class for string (utf8) array elements.
- """
- def as_py(self):
- """
- Return this value as a Python unicode string.
- """
- cdef CStringArray* ap = <CStringArray*> self.sp_array.get()
- return ap.GetString(self.index).decode('utf-8')
+def _datetime_from_int(int64_t value, TimeUnit unit, tzinfo=None):
+ if unit == TimeUnit_SECOND:
+ delta = datetime.timedelta(seconds=value)
+ elif unit == TimeUnit_MILLI:
+ delta = datetime.timedelta(milliseconds=value)
+ elif unit == TimeUnit_MICRO:
+ delta = datetime.timedelta(microseconds=value)
+ else:
+ # TimeUnit_NANO: prefer pandas timestamps if available
+ if _pandas_api.have_pandas:
+ return _pandas_api.pd.Timestamp(value, tz=tzinfo, unit='ns')
+ # otherwise safely truncate to microsecond resolution datetime
+ if value % 1000 != 0:
+ raise ValueError(
+ "Nanosecond resolution temporal type {} is not safely "
+ "convertible to microseconds to convert to datetime.datetime. "
+ "Install pandas to return as Timestamp with nanosecond "
+ "support or access the .value attribute.".format(value)
+ )
+ delta = datetime.timedelta(microseconds=value // 1000)
- def as_buffer(self):
- """
- Return a view over this value as a Buffer object.
- """
- cdef:
- CStringArray* ap = <CStringArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
+ dt = datetime.datetime(1970, 1, 1) + delta
+ # adjust timezone if set to the datatype
+ if tzinfo is not None:
+ dt = tzinfo.fromutc(dt)
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
+ return dt
-cdef class LargeStringValue(ArrayValue):
+cdef class Time32Scalar(Scalar):
"""
- Concrete class for large string (utf8) array elements.
+ Concrete class for time32 scalars.
"""
def as_py(self):
"""
- Return this value as a Python unicode string.
- """
- cdef CLargeStringArray* ap = <CLargeStringArray*> self.sp_array.get()
- return ap.GetString(self.index).decode('utf-8')
-
- def as_buffer(self):
- """
- Return a view over this value as a Buffer object.
+ Return this value as a Python datetime.timedelta instance.
"""
cdef:
- CLargeStringArray* ap = <CLargeStringArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
+ CTime32Scalar* sp = <CTime32Scalar*> self.wrapped.get()
+ CTime32Type* dtype = <CTime32Type*> sp.type.get()
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
+ if sp.is_valid:
+ return _datetime_from_int(sp.value, unit=dtype.unit()).time()
+ else:
+ return None
-cdef class BinaryValue(ArrayValue):
+cdef class Time64Scalar(Scalar):
"""
- Concrete class for variable-sized binary array elements.
+ Concrete class for time64 scalars.
"""
def as_py(self):
"""
- Return this value as a Python bytes object.
- """
- cdef:
- const uint8_t* ptr
- int32_t length
- CBinaryArray* ap = <CBinaryArray*> self.sp_array.get()
-
- ptr = ap.GetValue(self.index, &length)
- return cp.PyBytes_FromStringAndSize(<const char*>(ptr), length)
-
- def as_buffer(self):
- """
- Return a view over this value as a Buffer object.
+ Return this value as a Python datetime.timedelta instance.
"""
cdef:
- CBinaryArray* ap = <CBinaryArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
+ CTime64Scalar* sp = <CTime64Scalar*> self.wrapped.get()
+ CTime64Type* dtype = <CTime64Type*> sp.type.get()
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
+ if sp.is_valid:
+ return _datetime_from_int(sp.value, unit=dtype.unit()).time()
+ else:
+ return None
-cdef class LargeBinaryValue(ArrayValue):
+cdef class TimestampScalar(Scalar):
"""
- Concrete class for large variable-sized binary array elements.
+ Concrete class for timestamp scalars.
"""
+ @property
+ def value(self):
+ cdef CTimestampScalar* sp = <CTimestampScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
+
def as_py(self):
"""
- Return this value as a Python bytes object.
+ Return this value as a Pandas Timestamp instance (if available),
+ otherwise as a Python datetime.timedelta instance.
"""
cdef:
- const uint8_t* ptr
- int64_t length
- CLargeBinaryArray* ap = <CLargeBinaryArray*> self.sp_array.get()
+ CTimestampScalar* sp = <CTimestampScalar*> self.wrapped.get()
+ CTimestampType* dtype = <CTimestampType*> sp.type.get()
- ptr = ap.GetValue(self.index, &length)
- return cp.PyBytes_FromStringAndSize(<const char*>(ptr), length)
+ if not sp.is_valid:
+ return None
- def as_buffer(self):
- """
- Return a view over this value as a Buffer object.
- """
- cdef:
- CLargeBinaryArray* ap = <CLargeBinaryArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
+ if not dtype.timezone().empty():
+ tzinfo = string_to_tzinfo(frombytes(dtype.timezone()))
+ else:
+ tzinfo = None
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
+ return _datetime_from_int(sp.value, unit=dtype.unit(), tzinfo=tzinfo)
-cdef class ListValue(ArrayValue):
+cdef class DurationScalar(Scalar):
"""
- Concrete class for list array elements.
+ Concrete class for duration scalars.
"""
- def __len__(self):
- """
- Return the number of values.
- """
- return self.length()
-
- def __getitem__(self, i):
- """
- Return the value at the given index.
- """
- return self.getitem(_normalize_index(i, self.length()))
-
- def __iter__(self):
- """
- Iterate over this element's values.
- """
- for i in range(len(self)):
- yield self.getitem(i)
- raise StopIteration
-
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CListArray*> sp_array.get()
- self.value_type = pyarrow_wrap_data_type(self.ap.value_type())
-
- cdef getitem(self, int64_t i):
- cdef int64_t j = self.ap.value_offset(self.index) + i
- return box_scalar(self.value_type, self.ap.values(), j)
-
- cdef int64_t length(self):
- return self.ap.value_length(self.index)
+ @property
+ def value(self):
+ cdef CDurationScalar* sp = <CDurationScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
def as_py(self):
"""
- Return this value as a Python list.
+ Return this value as a Pandas Timestamp instance (if available),
+ otherwise as a Python datetime.timedelta instance.
"""
cdef:
- int64_t j
- list result = []
+ CDurationScalar* sp = <CDurationScalar*> self.wrapped.get()
+ CDurationType* dtype = <CDurationType*> sp.type.get()
+ TimeUnit unit = dtype.unit()
- for j in range(len(self)):
- result.append(self.getitem(j).as_py())
+ if not sp.is_valid:
+ return None
- return result
+ if unit == TimeUnit_SECOND:
+ return datetime.timedelta(seconds=sp.value)
+ elif unit == TimeUnit_MILLI:
+ return datetime.timedelta(milliseconds=sp.value)
+ elif unit == TimeUnit_MICRO:
+ return datetime.timedelta(microseconds=sp.value)
+ else:
+ # TimeUnit_NANO: prefer pandas timestamps if available
+ if _pandas_api.have_pandas:
+ return _pandas_api.pd.Timedelta(sp.value, unit='ns')
+ # otherwise safely truncate to microsecond resolution timedelta
+ if sp.value % 1000 != 0:
+ raise ValueError(
+ "Nanosecond duration {} is not safely convertible to "
+ "microseconds to convert to datetime.timedelta. Install "
+ "pandas to return as Timedelta with nanosecond support or "
+ "access the .value attribute.".format(sp.value)
+ )
+ return datetime.timedelta(microseconds=sp.value // 1000)
-cdef class LargeListValue(ArrayValue):
+cdef class BinaryScalar(Scalar):
"""
- Concrete class for large list array elements.
+ Concrete class for binary-like scalars.
"""
- def __len__(self):
- """
- Return the number of values.
- """
- return self.length()
-
- def __getitem__(self, i):
+ def as_buffer(self):
"""
- Return the value at the given index.
+ Return a view over this value as a Buffer object.
"""
- return self.getitem(_normalize_index(i, self.length()))
+ cdef CBinaryScalar* sp = <CBinaryScalar*> self.wrapped.get()
+ return pyarrow_wrap_buffer(sp.value) if sp.is_valid else None
- def __iter__(self):
+ def as_py(self):
"""
- Iterate over this element's values.
+ Return this value as a Python bytes.
"""
- for i in range(len(self)):
- yield self.getitem(i)
- raise StopIteration
+ buffer = self.as_buffer()
+ return None if buffer is None else buffer.to_pybytes()
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CLargeListArray*> sp_array.get()
- self.value_type = pyarrow_wrap_data_type(self.ap.value_type())
- cdef getitem(self, int64_t i):
- cdef int64_t j = self.ap.value_offset(self.index) + i
- return box_scalar(self.value_type, self.ap.values(), j)
+cdef class LargeBinaryScalar(BinaryScalar):
+ pass
- cdef int64_t length(self):
- return self.ap.value_length(self.index)
- def as_py(self):
- """
- Return this value as a Python list.
- """
- cdef:
- int64_t j
- list result = []
-
- for j in range(len(self)):
- result.append(self.getitem(j).as_py())
+cdef class FixedSizeBinaryScalar(BinaryScalar):
+ pass
- return result
-
-cdef class MapValue(ArrayValue):
+cdef class StringScalar(BinaryScalar):
"""
- Concrete class for map array elements.
+ Concrete class for string-like (utf8) scalars.
"""
- def __len__(self):
- """
- Return the number of values.
- """
- return self.length()
-
- def __getitem__(self, i):
- """
- Return the value at the given index.
- """
- return self.getitem(_normalize_index(i, self.length()))
-
- def __iter__(self):
- """
- Iterate over this element's values.
- """
- for i in range(len(self)):
- yield self.getitem(i)
- raise StopIteration
-
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CMapArray*> sp_array.get()
- self.key_type = pyarrow_wrap_data_type(self.ap.map_type().key_type())
- self.item_type = pyarrow_wrap_data_type(self.ap.map_type().item_type())
-
- cdef getitem(self, int64_t i):
- cdef int64_t j = self.ap.value_offset(self.index) + i
- return (box_scalar(self.key_type, self.ap.keys(), j),
- box_scalar(self.item_type, self.ap.items(), j))
-
- cdef int64_t length(self):
- return self.ap.value_length(self.index)
-
def as_py(self):
"""
- Return this value as a Python list of tuples, each containing a
- key and item.
+ Return this value as a Python string.
"""
- cdef:
- int64_t j
- list result = []
+ buffer = self.as_buffer()
+ return None if buffer is None else str(buffer, 'utf8')
- for j in range(len(self)):
- key, item = self.getitem(j)
- result.append((key.as_py(), item.as_py()))
- return result
+cdef class LargeStringScalar(StringScalar):
+ pass
-cdef class FixedSizeListValue(ArrayValue):
+cdef class ListScalar(Scalar):
"""
- Concrete class for fixed size list array elements.
+ Concrete class for list-like scalars.
"""
+ @property
+ def values(self):
+ cdef CListScalar* sp = <CListScalar*> self.wrapped.get()
+ if sp.is_valid:
+ return pyarrow_wrap_array(sp.value)
+ else:
+ return None
+
def __len__(self):
"""
Return the number of values.
"""
- return self.length()
+ return len(self.values)
def __getitem__(self, i):
"""
Return the value at the given index.
"""
- return self.getitem(_normalize_index(i, self.length()))
+ return self.values[_normalize_index(i, len(self))]
def __iter__(self):
"""
Iterate over this element's values.
"""
- for i in range(len(self)):
- yield self.getitem(i)
- raise StopIteration
-
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CFixedSizeListArray*> sp_array.get()
- self.value_type = pyarrow_wrap_data_type(self.ap.value_type())
-
- cdef getitem(self, int64_t i):
- cdef int64_t j = self.ap.value_offset(self.index) + i
- return box_scalar(self.value_type, self.ap.values(), j)
-
- cdef int64_t length(self):
- return self.ap.value_length(self.index)
+ return iter(self.values)
def as_py(self):
"""
Return this value as a Python list.
"""
- cdef:
- int64_t j
- list result = []
-
- for j in range(len(self)):
- result.append(self.getitem(j).as_py())
-
- return result
-
-
-cdef class UnionValue(ArrayValue):
- """
- Concrete class for union array elements.
- """
+ arr = self.values
+ return None if arr is None else arr.to_pylist()
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CUnionArray*> sp_array.get()
-
- cdef getitem(self, int64_t i):
- cdef int child_id = self.ap.child_id(i)
- cdef shared_ptr[CArray] child = self.ap.field(child_id)
- cdef CDenseUnionArray* dense
- if self.ap.mode() == _UnionMode_SPARSE:
- return box_scalar(self.type[child_id].type, child, i)
- else:
- dense = <CDenseUnionArray*> self.ap
- return box_scalar(self.type[child_id].type, child,
- dense.value_offset(i))
- def as_py(self):
- """
- Return this value as a Python object.
+cdef class FixedSizeListScalar(ListScalar):
+ pass
- The exact type depends on the underlying union member.
- """
- return self.getitem(self.index).as_py()
+cdef class LargeListScalar(ListScalar):
+ pass
-cdef class FixedSizeBinaryValue(ArrayValue):
- """
- Concrete class for fixed-size binary array elements.
- """
- def as_py(self):
- """
- Return this value as a Python bytes object.
- """
- cdef:
- CFixedSizeBinaryArray* ap
- CFixedSizeBinaryType* ap_type
- int32_t length
- const char* data
- ap = <CFixedSizeBinaryArray*> self.sp_array.get()
- ap_type = <CFixedSizeBinaryType*> ap.type().get()
- length = ap_type.byte_width()
- data = <const char*> ap.GetValue(self.index)
- return cp.PyBytes_FromStringAndSize(data, length)
-
-
-cdef class StructValue(ArrayValue):
+cdef class StructScalar(Scalar, collections.abc.Mapping):
"""
- Concrete class for struct array elements.
+ Concrete class for struct scalars.
"""
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
- self.ap = <CStructArray*> sp_array.get()
-
- def __getitem__(self, key):
- """
- Return the child value for the given field name.
- """
- cdef:
- CStructType* type
- int index
-
- type = <CStructType*> self.type.type
- index = type.GetFieldIndex(tobytes(key))
-
- if index < 0:
- raise KeyError(key)
-
- return pyarrow_wrap_array(self.ap.field(index))[self.index]
+ def __len__(self):
+ cdef CStructScalar* sp = <CStructScalar*> self.wrapped.get()
+ return sp.value.size()
- def as_py(self):
- """
- Return this value as a Python dict.
- """
+ def __iter__(self):
cdef:
- vector[shared_ptr[CField]] child_fields = self.type.type.fields()
-
- wrapped_arrays = [pyarrow_wrap_array(self.ap.field(i))
- for i in range(self.ap.num_fields())]
- child_names = [child.get().name() for child in child_fields]
- # Return the struct as a dict
- return {
- frombytes(name): child_array[self.index].as_py()
- for name, child_array in zip(child_names, wrapped_arrays)
- }
-
-
-cdef class DictionaryValue(ArrayValue):
- """
- Concrete class for dictionary-encoded array elements.
- """
-
- def as_py(self):
- """
- Return this value as a Python object.
-
- The exact type depends on the dictionary value type.
- """
- return self.dictionary_value.as_py()
-
- @property
- def index_value(self):
- """
- Return this value's underlying index as a ArrayValue of the right
- signed integer type.
- """
- cdef CDictionaryArray* darr = <CDictionaryArray*>(self.sp_array.get())
- indices = pyarrow_wrap_array(darr.indices())
- return indices[self.index]
-
- @property
- def dictionary_value(self):
- """
- Return this value's underlying dictionary value as a ArrayValue.
- """
- cdef CDictionaryArray* darr = <CDictionaryArray*>(self.sp_array.get())
- dictionary = pyarrow_wrap_array(darr.dictionary())
- return dictionary[self.index_value.as_py()]
-
-
-cdef dict _array_value_classes = {
- _Type_BOOL: BooleanValue,
- _Type_UINT8: UInt8Value,
- _Type_UINT16: UInt16Value,
- _Type_UINT32: UInt32Value,
- _Type_UINT64: UInt64Value,
- _Type_INT8: Int8Value,
- _Type_INT16: Int16Value,
- _Type_INT32: Int32Value,
- _Type_INT64: Int64Value,
- _Type_DATE32: Date32Value,
- _Type_DATE64: Date64Value,
- _Type_TIME32: Time32Value,
- _Type_TIME64: Time64Value,
- _Type_TIMESTAMP: TimestampValue,
- _Type_DURATION: DurationValue,
- _Type_HALF_FLOAT: HalfFloatValue,
- _Type_FLOAT: FloatValue,
- _Type_DOUBLE: DoubleValue,
- _Type_LIST: ListValue,
- _Type_LARGE_LIST: LargeListValue,
- _Type_MAP: MapValue,
- _Type_FIXED_SIZE_LIST: FixedSizeListValue,
- _Type_SPARSE_UNION: UnionValue,
- _Type_DENSE_UNION: UnionValue,
- _Type_BINARY: BinaryValue,
- _Type_STRING: StringValue,
- _Type_LARGE_BINARY: LargeBinaryValue,
- _Type_LARGE_STRING: LargeStringValue,
- _Type_FIXED_SIZE_BINARY: FixedSizeBinaryValue,
- _Type_DECIMAL: DecimalValue,
- _Type_STRUCT: StructValue,
- _Type_DICTIONARY: DictionaryValue,
-}
-
-cdef class ScalarValue(Scalar):
- """
- The base class for scalars.
- """
-
- def __init__(self):
- raise TypeError("Do not call {}'s constructor directly."
- .format(self.__class__.__name__))
-
- cdef void init(self, const shared_ptr[CScalar]& sp_scalar):
- self.sp_scalar = sp_scalar
-
- def __repr__(self):
- if hasattr(self, 'as_py'):
- return repr(self.as_py())
- else:
- return super(Scalar, self).__repr__()
+ CStructScalar* sp = <CStructScalar*> self.wrapped.get()
+ CStructType* dtype = <CStructType*> sp.type.get()
+ vector[shared_ptr[CField]] fields = dtype.fields()
- def __str__(self):
- if hasattr(self, 'as_py'):
- return str(self.as_py())
- else:
- return super(Scalar, self).__str__()
+ if sp.is_valid:
+ for i in range(dtype.num_fields()):
+ yield frombytes(fields[i].get().name())
- def __eq__(self, other):
- if hasattr(self, 'as_py'):
- if isinstance(other, ScalarValue):
- other = other.as_py()
- return self.as_py() == other
+ def __contains__(self, key):
+ try:
+ self[key]
+ except (KeyError, IndexError):
+ return False
else:
- raise NotImplemented(
- "Cannot compare scalars that don't support as_py()")
-
- def __hash__(self):
- return hash(self.as_py())
-
-
-cdef class NullScalar(ScalarValue):
- """
- Concrete class for null scalars.
- """
-
- def as_py(self):
- """
- Return this value as a Python None.
- """
- return None
-
-
-cdef class BooleanScalar(ScalarValue):
- """
- Concrete class for boolean scalars.
- """
+ return True
- def as_py(self):
- """
- Return this value as a Python bool.
+ def __getitem__(self, key):
"""
- cdef CBooleanScalar* sp = <CBooleanScalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
-
+ Return the child value for the given field.
-cdef class UInt8Scalar(ScalarValue):
- """
- Concrete class for uint8 scalars.
- """
+ Parameters
+ ----------
+ index : Union[int, str]
+ Index / position or name of the field.
- def as_py(self):
- """
- Return this value as a Python int.
+ Returns
+ -------
+ result : Scalar
"""
- cdef CUInt8Scalar* sp = <CUInt8Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
+ cdef:
+ CFieldRef ref
+ CStructScalar* sp = <CStructScalar*> self.wrapped.get()
+ if isinstance(key, (bytes, str)):
+ ref = CFieldRef(<c_string> tobytes(key))
+ elif isinstance(key, int):
+ ref = CFieldRef(<int> key)
+ else:
+ raise TypeError('Expected integer or string index')
-cdef class Int8Scalar(ScalarValue):
- """
- Concrete class for int8 scalars.
- """
+ try:
+ return Scalar.wrap(GetResultValue(sp.field(ref)))
+ except ArrowInvalid:
+ if isinstance(key, int):
+ raise IndexError(key)
+ else:
+ raise KeyError(key)
def as_py(self):
"""
- Return this value as a Python int.
+ Return this value as a Python dict.
"""
- cdef CInt8Scalar* sp = <CInt8Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
+ if self.is_valid:
+ return {k: v.as_py() for k, v in self.items()}
+ else:
+ return None
-cdef class UInt16Scalar(ScalarValue):
+cdef class MapScalar(ListScalar):
"""
- Concrete class for uint16 scalars.
+ Concrete class for map scalars.
"""
- def as_py(self):
+ def __getitem__(self, i):
"""
- Return this value as a Python int.
+ Return the value at the given index.
"""
- cdef CUInt16Scalar* sp = <CUInt16Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
-
-
-cdef class Int16Scalar(ScalarValue):
- """
- Concrete class for int16 scalars.
- """
+ arr = self.values
+ if arr is None:
+ raise IndexError(i)
+ dct = arr[_normalize_index(i, len(arr))]
+ return (dct['key'], dct['value'])
- def as_py(self):
+ def __iter__(self):
"""
- Return this value as a Python int.
+ Iterate over this element's values.
"""
- cdef CInt16Scalar* sp = <CInt16Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
-
-
-cdef class UInt32Scalar(ScalarValue):
- """
- Concrete class for uint32 scalars.
- """
+ arr = self.values
+ if arr is None:
+ return iter(zip(arr.field('key'), arr.field('value')))
+ else:
+ raise StopIteration
def as_py(self):
"""
- Return this value as a Python int.
+ Return this value as a Python list.
"""
- cdef CUInt32Scalar* sp = <CUInt32Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
+ arr = self.values
+ if arr is not None:
+ return list(zip(arr.field('key'), arr.field('value')))
+ else:
+ return None
-cdef class Int32Scalar(ScalarValue):
+cdef class DictionaryScalar(Scalar):
"""
- Concrete class for int32 scalars.
+ Concrete class for dictionary-encoded scalars.
"""
- def as_py(self):
+ @property
+ def index(self):
"""
- Return this value as a Python int.
+ Return this value's underlying index as a scalar.
"""
- cdef CInt32Scalar* sp = <CInt32Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
+ cdef CDictionaryScalar* sp = <CDictionaryScalar*> self.wrapped.get()
+ return Scalar.wrap(sp.value.index)
-
-cdef class UInt64Scalar(ScalarValue):
- """
- Concrete class for uint64 scalars.
- """
-
- def as_py(self):
+ @property
+ def value(self):
"""
- Return this value as a Python int.
+ Return the encoded value as a scalar.
"""
- cdef CUInt64Scalar* sp = <CUInt64Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
+ # TODO(kszucs): optimize it to spare uneccessary python object boxing
+ return self.dictionary[self.index.as_py()] if self.is_valid else None
-
-cdef class Int64Scalar(ScalarValue):
- """
- Concrete class for int64 scalars.
- """
+ @property
+ def dictionary(self):
+ cdef CDictionaryScalar* sp = <CDictionaryScalar*> self.wrapped.get()
+ return pyarrow_wrap_array(sp.value.dictionary)
def as_py(self):
"""
- Return this value as a Python int.
+ Return this encoded value as a Python object.
"""
- cdef CInt64Scalar* sp = <CInt64Scalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
-
+ value = self.value
+ return None if value is None else value.as_py()
-cdef class FloatScalar(ScalarValue):
- """
- Concrete class for float scalars.
- """
+ @property
+ def index_value(self):
+ warnings.warn("`dictionary_value` property is deprecated as of 1.0.0"
+ "please use the `value` property instead",
+ FutureWarning)
+ index = self.index
+ return None if index is None else self.index
- def as_py(self):
- """
- Return this value as a Python float.
- """
- cdef CFloatScalar* sp = <CFloatScalar*> self.sp_scalar.get()
- return sp.value if sp.is_valid else None
+ @property
+ def dictionary_value(self):
+ warnings.warn("`dictionary_value` property is deprecated as of 1.0.0, "
+ "please use the `value` property instead", FutureWarning)
+ value = self.value
+ return None if value is None else self.value
Review comment:
Updated.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] pitrou commented on a change in pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
pitrou commented on a change in pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#discussion_r450319790
##########
File path: cpp/src/arrow/scalar_test.cc
##########
@@ -451,6 +565,127 @@ TEST(TestStructScalar, FieldAccess) {
ASSERT_RAISES(Invalid, abc.field(5).status());
ASSERT_RAISES(Invalid, abc.field("c").status());
+
+ ASSERT_OK_AND_ASSIGN(auto d, abc.field("d"));
+ ASSERT_TRUE(d->Equals(MakeNullScalar(int64())));
+}
+
+TEST(TestDictionaryScalar, Basics) {
+ auto ty = dictionary(int8(), utf8());
+ auto dict = ArrayFromJSON(utf8(), "[\"alpha\", \"beta\", \"gamma\"]");
+
+ DictionaryScalar::ValueType alpha;
+ ASSERT_OK_AND_ASSIGN(alpha.index, MakeScalar(int8(), 0));
+ alpha.dictionary = dict;
+
+ DictionaryScalar::ValueType gamma;
+ ASSERT_OK_AND_ASSIGN(gamma.index, MakeScalar(int8(), 2));
+ gamma.dictionary = dict;
+
+ auto scalar_null = MakeNullScalar(ty);
+ auto scalar_alpha = DictionaryScalar(alpha, ty);
+ auto scalar_gamma = DictionaryScalar(gamma, ty);
+
+ ASSERT_OK_AND_ASSIGN(
+ auto encoded_null,
+ checked_cast<const DictionaryScalar&>(*scalar_null).GetEncodedValue());
+ ASSERT_TRUE(encoded_null->Equals(MakeNullScalar(utf8())));
+
+ ASSERT_OK_AND_ASSIGN(
+ auto encoded_alpha,
+ checked_cast<const DictionaryScalar&>(scalar_alpha).GetEncodedValue());
+ ASSERT_TRUE(encoded_alpha->Equals(MakeScalar("alpha")));
+
+ ASSERT_OK_AND_ASSIGN(
+ auto encoded_gamma,
+ checked_cast<const DictionaryScalar&>(scalar_gamma).GetEncodedValue());
+ ASSERT_TRUE(encoded_gamma->Equals(MakeScalar("gamma")));
+
+ // test Array.GetScalar
+ DictionaryArray arr(ty, ArrayFromJSON(int8(), "[2, 0, 1, null]"), dict);
+ ASSERT_OK_AND_ASSIGN(auto first, arr.GetScalar(0));
+ ASSERT_OK_AND_ASSIGN(auto second, arr.GetScalar(1));
+ ASSERT_OK_AND_ASSIGN(auto last, arr.GetScalar(3));
+
+ ASSERT_TRUE(first->Equals(scalar_gamma));
+ ASSERT_TRUE(second->Equals(scalar_alpha));
+ ASSERT_TRUE(last->Equals(scalar_null));
+}
+
+TEST(TestSparseUnionScalar, Basics) {
+ auto ty = sparse_union({field("string", utf8()), field("number", uint64())});
+
+ auto alpha = MakeScalar("alpha");
+ auto beta = MakeScalar("beta");
+ ASSERT_OK_AND_ASSIGN(auto two, MakeScalar(uint64(), 2));
+
+ auto scalar_alpha = SparseUnionScalar(alpha, ty);
+ auto scalar_beta = SparseUnionScalar(beta, ty);
+ auto scalar_two = SparseUnionScalar(two, ty);
+
+ // test Array.GetScalar
+ std::vector<std::shared_ptr<Array>> children{
+ ArrayFromJSON(utf8(), "[\"alpha\", \"\", \"beta\", null, \"gamma\"]"),
+ ArrayFromJSON(uint64(), "[1, 2, 11, 22, null]")};
+
+ auto type_ids = ArrayFromJSON(int8(), "[0, 1, 0, 0, 1]");
+ SparseUnionArray arr(ty, 5, children, type_ids->data()->buffers[1]);
+ ASSERT_OK(arr.ValidateFull());
+
+ ASSERT_OK_AND_ASSIGN(auto first, arr.GetScalar(0));
+ ASSERT_TRUE(first->Equals(scalar_alpha));
+
+ ASSERT_OK_AND_ASSIGN(auto second, arr.GetScalar(1));
+ ASSERT_TRUE(second->Equals(scalar_two));
+
+ ASSERT_OK_AND_ASSIGN(auto third, arr.GetScalar(2));
+ ASSERT_TRUE(third->Equals(scalar_beta));
+
+ ASSERT_OK_AND_ASSIGN(auto fourth, arr.GetScalar(3));
+ ASSERT_TRUE(fourth->Equals(MakeNullScalar(ty)));
Review comment:
Ah, I see. Yes, you're probably right. A pity we lose information about the value type, then.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] kszucs commented on a change in pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
kszucs commented on a change in pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#discussion_r450332885
##########
File path: cpp/src/arrow/scalar_test.cc
##########
@@ -451,6 +565,127 @@ TEST(TestStructScalar, FieldAccess) {
ASSERT_RAISES(Invalid, abc.field(5).status());
ASSERT_RAISES(Invalid, abc.field("c").status());
+
+ ASSERT_OK_AND_ASSIGN(auto d, abc.field("d"));
+ ASSERT_TRUE(d->Equals(MakeNullScalar(int64())));
+}
+
+TEST(TestDictionaryScalar, Basics) {
+ auto ty = dictionary(int8(), utf8());
+ auto dict = ArrayFromJSON(utf8(), "[\"alpha\", \"beta\", \"gamma\"]");
+
+ DictionaryScalar::ValueType alpha;
+ ASSERT_OK_AND_ASSIGN(alpha.index, MakeScalar(int8(), 0));
+ alpha.dictionary = dict;
+
+ DictionaryScalar::ValueType gamma;
+ ASSERT_OK_AND_ASSIGN(gamma.index, MakeScalar(int8(), 2));
+ gamma.dictionary = dict;
+
+ auto scalar_null = MakeNullScalar(ty);
+ auto scalar_alpha = DictionaryScalar(alpha, ty);
+ auto scalar_gamma = DictionaryScalar(gamma, ty);
+
+ ASSERT_OK_AND_ASSIGN(
+ auto encoded_null,
+ checked_cast<const DictionaryScalar&>(*scalar_null).GetEncodedValue());
+ ASSERT_TRUE(encoded_null->Equals(MakeNullScalar(utf8())));
+
+ ASSERT_OK_AND_ASSIGN(
+ auto encoded_alpha,
+ checked_cast<const DictionaryScalar&>(scalar_alpha).GetEncodedValue());
+ ASSERT_TRUE(encoded_alpha->Equals(MakeScalar("alpha")));
+
+ ASSERT_OK_AND_ASSIGN(
+ auto encoded_gamma,
+ checked_cast<const DictionaryScalar&>(scalar_gamma).GetEncodedValue());
+ ASSERT_TRUE(encoded_gamma->Equals(MakeScalar("gamma")));
+
+ // test Array.GetScalar
+ DictionaryArray arr(ty, ArrayFromJSON(int8(), "[2, 0, 1, null]"), dict);
+ ASSERT_OK_AND_ASSIGN(auto first, arr.GetScalar(0));
+ ASSERT_OK_AND_ASSIGN(auto second, arr.GetScalar(1));
+ ASSERT_OK_AND_ASSIGN(auto last, arr.GetScalar(3));
+
+ ASSERT_TRUE(first->Equals(scalar_gamma));
+ ASSERT_TRUE(second->Equals(scalar_alpha));
+ ASSERT_TRUE(last->Equals(scalar_null));
+}
+
+TEST(TestSparseUnionScalar, Basics) {
+ auto ty = sparse_union({field("string", utf8()), field("number", uint64())});
+
+ auto alpha = MakeScalar("alpha");
+ auto beta = MakeScalar("beta");
+ ASSERT_OK_AND_ASSIGN(auto two, MakeScalar(uint64(), 2));
+
+ auto scalar_alpha = SparseUnionScalar(alpha, ty);
+ auto scalar_beta = SparseUnionScalar(beta, ty);
+ auto scalar_two = SparseUnionScalar(two, ty);
+
+ // test Array.GetScalar
+ std::vector<std::shared_ptr<Array>> children{
+ ArrayFromJSON(utf8(), "[\"alpha\", \"\", \"beta\", null, \"gamma\"]"),
+ ArrayFromJSON(uint64(), "[1, 2, 11, 22, null]")};
+
+ auto type_ids = ArrayFromJSON(int8(), "[0, 1, 0, 0, 1]");
+ SparseUnionArray arr(ty, 5, children, type_ids->data()->buffers[1]);
+ ASSERT_OK(arr.ValidateFull());
+
+ ASSERT_OK_AND_ASSIGN(auto first, arr.GetScalar(0));
+ ASSERT_TRUE(first->Equals(scalar_alpha));
+
+ ASSERT_OK_AND_ASSIGN(auto second, arr.GetScalar(1));
+ ASSERT_TRUE(second->Equals(scalar_two));
+
+ ASSERT_OK_AND_ASSIGN(auto third, arr.GetScalar(2));
+ ASSERT_TRUE(third->Equals(scalar_beta));
+
+ ASSERT_OK_AND_ASSIGN(auto fourth, arr.GetScalar(3));
+ ASSERT_TRUE(fourth->Equals(MakeNullScalar(ty)));
Review comment:
Adding.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] pitrou commented on a change in pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
pitrou commented on a change in pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#discussion_r450314430
##########
File path: cpp/src/arrow/scalar_test.cc
##########
@@ -451,6 +565,127 @@ TEST(TestStructScalar, FieldAccess) {
ASSERT_RAISES(Invalid, abc.field(5).status());
ASSERT_RAISES(Invalid, abc.field("c").status());
+
+ ASSERT_OK_AND_ASSIGN(auto d, abc.field("d"));
+ ASSERT_TRUE(d->Equals(MakeNullScalar(int64())));
+}
+
+TEST(TestDictionaryScalar, Basics) {
+ auto ty = dictionary(int8(), utf8());
+ auto dict = ArrayFromJSON(utf8(), "[\"alpha\", \"beta\", \"gamma\"]");
+
+ DictionaryScalar::ValueType alpha;
+ ASSERT_OK_AND_ASSIGN(alpha.index, MakeScalar(int8(), 0));
+ alpha.dictionary = dict;
+
+ DictionaryScalar::ValueType gamma;
+ ASSERT_OK_AND_ASSIGN(gamma.index, MakeScalar(int8(), 2));
+ gamma.dictionary = dict;
+
+ auto scalar_null = MakeNullScalar(ty);
+ auto scalar_alpha = DictionaryScalar(alpha, ty);
+ auto scalar_gamma = DictionaryScalar(gamma, ty);
+
+ ASSERT_OK_AND_ASSIGN(
+ auto encoded_null,
+ checked_cast<const DictionaryScalar&>(*scalar_null).GetEncodedValue());
+ ASSERT_TRUE(encoded_null->Equals(MakeNullScalar(utf8())));
+
+ ASSERT_OK_AND_ASSIGN(
+ auto encoded_alpha,
+ checked_cast<const DictionaryScalar&>(scalar_alpha).GetEncodedValue());
+ ASSERT_TRUE(encoded_alpha->Equals(MakeScalar("alpha")));
+
+ ASSERT_OK_AND_ASSIGN(
+ auto encoded_gamma,
+ checked_cast<const DictionaryScalar&>(scalar_gamma).GetEncodedValue());
+ ASSERT_TRUE(encoded_gamma->Equals(MakeScalar("gamma")));
+
+ // test Array.GetScalar
+ DictionaryArray arr(ty, ArrayFromJSON(int8(), "[2, 0, 1, null]"), dict);
+ ASSERT_OK_AND_ASSIGN(auto first, arr.GetScalar(0));
+ ASSERT_OK_AND_ASSIGN(auto second, arr.GetScalar(1));
+ ASSERT_OK_AND_ASSIGN(auto last, arr.GetScalar(3));
+
+ ASSERT_TRUE(first->Equals(scalar_gamma));
+ ASSERT_TRUE(second->Equals(scalar_alpha));
+ ASSERT_TRUE(last->Equals(scalar_null));
+}
+
+TEST(TestSparseUnionScalar, Basics) {
+ auto ty = sparse_union({field("string", utf8()), field("number", uint64())});
+
+ auto alpha = MakeScalar("alpha");
+ auto beta = MakeScalar("beta");
+ ASSERT_OK_AND_ASSIGN(auto two, MakeScalar(uint64(), 2));
+
+ auto scalar_alpha = SparseUnionScalar(alpha, ty);
+ auto scalar_beta = SparseUnionScalar(beta, ty);
+ auto scalar_two = SparseUnionScalar(two, ty);
+
+ // test Array.GetScalar
+ std::vector<std::shared_ptr<Array>> children{
+ ArrayFromJSON(utf8(), "[\"alpha\", \"\", \"beta\", null, \"gamma\"]"),
+ ArrayFromJSON(uint64(), "[1, 2, 11, 22, null]")};
+
+ auto type_ids = ArrayFromJSON(int8(), "[0, 1, 0, 0, 1]");
+ SparseUnionArray arr(ty, 5, children, type_ids->data()->buffers[1]);
+ ASSERT_OK(arr.ValidateFull());
+
+ ASSERT_OK_AND_ASSIGN(auto first, arr.GetScalar(0));
+ ASSERT_TRUE(first->Equals(scalar_alpha));
+
+ ASSERT_OK_AND_ASSIGN(auto second, arr.GetScalar(1));
+ ASSERT_TRUE(second->Equals(scalar_two));
+
+ ASSERT_OK_AND_ASSIGN(auto third, arr.GetScalar(2));
+ ASSERT_TRUE(third->Equals(scalar_beta));
+
+ ASSERT_OK_AND_ASSIGN(auto fourth, arr.GetScalar(3));
+ ASSERT_TRUE(fourth->Equals(MakeNullScalar(ty)));
Review comment:
Hmm... interesting. Shouldn't it be `MakeNullScalar(utf8())`?
##########
File path: cpp/src/arrow/scalar_test.cc
##########
@@ -80,6 +84,15 @@ TYPED_TEST(TestNumericScalar, Basics) {
auto dyn_null_value = MakeNullScalar(expected_type);
ASSERT_EQ(*null_value, *dyn_null_value);
+
+ // test Array.GetScalar
+ auto arr = ArrayFromJSON(expected_type, "[null, 1, 2]");
+ ASSERT_OK_AND_ASSIGN(auto null, arr->GetScalar(0));
+ ASSERT_OK_AND_ASSIGN(auto one, arr->GetScalar(1));
+ ASSERT_OK_AND_ASSIGN(auto two, arr->GetScalar(2));
+ ASSERT_TRUE(null->Equals(*null_value));
+ ASSERT_TRUE(one->Equals(ScalarType(1)));
+ ASSERT_TRUE(two->Equals(ScalarType(2)));
Review comment:
You never check that `Equals` returns false?
##########
File path: cpp/src/arrow/scalar.cc
##########
@@ -185,6 +192,35 @@ DictionaryScalar::DictionaryScalar(std::shared_ptr<DataType> type)
0)
.ValueOrDie()} {}
+Result<std::shared_ptr<Scalar>> DictionaryScalar::GetEncodedValue() const {
+ const auto& dict_type = checked_cast<DictionaryType&>(*type);
+
+ if (!is_valid) {
+ return MakeNullScalar(dict_type.value_type());
+ }
+
+ int64_t index_value = 0;
+ switch (dict_type.index_type()->id()) {
+ case Type::INT8:
+ index_value = checked_cast<const Int8Scalar&>(*value.index).value;
+ break;
+ case Type::INT16:
+ index_value = checked_cast<const Int16Scalar&>(*value.index).value;
+ break;
+ case Type::INT32:
+ index_value = checked_cast<const Int32Scalar&>(*value.index).value;
+ break;
+ case Type::INT64:
+ index_value = checked_cast<const Int64Scalar&>(*value.index).value;
+ break;
+ default:
+ return Status::Invalid("Not implemented dictionary index type");
Review comment:
`Status::TypeError`?
##########
File path: cpp/src/arrow/scalar_test.cc
##########
@@ -451,6 +565,127 @@ TEST(TestStructScalar, FieldAccess) {
ASSERT_RAISES(Invalid, abc.field(5).status());
ASSERT_RAISES(Invalid, abc.field("c").status());
+
+ ASSERT_OK_AND_ASSIGN(auto d, abc.field("d"));
+ ASSERT_TRUE(d->Equals(MakeNullScalar(int64())));
+}
+
+TEST(TestDictionaryScalar, Basics) {
+ auto ty = dictionary(int8(), utf8());
+ auto dict = ArrayFromJSON(utf8(), "[\"alpha\", \"beta\", \"gamma\"]");
+
+ DictionaryScalar::ValueType alpha;
+ ASSERT_OK_AND_ASSIGN(alpha.index, MakeScalar(int8(), 0));
+ alpha.dictionary = dict;
+
+ DictionaryScalar::ValueType gamma;
+ ASSERT_OK_AND_ASSIGN(gamma.index, MakeScalar(int8(), 2));
+ gamma.dictionary = dict;
+
+ auto scalar_null = MakeNullScalar(ty);
+ auto scalar_alpha = DictionaryScalar(alpha, ty);
+ auto scalar_gamma = DictionaryScalar(gamma, ty);
+
+ ASSERT_OK_AND_ASSIGN(
+ auto encoded_null,
+ checked_cast<const DictionaryScalar&>(*scalar_null).GetEncodedValue());
+ ASSERT_TRUE(encoded_null->Equals(MakeNullScalar(utf8())));
+
+ ASSERT_OK_AND_ASSIGN(
+ auto encoded_alpha,
+ checked_cast<const DictionaryScalar&>(scalar_alpha).GetEncodedValue());
+ ASSERT_TRUE(encoded_alpha->Equals(MakeScalar("alpha")));
+
+ ASSERT_OK_AND_ASSIGN(
+ auto encoded_gamma,
+ checked_cast<const DictionaryScalar&>(scalar_gamma).GetEncodedValue());
+ ASSERT_TRUE(encoded_gamma->Equals(MakeScalar("gamma")));
+
+ // test Array.GetScalar
+ DictionaryArray arr(ty, ArrayFromJSON(int8(), "[2, 0, 1, null]"), dict);
+ ASSERT_OK_AND_ASSIGN(auto first, arr.GetScalar(0));
+ ASSERT_OK_AND_ASSIGN(auto second, arr.GetScalar(1));
+ ASSERT_OK_AND_ASSIGN(auto last, arr.GetScalar(3));
+
+ ASSERT_TRUE(first->Equals(scalar_gamma));
+ ASSERT_TRUE(second->Equals(scalar_alpha));
+ ASSERT_TRUE(last->Equals(scalar_null));
+}
+
+TEST(TestSparseUnionScalar, Basics) {
+ auto ty = sparse_union({field("string", utf8()), field("number", uint64())});
+
+ auto alpha = MakeScalar("alpha");
+ auto beta = MakeScalar("beta");
+ ASSERT_OK_AND_ASSIGN(auto two, MakeScalar(uint64(), 2));
+
+ auto scalar_alpha = SparseUnionScalar(alpha, ty);
+ auto scalar_beta = SparseUnionScalar(beta, ty);
+ auto scalar_two = SparseUnionScalar(two, ty);
+
+ // test Array.GetScalar
+ std::vector<std::shared_ptr<Array>> children{
+ ArrayFromJSON(utf8(), "[\"alpha\", \"\", \"beta\", null, \"gamma\"]"),
+ ArrayFromJSON(uint64(), "[1, 2, 11, 22, null]")};
+
+ auto type_ids = ArrayFromJSON(int8(), "[0, 1, 0, 0, 1]");
+ SparseUnionArray arr(ty, 5, children, type_ids->data()->buffers[1]);
+ ASSERT_OK(arr.ValidateFull());
+
+ ASSERT_OK_AND_ASSIGN(auto first, arr.GetScalar(0));
+ ASSERT_TRUE(first->Equals(scalar_alpha));
+
+ ASSERT_OK_AND_ASSIGN(auto second, arr.GetScalar(1));
+ ASSERT_TRUE(second->Equals(scalar_two));
+
+ ASSERT_OK_AND_ASSIGN(auto third, arr.GetScalar(2));
+ ASSERT_TRUE(third->Equals(scalar_beta));
+
+ ASSERT_OK_AND_ASSIGN(auto fourth, arr.GetScalar(3));
+ ASSERT_TRUE(fourth->Equals(MakeNullScalar(ty)));
+
+ ASSERT_OK_AND_ASSIGN(auto fifth, arr.GetScalar(4));
+ ASSERT_TRUE(fifth->Equals(MakeNullScalar(ty)));
Review comment:
Similarly, shouldn't it be `MakeNullScalar(uint64())`?
##########
File path: cpp/src/arrow/scalar_test.cc
##########
@@ -451,6 +565,127 @@ TEST(TestStructScalar, FieldAccess) {
ASSERT_RAISES(Invalid, abc.field(5).status());
ASSERT_RAISES(Invalid, abc.field("c").status());
+
+ ASSERT_OK_AND_ASSIGN(auto d, abc.field("d"));
+ ASSERT_TRUE(d->Equals(MakeNullScalar(int64())));
+}
+
+TEST(TestDictionaryScalar, Basics) {
+ auto ty = dictionary(int8(), utf8());
+ auto dict = ArrayFromJSON(utf8(), "[\"alpha\", \"beta\", \"gamma\"]");
+
+ DictionaryScalar::ValueType alpha;
+ ASSERT_OK_AND_ASSIGN(alpha.index, MakeScalar(int8(), 0));
+ alpha.dictionary = dict;
+
+ DictionaryScalar::ValueType gamma;
+ ASSERT_OK_AND_ASSIGN(gamma.index, MakeScalar(int8(), 2));
+ gamma.dictionary = dict;
+
+ auto scalar_null = MakeNullScalar(ty);
+ auto scalar_alpha = DictionaryScalar(alpha, ty);
+ auto scalar_gamma = DictionaryScalar(gamma, ty);
+
+ ASSERT_OK_AND_ASSIGN(
+ auto encoded_null,
+ checked_cast<const DictionaryScalar&>(*scalar_null).GetEncodedValue());
+ ASSERT_TRUE(encoded_null->Equals(MakeNullScalar(utf8())));
+
+ ASSERT_OK_AND_ASSIGN(
+ auto encoded_alpha,
+ checked_cast<const DictionaryScalar&>(scalar_alpha).GetEncodedValue());
+ ASSERT_TRUE(encoded_alpha->Equals(MakeScalar("alpha")));
+
+ ASSERT_OK_AND_ASSIGN(
+ auto encoded_gamma,
+ checked_cast<const DictionaryScalar&>(scalar_gamma).GetEncodedValue());
+ ASSERT_TRUE(encoded_gamma->Equals(MakeScalar("gamma")));
+
+ // test Array.GetScalar
+ DictionaryArray arr(ty, ArrayFromJSON(int8(), "[2, 0, 1, null]"), dict);
+ ASSERT_OK_AND_ASSIGN(auto first, arr.GetScalar(0));
+ ASSERT_OK_AND_ASSIGN(auto second, arr.GetScalar(1));
+ ASSERT_OK_AND_ASSIGN(auto last, arr.GetScalar(3));
+
+ ASSERT_TRUE(first->Equals(scalar_gamma));
+ ASSERT_TRUE(second->Equals(scalar_alpha));
+ ASSERT_TRUE(last->Equals(scalar_null));
+}
+
+TEST(TestSparseUnionScalar, Basics) {
+ auto ty = sparse_union({field("string", utf8()), field("number", uint64())});
+
+ auto alpha = MakeScalar("alpha");
+ auto beta = MakeScalar("beta");
+ ASSERT_OK_AND_ASSIGN(auto two, MakeScalar(uint64(), 2));
+
+ auto scalar_alpha = SparseUnionScalar(alpha, ty);
+ auto scalar_beta = SparseUnionScalar(beta, ty);
+ auto scalar_two = SparseUnionScalar(two, ty);
+
+ // test Array.GetScalar
+ std::vector<std::shared_ptr<Array>> children{
+ ArrayFromJSON(utf8(), "[\"alpha\", \"\", \"beta\", null, \"gamma\"]"),
+ ArrayFromJSON(uint64(), "[1, 2, 11, 22, null]")};
+
+ auto type_ids = ArrayFromJSON(int8(), "[0, 1, 0, 0, 1]");
+ SparseUnionArray arr(ty, 5, children, type_ids->data()->buffers[1]);
+ ASSERT_OK(arr.ValidateFull());
+
+ ASSERT_OK_AND_ASSIGN(auto first, arr.GetScalar(0));
+ ASSERT_TRUE(first->Equals(scalar_alpha));
+
+ ASSERT_OK_AND_ASSIGN(auto second, arr.GetScalar(1));
+ ASSERT_TRUE(second->Equals(scalar_two));
+
+ ASSERT_OK_AND_ASSIGN(auto third, arr.GetScalar(2));
+ ASSERT_TRUE(third->Equals(scalar_beta));
+
+ ASSERT_OK_AND_ASSIGN(auto fourth, arr.GetScalar(3));
+ ASSERT_TRUE(fourth->Equals(MakeNullScalar(ty)));
+
+ ASSERT_OK_AND_ASSIGN(auto fifth, arr.GetScalar(4));
+ ASSERT_TRUE(fifth->Equals(MakeNullScalar(ty)));
+}
+
+TEST(TestDenseUnionScalar, Basics) {
+ auto ty = dense_union({field("string", utf8()), field("number", uint64())});
+
+ auto alpha = MakeScalar("alpha");
+ auto beta = MakeScalar("beta");
+ ASSERT_OK_AND_ASSIGN(auto two, MakeScalar(uint64(), 2));
+ ASSERT_OK_AND_ASSIGN(auto three, MakeScalar(uint64(), 3));
+
+ auto scalar_alpha = DenseUnionScalar(alpha, ty);
+ auto scalar_beta = DenseUnionScalar(beta, ty);
+ auto scalar_two = DenseUnionScalar(two, ty);
+ auto scalar_three = DenseUnionScalar(three, ty);
+
+ // test Array.GetScalar
+ std::vector<std::shared_ptr<Array>> children = {
+ ArrayFromJSON(utf8(), "[\"alpha\", \"beta\", null]"),
+ ArrayFromJSON(uint64(), "[2, 3]")};
+
+ auto type_ids = ArrayFromJSON(int8(), "[0, 1, 0, 0, 1]");
+ auto offsets = ArrayFromJSON(int32(), "[0, 0, 1, 2, 1]");
+ DenseUnionArray arr(ty, 5, children, type_ids->data()->buffers[1],
+ offsets->data()->buffers[1]);
+ ASSERT_OK(arr.ValidateFull());
+
+ ASSERT_OK_AND_ASSIGN(auto first, arr.GetScalar(0));
+ ASSERT_TRUE(first->Equals(scalar_alpha));
+
+ ASSERT_OK_AND_ASSIGN(auto second, arr.GetScalar(1));
+ ASSERT_TRUE(second->Equals(scalar_two));
+
+ ASSERT_OK_AND_ASSIGN(auto third, arr.GetScalar(2));
+ ASSERT_TRUE(third->Equals(scalar_beta));
+
+ ASSERT_OK_AND_ASSIGN(auto fourth, arr.GetScalar(3));
+ ASSERT_TRUE(fourth->Equals(MakeNullScalar(ty)));
Review comment:
`MakeNullScalar(utf8)`?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] kszucs commented on a change in pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
kszucs commented on a change in pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#discussion_r445603562
##########
File path: python/pyarrow/util.py
##########
@@ -41,6 +41,24 @@ def wrapper(*args, **kwargs):
return wrapper
+def _deprecate_class(old_name, new_class, next_version,
+ instancecheck=True):
+ """
+ Raise warning if a deprecated class is used in an isinstance check.
Review comment:
The deprecated classes cannot be instantiated, so we don't need to worry about it for now - although we can add support for overriding `__init__` for later reuse.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] kszucs commented on a change in pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
kszucs commented on a change in pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#discussion_r446148546
##########
File path: python/pyarrow/scalar.pxi
##########
@@ -16,1198 +16,704 @@
# under the License.
-_NULL = NA = None
-
-
cdef class Scalar:
"""
- The base class for all array elements.
+ The base class for scalars.
"""
+ def __init__(self):
+ raise TypeError("Do not call {}'s constructor directly, use "
+ "pa.scalar() instead.".format(self.__class__.__name__))
-cdef class NullType(Scalar):
- """
- Singleton for null array elements.
- """
- # TODO rename this NullValue?
+ cdef void init(self, const shared_ptr[CScalar]& wrapped):
+ self.wrapped = wrapped
- def __cinit__(self):
- global NA
- if NA is not None:
- raise Exception('Cannot create multiple NAType instances')
+ @staticmethod
+ cdef wrap(const shared_ptr[CScalar]& wrapped):
+ cdef:
+ Scalar self
+ Type type_id = wrapped.get().type.get().id()
+
+ if type_id == _Type_NA:
+ return _NULL
+
+ typ = _scalar_classes[type_id]
+ self = typ.__new__(typ)
+ self.init(wrapped)
+
+ return self
+
+ cdef inline shared_ptr[CScalar] unwrap(self) nogil:
+ return self.wrapped
- self.type = null()
+ @property
+ def type(self):
+ return pyarrow_wrap_data_type(self.wrapped.get().type)
def __repr__(self):
- return 'NULL'
+ return '<pyarrow.{}: {!r}>'.format(
+ self.__class__.__name__, self.as_py()
+ )
- def as_py(self):
- """
- Return None
- """
- return None
+ def __str__(self):
+ return str(self.as_py())
def __eq__(self, other):
- return NA
+ # TODO(kszucs): use c++ Equals
+ if isinstance(other, Scalar):
+ other = other.as_py()
+ return self.as_py() == other
+ def __hash__(self):
+ # TODO(kszucs): use C++ hash if implemented for the type
+ return hash(self.as_py())
+
+ def as_py(self):
+ raise NotImplementedError()
-_NULL = NA = NullType()
+
+_NULL = NA = None
-cdef class ArrayValue(Scalar):
+cdef class NullScalar(Scalar):
"""
- The base class for non-null array elements.
+ Concrete class for null scalars.
"""
- def __init__(self):
- raise TypeError("Do not call {}'s constructor directly, use array "
- "subscription instead."
- .format(self.__class__.__name__))
-
- cdef void init(self, DataType type, const shared_ptr[CArray]& sp_array,
- int64_t index):
- self.type = type
- self.index = index
- self._set_array(sp_array)
+ def __cinit__(self):
+ global NA
+ if NA is not None:
+ raise Exception('Cannot create multiple NAType instances')
+ self.init(shared_ptr[CScalar](new CNullScalar()))
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
+ def __init__(self):
+ pass
- def __repr__(self):
- if hasattr(self, 'as_py'):
- return repr(self.as_py())
- else:
- return super(Scalar, self).__repr__()
+ def __eq__(self, other):
+ return NA
- def __str__(self):
- if hasattr(self, 'as_py'):
- return str(self.as_py())
- else:
- return super(Scalar, self).__str__()
+ def as_py(self):
+ """
+ Return this value as a Python None.
+ """
+ return None
- def __eq__(self, other):
- if hasattr(self, 'as_py'):
- if isinstance(other, ArrayValue):
- other = other.as_py()
- return self.as_py() == other
- else:
- raise NotImplementedError(
- "Cannot compare Arrow values that don't support as_py()")
- def __hash__(self):
- return hash(self.as_py())
+_NULL = NA = NullScalar()
-cdef class BooleanValue(ArrayValue):
+cdef class BooleanScalar(Scalar):
"""
- Concrete class for boolean array elements.
+ Concrete class for boolean scalars.
"""
def as_py(self):
"""
Return this value as a Python bool.
"""
- cdef CBooleanArray* ap = <CBooleanArray*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CBooleanScalar* sp = <CBooleanScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int8Value(ArrayValue):
+cdef class UInt8Scalar(Scalar):
"""
- Concrete class for int8 array elements.
+ Concrete class for uint8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt8Array* ap = <CInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt8Scalar* sp = <CUInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt8Value(ArrayValue):
+cdef class Int8Scalar(Scalar):
"""
- Concrete class for uint8 array elements.
+ Concrete class for int8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt8Array* ap = <CUInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt8Scalar* sp = <CInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int16Value(ArrayValue):
+cdef class UInt16Scalar(Scalar):
"""
- Concrete class for int16 array elements.
+ Concrete class for uint16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt16Array* ap = <CInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt16Scalar* sp = <CUInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt16Value(ArrayValue):
+cdef class Int16Scalar(Scalar):
"""
- Concrete class for uint16 array elements.
+ Concrete class for int16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt16Array* ap = <CUInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt16Scalar* sp = <CInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int32Value(ArrayValue):
+cdef class UInt32Scalar(Scalar):
"""
- Concrete class for int32 array elements.
+ Concrete class for uint32 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt32Array* ap = <CInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt32Scalar* sp = <CUInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt32Value(ArrayValue):
+cdef class Int32Scalar(Scalar):
"""
- Concrete class for uint32 array elements.
+ Concrete class for int32 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt32Array* ap = <CUInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt32Scalar* sp = <CInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int64Value(ArrayValue):
+cdef class UInt64Scalar(Scalar):
"""
- Concrete class for int64 array elements.
+ Concrete class for uint64 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt64Array* ap = <CInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt64Scalar* sp = <CUInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt64Value(ArrayValue):
+cdef class Int64Scalar(Scalar):
"""
- Concrete class for uint64 array elements.
+ Concrete class for int64 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt64Array* ap = <CUInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt64Scalar* sp = <CInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date32Value(ArrayValue):
+cdef class HalfFloatScalar(Scalar):
"""
- Concrete class for date32 array elements.
+ Concrete class for float scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python float.
"""
- cdef CDate32Array* ap = <CDate32Array*> self.sp_array.get()
+ cdef CHalfFloatScalar* sp = <CHalfFloatScalar*> self.wrapped.get()
+ return PyHalf_FromHalf(sp.value) if sp.is_valid else None
+
+
+cdef class FloatScalar(Scalar):
+ """
+ Concrete class for float scalars.
+ """
- # Shift to seconds since epoch
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(days=ap.Value(self.index)))
+ def as_py(self):
+ """
+ Return this value as a Python float.
+ """
+ cdef CFloatScalar* sp = <CFloatScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date64Value(ArrayValue):
+cdef class DoubleScalar(Scalar):
"""
- Concrete class for date64 array elements.
+ Concrete class for double scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python float.
"""
- cdef CDate64Array* ap = <CDate64Array*> self.sp_array.get()
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(
- days=ap.Value(self.index) / 86400000))
+ cdef CDoubleScalar* sp = <CDoubleScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Time32Value(ArrayValue):
+cdef class DecimalScalar(Scalar):
"""
- Concrete class for time32 array elements.
+ Concrete class for decimal128 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.timedelta instance.
+ Return this value as a Python Decimal.
"""
cdef:
- CTime32Array* ap = <CTime32Array*> self.sp_array.get()
- CTime32Type* dtype = <CTime32Type*> ap.type().get()
-
- if dtype.unit() == TimeUnit_SECOND:
- delta = datetime.timedelta(seconds=ap.Value(self.index))
- return (datetime.datetime(1970, 1, 1) + delta).time()
+ CDecimal128Scalar* sp = <CDecimal128Scalar*> self.wrapped.get()
+ CDecimal128Type* dtype = <CDecimal128Type*> sp.type.get()
+ if sp.is_valid:
+ return _pydecimal.Decimal(
+ frombytes(sp.value.ToString(dtype.scale()))
+ )
else:
- return _box_time_milli(ap.Value(self.index))
+ return None
-cdef class Time64Value(ArrayValue):
+cdef class Date32Scalar(Scalar):
"""
- Concrete class for time64 array elements.
+ Concrete class for date32 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.timedelta instance.
+ Return this value as a Python datetime.datetime instance.
"""
- cdef:
- CTime64Array* ap = <CTime64Array*> self.sp_array.get()
- CTime64Type* dtype = <CTime64Type*> ap.type().get()
+ cdef CDate32Scalar* sp = <CDate32Scalar*> self.wrapped.get()
- cdef int64_t val = ap.Value(self.index)
- if dtype.unit() == TimeUnit_MICRO:
- return _box_time_micro(val)
+ if sp.is_valid:
+ # shift to seconds since epoch
+ return (
+ datetime.date(1970, 1, 1) + datetime.timedelta(days=sp.value)
+ )
else:
- return (datetime.datetime(1970, 1, 1) +
- datetime.timedelta(microseconds=val / 1000)).time()
-
+ return None
-cpdef _box_time_milli(int64_t val):
- delta = datetime.timedelta(milliseconds=val)
- return (datetime.datetime(1970, 1, 1) + delta).time()
+cdef class Date64Scalar(Scalar):
+ """
+ Concrete class for date64 scalars.
+ """
-cpdef _box_time_micro(int64_t val):
- return (datetime.datetime(1970, 1, 1) +
- datetime.timedelta(microseconds=val)).time()
+ def as_py(self):
+ """
+ Return this value as a Python datetime.datetime instance.
+ """
+ cdef CDate64Scalar* sp = <CDate64Scalar*> self.wrapped.get()
+ if sp.is_valid:
+ return (
+ datetime.date(1970, 1, 1) +
+ datetime.timedelta(days=sp.value / 86400000)
+ )
+ else:
+ return None
-cdef dict _DATETIME_CONVERSION_FUNCTIONS = {}
-cdef c_bool _datetime_conversion_initialized = False
+def _datetime_from_int(int64_t value, TimeUnit unit, tzinfo=None):
+ if unit == TimeUnit_SECOND:
+ delta = datetime.timedelta(seconds=value)
+ elif unit == TimeUnit_MILLI:
+ delta = datetime.timedelta(milliseconds=value)
+ elif unit == TimeUnit_MICRO:
+ delta = datetime.timedelta(microseconds=value)
+ else:
+ # TimeUnit_NANO: prefer pandas timestamps if available
+ if _pandas_api.have_pandas:
+ return _pandas_api.pd.Timestamp(value, tz=tzinfo, unit='ns')
+ # otherwise safely truncate to microsecond resolution datetime
+ if value % 1000 != 0:
+ raise ValueError(
+ "Nanosecond resolution temporal type {} is not safely "
+ "convertible to microseconds to convert to datetime.datetime. "
+ "Install pandas to return as Timestamp with nanosecond "
+ "support or access the .value attribute.".format(value)
+ )
+ delta = datetime.timedelta(microseconds=value)
-cdef _add_micros_maybe_localize(dt, micros, tzinfo):
- import pytz
- dt = dt.replace(microsecond=micros)
+ dt = datetime.datetime(1970, 1, 1) + delta
+ # adjust timezone if set to the datatype
if tzinfo is not None:
- if not isinstance(tzinfo, datetime.tzinfo):
- tzinfo = string_to_tzinfo(tzinfo)
dt = tzinfo.fromutc(dt)
- return dt
-
-
-cdef _datetime_from_seconds(int64_t v):
- return datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=v)
-
-
-def _nanoseconds_to_datetime_safe(v, tzinfo):
- if v % 1000 != 0:
- raise ValueError("Nanosecond timestamp {} is not safely convertible "
- " to microseconds to convert to datetime.datetime."
- " Install pandas to return as Timestamp with "
- " nanosecond support or access the .value attribute.")
- v = v // 1000
- micros = v % 1_000_000
-
- dt = _datetime_from_seconds(v // 1_000_000)
- return _add_micros_maybe_localize(dt, micros, tzinfo)
-
-
-def _microseconds_to_datetime(v, tzinfo):
- micros = v % 1_000_000
- dt = _datetime_from_seconds(v // 1_000_000)
- return _add_micros_maybe_localize(dt, micros, tzinfo)
+ return dt
-def _millis_to_datetime(v, tzinfo):
- millis = v % 1_000
- dt = _datetime_from_seconds(v // 1000)
- return _add_micros_maybe_localize(dt, millis * 1000, tzinfo)
+cdef class Time32Scalar(Scalar):
+ """
+ Concrete class for time32 scalars.
+ """
-def _seconds_to_datetime(v, tzinfo):
- dt = _datetime_from_seconds(v)
- return _add_micros_maybe_localize(dt, 0, tzinfo)
+ def as_py(self):
+ """
+ Return this value as a Python datetime.timedelta instance.
+ """
+ cdef:
+ CTime32Scalar* sp = <CTime32Scalar*> self.wrapped.get()
+ CTime32Type* dtype = <CTime32Type*> sp.type.get()
+ if sp.is_valid:
+ return _datetime_from_int(sp.value, unit=dtype.unit()).time()
+ else:
+ return None
-def _datetime_conversion_functions():
- global _datetime_conversion_initialized
- if _datetime_conversion_initialized:
- return _DATETIME_CONVERSION_FUNCTIONS
- _DATETIME_CONVERSION_FUNCTIONS.update({
- TimeUnit_SECOND: _seconds_to_datetime,
- TimeUnit_MILLI: _millis_to_datetime,
- TimeUnit_MICRO: _microseconds_to_datetime,
- TimeUnit_NANO: _nanoseconds_to_datetime_safe
- })
+cdef class Time64Scalar(Scalar):
+ """
+ Concrete class for time64 scalars.
+ """
- try:
- import pandas as pd
- _DATETIME_CONVERSION_FUNCTIONS[TimeUnit_NANO] = (
- lambda x, tzinfo: pd.Timestamp(
- x, tz=tzinfo, unit='ns',
- )
- )
- except ImportError:
- pass
+ def as_py(self):
+ """
+ Return this value as a Python datetime.timedelta instance.
+ """
+ cdef:
+ CTime64Scalar* sp = <CTime64Scalar*> self.wrapped.get()
+ CTime64Type* dtype = <CTime64Type*> sp.type.get()
- _datetime_conversion_initialized = True
- return _DATETIME_CONVERSION_FUNCTIONS
+ if sp.is_valid:
+ return _datetime_from_int(sp.value, unit=dtype.unit()).time()
+ else:
+ return None
-cdef class TimestampValue(ArrayValue):
+cdef class TimestampScalar(Scalar):
"""
- Concrete class for timestamp array elements.
+ Concrete class for timestamp scalars.
"""
@property
def value(self):
- cdef CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
- cdef CTimestampType* dtype = <CTimestampType*> ap.type().get()
- return ap.Value(self.index)
+ cdef CTimestampScalar* sp = <CTimestampScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
def as_py(self):
"""
Return this value as a Pandas Timestamp instance (if available),
otherwise as a Python datetime.timedelta instance.
"""
- cdef CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
- cdef CTimestampType* dtype = <CTimestampType*> ap.type().get()
+ cdef:
+ CTimestampScalar* sp = <CTimestampScalar*> self.wrapped.get()
+ CTimestampType* dtype = <CTimestampType*> sp.type.get()
- value = self.value
+ if not sp.is_valid:
+ return None
if not dtype.timezone().empty():
tzinfo = string_to_tzinfo(frombytes(dtype.timezone()))
+ if not isinstance(tzinfo, datetime.tzinfo):
+ tzinfo = string_to_tzinfo(tzinfo)
else:
tzinfo = None
- try:
- converter = _datetime_conversion_functions()[dtype.unit()]
- except KeyError:
- raise ValueError(
- 'Cannot convert nanosecond timestamps without pandas'
- )
- return converter(value, tzinfo=tzinfo)
-
-
-cdef dict _TIMEDELTA_CONVERSION_FUNCTIONS = {}
-
-
-def _nanoseconds_to_timedelta_safe(v):
- if v % 1000 != 0:
- raise ValueError(
- "Nanosecond duration {} is not safely convertible to microseconds "
- "to convert to datetime.timedelta. Install pandas to return as "
- "Timedelta with nanosecond support or access the .value "
- "attribute.".format(v))
- micros = v // 1000
-
- return datetime.timedelta(microseconds=micros)
-
+ return _datetime_from_int(sp.value, unit=dtype.unit(), tzinfo=tzinfo)
-def _timedelta_conversion_functions():
- if _TIMEDELTA_CONVERSION_FUNCTIONS:
- return _TIMEDELTA_CONVERSION_FUNCTIONS
- _TIMEDELTA_CONVERSION_FUNCTIONS.update({
- TimeUnit_SECOND: lambda v: datetime.timedelta(seconds=v),
- TimeUnit_MILLI: lambda v: datetime.timedelta(milliseconds=v),
- TimeUnit_MICRO: lambda v: datetime.timedelta(microseconds=v),
- TimeUnit_NANO: _nanoseconds_to_timedelta_safe
- })
-
- try:
- import pandas as pd
- _TIMEDELTA_CONVERSION_FUNCTIONS[TimeUnit_NANO] = (
- lambda v: pd.Timedelta(v, unit='ns')
- )
- except ImportError:
- pass
-
- return _TIMEDELTA_CONVERSION_FUNCTIONS
-
-
-cdef class DurationValue(ArrayValue):
+cdef class DurationScalar(Scalar):
"""
- Concrete class for duration array elements.
+ Concrete class for duration scalars.
"""
@property
def value(self):
- cdef CDurationArray* ap = <CDurationArray*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CDurationScalar* sp = <CDurationScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
def as_py(self):
"""
Return this value as a Pandas Timestamp instance (if available),
otherwise as a Python datetime.timedelta instance.
"""
- cdef CDurationArray* ap = <CDurationArray*> self.sp_array.get()
- cdef CDurationType* dtype = <CDurationType*> ap.type().get()
-
- cdef int64_t value = ap.Value(self.index)
- converter = _timedelta_conversion_functions()[dtype.unit()]
- return converter(value)
-
+ cdef:
+ CDurationScalar* sp = <CDurationScalar*> self.wrapped.get()
+ CDurationType* dtype = <CDurationType*> sp.type.get()
+ TimeUnit unit = dtype.unit()
-cdef class HalfFloatValue(ArrayValue):
- """
- Concrete class for float16 array elements.
- """
+ if not sp.is_valid:
+ return None
- def as_py(self):
- """
- Return this value as a Python float.
- """
- cdef CHalfFloatArray* ap = <CHalfFloatArray*> self.sp_array.get()
- return PyHalf_FromHalf(ap.Value(self.index))
+ if unit == TimeUnit_SECOND:
+ return datetime.timedelta(seconds=sp.value)
+ elif unit == TimeUnit_MILLI:
+ return datetime.timedelta(milliseconds=sp.value)
+ elif unit == TimeUnit_MICRO:
+ return datetime.timedelta(microseconds=sp.value)
+ else:
+ # TimeUnit_NANO: prefer pandas timestamps if available
+ if _pandas_api.have_pandas:
+ return _pandas_api.pd.Timedelta(sp.value, unit='ns')
+ # otherwise safely truncate to microsecond resolution timedelta
+ if sp.value % 1000 != 0:
+ raise ValueError(
+ "Nanosecond duration {} is not safely convertible to "
+ "microseconds to convert to datetime.timedelta. Install "
+ "pandas to return as Timedelta with nanosecond support or "
+ "access the .value attribute.".format(sp.value)
+ )
+ return datetime.timedelta(microseconds=sp.value // 1000)
-cdef class FloatValue(ArrayValue):
+cdef class BinaryScalar(Scalar):
"""
- Concrete class for float32 array elements.
+ Concrete class for binary-like scalars.
"""
- def as_py(self):
+ def as_buffer(self):
"""
- Return this value as a Python float.
+ Return a view over this value as a Buffer object.
"""
- cdef CFloatArray* ap = <CFloatArray*> self.sp_array.get()
- return ap.Value(self.index)
-
-
-cdef class DoubleValue(ArrayValue):
- """
- Concrete class for float64 array elements.
- """
+ cdef CBinaryScalar* sp = <CBinaryScalar*> self.wrapped.get()
+ return pyarrow_wrap_buffer(sp.value) if sp.is_valid else None
def as_py(self):
"""
- Return this value as a Python float.
+ Return this value as a Python bytes.
"""
- cdef CDoubleArray* ap = <CDoubleArray*> self.sp_array.get()
- return ap.Value(self.index)
+ buffer = self.as_buffer()
+ if buffer is not None:
+ return self.as_buffer().to_pybytes()
+ else:
+ return None
-cdef class DecimalValue(ArrayValue):
- """
- Concrete class for decimal128 array elements.
- """
+cdef class LargeBinaryScalar(BinaryScalar):
+ pass
- def as_py(self):
- """
- Return this value as a Python Decimal.
- """
- cdef:
- CDecimal128Array* ap = <CDecimal128Array*> self.sp_array.get()
- c_string s = ap.FormatValue(self.index)
- return _pydecimal.Decimal(s.decode('utf8'))
+cdef class FixedSizeBinaryScalar(BinaryScalar):
+ pass
-cdef class StringValue(ArrayValue):
+
+cdef class StringScalar(BinaryScalar):
"""
- Concrete class for string (utf8) array elements.
+ Concrete class for string-like (utf8) scalars.
"""
def as_py(self):
"""
- Return this value as a Python unicode string.
+ Return this value as a Python string.
"""
- cdef CStringArray* ap = <CStringArray*> self.sp_array.get()
- return ap.GetString(self.index).decode('utf-8')
+ buffer = self.as_buffer()
+ if buffer is not None:
+ return frombytes(self.as_buffer().to_pybytes())
+ else:
+ return None
- def as_buffer(self):
- """
- Return a view over this value as a Buffer object.
- """
- cdef:
- CStringArray* ap = <CStringArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
+cdef class LargeStringScalar(StringScalar):
+ pass
-cdef class LargeStringValue(ArrayValue):
+cdef class ListScalar(Scalar):
"""
- Concrete class for large string (utf8) array elements.
+ Concrete class for list-like scalars.
"""
- def as_py(self):
+ cdef array(self):
+ cdef CListScalar* sp = <CListScalar*> self.wrapped.get()
+ if sp.is_valid:
+ return pyarrow_wrap_array(sp.value)
+ else:
+ return None
+
+ def __len__(self):
"""
- Return this value as a Python unicode string.
+ Return the number of values.
"""
- cdef CLargeStringArray* ap = <CLargeStringArray*> self.sp_array.get()
- return ap.GetString(self.index).decode('utf-8')
+ return len(self.array())
- def as_buffer(self):
+ def __getitem__(self, i):
"""
- Return a view over this value as a Buffer object.
+ Return the value at the given index.
"""
- cdef:
- CLargeStringArray* ap = <CLargeStringArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
-
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
+ return self.array()[_normalize_index(i, len(self))]
-
-cdef class BinaryValue(ArrayValue):
- """
- Concrete class for variable-sized binary array elements.
- """
+ def __iter__(self):
+ """
+ Iterate over this element's values.
+ """
+ return iter(self.array())
def as_py(self):
"""
- Return this value as a Python bytes object.
+ Return this value as a Python list.
"""
- cdef:
- const uint8_t* ptr
- int32_t length
- CBinaryArray* ap = <CBinaryArray*> self.sp_array.get()
+ arr = self.array()
+ return None if arr is None else arr.to_pylist()
- ptr = ap.GetValue(self.index, &length)
- return cp.PyBytes_FromStringAndSize(<const char*>(ptr), length)
- def as_buffer(self):
- """
- Return a view over this value as a Buffer object.
- """
- cdef:
- CBinaryArray* ap = <CBinaryArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
+cdef class FixedSizeListScalar(ListScalar):
+ pass
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
+cdef class LargeListScalar(ListScalar):
+ pass
-cdef class LargeBinaryValue(ArrayValue):
+
+cdef class StructScalar(Scalar):
"""
- Concrete class for large variable-sized binary array elements.
+ Concrete class for struct scalars.
"""
- def as_py(self):
+ def __len__(self):
+ cdef CStructScalar* sp = <CStructScalar*> self.wrapped.get()
+ return sp.value.size()
+
+ def __getitem__(self, key):
"""
- Return this value as a Python bytes object.
+ Return the child value for the given field.
+
+ Parameters
+ ----------
+ index : Union[int, str]
+ Index / position or name of the field.
+
+ Returns
+ -------
+ result : Scalar
"""
cdef:
- const uint8_t* ptr
- int64_t length
- CLargeBinaryArray* ap = <CLargeBinaryArray*> self.sp_array.get()
+ CFieldRef ref
+ CStructScalar* sp = <CStructScalar*> self.wrapped.get()
+
+ if isinstance(key, (bytes, str)):
+ ref = CFieldRef(<c_string> tobytes(key))
+ elif isinstance(key, int):
+ ref = CFieldRef(<int> key)
+ else:
+ raise TypeError('Expected integer or string index')
- ptr = ap.GetValue(self.index, &length)
- return cp.PyBytes_FromStringAndSize(<const char*>(ptr), length)
+ try:
+ return Scalar.wrap(GetResultValue(sp.field(ref)))
+ except ArrowInvalid:
+ raise IndexError(key)
- def as_buffer(self):
+ def as_py(self):
"""
- Return a view over this value as a Buffer object.
+ Return this value as a Python dict.
"""
cdef:
- CLargeBinaryArray* ap = <CLargeBinaryArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
+ CStructScalar* sp = <CStructScalar*> self.wrapped.get()
+ CStructType* dtype = <CStructType*> sp.type.get()
+ vector[shared_ptr[CField]] fields = dtype.fields()
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
+ if sp.is_valid:
+ return {frombytes(fields[i].get().name()): Scalar.wrap(sp.value[i])
Review comment:
Updated, also extended the StructScalar's API to match a python mapping.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] kszucs commented on a change in pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
kszucs commented on a change in pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#discussion_r448620980
##########
File path: python/pyarrow/scalar.pxi
##########
@@ -16,1198 +16,745 @@
# under the License.
-_NULL = NA = None
+import collections
cdef class Scalar:
"""
- The base class for all array elements.
+ The base class for scalars.
"""
+ def __init__(self):
+ raise TypeError("Do not call {}'s constructor directly, use "
+ "pa.scalar() instead.".format(self.__class__.__name__))
-cdef class NullType(Scalar):
- """
- Singleton for null array elements.
- """
- # TODO rename this NullValue?
+ cdef void init(self, const shared_ptr[CScalar]& wrapped):
+ self.wrapped = wrapped
- def __cinit__(self):
- global NA
- if NA is not None:
- raise Exception('Cannot create multiple NAType instances')
+ @staticmethod
+ cdef wrap(const shared_ptr[CScalar]& wrapped):
+ cdef:
+ Scalar self
+ Type type_id = wrapped.get().type.get().id()
+
+ if type_id == _Type_NA:
+ return _NULL
+
+ typ = _scalar_classes[type_id]
+ self = typ.__new__(typ)
+ self.init(wrapped)
+
+ return self
+
+ cdef inline shared_ptr[CScalar] unwrap(self) nogil:
+ return self.wrapped
+
+ @property
+ def type(self):
+ return pyarrow_wrap_data_type(self.wrapped.get().type)
- self.type = null()
+ @property
+ def is_valid(self):
+ return self.wrapped.get().is_valid
def __repr__(self):
- return 'NULL'
+ return '<pyarrow.{}: {!r}>'.format(
+ self.__class__.__name__, self.as_py()
+ )
- def as_py(self):
- """
- Return None
- """
- return None
+ def __str__(self):
+ return str(self.as_py())
+
+ def equals(self, Scalar other):
+ return self.wrapped.get().Equals(other.unwrap().get()[0])
def __eq__(self, other):
- return NA
+ try:
+ if not isinstance(other, Scalar):
+ other = scalar(other, type=self.type)
+ return self.equals(other)
+ except (TypeError, ValueError, ArrowInvalid):
+ return NotImplemented
+
+ def __hash__(self):
+ cdef CScalarHash hasher
+ return hasher(self.wrapped)
+
+ def as_py(self):
+ raise NotImplementedError()
-_NULL = NA = NullType()
+_NULL = NA = None
-cdef class ArrayValue(Scalar):
+cdef class NullScalar(Scalar):
"""
- The base class for non-null array elements.
+ Concrete class for null scalars.
"""
- def __init__(self):
- raise TypeError("Do not call {}'s constructor directly, use array "
- "subscription instead."
- .format(self.__class__.__name__))
+ def __cinit__(self):
+ global NA
+ if NA is not None:
+ raise Exception('Cannot create multiple NAType instances')
+ self.init(shared_ptr[CScalar](new CNullScalar()))
- cdef void init(self, DataType type, const shared_ptr[CArray]& sp_array,
- int64_t index):
- self.type = type
- self.index = index
- self._set_array(sp_array)
+ def __init__(self):
+ pass
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
+ def __eq__(self, other):
+ return NA
Review comment:
Indeed, the c++ impl should return the same.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] kszucs commented on a change in pull request #7519: ARROW-9017: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
kszucs commented on a change in pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#discussion_r449512190
##########
File path: python/pyarrow/tests/test_array.py
##########
@@ -583,12 +584,14 @@ def test_dictionary_from_numpy():
assert d2.dictionary.to_pylist() == dictionary.tolist()
for i in range(len(indices)):
+ assert d1[i].dictionary == dictionary
assert d1[i].as_py() == dictionary[indices[i]]
- if mask[i]:
- assert d2[i] is pa.NULL
- else:
- assert d2[i].as_py() == dictionary[indices[i]]
+ print(i, d2[i])
+ # if mask[i]:
+ # assert d2[i].as_py() is None
+ # else:
+ # assert d2[i].as_py() == dictionary[indices[i]]
Review comment:
Fixed.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [arrow] jorisvandenbossche commented on a change in pull request #7519: ARROW-9153: [C++][Python] Refactor scalar bindings
Posted by GitBox <gi...@apache.org>.
jorisvandenbossche commented on a change in pull request #7519:
URL: https://github.com/apache/arrow/pull/7519#discussion_r445523294
##########
File path: python/pyarrow/_dataset.pyx
##########
@@ -216,22 +216,18 @@ cdef class Expression:
@staticmethod
def _scalar(value):
cdef:
- shared_ptr[CScalar] scalar
-
- if value is None:
- scalar.reset(new CNullScalar())
- elif isinstance(value, bool):
- scalar = MakeScalar(<c_bool>value)
- elif isinstance(value, float):
- scalar = MakeScalar(<double>value)
- elif isinstance(value, int):
- scalar = MakeScalar(<int64_t>value)
- elif isinstance(value, (bytes, str)):
- scalar = MakeStringScalar(tobytes(value))
Review comment:
I think the `MakeStringScalar` included in libarrow.pxd can then be removed (I don't see any other usage of it)
##########
File path: python/pyarrow/tests/test_parquet.py
##########
@@ -2028,7 +2028,7 @@ def test_filters_invalid_pred_op(tempdir, use_legacy_dataset):
use_legacy_dataset=use_legacy_dataset)
assert dataset.read().num_rows == 0
- with pytest.raises(ValueError if use_legacy_dataset else TypeError):
+ with pytest.raises(ValueError if use_legacy_dataset else pa.ArrowInvalid):
# dataset API returns TypeError when trying create invalid comparison
Review comment:
```suggestion
with pytest.raises(ValueError):
```
ArrowInvalid is a ValueError, I think, so if this changed, the above should be sufficient
##########
File path: python/pyarrow/util.py
##########
@@ -41,6 +41,24 @@ def wrapper(*args, **kwargs):
return wrapper
+def _deprecate_class(old_name, new_class, next_version,
+ instancecheck=True):
+ """
+ Raise warning if a deprecated class is used in an isinstance check.
Review comment:
Shouldn't it also raise when instantiated?
##########
File path: python/pyarrow/scalar.pxi
##########
@@ -16,1198 +16,704 @@
# under the License.
-_NULL = NA = None
-
-
cdef class Scalar:
"""
- The base class for all array elements.
+ The base class for scalars.
"""
+ def __init__(self):
+ raise TypeError("Do not call {}'s constructor directly, use "
+ "pa.scalar() instead.".format(self.__class__.__name__))
-cdef class NullType(Scalar):
- """
- Singleton for null array elements.
- """
- # TODO rename this NullValue?
+ cdef void init(self, const shared_ptr[CScalar]& wrapped):
+ self.wrapped = wrapped
- def __cinit__(self):
- global NA
- if NA is not None:
- raise Exception('Cannot create multiple NAType instances')
+ @staticmethod
+ cdef wrap(const shared_ptr[CScalar]& wrapped):
+ cdef:
+ Scalar self
+ Type type_id = wrapped.get().type.get().id()
+
+ if type_id == _Type_NA:
+ return _NULL
+
+ typ = _scalar_classes[type_id]
+ self = typ.__new__(typ)
+ self.init(wrapped)
+
+ return self
+
+ cdef inline shared_ptr[CScalar] unwrap(self) nogil:
+ return self.wrapped
- self.type = null()
+ @property
+ def type(self):
+ return pyarrow_wrap_data_type(self.wrapped.get().type)
def __repr__(self):
- return 'NULL'
+ return '<pyarrow.{}: {!r}>'.format(
+ self.__class__.__name__, self.as_py()
+ )
- def as_py(self):
- """
- Return None
- """
- return None
+ def __str__(self):
+ return str(self.as_py())
def __eq__(self, other):
- return NA
+ # TODO(kszucs): use c++ Equals
+ if isinstance(other, Scalar):
+ other = other.as_py()
+ return self.as_py() == other
+ def __hash__(self):
+ # TODO(kszucs): use C++ hash if implemented for the type
+ return hash(self.as_py())
+
+ def as_py(self):
+ raise NotImplementedError()
-_NULL = NA = NullType()
+
+_NULL = NA = None
-cdef class ArrayValue(Scalar):
+cdef class NullScalar(Scalar):
"""
- The base class for non-null array elements.
+ Concrete class for null scalars.
"""
- def __init__(self):
- raise TypeError("Do not call {}'s constructor directly, use array "
- "subscription instead."
- .format(self.__class__.__name__))
-
- cdef void init(self, DataType type, const shared_ptr[CArray]& sp_array,
- int64_t index):
- self.type = type
- self.index = index
- self._set_array(sp_array)
+ def __cinit__(self):
+ global NA
+ if NA is not None:
+ raise Exception('Cannot create multiple NAType instances')
+ self.init(shared_ptr[CScalar](new CNullScalar()))
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
+ def __init__(self):
+ pass
- def __repr__(self):
- if hasattr(self, 'as_py'):
- return repr(self.as_py())
- else:
- return super(Scalar, self).__repr__()
+ def __eq__(self, other):
+ return NA
- def __str__(self):
- if hasattr(self, 'as_py'):
- return str(self.as_py())
- else:
- return super(Scalar, self).__str__()
+ def as_py(self):
+ """
+ Return this value as a Python None.
+ """
+ return None
- def __eq__(self, other):
- if hasattr(self, 'as_py'):
- if isinstance(other, ArrayValue):
- other = other.as_py()
- return self.as_py() == other
- else:
- raise NotImplementedError(
- "Cannot compare Arrow values that don't support as_py()")
- def __hash__(self):
- return hash(self.as_py())
+_NULL = NA = NullScalar()
-cdef class BooleanValue(ArrayValue):
+cdef class BooleanScalar(Scalar):
"""
- Concrete class for boolean array elements.
+ Concrete class for boolean scalars.
"""
def as_py(self):
"""
Return this value as a Python bool.
"""
- cdef CBooleanArray* ap = <CBooleanArray*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CBooleanScalar* sp = <CBooleanScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int8Value(ArrayValue):
+cdef class UInt8Scalar(Scalar):
"""
- Concrete class for int8 array elements.
+ Concrete class for uint8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt8Array* ap = <CInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt8Scalar* sp = <CUInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt8Value(ArrayValue):
+cdef class Int8Scalar(Scalar):
"""
- Concrete class for uint8 array elements.
+ Concrete class for int8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt8Array* ap = <CUInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt8Scalar* sp = <CInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int16Value(ArrayValue):
+cdef class UInt16Scalar(Scalar):
"""
- Concrete class for int16 array elements.
+ Concrete class for uint16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt16Array* ap = <CInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt16Scalar* sp = <CUInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt16Value(ArrayValue):
+cdef class Int16Scalar(Scalar):
"""
- Concrete class for uint16 array elements.
+ Concrete class for int16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt16Array* ap = <CUInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt16Scalar* sp = <CInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int32Value(ArrayValue):
+cdef class UInt32Scalar(Scalar):
"""
- Concrete class for int32 array elements.
+ Concrete class for uint32 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt32Array* ap = <CInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt32Scalar* sp = <CUInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt32Value(ArrayValue):
+cdef class Int32Scalar(Scalar):
"""
- Concrete class for uint32 array elements.
+ Concrete class for int32 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt32Array* ap = <CUInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt32Scalar* sp = <CInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int64Value(ArrayValue):
+cdef class UInt64Scalar(Scalar):
"""
- Concrete class for int64 array elements.
+ Concrete class for uint64 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt64Array* ap = <CInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt64Scalar* sp = <CUInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt64Value(ArrayValue):
+cdef class Int64Scalar(Scalar):
"""
- Concrete class for uint64 array elements.
+ Concrete class for int64 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt64Array* ap = <CUInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt64Scalar* sp = <CInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date32Value(ArrayValue):
+cdef class HalfFloatScalar(Scalar):
"""
- Concrete class for date32 array elements.
+ Concrete class for float scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python float.
"""
- cdef CDate32Array* ap = <CDate32Array*> self.sp_array.get()
+ cdef CHalfFloatScalar* sp = <CHalfFloatScalar*> self.wrapped.get()
+ return PyHalf_FromHalf(sp.value) if sp.is_valid else None
+
+
+cdef class FloatScalar(Scalar):
+ """
+ Concrete class for float scalars.
+ """
- # Shift to seconds since epoch
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(days=ap.Value(self.index)))
+ def as_py(self):
+ """
+ Return this value as a Python float.
+ """
+ cdef CFloatScalar* sp = <CFloatScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date64Value(ArrayValue):
+cdef class DoubleScalar(Scalar):
"""
- Concrete class for date64 array elements.
+ Concrete class for double scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python float.
"""
- cdef CDate64Array* ap = <CDate64Array*> self.sp_array.get()
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(
- days=ap.Value(self.index) / 86400000))
+ cdef CDoubleScalar* sp = <CDoubleScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Time32Value(ArrayValue):
+cdef class DecimalScalar(Scalar):
"""
- Concrete class for time32 array elements.
+ Concrete class for decimal128 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.timedelta instance.
+ Return this value as a Python Decimal.
"""
cdef:
- CTime32Array* ap = <CTime32Array*> self.sp_array.get()
- CTime32Type* dtype = <CTime32Type*> ap.type().get()
-
- if dtype.unit() == TimeUnit_SECOND:
- delta = datetime.timedelta(seconds=ap.Value(self.index))
- return (datetime.datetime(1970, 1, 1) + delta).time()
+ CDecimal128Scalar* sp = <CDecimal128Scalar*> self.wrapped.get()
+ CDecimal128Type* dtype = <CDecimal128Type*> sp.type.get()
+ if sp.is_valid:
+ return _pydecimal.Decimal(
+ frombytes(sp.value.ToString(dtype.scale()))
+ )
else:
- return _box_time_milli(ap.Value(self.index))
+ return None
-cdef class Time64Value(ArrayValue):
+cdef class Date32Scalar(Scalar):
"""
- Concrete class for time64 array elements.
+ Concrete class for date32 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.timedelta instance.
+ Return this value as a Python datetime.datetime instance.
"""
- cdef:
- CTime64Array* ap = <CTime64Array*> self.sp_array.get()
- CTime64Type* dtype = <CTime64Type*> ap.type().get()
+ cdef CDate32Scalar* sp = <CDate32Scalar*> self.wrapped.get()
- cdef int64_t val = ap.Value(self.index)
- if dtype.unit() == TimeUnit_MICRO:
- return _box_time_micro(val)
+ if sp.is_valid:
+ # shift to seconds since epoch
+ return (
+ datetime.date(1970, 1, 1) + datetime.timedelta(days=sp.value)
+ )
else:
- return (datetime.datetime(1970, 1, 1) +
- datetime.timedelta(microseconds=val / 1000)).time()
-
+ return None
-cpdef _box_time_milli(int64_t val):
- delta = datetime.timedelta(milliseconds=val)
- return (datetime.datetime(1970, 1, 1) + delta).time()
+cdef class Date64Scalar(Scalar):
+ """
+ Concrete class for date64 scalars.
+ """
-cpdef _box_time_micro(int64_t val):
- return (datetime.datetime(1970, 1, 1) +
- datetime.timedelta(microseconds=val)).time()
+ def as_py(self):
+ """
+ Return this value as a Python datetime.datetime instance.
+ """
+ cdef CDate64Scalar* sp = <CDate64Scalar*> self.wrapped.get()
+ if sp.is_valid:
+ return (
+ datetime.date(1970, 1, 1) +
+ datetime.timedelta(days=sp.value / 86400000)
+ )
+ else:
+ return None
-cdef dict _DATETIME_CONVERSION_FUNCTIONS = {}
-cdef c_bool _datetime_conversion_initialized = False
+def _datetime_from_int(int64_t value, TimeUnit unit, tzinfo=None):
+ if unit == TimeUnit_SECOND:
+ delta = datetime.timedelta(seconds=value)
+ elif unit == TimeUnit_MILLI:
+ delta = datetime.timedelta(milliseconds=value)
+ elif unit == TimeUnit_MICRO:
+ delta = datetime.timedelta(microseconds=value)
+ else:
+ # TimeUnit_NANO: prefer pandas timestamps if available
+ if _pandas_api.have_pandas:
+ return _pandas_api.pd.Timestamp(value, tz=tzinfo, unit='ns')
+ # otherwise safely truncate to microsecond resolution datetime
+ if value % 1000 != 0:
+ raise ValueError(
+ "Nanosecond resolution temporal type {} is not safely "
+ "convertible to microseconds to convert to datetime.datetime. "
+ "Install pandas to return as Timestamp with nanosecond "
+ "support or access the .value attribute.".format(value)
+ )
+ delta = datetime.timedelta(microseconds=value)
-cdef _add_micros_maybe_localize(dt, micros, tzinfo):
- import pytz
- dt = dt.replace(microsecond=micros)
+ dt = datetime.datetime(1970, 1, 1) + delta
+ # adjust timezone if set to the datatype
if tzinfo is not None:
- if not isinstance(tzinfo, datetime.tzinfo):
- tzinfo = string_to_tzinfo(tzinfo)
dt = tzinfo.fromutc(dt)
- return dt
-
-
-cdef _datetime_from_seconds(int64_t v):
- return datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=v)
-
-
-def _nanoseconds_to_datetime_safe(v, tzinfo):
- if v % 1000 != 0:
- raise ValueError("Nanosecond timestamp {} is not safely convertible "
- " to microseconds to convert to datetime.datetime."
- " Install pandas to return as Timestamp with "
- " nanosecond support or access the .value attribute.")
- v = v // 1000
- micros = v % 1_000_000
-
- dt = _datetime_from_seconds(v // 1_000_000)
- return _add_micros_maybe_localize(dt, micros, tzinfo)
-
-
-def _microseconds_to_datetime(v, tzinfo):
- micros = v % 1_000_000
- dt = _datetime_from_seconds(v // 1_000_000)
- return _add_micros_maybe_localize(dt, micros, tzinfo)
+ return dt
-def _millis_to_datetime(v, tzinfo):
- millis = v % 1_000
- dt = _datetime_from_seconds(v // 1000)
- return _add_micros_maybe_localize(dt, millis * 1000, tzinfo)
+cdef class Time32Scalar(Scalar):
+ """
+ Concrete class for time32 scalars.
+ """
-def _seconds_to_datetime(v, tzinfo):
- dt = _datetime_from_seconds(v)
- return _add_micros_maybe_localize(dt, 0, tzinfo)
+ def as_py(self):
+ """
+ Return this value as a Python datetime.timedelta instance.
+ """
+ cdef:
+ CTime32Scalar* sp = <CTime32Scalar*> self.wrapped.get()
+ CTime32Type* dtype = <CTime32Type*> sp.type.get()
+ if sp.is_valid:
+ return _datetime_from_int(sp.value, unit=dtype.unit()).time()
+ else:
+ return None
-def _datetime_conversion_functions():
- global _datetime_conversion_initialized
- if _datetime_conversion_initialized:
- return _DATETIME_CONVERSION_FUNCTIONS
- _DATETIME_CONVERSION_FUNCTIONS.update({
- TimeUnit_SECOND: _seconds_to_datetime,
- TimeUnit_MILLI: _millis_to_datetime,
- TimeUnit_MICRO: _microseconds_to_datetime,
- TimeUnit_NANO: _nanoseconds_to_datetime_safe
- })
+cdef class Time64Scalar(Scalar):
+ """
+ Concrete class for time64 scalars.
+ """
- try:
- import pandas as pd
- _DATETIME_CONVERSION_FUNCTIONS[TimeUnit_NANO] = (
- lambda x, tzinfo: pd.Timestamp(
- x, tz=tzinfo, unit='ns',
- )
- )
- except ImportError:
- pass
+ def as_py(self):
+ """
+ Return this value as a Python datetime.timedelta instance.
+ """
+ cdef:
+ CTime64Scalar* sp = <CTime64Scalar*> self.wrapped.get()
+ CTime64Type* dtype = <CTime64Type*> sp.type.get()
- _datetime_conversion_initialized = True
- return _DATETIME_CONVERSION_FUNCTIONS
+ if sp.is_valid:
+ return _datetime_from_int(sp.value, unit=dtype.unit()).time()
+ else:
+ return None
-cdef class TimestampValue(ArrayValue):
+cdef class TimestampScalar(Scalar):
"""
- Concrete class for timestamp array elements.
+ Concrete class for timestamp scalars.
"""
@property
def value(self):
- cdef CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
- cdef CTimestampType* dtype = <CTimestampType*> ap.type().get()
- return ap.Value(self.index)
+ cdef CTimestampScalar* sp = <CTimestampScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
def as_py(self):
"""
Return this value as a Pandas Timestamp instance (if available),
otherwise as a Python datetime.timedelta instance.
"""
- cdef CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
- cdef CTimestampType* dtype = <CTimestampType*> ap.type().get()
+ cdef:
+ CTimestampScalar* sp = <CTimestampScalar*> self.wrapped.get()
+ CTimestampType* dtype = <CTimestampType*> sp.type.get()
- value = self.value
+ if not sp.is_valid:
+ return None
if not dtype.timezone().empty():
tzinfo = string_to_tzinfo(frombytes(dtype.timezone()))
+ if not isinstance(tzinfo, datetime.tzinfo):
+ tzinfo = string_to_tzinfo(tzinfo)
else:
tzinfo = None
- try:
- converter = _datetime_conversion_functions()[dtype.unit()]
- except KeyError:
- raise ValueError(
- 'Cannot convert nanosecond timestamps without pandas'
- )
- return converter(value, tzinfo=tzinfo)
-
-
-cdef dict _TIMEDELTA_CONVERSION_FUNCTIONS = {}
-
-
-def _nanoseconds_to_timedelta_safe(v):
- if v % 1000 != 0:
- raise ValueError(
- "Nanosecond duration {} is not safely convertible to microseconds "
- "to convert to datetime.timedelta. Install pandas to return as "
- "Timedelta with nanosecond support or access the .value "
- "attribute.".format(v))
- micros = v // 1000
-
- return datetime.timedelta(microseconds=micros)
-
+ return _datetime_from_int(sp.value, unit=dtype.unit(), tzinfo=tzinfo)
-def _timedelta_conversion_functions():
- if _TIMEDELTA_CONVERSION_FUNCTIONS:
- return _TIMEDELTA_CONVERSION_FUNCTIONS
- _TIMEDELTA_CONVERSION_FUNCTIONS.update({
- TimeUnit_SECOND: lambda v: datetime.timedelta(seconds=v),
- TimeUnit_MILLI: lambda v: datetime.timedelta(milliseconds=v),
- TimeUnit_MICRO: lambda v: datetime.timedelta(microseconds=v),
- TimeUnit_NANO: _nanoseconds_to_timedelta_safe
- })
-
- try:
- import pandas as pd
- _TIMEDELTA_CONVERSION_FUNCTIONS[TimeUnit_NANO] = (
- lambda v: pd.Timedelta(v, unit='ns')
- )
- except ImportError:
- pass
-
- return _TIMEDELTA_CONVERSION_FUNCTIONS
-
-
-cdef class DurationValue(ArrayValue):
+cdef class DurationScalar(Scalar):
"""
- Concrete class for duration array elements.
+ Concrete class for duration scalars.
"""
@property
def value(self):
- cdef CDurationArray* ap = <CDurationArray*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CDurationScalar* sp = <CDurationScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
def as_py(self):
"""
Return this value as a Pandas Timestamp instance (if available),
otherwise as a Python datetime.timedelta instance.
"""
- cdef CDurationArray* ap = <CDurationArray*> self.sp_array.get()
- cdef CDurationType* dtype = <CDurationType*> ap.type().get()
-
- cdef int64_t value = ap.Value(self.index)
- converter = _timedelta_conversion_functions()[dtype.unit()]
- return converter(value)
-
+ cdef:
+ CDurationScalar* sp = <CDurationScalar*> self.wrapped.get()
+ CDurationType* dtype = <CDurationType*> sp.type.get()
+ TimeUnit unit = dtype.unit()
-cdef class HalfFloatValue(ArrayValue):
- """
- Concrete class for float16 array elements.
- """
+ if not sp.is_valid:
+ return None
- def as_py(self):
- """
- Return this value as a Python float.
- """
- cdef CHalfFloatArray* ap = <CHalfFloatArray*> self.sp_array.get()
- return PyHalf_FromHalf(ap.Value(self.index))
+ if unit == TimeUnit_SECOND:
+ return datetime.timedelta(seconds=sp.value)
+ elif unit == TimeUnit_MILLI:
+ return datetime.timedelta(milliseconds=sp.value)
+ elif unit == TimeUnit_MICRO:
+ return datetime.timedelta(microseconds=sp.value)
+ else:
+ # TimeUnit_NANO: prefer pandas timestamps if available
+ if _pandas_api.have_pandas:
+ return _pandas_api.pd.Timedelta(sp.value, unit='ns')
+ # otherwise safely truncate to microsecond resolution timedelta
+ if sp.value % 1000 != 0:
+ raise ValueError(
+ "Nanosecond duration {} is not safely convertible to "
+ "microseconds to convert to datetime.timedelta. Install "
+ "pandas to return as Timedelta with nanosecond support or "
+ "access the .value attribute.".format(sp.value)
+ )
+ return datetime.timedelta(microseconds=sp.value // 1000)
-cdef class FloatValue(ArrayValue):
+cdef class BinaryScalar(Scalar):
"""
- Concrete class for float32 array elements.
+ Concrete class for binary-like scalars.
"""
- def as_py(self):
+ def as_buffer(self):
"""
- Return this value as a Python float.
+ Return a view over this value as a Buffer object.
"""
- cdef CFloatArray* ap = <CFloatArray*> self.sp_array.get()
- return ap.Value(self.index)
-
-
-cdef class DoubleValue(ArrayValue):
- """
- Concrete class for float64 array elements.
- """
+ cdef CBinaryScalar* sp = <CBinaryScalar*> self.wrapped.get()
+ return pyarrow_wrap_buffer(sp.value) if sp.is_valid else None
def as_py(self):
"""
- Return this value as a Python float.
+ Return this value as a Python bytes.
"""
- cdef CDoubleArray* ap = <CDoubleArray*> self.sp_array.get()
- return ap.Value(self.index)
+ buffer = self.as_buffer()
+ if buffer is not None:
+ return self.as_buffer().to_pybytes()
+ else:
+ return None
-cdef class DecimalValue(ArrayValue):
- """
- Concrete class for decimal128 array elements.
- """
+cdef class LargeBinaryScalar(BinaryScalar):
+ pass
- def as_py(self):
- """
- Return this value as a Python Decimal.
- """
- cdef:
- CDecimal128Array* ap = <CDecimal128Array*> self.sp_array.get()
- c_string s = ap.FormatValue(self.index)
- return _pydecimal.Decimal(s.decode('utf8'))
+cdef class FixedSizeBinaryScalar(BinaryScalar):
+ pass
-cdef class StringValue(ArrayValue):
+
+cdef class StringScalar(BinaryScalar):
"""
- Concrete class for string (utf8) array elements.
+ Concrete class for string-like (utf8) scalars.
"""
def as_py(self):
"""
- Return this value as a Python unicode string.
+ Return this value as a Python string.
"""
- cdef CStringArray* ap = <CStringArray*> self.sp_array.get()
- return ap.GetString(self.index).decode('utf-8')
+ buffer = self.as_buffer()
+ if buffer is not None:
+ return frombytes(self.as_buffer().to_pybytes())
+ else:
+ return None
- def as_buffer(self):
- """
- Return a view over this value as a Buffer object.
- """
- cdef:
- CStringArray* ap = <CStringArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
+cdef class LargeStringScalar(StringScalar):
+ pass
-cdef class LargeStringValue(ArrayValue):
+cdef class ListScalar(Scalar):
"""
- Concrete class for large string (utf8) array elements.
+ Concrete class for list-like scalars.
"""
- def as_py(self):
+ cdef array(self):
Review comment:
do we want to expose this in some form publicly as well? (like you have the "raw" `value` for timestamps)
##########
File path: python/pyarrow/scalar.pxi
##########
@@ -16,1198 +16,704 @@
# under the License.
-_NULL = NA = None
-
-
cdef class Scalar:
"""
- The base class for all array elements.
+ The base class for scalars.
"""
+ def __init__(self):
+ raise TypeError("Do not call {}'s constructor directly, use "
+ "pa.scalar() instead.".format(self.__class__.__name__))
-cdef class NullType(Scalar):
- """
- Singleton for null array elements.
- """
- # TODO rename this NullValue?
+ cdef void init(self, const shared_ptr[CScalar]& wrapped):
+ self.wrapped = wrapped
- def __cinit__(self):
- global NA
- if NA is not None:
- raise Exception('Cannot create multiple NAType instances')
+ @staticmethod
+ cdef wrap(const shared_ptr[CScalar]& wrapped):
+ cdef:
+ Scalar self
+ Type type_id = wrapped.get().type.get().id()
+
+ if type_id == _Type_NA:
+ return _NULL
+
+ typ = _scalar_classes[type_id]
+ self = typ.__new__(typ)
+ self.init(wrapped)
+
+ return self
+
+ cdef inline shared_ptr[CScalar] unwrap(self) nogil:
+ return self.wrapped
- self.type = null()
+ @property
+ def type(self):
+ return pyarrow_wrap_data_type(self.wrapped.get().type)
def __repr__(self):
- return 'NULL'
+ return '<pyarrow.{}: {!r}>'.format(
Review comment:
Big +1 on this better repr! Just showing the repr of the as_py value was always very confusing :)
##########
File path: python/pyarrow/tests/test_scalars.py
##########
@@ -17,426 +17,395 @@
import datetime
import pytest
-import unittest
import numpy as np
import pyarrow as pa
-class TestScalars(unittest.TestCase):
-
- def test_null_singleton(self):
- with pytest.raises(Exception):
- pa.NAType()
+@pytest.mark.parametrize(['value', 'ty', 'klass', 'deprecated'], [
+ (None, None, pa.NullScalar, pa.NullType),
+ (False, None, pa.BooleanScalar, pa.BooleanValue),
+ (True, None, pa.BooleanScalar, pa.BooleanValue),
+ (1, None, pa.Int64Scalar, pa.Int64Value),
+ (-1, None, pa.Int64Scalar, pa.Int64Value),
+ (1, pa.int8(), pa.Int8Scalar, pa.Int8Value),
+ (1, pa.uint8(), pa.UInt8Scalar, pa.UInt8Value),
+ (1, pa.int16(), pa.Int16Scalar, pa.Int16Value),
+ (1, pa.uint16(), pa.UInt16Scalar, pa.UInt16Value),
+ (1, pa.int32(), pa.Int32Scalar, pa.Int32Value),
+ (1, pa.uint32(), pa.UInt32Scalar, pa.UInt32Value),
+ (1, pa.int64(), pa.Int64Scalar, pa.Int64Value),
+ (1, pa.uint64(), pa.UInt64Scalar, pa.UInt64Value),
+ (1.0, None, pa.DoubleScalar, pa.DoubleValue),
+ (np.float16(1.0), pa.float16(), pa.HalfFloatScalar, pa.HalfFloatValue),
+ (1.0, pa.float32(), pa.FloatScalar, pa.FloatValue),
+ ("string", None, pa.StringScalar, pa.StringValue),
+ (b"bytes", None, pa.BinaryScalar, pa.BinaryValue),
+ ([1, 2, 3], None, pa.ListScalar, pa.ListValue),
+ ([1, 2, 3, 4], pa.large_list(pa.int8()), pa.LargeListScalar,
+ pa.LargeListValue),
+ # date
+ # time
+])
+def test_type_inference(value, ty, klass, deprecated):
+ s = pa.scalar(value, type=ty)
+ assert isinstance(s, klass)
+ assert s == value
+ with pytest.warns(FutureWarning):
+ isinstance(s, deprecated)
+
+
+def test_null_singleton():
+ with pytest.raises(Exception):
+ pa.NullScalar()
+
+
+def test_nulls():
+ arr = pa.array([None, None])
+ for v in arr:
+ assert v is pa.NA
+ assert v.as_py() is None
+
+
+def test_null_equality():
+ assert (pa.NA == pa.NA) is pa.NA
+ assert (pa.NA == 1) is pa.NA
Review comment:
I don't know to what extent we want to fully work out the scalars (so can certainly be a follow-up), but so the typed null scalars (not pa.NA), should probably behave the same as pa.NA, eg when it comes to equality (`pa.NA == 1` gives pa.NA, but `pa.scalar(None, pa.int64()) == 1` gives False)
##########
File path: python/pyarrow/scalar.pxi
##########
@@ -16,1198 +16,704 @@
# under the License.
-_NULL = NA = None
-
-
cdef class Scalar:
"""
- The base class for all array elements.
+ The base class for scalars.
"""
+ def __init__(self):
+ raise TypeError("Do not call {}'s constructor directly, use "
+ "pa.scalar() instead.".format(self.__class__.__name__))
-cdef class NullType(Scalar):
- """
- Singleton for null array elements.
- """
- # TODO rename this NullValue?
+ cdef void init(self, const shared_ptr[CScalar]& wrapped):
+ self.wrapped = wrapped
- def __cinit__(self):
- global NA
- if NA is not None:
- raise Exception('Cannot create multiple NAType instances')
+ @staticmethod
+ cdef wrap(const shared_ptr[CScalar]& wrapped):
+ cdef:
+ Scalar self
+ Type type_id = wrapped.get().type.get().id()
+
+ if type_id == _Type_NA:
+ return _NULL
+
+ typ = _scalar_classes[type_id]
+ self = typ.__new__(typ)
+ self.init(wrapped)
+
+ return self
+
+ cdef inline shared_ptr[CScalar] unwrap(self) nogil:
+ return self.wrapped
- self.type = null()
+ @property
+ def type(self):
+ return pyarrow_wrap_data_type(self.wrapped.get().type)
def __repr__(self):
- return 'NULL'
+ return '<pyarrow.{}: {!r}>'.format(
+ self.__class__.__name__, self.as_py()
+ )
- def as_py(self):
- """
- Return None
- """
- return None
+ def __str__(self):
+ return str(self.as_py())
def __eq__(self, other):
- return NA
+ # TODO(kszucs): use c++ Equals
+ if isinstance(other, Scalar):
+ other = other.as_py()
+ return self.as_py() == other
+ def __hash__(self):
+ # TODO(kszucs): use C++ hash if implemented for the type
+ return hash(self.as_py())
+
+ def as_py(self):
+ raise NotImplementedError()
-_NULL = NA = NullType()
+
+_NULL = NA = None
-cdef class ArrayValue(Scalar):
+cdef class NullScalar(Scalar):
"""
- The base class for non-null array elements.
+ Concrete class for null scalars.
"""
- def __init__(self):
- raise TypeError("Do not call {}'s constructor directly, use array "
- "subscription instead."
- .format(self.__class__.__name__))
-
- cdef void init(self, DataType type, const shared_ptr[CArray]& sp_array,
- int64_t index):
- self.type = type
- self.index = index
- self._set_array(sp_array)
+ def __cinit__(self):
+ global NA
+ if NA is not None:
+ raise Exception('Cannot create multiple NAType instances')
+ self.init(shared_ptr[CScalar](new CNullScalar()))
- cdef void _set_array(self, const shared_ptr[CArray]& sp_array):
- self.sp_array = sp_array
+ def __init__(self):
+ pass
- def __repr__(self):
- if hasattr(self, 'as_py'):
- return repr(self.as_py())
- else:
- return super(Scalar, self).__repr__()
+ def __eq__(self, other):
+ return NA
- def __str__(self):
- if hasattr(self, 'as_py'):
- return str(self.as_py())
- else:
- return super(Scalar, self).__str__()
+ def as_py(self):
+ """
+ Return this value as a Python None.
+ """
+ return None
- def __eq__(self, other):
- if hasattr(self, 'as_py'):
- if isinstance(other, ArrayValue):
- other = other.as_py()
- return self.as_py() == other
- else:
- raise NotImplementedError(
- "Cannot compare Arrow values that don't support as_py()")
- def __hash__(self):
- return hash(self.as_py())
+_NULL = NA = NullScalar()
-cdef class BooleanValue(ArrayValue):
+cdef class BooleanScalar(Scalar):
"""
- Concrete class for boolean array elements.
+ Concrete class for boolean scalars.
"""
def as_py(self):
"""
Return this value as a Python bool.
"""
- cdef CBooleanArray* ap = <CBooleanArray*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CBooleanScalar* sp = <CBooleanScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int8Value(ArrayValue):
+cdef class UInt8Scalar(Scalar):
"""
- Concrete class for int8 array elements.
+ Concrete class for uint8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt8Array* ap = <CInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt8Scalar* sp = <CUInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt8Value(ArrayValue):
+cdef class Int8Scalar(Scalar):
"""
- Concrete class for uint8 array elements.
+ Concrete class for int8 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt8Array* ap = <CUInt8Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt8Scalar* sp = <CInt8Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int16Value(ArrayValue):
+cdef class UInt16Scalar(Scalar):
"""
- Concrete class for int16 array elements.
+ Concrete class for uint16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt16Array* ap = <CInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt16Scalar* sp = <CUInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt16Value(ArrayValue):
+cdef class Int16Scalar(Scalar):
"""
- Concrete class for uint16 array elements.
+ Concrete class for int16 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt16Array* ap = <CUInt16Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt16Scalar* sp = <CInt16Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int32Value(ArrayValue):
+cdef class UInt32Scalar(Scalar):
"""
- Concrete class for int32 array elements.
+ Concrete class for uint32 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt32Array* ap = <CInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt32Scalar* sp = <CUInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt32Value(ArrayValue):
+cdef class Int32Scalar(Scalar):
"""
- Concrete class for uint32 array elements.
+ Concrete class for int32 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt32Array* ap = <CUInt32Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt32Scalar* sp = <CInt32Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Int64Value(ArrayValue):
+cdef class UInt64Scalar(Scalar):
"""
- Concrete class for int64 array elements.
+ Concrete class for uint64 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CInt64Array* ap = <CInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CUInt64Scalar* sp = <CUInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class UInt64Value(ArrayValue):
+cdef class Int64Scalar(Scalar):
"""
- Concrete class for uint64 array elements.
+ Concrete class for int64 scalars.
"""
def as_py(self):
"""
Return this value as a Python int.
"""
- cdef CUInt64Array* ap = <CUInt64Array*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CInt64Scalar* sp = <CInt64Scalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date32Value(ArrayValue):
+cdef class HalfFloatScalar(Scalar):
"""
- Concrete class for date32 array elements.
+ Concrete class for float scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python float.
"""
- cdef CDate32Array* ap = <CDate32Array*> self.sp_array.get()
+ cdef CHalfFloatScalar* sp = <CHalfFloatScalar*> self.wrapped.get()
+ return PyHalf_FromHalf(sp.value) if sp.is_valid else None
+
+
+cdef class FloatScalar(Scalar):
+ """
+ Concrete class for float scalars.
+ """
- # Shift to seconds since epoch
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(days=ap.Value(self.index)))
+ def as_py(self):
+ """
+ Return this value as a Python float.
+ """
+ cdef CFloatScalar* sp = <CFloatScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Date64Value(ArrayValue):
+cdef class DoubleScalar(Scalar):
"""
- Concrete class for date64 array elements.
+ Concrete class for double scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.datetime instance.
+ Return this value as a Python float.
"""
- cdef CDate64Array* ap = <CDate64Array*> self.sp_array.get()
- return (datetime.date(1970, 1, 1) +
- datetime.timedelta(
- days=ap.Value(self.index) / 86400000))
+ cdef CDoubleScalar* sp = <CDoubleScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
-cdef class Time32Value(ArrayValue):
+cdef class DecimalScalar(Scalar):
"""
- Concrete class for time32 array elements.
+ Concrete class for decimal128 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.timedelta instance.
+ Return this value as a Python Decimal.
"""
cdef:
- CTime32Array* ap = <CTime32Array*> self.sp_array.get()
- CTime32Type* dtype = <CTime32Type*> ap.type().get()
-
- if dtype.unit() == TimeUnit_SECOND:
- delta = datetime.timedelta(seconds=ap.Value(self.index))
- return (datetime.datetime(1970, 1, 1) + delta).time()
+ CDecimal128Scalar* sp = <CDecimal128Scalar*> self.wrapped.get()
+ CDecimal128Type* dtype = <CDecimal128Type*> sp.type.get()
+ if sp.is_valid:
+ return _pydecimal.Decimal(
+ frombytes(sp.value.ToString(dtype.scale()))
+ )
else:
- return _box_time_milli(ap.Value(self.index))
+ return None
-cdef class Time64Value(ArrayValue):
+cdef class Date32Scalar(Scalar):
"""
- Concrete class for time64 array elements.
+ Concrete class for date32 scalars.
"""
def as_py(self):
"""
- Return this value as a Python datetime.timedelta instance.
+ Return this value as a Python datetime.datetime instance.
"""
- cdef:
- CTime64Array* ap = <CTime64Array*> self.sp_array.get()
- CTime64Type* dtype = <CTime64Type*> ap.type().get()
+ cdef CDate32Scalar* sp = <CDate32Scalar*> self.wrapped.get()
- cdef int64_t val = ap.Value(self.index)
- if dtype.unit() == TimeUnit_MICRO:
- return _box_time_micro(val)
+ if sp.is_valid:
+ # shift to seconds since epoch
+ return (
+ datetime.date(1970, 1, 1) + datetime.timedelta(days=sp.value)
+ )
else:
- return (datetime.datetime(1970, 1, 1) +
- datetime.timedelta(microseconds=val / 1000)).time()
-
+ return None
-cpdef _box_time_milli(int64_t val):
- delta = datetime.timedelta(milliseconds=val)
- return (datetime.datetime(1970, 1, 1) + delta).time()
+cdef class Date64Scalar(Scalar):
+ """
+ Concrete class for date64 scalars.
+ """
-cpdef _box_time_micro(int64_t val):
- return (datetime.datetime(1970, 1, 1) +
- datetime.timedelta(microseconds=val)).time()
+ def as_py(self):
+ """
+ Return this value as a Python datetime.datetime instance.
+ """
+ cdef CDate64Scalar* sp = <CDate64Scalar*> self.wrapped.get()
+ if sp.is_valid:
+ return (
+ datetime.date(1970, 1, 1) +
+ datetime.timedelta(days=sp.value / 86400000)
+ )
+ else:
+ return None
-cdef dict _DATETIME_CONVERSION_FUNCTIONS = {}
-cdef c_bool _datetime_conversion_initialized = False
+def _datetime_from_int(int64_t value, TimeUnit unit, tzinfo=None):
+ if unit == TimeUnit_SECOND:
+ delta = datetime.timedelta(seconds=value)
+ elif unit == TimeUnit_MILLI:
+ delta = datetime.timedelta(milliseconds=value)
+ elif unit == TimeUnit_MICRO:
+ delta = datetime.timedelta(microseconds=value)
+ else:
+ # TimeUnit_NANO: prefer pandas timestamps if available
+ if _pandas_api.have_pandas:
+ return _pandas_api.pd.Timestamp(value, tz=tzinfo, unit='ns')
+ # otherwise safely truncate to microsecond resolution datetime
+ if value % 1000 != 0:
+ raise ValueError(
+ "Nanosecond resolution temporal type {} is not safely "
+ "convertible to microseconds to convert to datetime.datetime. "
+ "Install pandas to return as Timestamp with nanosecond "
+ "support or access the .value attribute.".format(value)
+ )
+ delta = datetime.timedelta(microseconds=value)
-cdef _add_micros_maybe_localize(dt, micros, tzinfo):
- import pytz
- dt = dt.replace(microsecond=micros)
+ dt = datetime.datetime(1970, 1, 1) + delta
+ # adjust timezone if set to the datatype
if tzinfo is not None:
- if not isinstance(tzinfo, datetime.tzinfo):
- tzinfo = string_to_tzinfo(tzinfo)
dt = tzinfo.fromutc(dt)
- return dt
-
-
-cdef _datetime_from_seconds(int64_t v):
- return datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=v)
-
-
-def _nanoseconds_to_datetime_safe(v, tzinfo):
- if v % 1000 != 0:
- raise ValueError("Nanosecond timestamp {} is not safely convertible "
- " to microseconds to convert to datetime.datetime."
- " Install pandas to return as Timestamp with "
- " nanosecond support or access the .value attribute.")
- v = v // 1000
- micros = v % 1_000_000
-
- dt = _datetime_from_seconds(v // 1_000_000)
- return _add_micros_maybe_localize(dt, micros, tzinfo)
-
-
-def _microseconds_to_datetime(v, tzinfo):
- micros = v % 1_000_000
- dt = _datetime_from_seconds(v // 1_000_000)
- return _add_micros_maybe_localize(dt, micros, tzinfo)
+ return dt
-def _millis_to_datetime(v, tzinfo):
- millis = v % 1_000
- dt = _datetime_from_seconds(v // 1000)
- return _add_micros_maybe_localize(dt, millis * 1000, tzinfo)
+cdef class Time32Scalar(Scalar):
+ """
+ Concrete class for time32 scalars.
+ """
-def _seconds_to_datetime(v, tzinfo):
- dt = _datetime_from_seconds(v)
- return _add_micros_maybe_localize(dt, 0, tzinfo)
+ def as_py(self):
+ """
+ Return this value as a Python datetime.timedelta instance.
+ """
+ cdef:
+ CTime32Scalar* sp = <CTime32Scalar*> self.wrapped.get()
+ CTime32Type* dtype = <CTime32Type*> sp.type.get()
+ if sp.is_valid:
+ return _datetime_from_int(sp.value, unit=dtype.unit()).time()
+ else:
+ return None
-def _datetime_conversion_functions():
- global _datetime_conversion_initialized
- if _datetime_conversion_initialized:
- return _DATETIME_CONVERSION_FUNCTIONS
- _DATETIME_CONVERSION_FUNCTIONS.update({
- TimeUnit_SECOND: _seconds_to_datetime,
- TimeUnit_MILLI: _millis_to_datetime,
- TimeUnit_MICRO: _microseconds_to_datetime,
- TimeUnit_NANO: _nanoseconds_to_datetime_safe
- })
+cdef class Time64Scalar(Scalar):
+ """
+ Concrete class for time64 scalars.
+ """
- try:
- import pandas as pd
- _DATETIME_CONVERSION_FUNCTIONS[TimeUnit_NANO] = (
- lambda x, tzinfo: pd.Timestamp(
- x, tz=tzinfo, unit='ns',
- )
- )
- except ImportError:
- pass
+ def as_py(self):
+ """
+ Return this value as a Python datetime.timedelta instance.
+ """
+ cdef:
+ CTime64Scalar* sp = <CTime64Scalar*> self.wrapped.get()
+ CTime64Type* dtype = <CTime64Type*> sp.type.get()
- _datetime_conversion_initialized = True
- return _DATETIME_CONVERSION_FUNCTIONS
+ if sp.is_valid:
+ return _datetime_from_int(sp.value, unit=dtype.unit()).time()
+ else:
+ return None
-cdef class TimestampValue(ArrayValue):
+cdef class TimestampScalar(Scalar):
"""
- Concrete class for timestamp array elements.
+ Concrete class for timestamp scalars.
"""
@property
def value(self):
- cdef CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
- cdef CTimestampType* dtype = <CTimestampType*> ap.type().get()
- return ap.Value(self.index)
+ cdef CTimestampScalar* sp = <CTimestampScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
def as_py(self):
"""
Return this value as a Pandas Timestamp instance (if available),
otherwise as a Python datetime.timedelta instance.
"""
- cdef CTimestampArray* ap = <CTimestampArray*> self.sp_array.get()
- cdef CTimestampType* dtype = <CTimestampType*> ap.type().get()
+ cdef:
+ CTimestampScalar* sp = <CTimestampScalar*> self.wrapped.get()
+ CTimestampType* dtype = <CTimestampType*> sp.type.get()
- value = self.value
+ if not sp.is_valid:
+ return None
if not dtype.timezone().empty():
tzinfo = string_to_tzinfo(frombytes(dtype.timezone()))
+ if not isinstance(tzinfo, datetime.tzinfo):
+ tzinfo = string_to_tzinfo(tzinfo)
else:
tzinfo = None
- try:
- converter = _datetime_conversion_functions()[dtype.unit()]
- except KeyError:
- raise ValueError(
- 'Cannot convert nanosecond timestamps without pandas'
- )
- return converter(value, tzinfo=tzinfo)
-
-
-cdef dict _TIMEDELTA_CONVERSION_FUNCTIONS = {}
-
-
-def _nanoseconds_to_timedelta_safe(v):
- if v % 1000 != 0:
- raise ValueError(
- "Nanosecond duration {} is not safely convertible to microseconds "
- "to convert to datetime.timedelta. Install pandas to return as "
- "Timedelta with nanosecond support or access the .value "
- "attribute.".format(v))
- micros = v // 1000
-
- return datetime.timedelta(microseconds=micros)
-
+ return _datetime_from_int(sp.value, unit=dtype.unit(), tzinfo=tzinfo)
-def _timedelta_conversion_functions():
- if _TIMEDELTA_CONVERSION_FUNCTIONS:
- return _TIMEDELTA_CONVERSION_FUNCTIONS
- _TIMEDELTA_CONVERSION_FUNCTIONS.update({
- TimeUnit_SECOND: lambda v: datetime.timedelta(seconds=v),
- TimeUnit_MILLI: lambda v: datetime.timedelta(milliseconds=v),
- TimeUnit_MICRO: lambda v: datetime.timedelta(microseconds=v),
- TimeUnit_NANO: _nanoseconds_to_timedelta_safe
- })
-
- try:
- import pandas as pd
- _TIMEDELTA_CONVERSION_FUNCTIONS[TimeUnit_NANO] = (
- lambda v: pd.Timedelta(v, unit='ns')
- )
- except ImportError:
- pass
-
- return _TIMEDELTA_CONVERSION_FUNCTIONS
-
-
-cdef class DurationValue(ArrayValue):
+cdef class DurationScalar(Scalar):
"""
- Concrete class for duration array elements.
+ Concrete class for duration scalars.
"""
@property
def value(self):
- cdef CDurationArray* ap = <CDurationArray*> self.sp_array.get()
- return ap.Value(self.index)
+ cdef CDurationScalar* sp = <CDurationScalar*> self.wrapped.get()
+ return sp.value if sp.is_valid else None
def as_py(self):
"""
Return this value as a Pandas Timestamp instance (if available),
otherwise as a Python datetime.timedelta instance.
"""
- cdef CDurationArray* ap = <CDurationArray*> self.sp_array.get()
- cdef CDurationType* dtype = <CDurationType*> ap.type().get()
-
- cdef int64_t value = ap.Value(self.index)
- converter = _timedelta_conversion_functions()[dtype.unit()]
- return converter(value)
-
+ cdef:
+ CDurationScalar* sp = <CDurationScalar*> self.wrapped.get()
+ CDurationType* dtype = <CDurationType*> sp.type.get()
+ TimeUnit unit = dtype.unit()
-cdef class HalfFloatValue(ArrayValue):
- """
- Concrete class for float16 array elements.
- """
+ if not sp.is_valid:
+ return None
- def as_py(self):
- """
- Return this value as a Python float.
- """
- cdef CHalfFloatArray* ap = <CHalfFloatArray*> self.sp_array.get()
- return PyHalf_FromHalf(ap.Value(self.index))
+ if unit == TimeUnit_SECOND:
+ return datetime.timedelta(seconds=sp.value)
+ elif unit == TimeUnit_MILLI:
+ return datetime.timedelta(milliseconds=sp.value)
+ elif unit == TimeUnit_MICRO:
+ return datetime.timedelta(microseconds=sp.value)
+ else:
+ # TimeUnit_NANO: prefer pandas timestamps if available
+ if _pandas_api.have_pandas:
+ return _pandas_api.pd.Timedelta(sp.value, unit='ns')
+ # otherwise safely truncate to microsecond resolution timedelta
+ if sp.value % 1000 != 0:
+ raise ValueError(
+ "Nanosecond duration {} is not safely convertible to "
+ "microseconds to convert to datetime.timedelta. Install "
+ "pandas to return as Timedelta with nanosecond support or "
+ "access the .value attribute.".format(sp.value)
+ )
+ return datetime.timedelta(microseconds=sp.value // 1000)
-cdef class FloatValue(ArrayValue):
+cdef class BinaryScalar(Scalar):
"""
- Concrete class for float32 array elements.
+ Concrete class for binary-like scalars.
"""
- def as_py(self):
+ def as_buffer(self):
"""
- Return this value as a Python float.
+ Return a view over this value as a Buffer object.
"""
- cdef CFloatArray* ap = <CFloatArray*> self.sp_array.get()
- return ap.Value(self.index)
-
-
-cdef class DoubleValue(ArrayValue):
- """
- Concrete class for float64 array elements.
- """
+ cdef CBinaryScalar* sp = <CBinaryScalar*> self.wrapped.get()
+ return pyarrow_wrap_buffer(sp.value) if sp.is_valid else None
def as_py(self):
"""
- Return this value as a Python float.
+ Return this value as a Python bytes.
"""
- cdef CDoubleArray* ap = <CDoubleArray*> self.sp_array.get()
- return ap.Value(self.index)
+ buffer = self.as_buffer()
+ if buffer is not None:
+ return self.as_buffer().to_pybytes()
+ else:
+ return None
-cdef class DecimalValue(ArrayValue):
- """
- Concrete class for decimal128 array elements.
- """
+cdef class LargeBinaryScalar(BinaryScalar):
+ pass
- def as_py(self):
- """
- Return this value as a Python Decimal.
- """
- cdef:
- CDecimal128Array* ap = <CDecimal128Array*> self.sp_array.get()
- c_string s = ap.FormatValue(self.index)
- return _pydecimal.Decimal(s.decode('utf8'))
+cdef class FixedSizeBinaryScalar(BinaryScalar):
+ pass
-cdef class StringValue(ArrayValue):
+
+cdef class StringScalar(BinaryScalar):
"""
- Concrete class for string (utf8) array elements.
+ Concrete class for string-like (utf8) scalars.
"""
def as_py(self):
"""
- Return this value as a Python unicode string.
+ Return this value as a Python string.
"""
- cdef CStringArray* ap = <CStringArray*> self.sp_array.get()
- return ap.GetString(self.index).decode('utf-8')
+ buffer = self.as_buffer()
+ if buffer is not None:
+ return frombytes(self.as_buffer().to_pybytes())
+ else:
+ return None
- def as_buffer(self):
- """
- Return a view over this value as a Buffer object.
- """
- cdef:
- CStringArray* ap = <CStringArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
+cdef class LargeStringScalar(StringScalar):
+ pass
-cdef class LargeStringValue(ArrayValue):
+cdef class ListScalar(Scalar):
"""
- Concrete class for large string (utf8) array elements.
+ Concrete class for list-like scalars.
"""
- def as_py(self):
+ cdef array(self):
+ cdef CListScalar* sp = <CListScalar*> self.wrapped.get()
+ if sp.is_valid:
+ return pyarrow_wrap_array(sp.value)
+ else:
+ return None
+
+ def __len__(self):
"""
- Return this value as a Python unicode string.
+ Return the number of values.
"""
- cdef CLargeStringArray* ap = <CLargeStringArray*> self.sp_array.get()
- return ap.GetString(self.index).decode('utf-8')
+ return len(self.array())
- def as_buffer(self):
+ def __getitem__(self, i):
"""
- Return a view over this value as a Buffer object.
+ Return the value at the given index.
"""
- cdef:
- CLargeStringArray* ap = <CLargeStringArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
-
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
+ return self.array()[_normalize_index(i, len(self))]
-
-cdef class BinaryValue(ArrayValue):
- """
- Concrete class for variable-sized binary array elements.
- """
+ def __iter__(self):
+ """
+ Iterate over this element's values.
+ """
+ return iter(self.array())
def as_py(self):
"""
- Return this value as a Python bytes object.
+ Return this value as a Python list.
"""
- cdef:
- const uint8_t* ptr
- int32_t length
- CBinaryArray* ap = <CBinaryArray*> self.sp_array.get()
+ arr = self.array()
+ return None if arr is None else arr.to_pylist()
- ptr = ap.GetValue(self.index, &length)
- return cp.PyBytes_FromStringAndSize(<const char*>(ptr), length)
- def as_buffer(self):
- """
- Return a view over this value as a Buffer object.
- """
- cdef:
- CBinaryArray* ap = <CBinaryArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
+cdef class FixedSizeListScalar(ListScalar):
+ pass
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
+cdef class LargeListScalar(ListScalar):
+ pass
-cdef class LargeBinaryValue(ArrayValue):
+
+cdef class StructScalar(Scalar):
"""
- Concrete class for large variable-sized binary array elements.
+ Concrete class for struct scalars.
"""
- def as_py(self):
+ def __len__(self):
+ cdef CStructScalar* sp = <CStructScalar*> self.wrapped.get()
+ return sp.value.size()
+
+ def __getitem__(self, key):
"""
- Return this value as a Python bytes object.
+ Return the child value for the given field.
+
+ Parameters
+ ----------
+ index : Union[int, str]
+ Index / position or name of the field.
+
+ Returns
+ -------
+ result : Scalar
"""
cdef:
- const uint8_t* ptr
- int64_t length
- CLargeBinaryArray* ap = <CLargeBinaryArray*> self.sp_array.get()
+ CFieldRef ref
+ CStructScalar* sp = <CStructScalar*> self.wrapped.get()
+
+ if isinstance(key, (bytes, str)):
+ ref = CFieldRef(<c_string> tobytes(key))
+ elif isinstance(key, int):
+ ref = CFieldRef(<int> key)
+ else:
+ raise TypeError('Expected integer or string index')
- ptr = ap.GetValue(self.index, &length)
- return cp.PyBytes_FromStringAndSize(<const char*>(ptr), length)
+ try:
+ return Scalar.wrap(GetResultValue(sp.field(ref)))
+ except ArrowInvalid:
+ raise IndexError(key)
- def as_buffer(self):
+ def as_py(self):
"""
- Return a view over this value as a Buffer object.
+ Return this value as a Python dict.
"""
cdef:
- CLargeBinaryArray* ap = <CLargeBinaryArray*> self.sp_array.get()
- shared_ptr[CBuffer] buf
+ CStructScalar* sp = <CStructScalar*> self.wrapped.get()
+ CStructType* dtype = <CStructType*> sp.type.get()
+ vector[shared_ptr[CField]] fields = dtype.fields()
- buf = SliceBuffer(ap.value_data(), ap.value_offset(self.index),
- ap.value_length(self.index))
- return pyarrow_wrap_buffer(buf)
+ if sp.is_valid:
+ return {frombytes(fields[i].get().name()): Scalar.wrap(sp.value[i])
Review comment:
Should we directly take the `as_py` from the values in the dict as well?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org