You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@fory.apache.org by ch...@apache.org on 2026/09/20 04:22:05 UTC
(fory) branch main updated: fix(python): support named tuples and temporal array buffers (#4062)
This is an automated email from the ASF dual-hosted git repository.
chaokunyang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fory.git
The following commit(s) were added to refs/heads/main by this push:
new 31eb5ea5b fix(python): support named tuples and temporal array buffers (#4062)
31eb5ea5b is described below
commit 31eb5ea5b5c63e216edbbe97aec48ac8f3e77f74
Author: Shawn Yang <sh...@gmail.com>
AuthorDate: Sun Sep 20 12:22:01 2026 +0800
fix(python): support named tuples and temporal array buffers (#4062)
## What does this PR do?
Fix Python native named-tuple round trips by serializing their tuple
elements and reconstructing the concrete class with its field values.
Previously, named tuples were treated as ordinary objects with empty
slots, causing missing-constructor-argument errors or silently restoring
defaults. Preserve explicit registration, shared references,
instantiation policies, and graph-memory accounting.
Export contiguous NumPy datetime/timedelta arrays through a zero-copy
byte view so DataFrames with datetime indexes or columns work with
out-of-band buffers. Preserve dtype, shape, temporal units, and NaT
values, including scalar and empty arrays.
Fixes #4056.
Fixes #4057.
## Validation
- Reproduced both reports with Python 3.13.8, NumPy 2.3.3, and pandas
2.3.3 on main at `c0b0464f0594e9d2ef7520d756b1b58883a88ea1`.
- 52 focused regression cases passed in both Python and Cython modes.
- Full Python-mode suite: 883 passed, 35 skipped. Full Cython-mode
suite: 888 passed, 30 skipped.
- Ruff 0.15.22, Prettier, and both updated documentation examples
passed.
- Local numeric-array out-of-band round-trip microbenchmark: added dtype
dispatch cost about 0.02–0.05 microseconds (1.1–2.6%) in the measured
cases; temporal-array tests verify shared backing memory.
The changes affect Python native serialization only and do not change
the shared xlang protocol.
---
docs/object-serialization/python/native.md | 25 ++++++
docs/object-serialization/python/out-of-band.md | 13 ++-
python/pyfory/__init__.py | 2 +
python/pyfory/registry.py | 7 +-
python/pyfory/serializer.py | 31 +++++++
python/pyfory/tests/test_serializer.py | 115 +++++++++++++++++++++++-
6 files changed, 187 insertions(+), 6 deletions(-)
diff --git a/docs/object-serialization/python/native.md b/docs/object-serialization/python/native.md
index e77e24bbe..b1933b000 100644
--- a/docs/object-serialization/python/native.md
+++ b/docs/object-serialization/python/native.md
@@ -79,6 +79,31 @@ print(fory.loads(data)) # Person(name='Bob', age=25)
Use `dumps`/`loads` for pickle-style APIs, or `serialize`/`deserialize` when matching the xlang
API shape in code that switches modes explicitly.
+## Named Tuples
+
+Native mode supports both `typing.NamedTuple` and `collections.namedtuple`, preserving the
+concrete class and field values. Register the named tuple type on each peer when using strict mode:
+
+```python
+from typing import NamedTuple
+import pyfory
+
+class Record(NamedTuple):
+ name: str
+ values: tuple
+ count: int
+
+fory = pyfory.Fory(xlang=False, strict=True)
+fory.register(Record)
+
+record = Record("sample", (1.0, 2.0, 3.0), 42)
+restored = fory.loads(fory.dumps(record))
+assert type(restored) is Record
+assert restored == record
+```
+
+Writers and readers must use the same named tuple definition, including field order.
+
## Security And Dynamic Types
Native mode can reconstruct Python objects that execute import and construction logic during
diff --git a/docs/object-serialization/python/out-of-band.md b/docs/object-serialization/python/out-of-band.md
index 8db222fec..61010bd92 100644
--- a/docs/object-serialization/python/out-of-band.md
+++ b/docs/object-serialization/python/out-of-band.md
@@ -58,6 +58,10 @@ assert np.array_equal(array, deserialized_array)
## Out-of-Band with Pandas DataFrames
+Native mode supports DataFrames with datetime columns and a `DatetimeIndex`, preserving their
+datetime units and index metadata. NumPy `datetime64` and `timedelta64` arrays also support this
+buffer flow. Their C-contiguous storage is shared without copying; non-contiguous arrays are copied.
+
```python
import pyfory
import pandas as pd
@@ -65,12 +69,13 @@ import numpy as np
fory = pyfory.Fory(xlang=False, ref=False, strict=False)
-# Create a DataFrame with numeric columns
+# Create a DataFrame with a datetime index and column
+index = pd.date_range("2026-01-01", periods=1000, freq="D", name="date")
df = pd.DataFrame({
'a': np.arange(1000, dtype=np.float64),
'b': np.arange(1000, dtype=np.int64),
- 'c': ['text'] * 1000
-})
+ 'created_at': index
+}, index=index)
# Serialize with out-of-band buffers
buffer_objects = []
@@ -80,7 +85,7 @@ buffers = [obj.getbuffer() for obj in buffer_objects]
# Deserialize
deserialized_df = fory.deserialize(serialized_data, buffers=buffers)
-assert df.equals(deserialized_df)
+pd.testing.assert_frame_equal(df, deserialized_df)
```
## Selective Out-of-Band Serialization
diff --git a/python/pyfory/__init__.py b/python/pyfory/__init__.py
index 0f36562fb..dfaeee514 100644
--- a/python/pyfory/__init__.py
+++ b/python/pyfory/__init__.py
@@ -72,6 +72,7 @@ from pyfory.serializer import ( # pylint: disable=unused-import
CollectionSerializer,
ListSerializer,
TupleSerializer,
+ NamedTupleSerializer,
StringArraySerializer,
SetSerializer,
MapSerializer,
@@ -208,6 +209,7 @@ __all__ = [
"ThreadSafeFory",
"TimestampSerializer",
"TupleSerializer",
+ "NamedTupleSerializer",
"TypeId",
"TypeInfo",
"TypeSerializer",
diff --git a/python/pyfory/registry.py b/python/pyfory/registry.py
index 9cf00a443..e01f9e30e 100644
--- a/python/pyfory/registry.py
+++ b/python/pyfory/registry.py
@@ -67,6 +67,7 @@ from pyfory.serializer import (
BytesSerializer,
ListSerializer,
TupleSerializer,
+ NamedTupleSerializer,
MapSerializer,
SetSerializer,
NonExistEnum,
@@ -659,6 +660,8 @@ class TypeResolver:
and user_type_id in {None, NO_USER_TYPE_ID}
):
return self._types_info[cls]
+ if serializer is None and not self.xlang and issubclass(cls, tuple) and hasattr(cls, "_fields"):
+ serializer = self._create_serializer(cls)
n_params = len({typename, type_id, None}) - 1
if n_params == 0 and typename is None:
type_id = self._next_type_id()
@@ -865,7 +868,7 @@ class TypeResolver:
if not self.xlang:
if isinstance(serializer, EnumSerializer):
type_id = TypeId.NAMED_ENUM
- elif isinstance(serializer, (ObjectSerializer, StatefulSerializer)):
+ elif isinstance(serializer, (ObjectSerializer, StatefulSerializer, NamedTupleSerializer)):
type_id = TypeId.NAMED_EXT
elif self._internal_py_serializer_map.get(type(serializer)) is not None:
type_id = self._internal_py_serializer_map.get(type(serializer))[1]
@@ -985,6 +988,8 @@ class TypeResolver:
# Use StatefulSerializer for objects that support __getstate__ and __setstate__
serializer_cls = _DefaultPolicyStatefulSerializer if use_default_policy else StatefulSerializer
serializer = serializer_cls(serializer_type_resolver, cls)
+ elif not self.xlang and issubclass(cls, tuple) and hasattr(cls, "_fields"):
+ serializer = NamedTupleSerializer(serializer_type_resolver, cls)
elif hasattr(cls, "__dict__") or hasattr(cls, "__slots__"):
serializer_cls = _DefaultPolicyObjectSerializer if use_default_policy else ObjectSerializer
serializer = serializer_cls(serializer_type_resolver, cls)
diff --git a/python/pyfory/serializer.py b/python/pyfory/serializer.py
index cf025ab28..a7e074286 100644
--- a/python/pyfory/serializer.py
+++ b/python/pyfory/serializer.py
@@ -1184,10 +1184,40 @@ class NDArrayBufferObject(BufferObject):
def getbuffer(self) -> memoryview:
if self.array.flags.c_contiguous:
+ if self.dtype.kind in ("M", "m"):
+ # NumPy cannot export temporal dtypes through the buffer protocol.
+ # Their dtype and shape are already encoded separately; expose the
+ # same storage as bytes, including for zero-dimensional arrays.
+ return memoryview(self.array.reshape(-1).view(np.uint8))
return memoryview(self.array.data)
return memoryview(self.array.tobytes())
+class NamedTupleSerializer(Serializer):
+ """Serialize Python named tuples in field order and reconstruct their concrete type."""
+
+ def __init__(self, type_resolver, cls):
+ super().__init__(type_resolver, cls)
+ self._field_count = len(cls._fields)
+ self._graph_memory_bytes = _TUPLE_OWNER_BYTES + self._field_count * _REFERENCE_BYTES
+ self.read_data_always_advances = self._field_count > 0
+
+ def write(self, write_context, value):
+ # Named tuple fields live in tuple storage, not their empty __slots__.
+ for item in value:
+ write_context.write_ref(item)
+
+ def read(self, read_context):
+ if read_context.policy is not DEFAULT_POLICY:
+ read_context.policy.authorize_instantiation(self.type_)
+ read_context.reserve_graph_memory(self._graph_memory_bytes)
+ read_context.check_readable_bytes(self._field_count)
+ # Immutable fields must be supplied to __new__, rather than assigned
+ # after creating an empty instance as for ordinary Python objects.
+ fields = [read_context.read_ref() for _ in range(self._field_count)]
+ return self.type_(*fields)
+
+
class StatefulSerializer(Serializer):
"""
Serializer for objects that support __getstate__ and __setstate__.
@@ -2062,6 +2092,7 @@ __all__ = [
"CollectionSerializer",
"ListSerializer",
"TupleSerializer",
+ "NamedTupleSerializer",
"StringArraySerializer",
"SetSerializer",
"MapSerializer",
diff --git a/python/pyfory/tests/test_serializer.py b/python/pyfory/tests/test_serializer.py
index bb1d55cf0..1279b9de8 100644
--- a/python/pyfory/tests/test_serializer.py
+++ b/python/pyfory/tests/test_serializer.py
@@ -23,9 +23,10 @@ import io
import os
import pickle
import weakref
+from collections import namedtuple
from collections.abc import MutableSequence
from enum import Enum, IntEnum
-from typing import Any, List, Dict, Optional
+from typing import Any, List, Dict, NamedTuple, Optional
import numpy as np
import pandas as pd
@@ -81,6 +82,70 @@ def test_tuple():
assert ser_de(fory, (-1.0, 2)) == (-1.0, 2)
+class NamedRecord(NamedTuple):
+ name: str
+ values: tuple
+ count: int
+
+
+class DefaultNamedRecord(NamedTuple):
+ name: str = "default"
+ values: tuple = ()
+ count: int = 0
+
+
+NamedPoint = namedtuple("NamedPoint", ["x", "y"])
+EmptyNamedTuple = namedtuple("EmptyNamedTuple", [])
+
+
+@pytest.mark.parametrize("track_ref", [False, True])
+@pytest.mark.parametrize("strict", [False, True])
+@pytest.mark.parametrize("compatible", [False, True])
+@pytest.mark.parametrize(
+ "value",
+ [
+ NamedRecord("sample", (1.0, 2.0, 3.0), 42),
+ DefaultNamedRecord("sample", (1.0,), 42),
+ NamedPoint(1, None),
+ EmptyNamedTuple(),
+ ],
+)
+def test_namedtuple(value, track_ref, strict, compatible):
+ writer = Fory(xlang=False, ref=track_ref, strict=strict, compatible=compatible)
+ reader = writer
+ if strict:
+ reader = Fory(xlang=False, ref=track_ref, strict=True, compatible=compatible)
+ writer.register(type(value))
+ reader.register(type(value))
+ restored = reader.loads(writer.dumps([value, value]))
+ assert restored == [value, value]
+ assert type(restored[0]) is type(value)
+ if track_ref:
+ assert restored[0] is restored[1]
+
+
+def test_namedtuple_field_refs():
+ fory = Fory(xlang=False, ref=True, strict=False, compatible=False)
+ shared = [1, 2]
+ value = NamedRecord("shared", (shared, shared), 2)
+ restored = fory.loads(fory.dumps(value))
+ assert restored == value
+ assert restored.values[0] is restored.values[1]
+
+
+def test_namedtuple_policy():
+ class NamedTuplePolicy(pyfory.DeserializationPolicy):
+ def authorize_instantiation(self, cls, **kwargs):
+ if cls is NamedRecord:
+ raise ValueError("named tuple blocked")
+
+ fory = Fory(xlang=False, ref=False, strict=True, compatible=False, policy=NamedTuplePolicy())
+ fory.register(NamedRecord)
+ data = fory.dumps(NamedRecord("sample", (), 42))
+ with pytest.raises(ValueError, match="named tuple blocked"):
+ fory.loads(data)
+
+
def test_string():
fory = Fory(xlang=False, ref=True, compatible=False)
assert ser_de(fory, "hello") == "hello"
@@ -915,6 +980,54 @@ def test_pandas_dataframe():
assert df2.equals(df)
+@pytest.mark.parametrize("out_of_band", [False, True])
+@pytest.mark.parametrize("as_column", [False, True])
+def test_pandas_datetime(out_of_band, as_column):
+ fory = Fory(xlang=False, ref=False, strict=False, compatible=False)
+ rng = np.random.default_rng(0)
+ index = pd.date_range("2026-01-01", periods=10, freq="D", name="date")
+ df = pd.DataFrame({key: rng.random(10, dtype=np.float64) for key in "abc"}, index=index)
+ if as_column:
+ df = df.reset_index()
+ buffers = []
+ data = fory.dumps(df, buffer_callback=buffers.append if out_of_band else None)
+ views = [obj.getbuffer() for obj in buffers]
+ restored = fory.loads(data, buffers=views if out_of_band else None)
+ pd.testing.assert_frame_equal(restored, df)
+
+
+@pytest.mark.parametrize("dtype", ["datetime64[ns]", "datetime64[us]", "timedelta64[ns]"])
+@pytest.mark.parametrize("shape", [(), (0,), (6,), (2, 3)])
+def test_temporal_array_buffers(dtype, shape):
+ fory = Fory(xlang=False, ref=False, strict=True, compatible=False)
+ value = np.arange(int(np.prod(shape))).astype(dtype).reshape(shape)
+ if value.size:
+ value.flat[0] = "NaT"
+ value.setflags(write=False)
+ buffers = []
+ data = fory.dumps(value, buffer_callback=buffers.append)
+ assert len(buffers) == 1
+ view = buffers[0].getbuffer()
+ assert view.nbytes == value.nbytes
+ restored = fory.loads(data, buffers=[view])
+ assert restored.shape == value.shape
+ assert restored.dtype == value.dtype
+ np.testing.assert_array_equal(restored, value)
+ if value.size:
+ assert np.shares_memory(restored, value)
+
+
+@pytest.mark.parametrize("layout", ["strided", "fortran"])
+def test_temporal_array_layout(layout):
+ fory = Fory(xlang=False, ref=False, strict=True, compatible=False)
+ value = np.arange(12).astype("datetime64[ns]").reshape(3, 4)
+ value = value[:, ::2] if layout == "strided" else np.asfortranarray(value)
+ buffers = []
+ data = fory.dumps(value, buffer_callback=buffers.append)
+ restored = fory.loads(data, buffers=[obj.getbuffer() for obj in buffers])
+ np.testing.assert_array_equal(restored, value)
+
+
def test_unsupported_callback():
fory = Fory(xlang=False, ref=True, strict=False, compatible=False)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@fory.apache.org
For additional commands, e-mail: commits-help@fory.apache.org