You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@fory.apache.org by ch...@apache.org on 2025/07/14 17:34:41 UTC
(fory) branch main updated: feat: Implement ObjectSerializer for custom objects (#2413)
This is an automated email from the ASF dual-hosted git repository.
chaokunyang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fory.git
The following commit(s) were added to refs/heads/main by this push:
new f4718ac65 feat: Implement ObjectSerializer for custom objects (#2413)
f4718ac65 is described below
commit f4718ac652ef4fa21d900b691c8d21a96b18e1f8
Author: Emre Şafak <39...@users.noreply.github.com>
AuthorDate: Mon Jul 14 13:34:37 2025 -0400
feat: Implement ObjectSerializer for custom objects (#2413)
## What does this PR do?
* Introduce ObjectSerializer to handle serialization of arbitrary Python
objects. bjectSerializer serializes based on object's `__dict__` or
`__slots__`.
* Update `_registry.py` to correctly identify and register
ObjectSerializer.
## Related issues
Closes #2410
## Does this PR introduce any user-facing change?
- [ ] Does this PR introduce any public API change?
- [ ] Does this PR introduce any binary protocol compatibility change?
## Benchmark
Not done
---------
Co-authored-by: google-labs-jules[bot] <16...@users.noreply.github.com>
Co-authored-by: Emre Şafak <es...@users.noreply.github.com>
---
python/pyfory/_registry.py | 13 ++--
python/pyfory/serializer.py | 60 ++++++++++++++++-
python/pyfory/tests/test_serializer.py | 113 +++++++++++++++++++--------------
3 files changed, 134 insertions(+), 52 deletions(-)
diff --git a/python/pyfory/_registry.py b/python/pyfory/_registry.py
index 683f0a6bc..1fe214366 100644
--- a/python/pyfory/_registry.py
+++ b/python/pyfory/_registry.py
@@ -63,6 +63,7 @@ from pyfory.serializer import (
StatefulSerializer,
ReduceSerializer,
FunctionSerializer,
+ ObjectSerializer,
)
from pyfory.meta.metastring import MetaStringEncoder, MetaStringDecoder
from pyfory.type import (
@@ -467,7 +468,7 @@ class TypeResolver:
type_id = TypeId.NAMED_ENUM
elif type(serializer) is PickleSerializer:
type_id = PickleSerializer.PICKLE_TYPE_ID
- elif isinstance(serializer, (StatefulSerializer, ReduceSerializer)):
+ elif isinstance(serializer, (ObjectSerializer, StatefulSerializer, ReduceSerializer)):
type_id = TypeId.NAMED_EXT
if not self.require_registration:
if isinstance(serializer, DataClassSerializer):
@@ -493,8 +494,6 @@ class TypeResolver:
# Use PickleSerializer for function types (including lambdas)
serializer = PickleSerializer(self.fory, cls)
elif dataclasses.is_dataclass(cls):
- from pyfory import DataClassSerializer
-
serializer = DataClassSerializer(self.fory, cls)
elif issubclass(cls, enum.Enum):
serializer = EnumSerializer(self.fory, cls)
@@ -502,7 +501,7 @@ class TypeResolver:
hasattr(cls, "__reduce_ex__") and cls.__reduce_ex__ is not object.__reduce_ex__
):
# Use ReduceSerializer for objects that have custom __reduce__ or __reduce_ex__ methods
- # This has higher precedence than StatefulSerializer
+ # This has higher precedence than StatefulSerializer and ObjectSerializer
# Only use it for objects with custom reduce methods, not default ones from the object
module_name = getattr(cls, "__module__", "")
if module_name.startswith("pandas.") or module_name == "builtins" or cls.__name__ in ("type", "function", "method"):
@@ -519,6 +518,12 @@ class TypeResolver:
serializer = PickleSerializer(self.fory, cls)
else:
serializer = StatefulSerializer(self.fory, cls)
+ elif (
+ cls is not type
+ and (hasattr(cls, "__dict__") or hasattr(cls, "__slots__"))
+ and not (np and (issubclass(cls, np.dtype) or cls is type(np.dtype)))
+ ):
+ serializer = ObjectSerializer(self.fory, cls)
else:
serializer = PickleSerializer(self.fory, cls)
return serializer
diff --git a/python/pyfory/serializer.py b/python/pyfory/serializer.py
index d6db365e1..aba3809c4 100644
--- a/python/pyfory/serializer.py
+++ b/python/pyfory/serializer.py
@@ -302,7 +302,7 @@ class DataClassSerializer(Serializer):
self._xlang = xlang
# This will get superclass type hints too.
self._type_hints = typing.get_type_hints(clz)
- self._field_names = sorted(self._type_hints.keys())
+ self._field_names = self._get_field_names(clz)
self._has_slots = hasattr(clz, "__slots__")
if self._xlang:
@@ -335,6 +335,16 @@ class DataClassSerializer(Serializer):
self.write = self._generated_write_method
self.read = self._generated_read_method
+ def _get_field_names(self, clz):
+ if hasattr(clz, "__dict__"):
+ # Regular object with __dict__
+ # We can't know the fields without an instance, so we rely on type hints
+ return sorted(self._type_hints.keys())
+ elif hasattr(clz, "__slots__"):
+ # Object with __slots__
+ return sorted(clz.__slots__)
+ return []
+
def _gen_write_method(self):
context = {}
counter = itertools.count(0)
@@ -1178,6 +1188,54 @@ class PickleSerializer(Serializer):
return self.fory.handle_unsupported_read(buffer)
+class ObjectSerializer(Serializer):
+ """Serializer for regular Python objects.
+ It serializes objects based on `__dict__` or `__slots__`.
+ """
+
+ def __init__(self, fory, clz: type):
+ super().__init__(fory, clz)
+ # If the class defines __slots__, compute and store a sorted list once
+ slots = getattr(clz, "__slots__", None)
+ self._slot_field_names = None
+ if slots is not None:
+ # __slots__ can be a string or iterable of strings
+ if isinstance(slots, str):
+ slots = [slots]
+ self._slot_field_names = sorted(slots)
+
+ def write(self, buffer, value):
+ # Use precomputed slots if available, otherwise sort instance __dict__ keys
+ if self._slot_field_names is not None:
+ sorted_field_names = self._slot_field_names
+ else:
+ sorted_field_names = sorted(value.__dict__.keys())
+
+ buffer.write_varuint32(len(sorted_field_names))
+ for field_name in sorted_field_names:
+ buffer.write_string(field_name)
+ field_value = getattr(value, field_name)
+ self.fory.serialize_ref(buffer, field_value)
+
+ def read(self, buffer):
+ obj = self.type_.__new__(self.type_)
+ self.fory.ref_resolver.reference(obj)
+ num_fields = buffer.read_varuint32()
+ for _ in range(num_fields):
+ field_name = buffer.read_string()
+ field_value = self.fory.deserialize_ref(buffer)
+ setattr(obj, field_name, field_value)
+ return obj
+
+ def xwrite(self, buffer, value):
+ # for cross-language or minimal framing, reuse the same logic
+ return self.write(buffer, value)
+
+ def xread(self, buffer):
+ # symmetric to xwrite
+ return self.read(buffer)
+
+
class ComplexObjectSerializer(DataClassSerializer):
def __new__(cls, fory, clz):
warnings.warn(
diff --git a/python/pyfory/tests/test_serializer.py b/python/pyfory/tests/test_serializer.py
index 85f57067e..3f6cbcb49 100644
--- a/python/pyfory/tests/test_serializer.py
+++ b/python/pyfory/tests/test_serializer.py
@@ -109,7 +109,7 @@ def test_multi_chunk_complex_dict(track_ref):
fory = Fory(language=Language.PYTHON, ref_tracking=track_ref)
now = datetime.datetime.now()
day = datetime.date(2021, 11, 23)
- dict0 = {"a": "a", 1: 1, -1.0: -1.0, True: True, now: now, day: day}
+ dict0 = {"a": "a", 1: 1, -1.0: -1.0, True: True, now: now, day: day} # noqa: F601
assert ser_de(fory, dict0) == dict0
@@ -131,15 +131,11 @@ def test_big_chunk_dict(track_ref):
def test_basic_serializer(language):
fory = Fory(language=language, ref_tracking=True)
typeinfo = fory.type_resolver.get_typeinfo(datetime.datetime)
- assert isinstance(
- typeinfo.serializer, (TimestampSerializer, _serialization.TimestampSerializer)
- )
+ assert isinstance(typeinfo.serializer, (TimestampSerializer, _serialization.TimestampSerializer))
if language == Language.XLANG:
assert typeinfo.type_id == TypeId.TIMESTAMP
typeinfo = fory.type_resolver.get_typeinfo(datetime.date)
- assert isinstance(
- typeinfo.serializer, (DateSerializer, _serialization.DateSerializer)
- )
+ assert isinstance(typeinfo.serializer, (DateSerializer, _serialization.DateSerializer))
if language == Language.XLANG:
assert typeinfo.type_id == TypeId.LOCAL_DATE
assert ser_de(fory, True) is True
@@ -163,7 +159,7 @@ def test_basic_serializer(language):
assert ser_de(fory, list_) == list_
dict1_ = {"k1": "a", "k2": 1, "k3": -1.0, "k4": True, "k5": now, "k6": day}
assert ser_de(fory, dict1_) == dict1_
- dict2_ = {"a": "a", 1: 1, -1.0: -1.0, True: True, now: now, day: day}
+ dict2_ = {"a": "a", 1: 1, -1.0: -1.0, True: True, now: now, day: day} # noqa: F601
assert ser_de(fory, dict2_) == dict2_
set_ = {"a", 1, -1.0, True, now, day}
assert ser_de(fory, set_) == set_
@@ -364,9 +360,7 @@ def test_serialize_arrow_zero_copy():
buffer_objects = []
fory = Fory(language=Language.XLANG, ref_tracking=True)
serialized_data = Buffer.allocate(32)
- fory.serialize(
- record_batch, buffer=serialized_data, buffer_callback=buffer_objects.append
- )
+ fory.serialize(record_batch, buffer=serialized_data, buffer_callback=buffer_objects.append)
fory.serialize(table, buffer=serialized_data, buffer_callback=buffer_objects.append)
buffers = [o.to_buffer() for o in buffer_objects]
new_batch = fory.deserialize(serialized_data, buffers=buffers[:1])
@@ -409,9 +403,7 @@ class RegisterClass:
def test_register_py_serializer():
- fory = Fory(
- language=Language.PYTHON, ref_tracking=True, require_type_registration=False
- )
+ fory = Fory(language=Language.PYTHON, ref_tracking=True, require_type_registration=False)
class Serializer(pyfory.Serializer):
def write(self, buffer, value):
@@ -463,9 +455,7 @@ def test_register_type():
def test_pickle_fallback():
- fory = Fory(
- language=Language.PYTHON, ref_tracking=True, require_type_registration=False
- )
+ fory = Fory(language=Language.PYTHON, ref_tracking=True, require_type_registration=False)
o1 = [1, True, np.dtype(np.int32)]
data1 = fory.serialize(o1)
new_o1 = fory.deserialize(data1)
@@ -477,9 +467,7 @@ def test_pickle_fallback():
def test_unsupported_callback():
- fory = Fory(
- language=Language.PYTHON, ref_tracking=True, require_type_registration=False
- )
+ fory = Fory(language=Language.PYTHON, ref_tracking=True, require_type_registration=False)
def f1(x):
return x
@@ -498,27 +486,27 @@ def test_unsupported_callback():
def test_slice():
fory = Fory(language=Language.PYTHON, ref_tracking=True)
- assert fory.deserialize(fory.serialize(slice(1, None, "10"))) == slice(
- 1, None, "10"
- )
+ assert fory.deserialize(fory.serialize(slice(1, None, "10"))) == slice(1, None, "10")
assert fory.deserialize(fory.serialize(slice(1, 100, 10))) == slice(1, 100, 10)
assert fory.deserialize(fory.serialize(slice(1, None, 10))) == slice(1, None, 10)
assert fory.deserialize(fory.serialize(slice(10, 10, None))) == slice(10, 10, None)
- assert fory.deserialize(fory.serialize(slice(None, None, 10))) == slice(
- None, None, 10
- )
- assert fory.deserialize(fory.serialize(slice(None, None, None))) == slice(
- None, None, None
- )
- assert fory.deserialize(
- fory.serialize([1, 2, slice(1, 100, 10), slice(1, 100, 10)])
- ) == [1, 2, slice(1, 100, 10), slice(1, 100, 10)]
- assert fory.deserialize(
- fory.serialize([1, slice(1, None, 10), False, [], slice(1, 100, 10)])
- ) == [1, slice(1, None, 10), False, [], slice(1, 100, 10)]
- assert fory.deserialize(
- fory.serialize([1, slice(1, None, "10"), False, [], slice(1, 100, "10")])
- ) == [1, slice(1, None, "10"), False, [], slice(1, 100, "10")]
+ assert fory.deserialize(fory.serialize(slice(None, None, 10))) == slice(None, None, 10)
+ assert fory.deserialize(fory.serialize(slice(None, None, None))) == slice(None, None, None)
+ assert fory.deserialize(fory.serialize([1, 2, slice(1, 100, 10), slice(1, 100, 10)])) == [1, 2, slice(1, 100, 10), slice(1, 100, 10)]
+ assert fory.deserialize(fory.serialize([1, slice(1, None, 10), False, [], slice(1, 100, 10)])) == [
+ 1,
+ slice(1, None, 10),
+ False,
+ [],
+ slice(1, 100, 10),
+ ]
+ assert fory.deserialize(fory.serialize([1, slice(1, None, "10"), False, [], slice(1, 100, "10")])) == [
+ 1,
+ slice(1, None, "10"),
+ False,
+ [],
+ slice(1, 100, "10"),
+ ]
class EnumClass(Enum):
@@ -561,12 +549,8 @@ def test_cache_serializer():
def test_pandas_range_index():
- fory = Fory(
- language=Language.PYTHON, ref_tracking=True, require_type_registration=False
- )
- fory.register_type(
- pd.RangeIndex, serializer=pyfory.PandasRangeIndexSerializer(fory)
- )
+ fory = Fory(language=Language.PYTHON, ref_tracking=True, require_type_registration=False)
+ fory.register_type(pd.RangeIndex, serializer=pyfory.PandasRangeIndexSerializer(fory))
index = pd.RangeIndex(1, 100, 2, name="a")
new_index = ser_de(fory, index)
pd.testing.assert_index_equal(new_index, new_index)
@@ -590,9 +574,7 @@ def test_py_serialize_dataclass(track_ref):
ref_tracking=track_ref,
require_type_registration=False,
)
- obj1 = PyDataClass1(
- f1=1, f2=-2.0, f3="abc", f4=True, f5="xyz", f6=[1, 2], f7={"k1": "v1"}
- )
+ obj1 = PyDataClass1(f1=1, f2=-2.0, f3="abc", f4=True, f5="xyz", f6=[1, 2], f7={"k1": "v1"})
assert ser_de(fory, obj1) == obj1
obj2 = PyDataClass1(f1=None, f2=-2.0, f3="abc", f4=None, f5="xyz", f6=None, f7=None)
assert ser_de(fory, obj2) == obj2
@@ -649,6 +631,8 @@ def test_map_fields_chunk_serializer(track_ref):
tuple_key_dict = {(1, 2): {1, 2, 3}, (3, 4): {4, 5, 6}}
class CustomClass:
+ __slots__ = ["value"]
+
def __init__(self, value):
self.value = value
@@ -683,5 +667,40 @@ def test_map_fields_chunk_serializer(track_ref):
assert map_fields_object.single_key_dict == deserialized.single_key_dict
+class SomeTestObject:
+ def __init__(self, f1: int, f2: str):
+ self.f1 = f1
+ self.f2 = f2
+
+ def __eq__(self, other):
+ return self.f1 == other.f1 and self.f2 == other.f2
+
+
+class SomeTestSlotsObject:
+ __slots__ = ["f1", "f2"]
+
+ def __init__(self, f1: int, f2: str):
+ self.f1 = f1
+ self.f2 = f2
+
+ def __eq__(self, other):
+ return self.f1 == other.f1 and self.f2 == other.f2
+
+
+@pytest.mark.parametrize("track_ref", [False, True])
+def test_py_serialize_object(track_ref):
+ fory = Fory(
+ language=Language.PYTHON,
+ ref_tracking=track_ref,
+ require_type_registration=False,
+ )
+ fory.register_type(SomeTestObject)
+ fory.register_type(SomeTestSlotsObject)
+ obj1 = SomeTestObject(f1=1, f2="abc")
+ assert ser_de(fory, obj1) == obj1
+ obj2 = SomeTestSlotsObject(f1=1, f2="abc")
+ assert ser_de(fory, obj2) == obj2
+
+
if __name__ == "__main__":
test_string()
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@fory.apache.org
For additional commands, e-mail: commits-help@fory.apache.org