You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@fory.apache.org by ch...@apache.org on 2025/07/14 17:34:41 UTC

(fory) branch main updated: feat: Implement ObjectSerializer for custom objects (#2413)

This is an automated email from the ASF dual-hosted git repository.

chaokunyang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fory.git


The following commit(s) were added to refs/heads/main by this push:
     new f4718ac65 feat: Implement ObjectSerializer for custom objects (#2413)
f4718ac65 is described below

commit f4718ac652ef4fa21d900b691c8d21a96b18e1f8
Author: Emre Şafak <39...@users.noreply.github.com>
AuthorDate: Mon Jul 14 13:34:37 2025 -0400

    feat: Implement ObjectSerializer for custom objects (#2413)
    
    ## What does this PR do?
    
    * Introduce ObjectSerializer to handle serialization of arbitrary Python
    objects. bjectSerializer serializes based on object's `__dict__` or
    `__slots__`.
    * Update `_registry.py` to correctly identify and register
    ObjectSerializer.
    
    ## Related issues
    
    Closes #2410
    
    ## Does this PR introduce any user-facing change?
    
    - [ ] Does this PR introduce any public API change?
    - [ ] Does this PR introduce any binary protocol compatibility change?
    
    ## Benchmark
    
    Not done
    
    ---------
    
    Co-authored-by: google-labs-jules[bot] <16...@users.noreply.github.com>
    Co-authored-by: Emre Şafak <es...@users.noreply.github.com>
---
 python/pyfory/_registry.py             |  13 ++--
 python/pyfory/serializer.py            |  60 ++++++++++++++++-
 python/pyfory/tests/test_serializer.py | 113 +++++++++++++++++++--------------
 3 files changed, 134 insertions(+), 52 deletions(-)

diff --git a/python/pyfory/_registry.py b/python/pyfory/_registry.py
index 683f0a6bc..1fe214366 100644
--- a/python/pyfory/_registry.py
+++ b/python/pyfory/_registry.py
@@ -63,6 +63,7 @@ from pyfory.serializer import (
     StatefulSerializer,
     ReduceSerializer,
     FunctionSerializer,
+    ObjectSerializer,
 )
 from pyfory.meta.metastring import MetaStringEncoder, MetaStringDecoder
 from pyfory.type import (
@@ -467,7 +468,7 @@ class TypeResolver:
                 type_id = TypeId.NAMED_ENUM
             elif type(serializer) is PickleSerializer:
                 type_id = PickleSerializer.PICKLE_TYPE_ID
-            elif isinstance(serializer, (StatefulSerializer, ReduceSerializer)):
+            elif isinstance(serializer, (ObjectSerializer, StatefulSerializer, ReduceSerializer)):
                 type_id = TypeId.NAMED_EXT
             if not self.require_registration:
                 if isinstance(serializer, DataClassSerializer):
@@ -493,8 +494,6 @@ class TypeResolver:
                 # Use PickleSerializer for function types (including lambdas)
                 serializer = PickleSerializer(self.fory, cls)
             elif dataclasses.is_dataclass(cls):
-                from pyfory import DataClassSerializer
-
                 serializer = DataClassSerializer(self.fory, cls)
             elif issubclass(cls, enum.Enum):
                 serializer = EnumSerializer(self.fory, cls)
@@ -502,7 +501,7 @@ class TypeResolver:
                 hasattr(cls, "__reduce_ex__") and cls.__reduce_ex__ is not object.__reduce_ex__
             ):
                 # Use ReduceSerializer for objects that have custom __reduce__ or __reduce_ex__ methods
-                # This has higher precedence than StatefulSerializer
+                # This has higher precedence than StatefulSerializer and ObjectSerializer
                 # Only use it for objects with custom reduce methods, not default ones from the object
                 module_name = getattr(cls, "__module__", "")
                 if module_name.startswith("pandas.") or module_name == "builtins" or cls.__name__ in ("type", "function", "method"):
@@ -519,6 +518,12 @@ class TypeResolver:
                     serializer = PickleSerializer(self.fory, cls)
                 else:
                     serializer = StatefulSerializer(self.fory, cls)
+            elif (
+                cls is not type
+                and (hasattr(cls, "__dict__") or hasattr(cls, "__slots__"))
+                and not (np and (issubclass(cls, np.dtype) or cls is type(np.dtype)))
+            ):
+                serializer = ObjectSerializer(self.fory, cls)
             else:
                 serializer = PickleSerializer(self.fory, cls)
         return serializer
diff --git a/python/pyfory/serializer.py b/python/pyfory/serializer.py
index d6db365e1..aba3809c4 100644
--- a/python/pyfory/serializer.py
+++ b/python/pyfory/serializer.py
@@ -302,7 +302,7 @@ class DataClassSerializer(Serializer):
         self._xlang = xlang
         # This will get superclass type hints too.
         self._type_hints = typing.get_type_hints(clz)
-        self._field_names = sorted(self._type_hints.keys())
+        self._field_names = self._get_field_names(clz)
         self._has_slots = hasattr(clz, "__slots__")
 
         if self._xlang:
@@ -335,6 +335,16 @@ class DataClassSerializer(Serializer):
                 self.write = self._generated_write_method
                 self.read = self._generated_read_method
 
+    def _get_field_names(self, clz):
+        if hasattr(clz, "__dict__"):
+            # Regular object with __dict__
+            # We can't know the fields without an instance, so we rely on type hints
+            return sorted(self._type_hints.keys())
+        elif hasattr(clz, "__slots__"):
+            # Object with __slots__
+            return sorted(clz.__slots__)
+        return []
+
     def _gen_write_method(self):
         context = {}
         counter = itertools.count(0)
@@ -1178,6 +1188,54 @@ class PickleSerializer(Serializer):
         return self.fory.handle_unsupported_read(buffer)
 
 
+class ObjectSerializer(Serializer):
+    """Serializer for regular Python objects.
+    It serializes objects based on `__dict__` or `__slots__`.
+    """
+
+    def __init__(self, fory, clz: type):
+        super().__init__(fory, clz)
+        # If the class defines __slots__, compute and store a sorted list once
+        slots = getattr(clz, "__slots__", None)
+        self._slot_field_names = None
+        if slots is not None:
+            # __slots__ can be a string or iterable of strings
+            if isinstance(slots, str):
+                slots = [slots]
+            self._slot_field_names = sorted(slots)
+
+    def write(self, buffer, value):
+        # Use precomputed slots if available, otherwise sort instance __dict__ keys
+        if self._slot_field_names is not None:
+            sorted_field_names = self._slot_field_names
+        else:
+            sorted_field_names = sorted(value.__dict__.keys())
+
+        buffer.write_varuint32(len(sorted_field_names))
+        for field_name in sorted_field_names:
+            buffer.write_string(field_name)
+            field_value = getattr(value, field_name)
+            self.fory.serialize_ref(buffer, field_value)
+
+    def read(self, buffer):
+        obj = self.type_.__new__(self.type_)
+        self.fory.ref_resolver.reference(obj)
+        num_fields = buffer.read_varuint32()
+        for _ in range(num_fields):
+            field_name = buffer.read_string()
+            field_value = self.fory.deserialize_ref(buffer)
+            setattr(obj, field_name, field_value)
+        return obj
+
+    def xwrite(self, buffer, value):
+        # for cross-language or minimal framing, reuse the same logic
+        return self.write(buffer, value)
+
+    def xread(self, buffer):
+        # symmetric to xwrite
+        return self.read(buffer)
+
+
 class ComplexObjectSerializer(DataClassSerializer):
     def __new__(cls, fory, clz):
         warnings.warn(
diff --git a/python/pyfory/tests/test_serializer.py b/python/pyfory/tests/test_serializer.py
index 85f57067e..3f6cbcb49 100644
--- a/python/pyfory/tests/test_serializer.py
+++ b/python/pyfory/tests/test_serializer.py
@@ -109,7 +109,7 @@ def test_multi_chunk_complex_dict(track_ref):
     fory = Fory(language=Language.PYTHON, ref_tracking=track_ref)
     now = datetime.datetime.now()
     day = datetime.date(2021, 11, 23)
-    dict0 = {"a": "a", 1: 1, -1.0: -1.0, True: True, now: now, day: day}
+    dict0 = {"a": "a", 1: 1, -1.0: -1.0, True: True, now: now, day: day}  # noqa: F601
     assert ser_de(fory, dict0) == dict0
 
 
@@ -131,15 +131,11 @@ def test_big_chunk_dict(track_ref):
 def test_basic_serializer(language):
     fory = Fory(language=language, ref_tracking=True)
     typeinfo = fory.type_resolver.get_typeinfo(datetime.datetime)
-    assert isinstance(
-        typeinfo.serializer, (TimestampSerializer, _serialization.TimestampSerializer)
-    )
+    assert isinstance(typeinfo.serializer, (TimestampSerializer, _serialization.TimestampSerializer))
     if language == Language.XLANG:
         assert typeinfo.type_id == TypeId.TIMESTAMP
     typeinfo = fory.type_resolver.get_typeinfo(datetime.date)
-    assert isinstance(
-        typeinfo.serializer, (DateSerializer, _serialization.DateSerializer)
-    )
+    assert isinstance(typeinfo.serializer, (DateSerializer, _serialization.DateSerializer))
     if language == Language.XLANG:
         assert typeinfo.type_id == TypeId.LOCAL_DATE
     assert ser_de(fory, True) is True
@@ -163,7 +159,7 @@ def test_basic_serializer(language):
     assert ser_de(fory, list_) == list_
     dict1_ = {"k1": "a", "k2": 1, "k3": -1.0, "k4": True, "k5": now, "k6": day}
     assert ser_de(fory, dict1_) == dict1_
-    dict2_ = {"a": "a", 1: 1, -1.0: -1.0, True: True, now: now, day: day}
+    dict2_ = {"a": "a", 1: 1, -1.0: -1.0, True: True, now: now, day: day}  # noqa: F601
     assert ser_de(fory, dict2_) == dict2_
     set_ = {"a", 1, -1.0, True, now, day}
     assert ser_de(fory, set_) == set_
@@ -364,9 +360,7 @@ def test_serialize_arrow_zero_copy():
     buffer_objects = []
     fory = Fory(language=Language.XLANG, ref_tracking=True)
     serialized_data = Buffer.allocate(32)
-    fory.serialize(
-        record_batch, buffer=serialized_data, buffer_callback=buffer_objects.append
-    )
+    fory.serialize(record_batch, buffer=serialized_data, buffer_callback=buffer_objects.append)
     fory.serialize(table, buffer=serialized_data, buffer_callback=buffer_objects.append)
     buffers = [o.to_buffer() for o in buffer_objects]
     new_batch = fory.deserialize(serialized_data, buffers=buffers[:1])
@@ -409,9 +403,7 @@ class RegisterClass:
 
 
 def test_register_py_serializer():
-    fory = Fory(
-        language=Language.PYTHON, ref_tracking=True, require_type_registration=False
-    )
+    fory = Fory(language=Language.PYTHON, ref_tracking=True, require_type_registration=False)
 
     class Serializer(pyfory.Serializer):
         def write(self, buffer, value):
@@ -463,9 +455,7 @@ def test_register_type():
 
 
 def test_pickle_fallback():
-    fory = Fory(
-        language=Language.PYTHON, ref_tracking=True, require_type_registration=False
-    )
+    fory = Fory(language=Language.PYTHON, ref_tracking=True, require_type_registration=False)
     o1 = [1, True, np.dtype(np.int32)]
     data1 = fory.serialize(o1)
     new_o1 = fory.deserialize(data1)
@@ -477,9 +467,7 @@ def test_pickle_fallback():
 
 
 def test_unsupported_callback():
-    fory = Fory(
-        language=Language.PYTHON, ref_tracking=True, require_type_registration=False
-    )
+    fory = Fory(language=Language.PYTHON, ref_tracking=True, require_type_registration=False)
 
     def f1(x):
         return x
@@ -498,27 +486,27 @@ def test_unsupported_callback():
 
 def test_slice():
     fory = Fory(language=Language.PYTHON, ref_tracking=True)
-    assert fory.deserialize(fory.serialize(slice(1, None, "10"))) == slice(
-        1, None, "10"
-    )
+    assert fory.deserialize(fory.serialize(slice(1, None, "10"))) == slice(1, None, "10")
     assert fory.deserialize(fory.serialize(slice(1, 100, 10))) == slice(1, 100, 10)
     assert fory.deserialize(fory.serialize(slice(1, None, 10))) == slice(1, None, 10)
     assert fory.deserialize(fory.serialize(slice(10, 10, None))) == slice(10, 10, None)
-    assert fory.deserialize(fory.serialize(slice(None, None, 10))) == slice(
-        None, None, 10
-    )
-    assert fory.deserialize(fory.serialize(slice(None, None, None))) == slice(
-        None, None, None
-    )
-    assert fory.deserialize(
-        fory.serialize([1, 2, slice(1, 100, 10), slice(1, 100, 10)])
-    ) == [1, 2, slice(1, 100, 10), slice(1, 100, 10)]
-    assert fory.deserialize(
-        fory.serialize([1, slice(1, None, 10), False, [], slice(1, 100, 10)])
-    ) == [1, slice(1, None, 10), False, [], slice(1, 100, 10)]
-    assert fory.deserialize(
-        fory.serialize([1, slice(1, None, "10"), False, [], slice(1, 100, "10")])
-    ) == [1, slice(1, None, "10"), False, [], slice(1, 100, "10")]
+    assert fory.deserialize(fory.serialize(slice(None, None, 10))) == slice(None, None, 10)
+    assert fory.deserialize(fory.serialize(slice(None, None, None))) == slice(None, None, None)
+    assert fory.deserialize(fory.serialize([1, 2, slice(1, 100, 10), slice(1, 100, 10)])) == [1, 2, slice(1, 100, 10), slice(1, 100, 10)]
+    assert fory.deserialize(fory.serialize([1, slice(1, None, 10), False, [], slice(1, 100, 10)])) == [
+        1,
+        slice(1, None, 10),
+        False,
+        [],
+        slice(1, 100, 10),
+    ]
+    assert fory.deserialize(fory.serialize([1, slice(1, None, "10"), False, [], slice(1, 100, "10")])) == [
+        1,
+        slice(1, None, "10"),
+        False,
+        [],
+        slice(1, 100, "10"),
+    ]
 
 
 class EnumClass(Enum):
@@ -561,12 +549,8 @@ def test_cache_serializer():
 
 
 def test_pandas_range_index():
-    fory = Fory(
-        language=Language.PYTHON, ref_tracking=True, require_type_registration=False
-    )
-    fory.register_type(
-        pd.RangeIndex, serializer=pyfory.PandasRangeIndexSerializer(fory)
-    )
+    fory = Fory(language=Language.PYTHON, ref_tracking=True, require_type_registration=False)
+    fory.register_type(pd.RangeIndex, serializer=pyfory.PandasRangeIndexSerializer(fory))
     index = pd.RangeIndex(1, 100, 2, name="a")
     new_index = ser_de(fory, index)
     pd.testing.assert_index_equal(new_index, new_index)
@@ -590,9 +574,7 @@ def test_py_serialize_dataclass(track_ref):
         ref_tracking=track_ref,
         require_type_registration=False,
     )
-    obj1 = PyDataClass1(
-        f1=1, f2=-2.0, f3="abc", f4=True, f5="xyz", f6=[1, 2], f7={"k1": "v1"}
-    )
+    obj1 = PyDataClass1(f1=1, f2=-2.0, f3="abc", f4=True, f5="xyz", f6=[1, 2], f7={"k1": "v1"})
     assert ser_de(fory, obj1) == obj1
     obj2 = PyDataClass1(f1=None, f2=-2.0, f3="abc", f4=None, f5="xyz", f6=None, f7=None)
     assert ser_de(fory, obj2) == obj2
@@ -649,6 +631,8 @@ def test_map_fields_chunk_serializer(track_ref):
     tuple_key_dict = {(1, 2): {1, 2, 3}, (3, 4): {4, 5, 6}}
 
     class CustomClass:
+        __slots__ = ["value"]
+
         def __init__(self, value):
             self.value = value
 
@@ -683,5 +667,40 @@ def test_map_fields_chunk_serializer(track_ref):
     assert map_fields_object.single_key_dict == deserialized.single_key_dict
 
 
+class SomeTestObject:
+    def __init__(self, f1: int, f2: str):
+        self.f1 = f1
+        self.f2 = f2
+
+    def __eq__(self, other):
+        return self.f1 == other.f1 and self.f2 == other.f2
+
+
+class SomeTestSlotsObject:
+    __slots__ = ["f1", "f2"]
+
+    def __init__(self, f1: int, f2: str):
+        self.f1 = f1
+        self.f2 = f2
+
+    def __eq__(self, other):
+        return self.f1 == other.f1 and self.f2 == other.f2
+
+
+@pytest.mark.parametrize("track_ref", [False, True])
+def test_py_serialize_object(track_ref):
+    fory = Fory(
+        language=Language.PYTHON,
+        ref_tracking=track_ref,
+        require_type_registration=False,
+    )
+    fory.register_type(SomeTestObject)
+    fory.register_type(SomeTestSlotsObject)
+    obj1 = SomeTestObject(f1=1, f2="abc")
+    assert ser_de(fory, obj1) == obj1
+    obj2 = SomeTestSlotsObject(f1=1, f2="abc")
+    assert ser_de(fory, obj2) == obj2
+
+
 if __name__ == "__main__":
     test_string()


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@fory.apache.org
For additional commands, e-mail: commits-help@fory.apache.org