You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@fory.apache.org by ch...@apache.org on 2025/09/26 05:24:04 UTC
(fory) branch main updated: feat(python): support local py class serialization (#2665)
This is an automated email from the ASF dual-hosted git repository.
chaokunyang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fory.git
The following commit(s) were added to refs/heads/main by this push:
new 71c9a000e feat(python): support local py class serialization (#2665)
71c9a000e is described below
commit 71c9a000e110086b95776e66395b17de8f4f9ce0
Author: Shawn Yang <sh...@gmail.com>
AuthorDate: Fri Sep 26 13:23:58 2025 +0800
feat(python): support local py class serialization (#2665)
## Why?
<!-- Describe the purpose of this PR. -->
## What does this PR do?
1. support local py class serialization
2. this pr also fix a crash when deserialize null values
## Related issues
Closes #2659
## Does this PR introduce any user-facing change?
<!--
If any user-facing interface changes, please [open an
issue](https://github.com/apache/fory/issues/new/choose) describing the
need to do so and update the document if necessary.
Delete section if not applicable.
-->
- [ ] Does this PR introduce any public API change?
- [ ] Does this PR introduce any binary protocol compatibility change?
## Benchmark
<!--
When the PR has an impact on performance (if you don't know whether the
PR will have an impact on performance, you can submit the PR first, and
if it will have impact on performance, the code reviewer will explain
it), be sure to attach a benchmark data here.
Delete section if not applicable.
-->
---
AGENTS.md | 2 +-
python/pyfory/_serialization.pyx | 24 +-
python/pyfory/resolver.py | 14 ++
python/pyfory/serializer.py | 126 +++++++++--
python/pyfory/tests/test_class_serializer.py | 321 +++++++++++++++++++++++++++
5 files changed, 466 insertions(+), 21 deletions(-)
diff --git a/AGENTS.md b/AGENTS.md
index a69285288..46a8d951f 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -86,7 +86,7 @@ ruff check --fix .
# Install
pip install -v -e .
-# Build native extension
+# Build native extension when cython code changed
bazel build //:cp_fory_so --config=x86_64 # For x86_64
bazel build //:cp_fory_so --copt=-fsigned-char # For arm64 and aarch64
diff --git a/python/pyfory/_serialization.pyx b/python/pyfory/_serialization.pyx
index 00fd2e929..b468a485a 100644
--- a/python/pyfory/_serialization.pyx
+++ b/python/pyfory/_serialization.pyx
@@ -140,10 +140,14 @@ cdef class MapRefResolver:
if not self.ref_tracking:
return head_flag
cdef int32_t ref_id
+ cdef PyObject * obj
if head_flag == REF_FLAG:
# read reference id and get object from reference resolver
ref_id = buffer.read_varuint32()
- self.read_object = <object> (self.read_objects[ref_id])
+ assert 0 <= ref_id < self.read_objects.size(), f"Invalid ref id {ref_id}, current size {self.read_objects.size()}"
+ obj = self.read_objects[ref_id]
+ assert obj != NULL, f"Invalid ref id {ref_id}, current size {self.read_objects.size()}"
+ self.read_object = <object> obj
return REF_FLAG
else:
self.read_object = None
@@ -163,10 +167,16 @@ cdef class MapRefResolver:
# `refId >= NOT_NULL_VALUE_FLAG` to read data.
return buffer.read_int8()
head_flag = buffer.read_int8()
+ cdef int32_t ref_id
+ cdef PyObject *obj
if head_flag == REF_FLAG:
# read reference id and get object from reference resolver
ref_id = buffer.read_varuint32()
- self.read_object = <object> (self.read_objects[ref_id])
+ # avoid wrong id cause crash
+ assert 0 <= ref_id < self.read_objects.size(), f"Invalid ref id {ref_id}, current size {self.read_objects.size()}"
+ obj = self.read_objects[ref_id]
+ assert obj != NULL, f"Invalid ref id {ref_id}, current size {self.read_objects.size()}"
+ self.read_object = <object> obj
# `head_flag` except `REF_FLAG` can be used as stub reference id because
# we use `refId >= NOT_NULL_VALUE_FLAG` to read data.
return head_flag
@@ -176,6 +186,11 @@ cdef class MapRefResolver:
return self.preserve_ref_id()
return head_flag
+ cpdef inline int32_t last_preserved_ref_id(self):
+ cdef int32_t length = self.read_ref_ids.size()
+ assert length > 0
+ return self.read_ref_ids[length - 1]
+
cpdef inline reference(self, obj):
if not self.ref_tracking:
return
@@ -192,7 +207,10 @@ cdef class MapRefResolver:
if id_ is None:
return self.read_object
cdef int32_t ref_id = id_
- return <object> (self.read_objects[ref_id])
+ cdef PyObject * obj = self.read_objects[ref_id]
+ if obj == NULL:
+ return None
+ return <object> obj
cpdef inline set_read_object(self, int32_t ref_id, obj):
if not self.ref_tracking:
diff --git a/python/pyfory/resolver.py b/python/pyfory/resolver.py
index 1dbec953e..710efb7bd 100644
--- a/python/pyfory/resolver.py
+++ b/python/pyfory/resolver.py
@@ -84,6 +84,14 @@ class RefResolver(ABC):
a reference id
"""
+ @abstractmethod
+ def last_preserved_ref_id(self) -> int:
+ """
+ Returns
+ -------
+ the last preserved reference id.
+ """
+
@abstractmethod
def reference(self, obj):
"""
@@ -188,6 +196,9 @@ class MapRefResolver(RefResolver):
# `refId >= NOT_NULL_VALUE_FLAG` to read data.
return head_flag
+ def last_preserved_ref_id(self) -> int:
+ return self.read_ref_ids[-1]
+
def reference(self, obj):
ref_id = self.read_ref_ids.pop()
self.set_read_object(ref_id, obj)
@@ -236,6 +247,9 @@ class NoRefResolver(RefResolver):
# `refId >= NOT_NULL_VALUE_FLAG` to read data.
return buffer.read_int8()
+ def last_preserved_ref_id(self) -> int:
+ return -1
+
def reference(self, obj):
pass
diff --git a/python/pyfory/serializer.py b/python/pyfory/serializer.py
index d8b403968..26dc92d1b 100644
--- a/python/pyfory/serializer.py
+++ b/python/pyfory/serializer.py
@@ -137,31 +137,99 @@ class NoneSerializer(Serializer):
return None
+__skip_class_attr_names__ = ("__module__", "__qualname__", "__dict__", "__weakref__")
+
+
class TypeSerializer(Serializer):
- """Serializer for Python type objects (classes)."""
+ """Serializer for Python type objects (classes), including local classes."""
def __init__(self, fory, cls):
super().__init__(fory, cls)
self.cls = cls
def write(self, buffer, value):
- # Serialize the type by its module and name
- module_name = getattr(value, "__module__", "")
- type_name = getattr(value, "__name__", "")
- buffer.write_string(module_name)
- buffer.write_string(type_name)
+ module_name = value.__module__
+ qualname = value.__qualname__
+
+ if module_name == "__main__" or "<locals>" in qualname:
+ # Local class - serialize full context
+ buffer.write_int8(1) # Local class marker
+ self._serialize_local_class(buffer, value)
+ else:
+ buffer.write_int8(0) # Global class marker
+ buffer.write_string(module_name)
+ buffer.write_string(qualname)
def read(self, buffer):
- module_name = buffer.read_string()
- type_name = buffer.read_string()
+ class_type = buffer.read_int8()
- # Import the module and get the type
- if module_name and module_name != "builtins":
- module = __import__(module_name, fromlist=[type_name])
- return getattr(module, type_name)
+ if class_type == 1:
+ # Local class - deserialize from full context
+ return self._deserialize_local_class(buffer)
else:
- # Handle built-in types
- return getattr(builtins, type_name, type)
+ # Global class - import by module and name
+ module_name = buffer.read_string()
+ qualname = buffer.read_string()
+ cls = importlib.import_module(module_name)
+ for name in qualname.split("."):
+ cls = getattr(cls, name)
+ return cls
+
+ def _serialize_local_class(self, buffer, cls):
+ """Serialize a local class by capturing its creation context."""
+ assert self.fory.ref_tracking, "Reference tracking must be enabled for local classes serialization"
+ # Basic class information
+ module = cls.__module__
+ qualname = cls.__qualname__
+ buffer.write_string(module)
+ buffer.write_string(qualname)
+
+ # Serialize base classes
+ # Let Fory's normal serialization handle bases (including other local classes)
+ bases = cls.__bases__
+ buffer.write_varuint32(len(bases))
+ for base in bases:
+ self.fory.serialize_ref(buffer, base)
+
+ # Serialize class dictionary (excluding special attributes)
+ # FunctionSerializer will automatically handle methods with closures
+ class_dict = {}
+ for attr_name, attr_value in cls.__dict__.items():
+ # Skip special attributes that are handled by type() constructor
+ if attr_name in __skip_class_attr_names__:
+ continue
+ class_dict[attr_name] = attr_value
+
+ # Let Fory's normal serialization handle the class dict
+ # This will use FunctionSerializer for methods, which handles closures properly
+ self.fory.serialize_ref(buffer, class_dict)
+
+ def _deserialize_local_class(self, buffer):
+ """Deserialize a local class by recreating it with the captured context."""
+ assert self.fory.ref_tracking, "Reference tracking must be enabled for local classes deserialization"
+ # Read basic class information
+ module = buffer.read_string()
+ qualname = buffer.read_string()
+ name = qualname.rsplit(".", 1)[-1]
+ ref_id = self.fory.ref_resolver.last_preserved_ref_id()
+
+ # Read base classes
+ num_bases = buffer.read_varuint32()
+ bases = tuple([self.fory.deserialize_ref(buffer) for _ in range(num_bases)])
+ # Create the class using type() constructor
+ cls = type(name, bases, {})
+ # `class_dict` may reference to `cls`, which is a circular reference
+ self.fory.ref_resolver.set_read_object(ref_id, cls)
+ # Read class dictionary
+ # Fory's normal deserialization will handle methods via FunctionSerializer
+ class_dict = self.fory.deserialize_ref(buffer)
+ for k, v in class_dict.items():
+ setattr(cls, k, v)
+
+ # Set module and qualname
+ cls.__module__ = module
+ cls.__qualname__ = qualname
+ return cls
class PandasRangeIndexSerializer(Serializer):
@@ -244,7 +312,14 @@ from pyfory._struct import _get_hash, _sort_fields, StructFieldSerializerVisitor
class DataClassSerializer(Serializer):
- def __init__(self, fory, clz: type, xlang: bool = False, field_names: List[str] = None, serializers: List[Serializer] = None):
+ def __init__(
+ self,
+ fory,
+ clz: type,
+ xlang: bool = False,
+ field_names: List[str] = None,
+ serializers: List[Serializer] = None,
+ ):
super().__init__(fory, clz)
self._xlang = xlang
# This will get superclass type hints too.
@@ -915,7 +990,14 @@ class ReduceSerializer(CrossLanguageCompatibleSerializer):
elif len(reduce_result) == 5:
# Case 5: (callable, args, state, listitems, dictitems)
callable_obj, args, state, listitems, dictitems = reduce_result
- reduce_data = ("callable", callable_obj, args, state, listitems, dictitems)
+ reduce_data = (
+ "callable",
+ callable_obj,
+ args,
+ state,
+ listitems,
+ dictitems,
+ )
else:
raise ValueError(f"Invalid __reduce__ result length: {len(reduce_result)}")
else:
@@ -1003,7 +1085,17 @@ class FunctionSerializer(CrossLanguageCompatibleSerializer):
"""
# Cache for function attributes that are handled separately
- _FUNCTION_ATTRS = frozenset(("__code__", "__name__", "__defaults__", "__closure__", "__globals__", "__module__", "__qualname__"))
+ _FUNCTION_ATTRS = frozenset(
+ (
+ "__code__",
+ "__name__",
+ "__defaults__",
+ "__closure__",
+ "__globals__",
+ "__module__",
+ "__qualname__",
+ )
+ )
def _serialize_function(self, buffer, func):
"""Serialize a function by capturing all its components."""
diff --git a/python/pyfory/tests/test_class_serializer.py b/python/pyfory/tests/test_class_serializer.py
new file mode 100644
index 000000000..2f903ef9d
--- /dev/null
+++ b/python/pyfory/tests/test_class_serializer.py
@@ -0,0 +1,321 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from pyfory import Fory
+
+
+def test_local_class_serialization():
+ """Test serialization and deserialization of local classes."""
+
+ def create_local_class():
+ """Function that creates a local class."""
+
+ class LocalTestClass:
+ def __init__(self, value):
+ self.value = value
+
+ def get_value(self):
+ return self.value
+
+ def __eq__(self, other):
+ return isinstance(other, LocalTestClass) and self.value == other.value
+
+ return LocalTestClass
+
+ # Create an instance of the local class
+ LocalClass = create_local_class()
+
+ # Test basic serialization of the class type itself
+ fory = Fory(ref_tracking=True, require_type_registration=False)
+
+ # Serialize the class type
+ serialized = fory.serialize(LocalClass)
+ assert len(serialized) > 0
+
+ # Deserialize the class type
+ deserialized_class = fory.deserialize(serialized)
+ assert deserialized_class.__name__ == LocalClass.__name__
+ assert deserialized_class.__qualname__ == LocalClass.__qualname__
+
+ # Test that we can create instances of the deserialized class
+ instance1 = LocalClass(42)
+ instance2 = deserialized_class(42)
+
+ assert instance1.get_value() == instance2.get_value()
+ assert instance1.get_value() == 42
+
+
+def test_local_class_with_closure():
+ """Test serialization of a local class that uses closure variables."""
+
+ def create_local_class_with_closure(multiplier):
+ """Function that creates a local class using closure variables."""
+
+ class LocalClassWithClosure:
+ def __init__(self, value):
+ self.value = value
+
+ def get_multiplied_value(self):
+ return self.value * multiplier # Uses closure variable
+
+ return LocalClassWithClosure
+
+ # Create a local class with closure
+ LocalClassWithClosure = create_local_class_with_closure(3)
+
+ fory = Fory(ref_tracking=True, require_type_registration=False)
+
+ # Serialize the class type
+ serialized = fory.serialize(LocalClassWithClosure)
+ assert len(serialized) > 0
+
+ # Deserialize the class type
+ deserialized_class = fory.deserialize(serialized)
+ assert deserialized_class.__name__ == LocalClassWithClosure.__name__
+
+ # Test that the closure variable works
+ instance1 = LocalClassWithClosure(10)
+ instance2 = deserialized_class(10)
+
+ assert instance1.get_multiplied_value() == 30
+ assert instance2.get_multiplied_value() == 30
+ assert instance1.get_multiplied_value() == instance2.get_multiplied_value()
+
+
+def test_local_class_with_inheritance():
+ """Test local class with inheritance"""
+
+ def create_local_class_with_inheritance():
+ class BaseClass:
+ def base_method(self):
+ return "base"
+
+ class LocalDerivedClass(BaseClass):
+ def __init__(self, value):
+ self.value = value
+
+ def get_value(self):
+ return self.value
+
+ return LocalDerivedClass
+
+ LocalClass = create_local_class_with_inheritance()
+ fory = Fory(ref_tracking=True, require_type_registration=False)
+
+ # Serialize and deserialize the class
+ serialized = fory.serialize(LocalClass)
+ deserialized = fory.deserialize(serialized)
+
+ # Test inheritance works
+ instance1 = LocalClass(42)
+ instance2 = deserialized(100)
+
+ assert instance1.get_value() == 42
+ assert instance1.base_method() == "base"
+ assert instance2.get_value() == 100
+ assert instance2.base_method() == "base"
+
+
+def test_local_class_with_class_variables():
+ """Test local class with class variables"""
+
+ def create_class_with_vars():
+ class LocalClassWithVars:
+ class_var = "shared_value"
+ counter = 0
+
+ def __init__(self, value):
+ self.value = value
+ self.counter += 1
+ LocalClassWithVars.counter += 1
+
+ def get_value(self):
+ return self.value
+
+ def get_class_var(self):
+ return LocalClassWithVars.class_var
+
+ def get_info(self):
+ return f"value={self.value}, class_var={self.class_var}, counter={self.counter}"
+
+ return LocalClassWithVars
+
+ LocalClass = create_class_with_vars()
+ fory = Fory(ref_tracking=True, require_type_registration=False)
+
+ # Create some instances to modify class state
+ LocalClass(1) # This increments the counter
+ LocalClass(2) # This increments the counter again
+
+ # Serialize and deserialize the class
+ serialized = fory.serialize(LocalClass)
+ deserialized = fory.deserialize(serialized)
+
+ # Test class variables are preserved
+ instance3 = deserialized(3)
+
+ assert instance3.get_value() == 3
+ assert instance3.get_class_var() == "shared_value"
+ assert deserialized.class_var == LocalClass.class_var == "shared_value"
+ # Counter should be preserved from when class was serialized
+ assert deserialized.counter >= 0
+ # Test that the get_info method works with natural class references
+ info = instance3.get_info()
+ assert "value=3" in info
+ assert "class_var=shared_value" in info
+
+
+def test_nested_global_classes():
+ """Test serialization of nested global classes"""
+
+ # Define a nested global class (defined at module level but nested in structure)
+ class OuterGlobalClass:
+ class InnerGlobalClass:
+ def __init__(self, inner_value):
+ self.inner_value = inner_value
+
+ def get_inner(self):
+ return self.inner_value
+
+ def __init__(self, outer_value):
+ self.outer_value = outer_value
+
+ def get_outer(self):
+ return self.outer_value
+
+ def create_inner(self, inner_val):
+ return self.InnerGlobalClass(inner_val)
+
+ fory = Fory(ref_tracking=True, require_type_registration=False)
+
+ # Test serializing the outer class
+ serialized_outer = fory.serialize(OuterGlobalClass)
+ deserialized_outer = fory.deserialize(serialized_outer)
+
+ # Test serializing the inner class
+ serialized_inner = fory.serialize(OuterGlobalClass.InnerGlobalClass)
+ deserialized_inner = fory.deserialize(serialized_inner)
+
+ # Test functionality
+ outer_instance1 = OuterGlobalClass(100)
+ outer_instance2 = deserialized_outer(200)
+
+ inner_instance1 = OuterGlobalClass.InnerGlobalClass(50)
+ inner_instance2 = deserialized_inner(75)
+
+ assert outer_instance1.get_outer() == 100
+ assert outer_instance2.get_outer() == 200
+ assert inner_instance1.get_inner() == 50
+ assert inner_instance2.get_inner() == 75
+
+ # Test that deserialized outer class can create inner instances
+ inner_from_deserialized = outer_instance2.create_inner(300)
+ assert inner_from_deserialized.get_inner() == 300
+
+
+def test_complex_local_class_scenarios():
+ """Test complex local class scenarios including nested local classes"""
+
+ def create_complex_local_scenario(outer_multiplier):
+ def inner_helper(x):
+ return x * outer_multiplier
+
+ class OuterLocalClass:
+ shared_value = "outer_shared"
+
+ def __init__(self, value):
+ self.value = value
+
+ def outer_method(self):
+ return inner_helper(self.value)
+
+ def create_inner(self, inner_val):
+ def another_helper(y):
+ return y + outer_multiplier
+
+ class InnerLocalClass:
+ def __init__(self, inner_value):
+ self.inner_value = inner_value
+
+ def inner_method(self):
+ return another_helper(self.inner_value)
+
+ return InnerLocalClass(inner_val)
+
+ return OuterLocalClass
+
+ fory = Fory(ref_tracking=True, require_type_registration=False)
+
+ # Create complex local class with nested closures
+ ComplexLocalClass = create_complex_local_scenario(5)
+
+ # Serialize and deserialize
+ serialized = fory.serialize(ComplexLocalClass)
+ deserialized = fory.deserialize(serialized)
+
+ # Test functionality
+ instance1 = ComplexLocalClass(10)
+ instance2 = deserialized(20)
+
+ assert instance1.outer_method() == 50 # 10 * 5
+ assert instance2.outer_method() == 100 # 20 * 5
+
+ # Test nested inner class creation and methods
+ inner1 = instance1.create_inner(8)
+ inner2 = instance2.create_inner(12)
+
+ assert inner1.inner_method() == 13 # 8 + 5
+ assert inner2.inner_method() == 17 # 12 + 5
+
+
+def test_local_class_with_multiple_inheritance():
+ """Test local class with multiple inheritance"""
+
+ def create_local_class_with_multiple_inheritance():
+ class MixinA:
+ def method_a(self):
+ return "A"
+
+ class MixinB:
+ def method_b(self):
+ return "B"
+
+ class LocalMultipleInheritanceClass(MixinA, MixinB):
+ def __init__(self, value):
+ self.value = value
+
+ def combined_method(self):
+ return f"{self.method_a()}{self.method_b()}{self.value}"
+
+ return LocalMultipleInheritanceClass
+
+ fory = Fory(ref_tracking=True, require_type_registration=False)
+
+ LocalClass = create_local_class_with_multiple_inheritance()
+
+ # Serialize and deserialize
+ serialized = fory.serialize(LocalClass)
+ deserialized = fory.deserialize(serialized)
+
+ # Test functionality
+ instance1 = LocalClass("_test")
+ instance2 = deserialized("_check")
+
+ assert instance1.combined_method() == "AB_test"
+ assert instance2.combined_method() == "AB_check"
+ assert instance1.method_a() == "A"
+ assert instance2.method_b() == "B"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@fory.apache.org
For additional commands, e-mail: commits-help@fory.apache.org