You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@fory.apache.org by ch...@apache.org on 2025/10/20 10:52:50 UTC
(fory) branch main updated: feat(python): support dataclass compatible mode for python native mode (#2784)
This is an automated email from the ASF dual-hosted git repository.
chaokunyang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fory.git
The following commit(s) were added to refs/heads/main by this push:
new 90bc82393 feat(python): support dataclass compatible mode for python native mode (#2784)
90bc82393 is described below
commit 90bc82393ada55ef57fedb30f5f88b1fed8a1854
Author: Shawn Yang <sh...@gmail.com>
AuthorDate: Mon Oct 20 18:52:48 2025 +0800
feat(python): support dataclass compatible mode for python native mode (#2784)
## Why?
<!-- Describe the purpose of this PR. -->
support dataclass compatible mode for python native mode
<!-- Describe the details of this PR. -->
## Related issues
<!--
Is there any related issue? If this PR closes them you say say
fix/closes:
- #xxxx0
- #xxxx1
- Fixes #xxxx2
-->
## Does this PR introduce any user-facing change?
<!--
If any user-facing interface changes, please [open an
issue](https://github.com/apache/fory/issues/new/choose) describing the
need to do so and update the document if necessary.
Delete section if not applicable.
-->
- [ ] Does this PR introduce any public API change?
- [ ] Does this PR introduce any binary protocol compatibility change?
## Benchmark
<!--
When the PR has an impact on performance (if you don't know whether the
PR will have an impact on performance, you can submit the PR first, and
if it will have impact on performance, the code reviewer will explain
it), be sure to attach a benchmark data here.
Delete section if not applicable.
-->
---
python/pyfory/_fory.py | 8 +-
python/pyfory/_serialization.pyx | 6 +
python/pyfory/meta/typedef.py | 7 +-
python/pyfory/serializer.py | 448 +++++++++++++++++++++++----------
python/pyfory/tests/test_serializer.py | 54 +++-
python/pyfory/tests/test_struct.py | 193 +++++++++++++-
python/pyfory/type.py | 8 +-
7 files changed, 577 insertions(+), 147 deletions(-)
diff --git a/python/pyfory/_fory.py b/python/pyfory/_fory.py
index e89f8ffa7..f283bc4ea 100644
--- a/python/pyfory/_fory.py
+++ b/python/pyfory/_fory.py
@@ -121,6 +121,7 @@ class Fory:
"_peer_language",
"max_depth",
"depth",
+ "field_nullable",
)
def __init__(
@@ -130,6 +131,7 @@ class Fory:
strict: bool = True,
compatible: bool = False,
max_depth: int = 50,
+ field_nullable: bool = False,
**kwargs,
):
"""
@@ -157,11 +159,14 @@ class Fory:
you disable this option.
:param compatible:
Whether to enable compatible mode for cross-language serialization.
- When enabled, type forward/backward compatibility for struct fields will be enabled.
+ When enabled, type forward/backward compatibility for dataclass fields will be enabled.
:param max_depth:
The maximum depth of the deserialization data.
If the depth exceeds the maximum depth, an exception will be raised.
The default value is 50.
+ :param field_nullable:
+ Whether dataclass fields are nullable for python native mode(xlang=False). When enabled, dataclass fields
+ are always treated as nullable whether or not they are annotated with `Optional`.
"""
self.language = Language.XLANG if xlang else Language.PYTHON
if kwargs.get("language") is not None:
@@ -178,6 +183,7 @@ class Fory:
strict = kwargs.get("require_type_registration")
self.strict = _ENABLE_TYPE_REGISTRATION_FORCIBLY or strict
self.compatible = compatible
+ self.field_nullable = field_nullable if self.is_py else False
from pyfory._serialization import MetaStringResolver, SerializationContext
from pyfory._registry import TypeResolver
diff --git a/python/pyfory/_serialization.pyx b/python/pyfory/_serialization.pyx
index 8be215235..9b163fdf4 100644
--- a/python/pyfory/_serialization.pyx
+++ b/python/pyfory/_serialization.pyx
@@ -804,6 +804,7 @@ cdef class Fory:
cdef readonly c_bool strict
cdef readonly c_bool is_py
cdef readonly c_bool compatible
+ cdef readonly c_bool field_nullable
cdef readonly MapRefResolver ref_resolver
cdef readonly TypeResolver type_resolver
cdef readonly MetaStringResolver metastring_resolver
@@ -824,6 +825,7 @@ cdef class Fory:
strict: bool = True,
compatible: bool = False,
max_depth: int = 50,
+ field_nullable: bool = False,
**kwargs,
):
"""
@@ -856,6 +858,9 @@ cdef class Fory:
The maximum depth of the deserialization data.
If the depth exceeds the maximum depth, an exception will be raised.
The default value is 50.
+ :param field_nullable:
+ Whether dataclass fields are nullable for python native mode(xlang=False). When enabled, dataclass fields
+ are always treated as nullable whether or not they are annotated with `Optional`.
"""
self.language = Language.XLANG if xlang else Language.PYTHON
if kwargs.get("language") is not None:
@@ -872,6 +877,7 @@ cdef class Fory:
self.ref_tracking = ref
self.ref_resolver = MapRefResolver(ref)
self.is_py = self.language == Language.PYTHON
+ self.field_nullable = field_nullable if self.is_py else False
self.metastring_resolver = MetaStringResolver()
self.type_resolver = TypeResolver(self, meta_share=compatible)
self.serialization_context = SerializationContext(fory=self, scoped_meta_share_enabled=compatible)
diff --git a/python/pyfory/meta/typedef.py b/python/pyfory/meta/typedef.py
index 252a5535d..b3108f95b 100644
--- a/python/pyfory/meta/typedef.py
+++ b/python/pyfory/meta/typedef.py
@@ -59,7 +59,8 @@ class TypeDef:
self.is_compressed = is_compressed
def create_fields_serializer(self, resolver):
- field_types = infer_field_types(self.cls)
+ field_nullable = resolver.fory.field_nullable
+ field_types = infer_field_types(self.cls, field_nullable=field_nullable)
serializers = [field_info.field_type.create_serializer(resolver, field_types.get(field_info.name, None)) for field_info in self.fields]
return serializers
@@ -252,10 +253,10 @@ def build_field_infos(type_resolver, cls):
field_infos = []
nullable_map = {}
visitor = StructTypeIdVisitor(type_resolver.fory, cls)
-
+ field_nullable = type_resolver.fory.field_nullable
for field_name in field_names:
field_type_hint = type_hints.get(field_name, typing.Any)
- unwrapped_type, is_nullable = unwrap_optional(field_type_hint)
+ unwrapped_type, is_nullable = unwrap_optional(field_type_hint, field_nullable=field_nullable)
is_nullable = is_nullable or not is_primitive_type(unwrapped_type)
nullable_map[field_name] = is_nullable
field_type = build_field_type(type_resolver, field_name, unwrapped_type, visitor, is_nullable)
diff --git a/python/pyfory/serializer.py b/python/pyfory/serializer.py
index 7b66fa5bb..b009a0bda 100644
--- a/python/pyfory/serializer.py
+++ b/python/pyfory/serializer.py
@@ -377,13 +377,17 @@ class DataClassSerializer(Serializer):
self._field_names = field_names or self._get_field_names(clz)
self._has_slots = hasattr(clz, "__slots__")
self._nullable_fields = nullable_fields or {}
+ field_nullable = fory.field_nullable
if self._field_names and not self._nullable_fields:
for field_name in self._field_names:
if field_name in self._type_hints:
- unwrapped_type, is_nullable = unwrap_optional(self._type_hints[field_name])
+ unwrapped_type, is_nullable = unwrap_optional(self._type_hints[field_name], field_nullable=field_nullable)
is_nullable = is_nullable or not is_primitive_type(unwrapped_type)
self._nullable_fields[field_name] = is_nullable
+ # Cache unwrapped type hints
+ self._unwrapped_hints = self._compute_unwrapped_hints()
+
if self._xlang:
self._serializers = serializers or [None] * len(self._field_names)
if serializers is None:
@@ -407,8 +411,20 @@ class DataClassSerializer(Serializer):
clz,
)
else:
- # TODO compute hash for non-xlang mode more robustly
- self._hash = len(self._field_names)
+ # For non-xlang mode, use same infrastructure as xlang mode
+ # Python dataclass serialization follows the same spec as xlang
+ self._serializers = serializers or [None] * len(self._field_names)
+ if serializers is None:
+ visitor = StructFieldSerializerVisitor(fory)
+ for index, key in enumerate(self._field_names):
+ unwrapped_type, _ = unwrap_optional(self._type_hints[key])
+ serializer = infer_field(key, unwrapped_type, visitor, types_path=[])
+ self._serializers[index] = serializer
+ # In compatible mode, maintain stable field ordering (don't sort)
+ # In non-compatible mode, sort fields for consistent serialization
+ if not fory.compatible:
+ self._field_names, self._serializers = _sort_fields(fory.type_resolver, self._field_names, self._serializers, self._nullable_fields)
+ self._hash = 0 # Will be computed on first write/read
self._generated_write_method = self._gen_write_method()
self._generated_read_method = self._gen_read_method()
if _ENABLE_FORY_PYTHON_JIT:
@@ -419,41 +435,214 @@ class DataClassSerializer(Serializer):
def _get_field_names(self, clz):
if hasattr(clz, "__dict__"):
# Regular object with __dict__
- # We can't know the fields without an instance, so we rely on type hints
+ # For dataclasses, preserve field definition order
+ # In compatible mode, stable field ordering is critical for schema evolution
+ if dataclasses.is_dataclass(clz):
+ # Use dataclasses.fields() to get fields in definition order
+ return [field.name for field in dataclasses.fields(clz)]
+ # For non-dataclass objects, sort by key names for consistency
return sorted(self._type_hints.keys())
elif hasattr(clz, "__slots__"):
# Object with __slots__
return sorted(clz.__slots__)
return []
+ def _compute_unwrapped_hints(self):
+ """Compute unwrapped type hints once and cache."""
+ from pyfory.type import unwrap_optional
+
+ return {field_name: unwrap_optional(hint)[0] for field_name, hint in self._type_hints.items()}
+
+ def _ensure_hash_computed(self):
+ """Lazily compute and cache the hash if not already computed."""
+ if self._hash == 0:
+ self._hash = _get_hash(self.fory, self._field_names, self._unwrapped_hints)
+ return self._hash
+
+ def _write_header(self, buffer):
+ """Write serialization header (hash or field count based on compatible mode)."""
+ if not self.fory.compatible:
+ buffer.write_int32(self._ensure_hash_computed())
+ else:
+ buffer.write_varuint32(len(self._field_names))
+
+ def _read_header(self, buffer):
+ """Read serialization header and return number of fields written.
+
+ Returns:
+ int: Number of fields that were written
+
+ Raises:
+ TypeNotCompatibleError: If hash doesn't match in non-compatible mode
+ """
+ if not self.fory.compatible:
+ hash_ = buffer.read_int32()
+ expected_hash = self._ensure_hash_computed()
+ if hash_ != expected_hash:
+ raise TypeNotCompatibleError(f"Hash {hash_} is not consistent with {expected_hash} for type {self.type_}")
+ return len(self._field_names)
+ else:
+ return buffer.read_varuint32()
+
+ def _get_write_stmt_for_codegen(self, serializer, buffer, field_value):
+ """Generate write statement for code generation based on serializer type."""
+ if isinstance(serializer, BooleanSerializer):
+ return f"{buffer}.write_bool({field_value})"
+ elif isinstance(serializer, ByteSerializer):
+ return f"{buffer}.write_int8({field_value})"
+ elif isinstance(serializer, Int16Serializer):
+ return f"{buffer}.write_int16({field_value})"
+ elif isinstance(serializer, Int32Serializer):
+ return f"{buffer}.write_varint32({field_value})"
+ elif isinstance(serializer, Int64Serializer):
+ return f"{buffer}.write_varint64({field_value})"
+ elif isinstance(serializer, Float32Serializer):
+ return f"{buffer}.write_float32({field_value})"
+ elif isinstance(serializer, Float64Serializer):
+ return f"{buffer}.write_float64({field_value})"
+ elif isinstance(serializer, StringSerializer):
+ return f"{buffer}.write_string({field_value})"
+ else:
+ return None # Complex type, needs ref handling
+
+ def _get_read_stmt_for_codegen(self, serializer, buffer, field_value):
+ """Generate read statement for code generation based on serializer type."""
+ if isinstance(serializer, BooleanSerializer):
+ return f"{field_value} = {buffer}.read_bool()"
+ elif isinstance(serializer, ByteSerializer):
+ return f"{field_value} = {buffer}.read_int8()"
+ elif isinstance(serializer, Int16Serializer):
+ return f"{field_value} = {buffer}.read_int16()"
+ elif isinstance(serializer, Int32Serializer):
+ return f"{field_value} = {buffer}.read_varint32()"
+ elif isinstance(serializer, Int64Serializer):
+ return f"{field_value} = {buffer}.read_varint64()"
+ elif isinstance(serializer, Float32Serializer):
+ return f"{field_value} = {buffer}.read_float32()"
+ elif isinstance(serializer, Float64Serializer):
+ return f"{field_value} = {buffer}.read_float64()"
+ elif isinstance(serializer, StringSerializer):
+ return f"{field_value} = {buffer}.read_string()"
+ else:
+ return None # Complex type, needs ref handling
+
+ def _write_non_nullable_field(self, buffer, field_value, serializer):
+ """Write a non-nullable field value at runtime."""
+ if isinstance(serializer, BooleanSerializer):
+ buffer.write_bool(field_value)
+ elif isinstance(serializer, ByteSerializer):
+ buffer.write_int8(field_value)
+ elif isinstance(serializer, Int16Serializer):
+ buffer.write_int16(field_value)
+ elif isinstance(serializer, Int32Serializer):
+ buffer.write_varint32(field_value)
+ elif isinstance(serializer, Int64Serializer):
+ buffer.write_varint64(field_value)
+ elif isinstance(serializer, Float32Serializer):
+ buffer.write_float32(field_value)
+ elif isinstance(serializer, Float64Serializer):
+ buffer.write_float64(field_value)
+ elif isinstance(serializer, StringSerializer):
+ buffer.write_string(field_value)
+ else:
+ self.fory.write_ref_pyobject(buffer, field_value)
+
+ def _read_non_nullable_field(self, buffer, serializer):
+ """Read a non-nullable field value at runtime."""
+ if isinstance(serializer, BooleanSerializer):
+ return buffer.read_bool()
+ elif isinstance(serializer, ByteSerializer):
+ return buffer.read_int8()
+ elif isinstance(serializer, Int16Serializer):
+ return buffer.read_int16()
+ elif isinstance(serializer, Int32Serializer):
+ return buffer.read_varint32()
+ elif isinstance(serializer, Int64Serializer):
+ return buffer.read_varint64()
+ elif isinstance(serializer, Float32Serializer):
+ return buffer.read_float32()
+ elif isinstance(serializer, Float64Serializer):
+ return buffer.read_float64()
+ elif isinstance(serializer, StringSerializer):
+ return buffer.read_string()
+ else:
+ return self.fory.read_ref_pyobject(buffer)
+
+ def _write_nullable_field(self, buffer, field_value, serializer):
+ """Write a nullable field value at runtime."""
+ if field_value is None:
+ buffer.write_int8(NULL_FLAG)
+ else:
+ buffer.write_int8(NOT_NULL_VALUE_FLAG)
+ if isinstance(serializer, StringSerializer):
+ buffer.write_string(field_value)
+ else:
+ self.fory.write_ref_pyobject(buffer, field_value)
+
+ def _read_nullable_field(self, buffer, serializer):
+ """Read a nullable field value at runtime."""
+ flag = buffer.read_int8()
+ if flag == NULL_FLAG:
+ return None
+ else:
+ if isinstance(serializer, StringSerializer):
+ return buffer.read_string()
+ else:
+ return self.fory.read_ref_pyobject(buffer)
+
def _gen_write_method(self):
context = {}
counter = itertools.count(0)
buffer, fory, value, value_dict = "buffer", "fory", "value", "value_dict"
context[fory] = self.fory
+ context["_serializers"] = self._serializers
+
stmts = [
f'"""write method for {self.type_}"""',
- f"{buffer}.write_int32({self._hash})",
]
+
+ # Write hash only in non-compatible mode; in compatible mode, write field count
+ self._ensure_hash_computed()
+ if not self.fory.compatible:
+ stmts.append(f"{buffer}.write_int32({self._hash})")
+ else:
+ stmts.append(f"{buffer}.write_varuint32({len(self._field_names)})")
+
if not self._has_slots:
stmts.append(f"{value_dict} = {value}.__dict__")
- for field_name in self._field_names:
- field_type = self._type_hints[field_name]
+
+ # Write field values in order
+ for index, field_name in enumerate(self._field_names):
field_value = f"field_value{next(counter)}"
+ serializer_var = f"serializer{index}"
+ serializer = self._serializers[index]
+ context[serializer_var] = serializer
+
if not self._has_slots:
stmts.append(f"{field_value} = {value_dict}['{field_name}']")
else:
stmts.append(f"{field_value} = {value}.{field_name}")
- if field_type is bool:
- stmts.extend(gen_write_nullable_basic_stmts(buffer, field_value, bool))
- elif field_type is int:
- stmts.extend(gen_write_nullable_basic_stmts(buffer, field_value, int))
- elif field_type is float:
- stmts.extend(gen_write_nullable_basic_stmts(buffer, field_value, float))
- elif field_type is str:
- stmts.extend(gen_write_nullable_basic_stmts(buffer, field_value, str))
+
+ is_nullable = self._nullable_fields.get(field_name, False)
+ if is_nullable:
+ # Use gen_write_nullable_basic_stmts for nullable basic types
+ if isinstance(serializer, BooleanSerializer):
+ stmts.extend(gen_write_nullable_basic_stmts(buffer, field_value, bool))
+ elif isinstance(serializer, (ByteSerializer, Int16Serializer, Int32Serializer, Int64Serializer)):
+ stmts.extend(gen_write_nullable_basic_stmts(buffer, field_value, int))
+ elif isinstance(serializer, (Float32Serializer, Float64Serializer)):
+ stmts.extend(gen_write_nullable_basic_stmts(buffer, field_value, float))
+ elif isinstance(serializer, StringSerializer):
+ stmts.extend(gen_write_nullable_basic_stmts(buffer, field_value, str))
+ else:
+ # For complex types, use write_ref_pyobject
+ stmts.append(f"{fory}.write_ref_pyobject({buffer}, {field_value})")
else:
- stmts.append(f"{fory}.write_ref_pyobject({buffer}, {field_value})")
+ stmt = self._get_write_stmt_for_codegen(serializer, buffer, field_value)
+ if stmt is None:
+ stmt = f"{fory}.write_ref_pyobject({buffer}, {field_value})"
+ stmts.append(stmt)
+
self._write_method_code, func = compile_function(
f"write_{self.type_.__module__}_{self.type_.__qualname__}".replace(".", "_"),
[buffer, value],
@@ -475,36 +664,87 @@ class DataClassSerializer(Serializer):
context[fory] = self.fory
context[obj_class] = self.type_
context[ref_resolver] = self.fory.ref_resolver
+ context["_serializers"] = self._serializers
+ current_class_field_names = set(self._get_field_names(self.type_))
+
stmts = [
f'"""read method for {self.type_}"""',
- f"{obj} = {obj_class}.__new__({obj_class})",
- f"{ref_resolver}.reference({obj})",
- f"read_hash = {buffer}.read_int32()",
- f"if read_hash != {self._hash}:",
- f""" raise TypeNotCompatibleError(
- "Hash read_hash is not consistent with {self._hash} for {self.type_}")""",
]
+
+ # Read hash only in non-compatible mode; in compatible mode, read field count
+ self._ensure_hash_computed()
+ if not self.fory.compatible:
+ stmts.extend(
+ [
+ f"read_hash = {buffer}.read_int32()",
+ f"if read_hash != {self._hash}:",
+ f""" raise TypeNotCompatibleError(
+ f"Hash {{read_hash}} is not consistent with {self._hash} for type {self.type_}")""",
+ ]
+ )
+ else:
+ stmts.append(f"num_fields_written = {buffer}.read_varuint32()")
+
+ stmts.extend(
+ [
+ f"{obj} = {obj_class}.__new__({obj_class})",
+ f"{ref_resolver}.reference({obj})",
+ ]
+ )
+
if not self._has_slots:
stmts.append(f"{obj_dict} = {obj}.__dict__")
- def set_action(value: str):
- if not self._has_slots:
- return f"{obj_dict}['{field_name}'] = {value}"
+ # Read field values in order
+ for index, field_name in enumerate(self._field_names):
+ serializer_var = f"serializer{index}"
+ serializer = self._serializers[index]
+ context[serializer_var] = serializer
+ field_value = f"field_value{index}"
+ is_nullable = self._nullable_fields.get(field_name, False)
+
+ # Build field reading statements
+ field_stmts = []
+
+ if is_nullable:
+ # Use gen_read_nullable_basic_stmts for nullable basic types
+ if isinstance(serializer, BooleanSerializer):
+ field_stmts.extend(gen_read_nullable_basic_stmts(buffer, bool, lambda v: f"{field_value} = {v}"))
+ elif isinstance(serializer, (ByteSerializer, Int16Serializer, Int32Serializer, Int64Serializer)):
+ field_stmts.extend(gen_read_nullable_basic_stmts(buffer, int, lambda v: f"{field_value} = {v}"))
+ elif isinstance(serializer, (Float32Serializer, Float64Serializer)):
+ field_stmts.extend(gen_read_nullable_basic_stmts(buffer, float, lambda v: f"{field_value} = {v}"))
+ elif isinstance(serializer, StringSerializer):
+ field_stmts.extend(gen_read_nullable_basic_stmts(buffer, str, lambda v: f"{field_value} = {v}"))
+ else:
+ # For complex types, use read_ref_pyobject
+ field_stmts.append(f"{field_value} = {fory}.read_ref_pyobject({buffer})")
+ else:
+ stmt = self._get_read_stmt_for_codegen(serializer, buffer, field_value)
+ if stmt is None:
+ stmt = f"{field_value} = {fory}.read_ref_pyobject({buffer})"
+ field_stmts.append(stmt)
+
+ # Set field value if it exists in current class
+ if field_name not in current_class_field_names:
+ field_stmts.append(f"# {field_name} is not in {self.type_}")
else:
- return f"{obj}.{field_name} = {value}"
-
- for field_name in self._field_names:
- field_type = self._type_hints[field_name]
- if field_type is bool:
- stmts.extend(gen_read_nullable_basic_stmts(buffer, bool, set_action))
- elif field_type is int:
- stmts.extend(gen_read_nullable_basic_stmts(buffer, int, set_action))
- elif field_type is float:
- stmts.extend(gen_read_nullable_basic_stmts(buffer, float, set_action))
- elif field_type is str:
- stmts.extend(gen_read_nullable_basic_stmts(buffer, str, set_action))
+ if not self._has_slots:
+ field_stmts.append(f"{obj_dict}['{field_name}'] = {field_value}")
+ else:
+ field_stmts.append(f"{obj}.{field_name} = {field_value}")
+
+ # In compatible mode, wrap field reading in a check
+ if self.fory.compatible:
+ stmts.append(f"if {index} < num_fields_written:")
+ # Indent all field statements
+ from pyfory.codegen import ident_lines
+
+ field_stmts = ident_lines(field_stmts)
+ stmts.extend(field_stmts)
else:
- stmts.append(f"{obj}.{field_name} = {fory}.read_ref_pyobject({buffer})")
+ stmts.extend(field_stmts)
+
stmts.append(f"return {obj}")
self._read_method_code, func = compile_function(
f"read_{self.type_.__module__}_{self.type_.__qualname__}".replace(".", "_"),
@@ -518,24 +758,13 @@ class DataClassSerializer(Serializer):
context = {}
counter = itertools.count(0)
buffer, fory, value, value_dict = "buffer", "fory", "value", "value_dict"
- get_hash_func = "_get_hash"
context[fory] = self.fory
- context[get_hash_func] = _get_hash
- context["_field_names"] = self._field_names
- from pyfory.type import unwrap_optional
-
- unwrapped_hints = {}
- for field_name, hint in self._type_hints.items():
- unwrapped, _ = unwrap_optional(hint)
- unwrapped_hints[field_name] = unwrapped
- context["_type_hints"] = unwrapped_hints
context["_serializers"] = self._serializers
stmts = [
f'"""xwrite method for {self.type_}"""',
]
if not self.fory.compatible:
- if self._hash == 0:
- self._hash = _get_hash(self.fory, self._field_names, unwrapped_hints)
+ self._ensure_hash_computed()
stmts.append(f"{buffer}.write_int32({self._hash})")
if not self._has_slots:
stmts.append(f"{value_dict} = {value}.__dict__")
@@ -563,21 +792,8 @@ class DataClassSerializer(Serializer):
else:
stmts.append(f"{fory}.xwrite_ref({buffer}, {field_value}, serializer={serializer_var})")
else:
- if isinstance(serializer, BooleanSerializer):
- stmt = f"{buffer}.write_bool({field_value})"
- elif isinstance(serializer, ByteSerializer):
- stmt = f"{buffer}.write_int8({field_value})"
- elif isinstance(serializer, Int16Serializer):
- stmt = f"{buffer}.write_int16({field_value})"
- elif isinstance(serializer, Int32Serializer):
- stmt = f"{buffer}.write_varint32({field_value})"
- elif isinstance(serializer, Int64Serializer):
- stmt = f"{buffer}.write_varint64({field_value})"
- elif isinstance(serializer, Float32Serializer):
- stmt = f"{buffer}.write_float32({field_value})"
- elif isinstance(serializer, Float64Serializer):
- stmt = f"{buffer}.write_float64({field_value})"
- else:
+ stmt = self._get_write_stmt_for_codegen(serializer, buffer, field_value)
+ if stmt is None:
stmt = f"{fory}.xwrite_no_ref({buffer}, {field_value}, serializer={serializer_var})"
stmts.append(stmt)
self._xwrite_method_code, func = compile_function(
@@ -598,27 +814,16 @@ class DataClassSerializer(Serializer):
"obj_dict",
)
ref_resolver = "ref_resolver"
- get_hash_func = "_get_hash"
context[fory] = self.fory
context[obj_class] = self.type_
context[ref_resolver] = self.fory.ref_resolver
- context[get_hash_func] = _get_hash
- context["_field_names"] = self._field_names
- from pyfory.type import unwrap_optional
-
- unwrapped_hints = {}
- for field_name, hint in self._type_hints.items():
- unwrapped, _ = unwrap_optional(hint)
- unwrapped_hints[field_name] = unwrapped
- context["_type_hints"] = unwrapped_hints
context["_serializers"] = self._serializers
current_class_field_names = set(self._get_field_names(self.type_))
stmts = [
f'"""xread method for {self.type_}"""',
]
if not self.fory.compatible:
- if self._hash == 0:
- self._hash = _get_hash(self.fory, self._field_names, unwrapped_hints)
+ self._ensure_hash_computed()
stmts.extend(
[
f"read_hash = {buffer}.read_int32()",
@@ -656,21 +861,8 @@ class DataClassSerializer(Serializer):
else:
stmts.append(f"{field_value} = {fory}.xread_ref({buffer}, serializer={serializer_var})")
else:
- if isinstance(serializer, BooleanSerializer):
- stmt = f"{field_value} = {buffer}.read_bool()"
- elif isinstance(serializer, ByteSerializer):
- stmt = f"{field_value} = {buffer}.read_int8()"
- elif isinstance(serializer, Int16Serializer):
- stmt = f"{field_value} = {buffer}.read_int16()"
- elif isinstance(serializer, Int32Serializer):
- stmt = f"{field_value} = {buffer}.read_varint32()"
- elif isinstance(serializer, Int64Serializer):
- stmt = f"{field_value} = {buffer}.read_varint64()"
- elif isinstance(serializer, Float32Serializer):
- stmt = f"{field_value} = {buffer}.read_float32()"
- elif isinstance(serializer, Float64Serializer):
- stmt = f"{field_value} = {buffer}.read_float64()"
- else:
+ stmt = self._get_read_stmt_for_codegen(serializer, buffer, field_value)
+ if stmt is None:
stmt = f"{field_value} = {fory}.xread_no_ref({buffer}, serializer={serializer_var})"
stmts.append(stmt)
if field_name not in current_class_field_names:
@@ -690,39 +882,49 @@ class DataClassSerializer(Serializer):
return func
def write(self, buffer, value):
- buffer.write_int32(self._hash)
- for field_name in self._field_names:
+ """Write dataclass instance to buffer in Python native format."""
+ self._write_header(buffer)
+
+ for index, field_name in enumerate(self._field_names):
field_value = getattr(value, field_name)
- self.fory.serialize_ref(buffer, field_value)
+ serializer = self._serializers[index]
+ is_nullable = self._nullable_fields.get(field_name, False)
+
+ if is_nullable:
+ self._write_nullable_field(buffer, field_value, serializer)
+ else:
+ self._write_non_nullable_field(buffer, field_value, serializer)
def read(self, buffer):
- hash_ = buffer.read_int32()
- if hash_ != self._hash:
- raise TypeNotCompatibleError(
- f"Hash {hash_} is not consistent with {self._hash} for type {self.type_}",
- )
+ """Read dataclass instance from buffer in Python native format."""
+ num_fields_written = self._read_header(buffer)
+
obj = self.type_.__new__(self.type_)
self.fory.ref_resolver.reference(obj)
- for field_name in self._field_names:
- field_value = self.fory.read_ref(buffer)
- setattr(
- obj,
- field_name,
- field_value,
- )
+ current_class_field_names = set(self._get_field_names(self.type_))
+
+ for index, field_name in enumerate(self._field_names):
+ # Only read if this field was written
+ if index >= num_fields_written:
+ break
+
+ serializer = self._serializers[index]
+ is_nullable = self._nullable_fields.get(field_name, False)
+
+ if is_nullable:
+ field_value = self._read_nullable_field(buffer, serializer)
+ else:
+ field_value = self._read_non_nullable_field(buffer, serializer)
+
+ if field_name in current_class_field_names:
+ setattr(obj, field_name, field_value)
return obj
def xwrite(self, buffer: Buffer, value):
+ """Write dataclass instance to buffer in cross-language format."""
if not self._xlang:
raise TypeError("xwrite can only be called when DataClassSerializer is in xlang mode")
- if self._hash == 0:
- from pyfory.type import unwrap_optional
-
- unwrapped_hints = {}
- for field_name, hint in self._type_hints.items():
- unwrapped, _ = unwrap_optional(hint)
- unwrapped_hints[field_name] = unwrapped
- self._hash = _get_hash(self.fory, self._field_names, unwrapped_hints)
+ self._ensure_hash_computed()
if not self.fory.compatible:
buffer.write_int32(self._hash)
for index, field_name in enumerate(self._field_names):
@@ -735,16 +937,10 @@ class DataClassSerializer(Serializer):
self.fory.xwrite_ref(buffer, field_value, serializer=serializer)
def xread(self, buffer):
+ """Read dataclass instance from buffer in cross-language format."""
if not self._xlang:
raise TypeError("xread can only be called when DataClassSerializer is in xlang mode")
- if self._hash == 0:
- from pyfory.type import unwrap_optional
-
- unwrapped_hints = {}
- for field_name, hint in self._type_hints.items():
- unwrapped, _ = unwrap_optional(hint)
- unwrapped_hints[field_name] = unwrapped
- self._hash = _get_hash(self.fory, self._field_names, unwrapped_hints)
+ self._ensure_hash_computed()
if not self.fory.compatible:
hash_ = buffer.read_int32()
if hash_ != self._hash:
@@ -767,11 +963,7 @@ class DataClassSerializer(Serializer):
else:
field_value = self.fory.xread_ref(buffer, serializer=serializer)
if field_name in current_class_field_names:
- setattr(
- obj,
- field_name,
- field_value,
- )
+ setattr(obj, field_name, field_value)
return obj
diff --git a/python/pyfory/tests/test_serializer.py b/python/pyfory/tests/test_serializer.py
index bcff60272..7b9f6197f 100644
--- a/python/pyfory/tests/test_serializer.py
+++ b/python/pyfory/tests/test_serializer.py
@@ -23,7 +23,7 @@ import os
import pickle
import weakref
from enum import Enum
-from typing import Any, List, Dict
+from typing import Any, List, Dict, Optional
import numpy as np
import pandas as pd
@@ -558,13 +558,13 @@ def test_pandas_range_index():
@dataclass(unsafe_hash=True)
class PyDataClass1:
- f1: int
+ f1: Optional[int]
f2: float
f3: str
- f4: bool
+ f4: Optional[bool]
f5: Any
f6: List
- f7: Dict
+ f7: Optional[Dict]
@pytest.mark.parametrize("track_ref", [False, True])
@@ -580,6 +580,52 @@ def test_py_serialize_dataclass(track_ref):
assert ser_de(fory, obj2) == obj2
+@dataclass(unsafe_hash=True)
+class PyDataClass2:
+ f1: int
+ f2: float
+ f3: str
+ f4: bool
+ f5: Any
+ f6: List
+ f7: Dict
+
+
+@pytest.mark.parametrize("track_ref", [False, True])
+@pytest.mark.parametrize("compatible", [False, True])
+def test_py_serialize_dataclass_nullable_global(track_ref, compatible):
+ fory = Fory(
+ xlang=False,
+ ref=track_ref,
+ compatible=compatible,
+ strict=False,
+ field_nullable=True,
+ )
+ fory.register(PyDataClass2)
+ obj1 = PyDataClass2(f1=1, f2=-2.0, f3="abc", f4=True, f5="xyz", f6=[1, 2], f7={"k1": "v1"})
+ assert ser_de(fory, obj1) == obj1
+ obj2 = PyDataClass2(f1=None, f2=-2.0, f3="abc", f4=None, f5="xyz", f6=None, f7=None)
+ assert ser_de(fory, obj2) == obj2
+
+ if compatible:
+ fory2 = Fory(
+ xlang=False,
+ ref=track_ref,
+ compatible=compatible,
+ strict=False,
+ field_nullable=True,
+ )
+
+ @dataclass(unsafe_hash=True)
+ class PyDataClass3:
+ f1: int
+ f2: float
+
+ fory2.register(PyDataClass3)
+ assert fory2.deserialize(fory.serialize(obj1)) == PyDataClass3(f1=1, f2=-2.0)
+ assert fory2.deserialize(fory.serialize(obj2)) == PyDataClass3(f1=None, f2=-2.0)
+
+
@pytest.mark.parametrize("track_ref", [False, True])
def test_function(track_ref):
fory = Fory(
diff --git a/python/pyfory/tests/test_struct.py b/python/pyfory/tests/test_struct.py
index cf2536527..093f8396b 100644
--- a/python/pyfory/tests/test_struct.py
+++ b/python/pyfory/tests/test_struct.py
@@ -414,10 +414,12 @@ class OptionalFieldsObject:
f5: str = ""
+@pytest.mark.parametrize("xlang", [False, True])
@pytest.mark.parametrize("compatible", [False, True])
-def test_optional_fields(compatible):
- fory = Fory(xlang=True, ref=True, compatible=compatible)
- fory.register_type(OptionalFieldsObject, typename="example.OptionalFieldsObject")
+def test_optional_fields(xlang, compatible):
+ fory = Fory(xlang=xlang, ref=True, compatible=compatible, strict=False)
+ if xlang:
+ fory.register_type(OptionalFieldsObject, typename="example.OptionalFieldsObject")
obj_with_none = OptionalFieldsObject(f1=None, f2=None, f3=None, f4=42, f5="test")
result = ser_de(fory, obj_with_none)
@@ -451,11 +453,13 @@ class NestedOptionalObject:
f3: str = ""
+@pytest.mark.parametrize("xlang", [False, True])
@pytest.mark.parametrize("compatible", [False, True])
-def test_nested_optional_fields(compatible):
- fory = Fory(xlang=True, ref=True, compatible=compatible)
- fory.register_type(ComplexObject, typename="example.ComplexObject")
- fory.register_type(NestedOptionalObject, typename="example.NestedOptionalObject")
+def test_nested_optional_fields(xlang, compatible):
+ fory = Fory(xlang=xlang, ref=True, compatible=compatible, strict=False)
+ if xlang:
+ fory.register_type(ComplexObject, typename="example.ComplexObject")
+ fory.register_type(NestedOptionalObject, typename="example.NestedOptionalObject")
obj_with_none = NestedOptionalObject(f1=None, f2=None, f3="test")
result = ser_de(fory, obj_with_none)
@@ -493,6 +497,181 @@ class OptionalV3:
f2: str = ""
+@dataclass
+class CompatibleV1:
+ f1: int = 0
+ f2: str = ""
+ f3: float = 0.0
+
+
+@dataclass
+class CompatibleV2:
+ f1: int = 0
+ f2: str = ""
+ f3: float = 0.0
+ f4: bool = False
+
+
+@dataclass
+class CompatibleV3:
+ f1: int = 0
+ f2: str = ""
+
+
+@pytest.mark.parametrize("xlang", [False, True])
+def test_compatible_mode_add_field(xlang):
+ """Test that adding a field with default value works in compatible mode."""
+ fory_v1 = Fory(xlang=xlang, ref=True, compatible=True, strict=False)
+ fory_v2 = Fory(xlang=xlang, ref=True, compatible=True, strict=False)
+
+ fory_v1.register_type(CompatibleV1, typename="example.Compatible")
+ fory_v2.register_type(CompatibleV2, typename="example.Compatible")
+
+ # V1 object serialized
+ v1_obj = CompatibleV1(f1=100, f2="test", f3=3.14)
+ v1_binary = fory_v1.serialize(v1_obj)
+
+ # V2 can read V1 data, new field gets default value
+ v2_result = fory_v2.deserialize(v1_binary)
+ assert v2_result.f1 == 100
+ assert v2_result.f2 == "test"
+ assert v2_result.f3 == 3.14
+ assert v2_result.f4 is False # Default value
+
+
+@pytest.mark.parametrize("xlang", [False, True])
+def test_compatible_mode_remove_field(xlang):
+ """Test that removing a field works in compatible mode."""
+ fory_v2 = Fory(xlang=xlang, ref=True, compatible=True, strict=False)
+ fory_v3 = Fory(xlang=xlang, ref=True, compatible=True, strict=False)
+
+ fory_v2.register_type(CompatibleV2, typename="example.Compatible")
+ fory_v3.register_type(CompatibleV3, typename="example.Compatible")
+
+ # V2 object with all fields
+ v2_obj = CompatibleV2(f1=200, f2="hello", f3=2.71, f4=True)
+ v2_binary = fory_v2.serialize(v2_obj)
+
+ # V3 can read V2 data, extra fields are ignored
+ v3_result = fory_v3.deserialize(v2_binary)
+ assert v3_result.f1 == 200
+ assert v3_result.f2 == "hello"
+ # f3 and f4 from V2 are ignored
+
+
+@pytest.mark.parametrize("xlang", [False, True])
+def test_compatible_mode_bidirectional(xlang):
+ """Test bidirectional compatible serialization."""
+ fory_v1 = Fory(xlang=xlang, ref=True, compatible=True, strict=False)
+ fory_v2 = Fory(xlang=xlang, ref=True, compatible=True, strict=False)
+
+ fory_v1.register_type(CompatibleV1, typename="example.Compatible")
+ fory_v2.register_type(CompatibleV2, typename="example.Compatible")
+
+ # V1 -> V2
+ v1_obj = CompatibleV1(f1=100, f2="test", f3=3.14)
+ v1_binary = fory_v1.serialize(v1_obj)
+ v2_result = fory_v2.deserialize(v1_binary)
+ assert v2_result.f1 == 100
+ assert v2_result.f2 == "test"
+ assert v2_result.f3 == 3.14
+ assert v2_result.f4 is False
+
+ # V2 -> V1
+ v2_obj = CompatibleV2(f1=200, f2="hello", f3=2.71, f4=True)
+ v2_binary = fory_v2.serialize(v2_obj)
+ v1_result = fory_v1.deserialize(v2_binary)
+ assert v1_result.f1 == 200
+ assert v1_result.f2 == "hello"
+ assert v1_result.f3 == 2.71
+
+
+@dataclass
+class CompatibleWithOptional:
+ f1: Optional[int] = None
+ f2: str = ""
+ f3: Optional[List[int]] = None
+
+
+@dataclass
+class CompatibleWithOptionalV2:
+ f1: Optional[int] = None
+ f2: str = ""
+ f3: Optional[List[int]] = None
+ f4: Optional[str] = None
+
+
+@pytest.mark.parametrize("xlang", [False, True])
+def test_compatible_mode_with_optional_fields(xlang):
+ """Test compatible mode with optional fields."""
+ fory_v1 = Fory(xlang=xlang, ref=True, compatible=True, strict=False)
+ fory_v2 = Fory(xlang=xlang, ref=True, compatible=True, strict=False)
+
+ fory_v1.register_type(CompatibleWithOptional, typename="example.CompatibleOptional")
+ fory_v2.register_type(CompatibleWithOptionalV2, typename="example.CompatibleOptional")
+
+ # V1 with None values
+ v1_obj = CompatibleWithOptional(f1=None, f2="test", f3=None)
+ v1_binary = fory_v1.serialize(v1_obj)
+ v2_result = fory_v2.deserialize(v1_binary)
+ assert v2_result.f1 is None
+ assert v2_result.f2 == "test"
+ assert v2_result.f3 is None
+ assert v2_result.f4 is None
+
+ # V1 with values
+ v1_obj2 = CompatibleWithOptional(f1=100, f2="test", f3=[1, 2, 3])
+ v1_binary2 = fory_v1.serialize(v1_obj2)
+ v2_result2 = fory_v2.deserialize(v1_binary2)
+ assert v2_result2.f1 == 100
+ assert v2_result2.f2 == "test"
+ assert v2_result2.f3 == [1, 2, 3]
+ assert v2_result2.f4 is None
+
+
+@dataclass
+class CompatibleAllTypes:
+ f_int: int = 0
+ f_str: str = ""
+ f_float: float = 0.0
+ f_bool: bool = False
+ f_list: List[int] = None
+ f_dict: Dict[str, int] = None
+
+
+@dataclass
+class CompatibleAllTypesV2:
+ f_int: int = 0
+ f_str: str = ""
+ f_float: float = 0.0
+ f_bool: bool = False
+ f_list: List[int] = None
+ f_dict: Dict[str, int] = None
+ f_new: str = "default"
+
+
+@pytest.mark.parametrize("xlang", [False, True])
+def test_compatible_mode_all_basic_types(xlang):
+ """Test compatible mode with all basic types."""
+ fory_v1 = Fory(xlang=xlang, ref=True, compatible=True, strict=False)
+ fory_v2 = Fory(xlang=xlang, ref=True, compatible=True, strict=False)
+
+ fory_v1.register_type(CompatibleAllTypes, typename="example.CompatibleAllTypes")
+ fory_v2.register_type(CompatibleAllTypesV2, typename="example.CompatibleAllTypes")
+
+ v1_obj = CompatibleAllTypes(f_int=42, f_str="hello", f_float=3.14, f_bool=True, f_list=[1, 2, 3], f_dict={"a": 1, "b": 2})
+ v1_binary = fory_v1.serialize(v1_obj)
+ v2_result = fory_v2.deserialize(v1_binary)
+
+ assert v2_result.f_int == 42
+ assert v2_result.f_str == "hello"
+ assert v2_result.f_float == 3.14
+ assert v2_result.f_bool is True
+ assert v2_result.f_list == [1, 2, 3]
+ assert v2_result.f_dict == {"a": 1, "b": 2}
+ assert v2_result.f_new == "default"
+
+
def test_optional_compatible_mode_evolution():
fory_v1 = Fory(xlang=True, ref=True, compatible=True)
fory_v2 = Fory(xlang=True, ref=True, compatible=True)
diff --git a/python/pyfory/type.py b/python/pyfory/type.py
index ecd505d13..f41c7841d 100644
--- a/python/pyfory/type.py
+++ b/python/pyfory/type.py
@@ -415,14 +415,14 @@ class TypeVisitor(ABC):
pass
-def infer_field_types(type_):
+def infer_field_types(type_, field_nullable=False):
type_hints = typing.get_type_hints(type_)
from pyfory._struct import StructTypeVisitor
visitor = StructTypeVisitor(type_)
result = {}
for name, hint in sorted(type_hints.items()):
- unwrapped, _ = unwrap_optional(hint)
+ unwrapped, _ = unwrap_optional(hint, field_nullable=field_nullable)
result[name] = infer_field(name, unwrapped, visitor)
return result
@@ -435,9 +435,9 @@ def is_optional_type(type_):
return False
-def unwrap_optional(type_):
+def unwrap_optional(type_, field_nullable=False):
if not is_optional_type(type_):
- return type_, False
+ return type_, False or field_nullable
args = typing.get_args(type_) if hasattr(typing, "get_args") else getattr(type_, "__args__", ())
non_none_types = [arg for arg in args if arg is not type(None)]
if len(non_none_types) == 1:
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@fory.apache.org
For additional commands, e-mail: commits-help@fory.apache.org