You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@fory.apache.org by ch...@apache.org on 2026/10/04 10:21:47 UTC

(fory) branch main updated: fix(compiler): escape Python field names that shadow generated default helpers (#4107)

This is an automated email from the ASF dual-hosted git repository.

chaokunyang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fory.git


The following commit(s) were added to refs/heads/main by this push:
     new 57ed53124 fix(compiler): escape Python field names that shadow generated default helpers (#4107)
57ed53124 is described below

commit 57ed53124e61edbd0a88b7cb8897c3f21907304e
Author: Ayush Kumar <ay...@gmail.com>
AuthorDate: Sun Oct 4 15:51:42 2026 +0530

    fix(compiler): escape Python field names that shadow generated default helpers (#4107)
    
    ## What does this PR do?
    
    A schema such as `message Tricky { string pyfory = 1; list<string> tags
    = 2; }` generates a dataclass whose `pyfory` attribute shadows the
    module-level `pyfory` import, so later field defaults like
    `pyfory.field(id=2, ...)` raise `AttributeError` when the generated
    module is imported. The same shadowing happens for fields named `field`,
    `decimal`, `list`, or `dict`, which are all referenced by generated
    default expressions. This PR escapes those field names with a trailing
    underscore, reusing the existing keyword-escaping convention. The wire
    field name is unchanged because pyfory's `_to_snake_case` strips the
    trailing underscore when deriving the serialized field name.
    
    Verification: the new regression test fails without the generator change
    and passes with it; the full compiler suite (`python3 -m pytest
    fory_compiler/tests`) passes (492 passed, 1 skipped).
    
    ---------
    
    Co-authored-by: chaokunyang <sh...@gmail.com>
---
 compiler/fory_compiler/generators/python.py        |  22 +++-
 .../fory_compiler/tests/test_python_generator.py   | 132 +++++++++++++++++++++
 docs/compiler/generated-code/python.md             |   9 ++
 3 files changed, 161 insertions(+), 2 deletions(-)

diff --git a/compiler/fory_compiler/generators/python.py b/compiler/fory_compiler/generators/python.py
index 442c3c7ba..212e8a553 100644
--- a/compiler/fory_compiler/generators/python.py
+++ b/compiler/fory_compiler/generators/python.py
@@ -136,12 +136,30 @@ class PythonGenerator(PythonServiceGeneratorMixin, BaseGenerator):
         (PrimitiveKind.UINT64, "tagged"): "pyfory.TaggedUInt64",
     }
 
+    # Names referenced by generated field default expressions inside a
+    # dataclass body (`field(...)`, `pyfory.field(...)`, `decimal.Decimal(...)`,
+    # `default_factory=list/dict`). A dataclass attribute bound to one of these
+    # names would shadow the helper for later fields in the same class body, so
+    # such field names get the same trailing-underscore escape as keywords;
+    # pyfory strips the trailing underscore for the wire field name.
+    FIELD_DEFAULT_HELPER_NAMES = frozenset(
+        {"field", "pyfory", "decimal", "list", "dict"}
+    )
+
     def safe_name(self, name: str) -> str:
         """Return a Python-safe identifier."""
         if keyword.iskeyword(name):
             return f"{name}_"
         return name
 
+    def safe_field_name(self, name: str) -> str:
+        """Return a dataclass-field-safe identifier."""
+        # Shift the whole suffix family so field, field_, and field__ stay
+        # distinct instead of overwriting a generated attribute and its tag.
+        if name.rstrip("_") in self.FIELD_DEFAULT_HELPER_NAMES:
+            return f"{name}_"
+        return self.safe_name(name)
+
     def is_imported_type(self, type_def: object) -> bool:
         """Return True if a type definition comes from an imported IDL file."""
         if not self.schema.source_file:
@@ -601,7 +619,7 @@ class PythonGenerator(PythonServiceGeneratorMixin, BaseGenerator):
             field.element_ref,
             parent_stack,
         )
-        field_name = self.safe_name(self.to_snake_case(field.name))
+        field_name = self.safe_field_name(self.to_snake_case(field.name))
         default_factory = self.get_default_factory(field)
         default = self.get_default_value(field.field_type, field.optional)
         default_expr = default
@@ -675,7 +693,7 @@ class PythonGenerator(PythonServiceGeneratorMixin, BaseGenerator):
             return lines
         lines.append(f"{ind}        parts = []")
         for field in message.fields:
-            field_name = self.safe_name(self.to_snake_case(field.name))
+            field_name = self.safe_field_name(self.to_snake_case(field.name))
             if self.field_needs_safe_repr(field):
                 type_label = self.format_idl_type(field.field_type)
                 placeholder = f"{type_label}(...)"
diff --git a/compiler/fory_compiler/tests/test_python_generator.py b/compiler/fory_compiler/tests/test_python_generator.py
new file mode 100644
index 000000000..a1d94a4f2
--- /dev/null
+++ b/compiler/fory_compiler/tests/test_python_generator.py
@@ -0,0 +1,132 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Tests for Python code generation."""
+
+import dataclasses
+import sys
+import types
+from pathlib import Path
+from textwrap import dedent
+
+import pytest
+
+from fory_compiler.frontend.fdl.lexer import Lexer
+from fory_compiler.frontend.fdl.parser import Parser
+from fory_compiler.generators.base import GeneratorOptions
+from fory_compiler.generators.python import PythonGenerator
+from fory_compiler.ir.ast import Schema
+
+
+def parse_fdl(source: str) -> Schema:
+    return Parser(Lexer(source).tokenize()).parse()
+
+
+def generate_python(source: str) -> str:
+    schema = parse_fdl(source)
+    generator = PythonGenerator(schema, GeneratorOptions(output_dir=Path("/tmp")))
+    files = generator.generate()
+    assert len(files) == 1
+    return files[0].content
+
+
+def exec_generated_module(content: str) -> dict:
+    """Execute generated Python against a minimal pyfory stub.
+
+    The stub keeps dataclass semantics real so class-body evaluation errors
+    (such as a field shadowing a helper name) surface exactly as they would
+    with the real pyfory package.
+    """
+
+    def stub_field(id=None, *, nullable=False, ref=False, **kwargs):
+        return dataclasses.field(metadata={"id": id}, **kwargs)
+
+    def stub_dataclass(cls=None, **kwargs):
+        if cls is None:
+            return dataclasses.dataclass
+        return dataclasses.dataclass(cls)
+
+    stub = types.ModuleType("pyfory")
+    stub.field = stub_field
+    stub.dataclass = stub_dataclass
+    stub.Fory = object
+    stub.ThreadSafeFory = object
+    previous = sys.modules.get("pyfory")
+    sys.modules["pyfory"] = stub
+    module = types.ModuleType("generated_module")
+    sys.modules["generated_module"] = module
+    try:
+        exec(compile(content, "generated_module.py", "exec"), module.__dict__)
+        return module.__dict__
+    finally:
+        del sys.modules["generated_module"]
+        if previous is None:
+            del sys.modules["pyfory"]
+        else:
+            sys.modules["pyfory"] = previous
+
+
+def test_field_names_shadowing_default_helpers_are_escaped():
+    """Fields named after helpers used by later defaults must not break the class body."""
+    source = dedent(
+        """
+        package example;
+
+        message Tricky [id=100] {
+            string pyfory = 1;
+            string field = 2;
+            list<string> tags = 3;
+            map<string, string> attrs = 4;
+        }
+        """
+    )
+    output = generate_python(source)
+
+    # The escaped attribute names keep the wire names: pyfory strips the
+    # trailing underscore when computing the snake_case field name.
+    assert "pyfory_: str = pyfory.field(id=1" in output
+    assert "field_: str = pyfory.field(id=2" in output
+
+    namespace = exec_generated_module(output)
+    instance = namespace["Tricky"]()
+    assert instance.pyfory_ == ""
+    assert instance.field_ == ""
+    assert instance.tags == []
+    assert instance.attrs == {}
+
+
+@pytest.mark.parametrize("name", ["field", "pyfory", "decimal", "List", "dict"])
+@pytest.mark.parametrize("reverse", [False, True])
+@pytest.mark.parametrize("field_type", ["string", "any"])
+def test_helper_name_suffixes(name, reverse, field_type):
+    fields = [f"{field_type} {name}{'_' * i} = {i + 1};" for i in range(3)]
+    if reverse:
+        fields.reverse()
+    output = generate_python("message Tricky [id=100] {" + "\n".join(fields) + "}")
+    namespace = exec_generated_module(output)
+    cls = namespace["Tricky"]
+    names = [name.lower() + "_" * (i + 1) for i in range(3)]
+
+    assert {f.name: f.metadata["id"] for f in dataclasses.fields(cls)} == {
+        field_name: i + 1 for i, field_name in enumerate(names)
+    }
+    values = {field_name: f"value-{i}" for i, field_name in enumerate(names)}
+    instance = cls(**values)
+    assert dataclasses.asdict(instance) == values
+    for field_name, value in values.items():
+        rendered = repr(value) if field_type == "string" else "any(...)"
+        assert f"{field_name}={rendered}" in repr(instance)
diff --git a/docs/compiler/generated-code/python.md b/docs/compiler/generated-code/python.md
index c27120e43..d35a456b6 100644
--- a/docs/compiler/generated-code/python.md
+++ b/docs/compiler/generated-code/python.md
@@ -70,6 +70,15 @@ class Person:
     def from_bytes(cls, data: bytes) -> "Person": ...
 ```
 
+## Field Names
+
+Generated Python attributes use `snake_case`, with a trailing underscore for
+Python keywords. The names `field`, `pyfory`, `decimal`, `list`, and `dict` also
+receive a trailing underscore. Names consisting of one of these five names
+followed by underscores receive one additional underscore so fields remain
+distinct: `field`, `field_`, and `field__` become `field_`, `field__`, and
+`field___`. Field tag IDs remain unchanged.
+
 ## Registration
 
 Generated registration function:


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@fory.apache.org
For additional commands, e-mail: commits-help@fory.apache.org