You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ho...@apache.org on 2026/08/03 16:13:56 UTC

(spark) branch master updated: [SPARK-55206][PYTHON][SQL] Transpilation minimal functional implementation with python

This is an automated email from the ASF dual-hosted git repository.

asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new 16801417c49d [SPARK-55206][PYTHON][SQL] Transpilation minimal functional implementation with python
16801417c49d is described below

commit 16801417c49de8211a821a99962623ca60136bce
Author: Holden Karau <ho...@pigscanfly.ca>
AuthorDate: Mon Aug 3 09:13:44 2026 -0700

    [SPARK-55206][PYTHON][SQL] Transpilation minimal functional implementation with python
    
    ### What changes were proposed in this pull request?
    
    This is the first PR adding initial transpilation from Python to Catalyst and the framework for others to plug in different transpilers which can target other targets. This replaces https://github.com/apache/spark/pull/53547 and is the first PR as part of the transpilation SPIP https://docs.google.com/document/d/1cHc6tiR4yO3hppTzrK1F1w9RwyEPMvaeEuL2ub2LURg/edit?tab=t.0#heading=h.iz92aps6qbo5
    
    ### Why are the changes needed?
    
    Python UDF performance has been a perinial challenge in Spark, and while PyArrow makes the data copy slightly less expensive it's still much slower than native code. Additionally, Python UDF usage has increased substantailly with Pandas on Spark and it is especially hard for folks to write catalyst expressions here.
    
    ### Does this PR introduce _any_ user-facing change?
    
    New configuration flags are user visible (default to off).
    
    ### How was this patch tested?
    
    New unit tests that trigger always and hypothesis tests that trigger only selectively.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Hypothesis teests were generated by Claude 4.7, eq semantics null and truthiness w/claude 4.6, GH code review (unknown backing model) on itiial draft). snake camel swap with sonnet.
    
    Closes #56327 from holdenk/r3-SPARK-55206-transpilation-minimal-functional-impl-python.
    
    Lead-authored-by: Holden Karau <ho...@pigscanfly.ca>
    Co-authored-by: Holden Karau <ho...@snowflake.com>
    Signed-off-by: Holden Karau <ho...@snowflake.com>
---
 .github/workflows/build_and_test.yml               |   16 +-
 .../src/main/resources/error/error-conditions.json |   12 +
 dev/requirements.txt                               |    3 +
 dev/sparktestsupport/modules.py                    |    3 +
 pyproject.toml                                     |    3 +
 python/pyspark/errors/error-conditions.json        |    2 +-
 python/pyspark/sql/classic/column.py               |   13 +-
 python/pyspark/sql/connect/column.py               |    6 +-
 .../sql/tests/test_udf_transpile_hypothesis.py     |  711 ++++++++
 .../pyspark/sql/tests/test_udf_transpile_parity.py |  110 ++
 .../pyspark/sql/tests/test_udf_transpile_unit.py   | 1736 ++++++++++++++++++++
 python/pyspark/sql/transpile.py                    | 1098 +++++++++++++
 python/pyspark/sql/udf.py                          |  151 +-
 .../spark/sql/catalyst/analysis/Analyzer.scala     |    3 +-
 .../ResolveTranspiledPythonUDFOptions.scala        |   97 ++
 .../sql/catalyst/expressions/Expression.scala      |    3 +
 .../spark/sql/catalyst/expressions/PythonUDF.scala |   38 +-
 .../spark/sql/catalyst/optimizer/Optimizer.scala   |   85 +
 .../sql/catalyst/rules/RuleIdCollection.scala      |    1 +
 .../spark/sql/catalyst/trees/TreePatterns.scala    |    1 +
 .../apache/spark/sql/catalyst/util/package.scala   |    5 +
 .../spark/sql/errors/QueryCompilationErrors.scala  |   16 +
 .../org/apache/spark/sql/internal/SQLConf.scala    |   28 +
 .../ResolveTranspiledPythonUDFOptionsSuite.scala   |   95 ++
 .../optimizer/ConvertToCatalystSuite.scala         |  237 +++
 .../python/UserDefinedPythonFunction.scala         |   91 +-
 26 files changed, 4550 insertions(+), 14 deletions(-)

diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index 428a53745a45..d5beb9028a8d 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -113,6 +113,9 @@ jobs:
             docker=`./dev/is-changed.py -m docker-integration-tests`
             # Skip PySpark, SparkR, TPC-DS when only static UI resources (JS/CSS/HTML) changed.
             # These tests are unaffected by UI static resource modifications.
+            # Also we only run the very slow transpile changes when that logic changes
+            transpile=false
+            static_only=false
             changed_files=$(git diff --name-only "$APACHE_SPARK_REF" HEAD 2>/dev/null)
             if [ -n "$changed_files" ]; then
               static_only=true
@@ -122,8 +125,11 @@ jobs:
                   *) static_only=false; break ;;
                 esac
               done
-            else
-              static_only=false
+              for f in $changed_files; do
+                case "$f" in
+                  *transpile*) transpile=true; break ;;
+                esac
+              done
             fi
             if [ "$static_only" = "true" ]; then
               pyspark=false
@@ -170,6 +176,7 @@ jobs:
               \"k8s-integration-tests\" : \"$kubernetes\",
               \"buf\" : \"$buf\",
               \"ui\" : \"$ui\",
+              \"transpile\": \"$transpile\",
             }"
           echo $precondition # For debugging
           # Remove `\n` to avoid "Invalid format" error
@@ -678,6 +685,11 @@ jobs:
       METASPACE_SIZE: 1g
       BRANCH: ${{ inputs.branch }}
       PYSPARK_TEST_TIMEOUT: 450
+      RUN_HYPOTHESIS: ${{ fromJson(needs.precondition.outputs.required).transpile }}
+      # Each generated example runs two full Spark jobs (transpiled vs. interpreted
+      # differential), so the local default of 1000 blows past PYSPARK_TEST_TIMEOUT.
+      # Cap CI fuzz volume; the explicit @example edge seeds still always run.
+      RUN_HYPOTHESIS_MAX_EXAMPLES: 50
     steps:
     - name: Checkout Spark repository
       uses: actions/checkout@v6
diff --git a/common/utils/src/main/resources/error/error-conditions.json b/common/utils/src/main/resources/error/error-conditions.json
index d9f900d182bb..e7fc8c95a49b 100644
--- a/common/utils/src/main/resources/error/error-conditions.json
+++ b/common/utils/src/main/resources/error/error-conditions.json
@@ -5281,6 +5281,18 @@
     ],
     "sqlState" : "38000"
   },
+  "INVALID_UDF_PARAMETER_PLACEHOLDER" : {
+    "message" : [
+      "Invalid Python UDF transpiled-expression parameter placeholder: <placeholder>. Placeholders must be of the form `_udf_param_N` where N is a non-negative integer index into the UDF's positional arguments. This is an internal error in the Python UDF transpiler."
+    ],
+    "sqlState" : "42000"
+  },
+  "INVALID_UDF_PARAMETER_PLACEHOLDER_INDEX" : {
+    "message" : [
+      "Python UDF transpiled-expression parameter placeholder `_udf_param_<index>` references position <index> but the UDF only has <numParams> argument(s). This is an internal error in the Python UDF transpiler."
+    ],
+    "sqlState" : "42000"
+  },
   "INVALID_URL" : {
     "message" : [
       "The url is invalid: <url>. Use `try_parse_url` to tolerate invalid URL and return NULL instead."
diff --git a/dev/requirements.txt b/dev/requirements.txt
index a5ac9f06a668..edcaf053a449 100644
--- a/dev/requirements.txt
+++ b/dev/requirements.txt
@@ -21,6 +21,9 @@ openpyxl
 asv
 coverage
 tabulate
+# Used by the opt-in UDF transpile hypothesis test suite (gated on
+# RUN_HYPOTHESIS in the environment). See SPARK-54783.
+hypothesis>=6,<7
 
 # Kafka streaming test dependencies (optional)
 # Required for running Kafka integration tests with Docker test containers
diff --git a/dev/sparktestsupport/modules.py b/dev/sparktestsupport/modules.py
index 9763591f588e..79e2e5a7bc2e 100644
--- a/dev/sparktestsupport/modules.py
+++ b/dev/sparktestsupport/modules.py
@@ -642,6 +642,9 @@ pyspark_sql = Module(
         "pyspark.sql.tests.test_udf",
         "pyspark.sql.tests.test_udf_combinations",
         "pyspark.sql.tests.test_udf_profiler",
+        "pyspark.sql.tests.test_udf_transpile_hypothesis",
+        "pyspark.sql.tests.test_udf_transpile_parity",
+        "pyspark.sql.tests.test_udf_transpile_unit",
         "pyspark.sql.tests.test_unified_udf",
         "pyspark.sql.tests.test_udtf",
         "pyspark.sql.tests.test_tvf",
diff --git a/pyproject.toml b/pyproject.toml
index 35de692cf5b0..790a3268daf8 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -170,6 +170,9 @@ _test = [
     "lxml",
     "testcontainers[kafka]>=3.7.0",
     "kafka-python-ng>=2.0.2",
+    # Used by the opt-in UDF transpile hypothesis test suite (gated on
+    # the RUN_HYPOTHESIS env var); the suite skips cleanly if absent.
+    "hypothesis>=6,<7",
 ]
 
 _docs = [
diff --git a/python/pyspark/errors/error-conditions.json b/python/pyspark/errors/error-conditions.json
index bd1e5fe9ad2a..580f5a8f16e9 100644
--- a/python/pyspark/errors/error-conditions.json
+++ b/python/pyspark/errors/error-conditions.json
@@ -72,7 +72,7 @@
   },
   "CANNOT_CONVERT_COLUMN_INTO_BOOL": {
     "message": [
-      "Cannot convert column into bool: please use '&' for 'and', '|' for 'or', '~' for 'not' when building DataFrame boolean expressions."
+      "Cannot convert column into bool (offending column: <column>): please use '&' for 'and', '|' for 'or', '~' for 'not' when building DataFrame boolean expressions."
     ]
   },
   "CANNOT_CONVERT_TYPE": {
diff --git a/python/pyspark/sql/classic/column.py b/python/pyspark/sql/classic/column.py
index 5f54443b69e4..a313ddd13f8d 100644
--- a/python/pyspark/sql/classic/column.py
+++ b/python/pyspark/sql/classic/column.py
@@ -62,6 +62,13 @@ def _create_column_from_name(name: str) -> "JavaObject":
     return cast(JVMView, sc._jvm).functions.col(name)
 
 
+def _to_java_column_opt(col: Optional["ColumnOrName"]) -> Optional["JavaObject"]:
+    if col is None:
+        return None
+    else:
+        return _to_java_column(col)
+
+
 def _to_java_column(col: "ColumnOrName") -> "JavaObject":
     if isinstance(col, Column):
         jcol = col._jc
@@ -650,9 +657,13 @@ class Column(ParentColumn):
         return Column(jc)
 
     def __nonzero__(self) -> None:
+        try:
+            column_repr = self._jc.toString()
+        except Exception:
+            column_repr = "<unknown>"
         raise PySparkValueError(
             errorClass="CANNOT_CONVERT_COLUMN_INTO_BOOL",
-            messageParameters={},
+            messageParameters={"column": column_repr},
         )
 
     __bool__ = __nonzero__
diff --git a/python/pyspark/sql/connect/column.py b/python/pyspark/sql/connect/column.py
index 0eece36c950d..69a49f1e27cd 100644
--- a/python/pyspark/sql/connect/column.py
+++ b/python/pyspark/sql/connect/column.py
@@ -622,9 +622,13 @@ class Column(ParentColumn):
         )
 
     def __nonzero__(self) -> None:
+        try:
+            column_repr = repr(self._expr)
+        except Exception:
+            column_repr = "<unknown>"
         raise PySparkValueError(
             errorClass="CANNOT_CONVERT_COLUMN_INTO_BOOL",
-            messageParameters={},
+            messageParameters={"column": column_repr},
         )
 
     __bool__ = __nonzero__
diff --git a/python/pyspark/sql/tests/test_udf_transpile_hypothesis.py b/python/pyspark/sql/tests/test_udf_transpile_hypothesis.py
new file mode 100644
index 000000000000..a25d4318259a
--- /dev/null
+++ b/python/pyspark/sql/tests/test_udf_transpile_hypothesis.py
@@ -0,0 +1,711 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""
+Differential / property-based tests for UDF transpilation (SPARK-54783).
+
+These tests run a fixed set of small Python UDFs twice -- once with
+``spark.sql.experimental.optimizer.transpilePyUDFs`` enabled (so the catalyst
+transpiler in :mod:`pyspark.sql.transpile` rewrites them into native
+expressions) and once without -- and assert that the two runs produce the
+same results for inputs generated by Hypothesis.
+
+The transpiler is intentionally minimal at this point so we expect this
+suite to surface bugs (e.g. truthiness / NULL-handling mismatches between
+Python's ``if x:`` semantics and SQL's ``CASE WHEN``). Failures here should
+be treated as real correctness gaps in the transpiler, not as test bugs to
+silence.
+
+The suite is gated on two things, both required:
+
+* the ``RUN_HYPOTHESIS`` env var must be set to a truthy value
+  (``1``, ``true``, or ``yes``, case-insensitive), and
+* the ``hypothesis`` package must be installed.
+
+The gate is value-based rather than presence-based because CI always sets
+``RUN_HYPOTHESIS`` (to ``"true"`` or ``"false"``) via the transpile
+precondition in ``build_and_test.yml``; mere presence must not opt in, or
+the slow suite would run on every PySpark job.
+
+If either gate is unmet the entire suite is skipped cleanly so it never
+becomes a CI tax for folks who haven't opted in. In CI, this opt-in suite
+is wired through ``.github/workflows/build_and_test.yml``, which flips both
+gates on for the relevant job when PR changes touch the transpiler or this
+test file.
+
+To run locally::
+
+    pip install hypothesis
+    RUN_HYPOTHESIS=1 RUN_HYPOTHESIS_MAX_EXAMPLES=1000 \
+        python/run-tests --testnames pyspark.sql.tests.test_udf_transpile_hypothesis
+
+Set ``RUN_HYPOTHESIS_MAX_EXAMPLES`` to override the per-test example count
+(default 1000). Each generated example runs two full Spark jobs (a
+transpiled-vs-interpreted differential), so CI caps this at 50 via
+``build_and_test.yml`` to stay under ``PYSPARK_TEST_TIMEOUT``; the explicit
+``@example`` edge seeds always run on top of the generated ones regardless.
+"""
+
+import os
+import unittest
+import warnings
+from typing import Optional
+
+from pyspark.sql import Row
+from pyspark.sql.types import (
+    BooleanType,
+    LongType,
+    StructField,
+    StructType,
+)
+from pyspark.sql.udf import UserDefinedFunction
+from pyspark.testing.sqlutils import ReusedSQLTestCase
+from pyspark.testing.utils import have_package
+from pyspark.util import is_remote_only
+
+
+# Sentinel value used by ``_run`` to mark "this side raised". A unique
+# object is sufficient because we only ever compare it against itself
+# inside the helper.
+_SENTINEL_RAISED = object()
+
+
+_HYPOTHESIS_ENV = "RUN_HYPOTHESIS"
+_have_hypothesis = have_package("hypothesis")
+
+
+def _env_opts_in(value: Optional[str]) -> bool:
+    """Value-based opt-in: only 1/true/yes (case-insensitive) enable the suite.
+
+    CI always sets ``RUN_HYPOTHESIS`` -- to ``"true"`` or ``"false"`` -- via the
+    transpile precondition in ``build_and_test.yml``, so a presence-based check
+    would run this very slow suite on every PySpark job regardless of the
+    gating decision.
+    """
+    return value is not None and value.strip().lower() in ("1", "true", "yes")
+
+
+_hypothesis_enabled = _env_opts_in(os.environ.get(_HYPOTHESIS_ENV))
+# Transpilation is only supported in regular (non-Connect) Spark for now,
+# so the hypothesis suite skips cleanly under a pyspark-client-only install.
+_regular_spark = not is_remote_only()
+_skip_reason = (
+    f"Set {_HYPOTHESIS_ENV}=1 (or true/yes) to run; hypothesis must also be installed, "
+    "and the suite only runs under regular (non-Connect) Spark."
+)
+
+
+if _have_hypothesis:
+    from hypothesis import HealthCheck, example, given, settings, strategies as st
+
+    _DEFAULT_MAX_EXAMPLES = int(os.environ.get("RUN_HYPOTHESIS_MAX_EXAMPLES", "1000"))
+
+    # The ``function_scoped_fixture` health check is suppressed because we intentionally reuse the
+    # class-level SparkSession across examples; the per-example ``deadline`` is disabled because
+    # Spark task execution is much slower than hypothesis's default budget.
+    _hyp_settings = settings(
+        max_examples=_DEFAULT_MAX_EXAMPLES,
+        deadline=None,
+        suppress_health_check=[HealthCheck.function_scoped_fixture],
+    )
+
+    # Full 64-bit signed range -- used by comparison and equality tests where
+    # no arithmetic can overflow.
+    _LONG_BOUND = 2**63 - 1
+    _long_strategy = st.one_of(
+        st.none(), st.integers(min_value=-_LONG_BOUND, max_value=_LONG_BOUND)
+    )
+
+    # Narrower range for arithmetic tests (+4, -2, *3, +7, x+y).  Python's
+    # arithmetic never overflows, but Spark's ANSI mode raises on LongType
+    # overflow.  Worse, the Python UDF runner silently wraps out-of-range
+    # return values even with ANSI=True, so "both raised" never fires for
+    # overflow values and the test sees a spurious mismatch instead.
+    # 2**61 is safe for all operations: (2**61)*3 ~= 6.9e18 < 9.2e18 = Long.MAX.
+    _LONG_ARITH_BOUND = 2**61
+    _long_arith_strategy = st.one_of(
+        st.none(),
+        st.integers(min_value=-_LONG_ARITH_BOUND, max_value=_LONG_ARITH_BOUND),
+    )
+
+    _bool_strategy = st.one_of(st.none(), st.booleans())
+
+    # 32-bit signed boundaries. Values round-trip through LongType, but the
+    # int32 limits are where narrowing / off-by-one bugs in parameter-index
+    # plumbing and boundary handling tend to hide, so we always seed them.
+    _INT32_MAX = 2**31 - 1
+    _INT32_MIN = -(2**31)
+
+    # ---- Edge-case seeds (scalacheck-style) -----------------------------
+    #
+    # Hypothesis already biases toward "interesting" boundary values, but
+    # explicit ``@example`` decorators make a regression on a specific
+    # value -- e.g. NULL, zero, the type's max -- deterministic across
+    # runs. These are the values we always want to try, before random
+    # generation kicks in.
+    _LONG_EDGES = (None, 0, 1, -1, 7, -7, _INT32_MAX, _INT32_MIN, _LONG_BOUND, -_LONG_BOUND)
+    _LONG_ARITH_EDGES = (None, 0, 1, -1, 7, -7, _LONG_ARITH_BOUND, -_LONG_ARITH_BOUND)
+    # Bool space is exhaustive (only three values) so the @example
+    # decorators here serve more as documentation of the NULL handling
+    # we care about than as new coverage on top of hypothesis's
+    # generator.
+    _BOOL_EDGES = (None, True, False)
+    # Multi-arg edges -- nulls plus the four sign-combos for non-zero
+    # values. Catches off-by-one errors in parameter-index plumbing
+    # better than random generation alone.
+    _LONG_PAIR_EDGES = (
+        (None, None),
+        (None, 0),
+        (0, None),
+        (0, 0),
+        (1, -1),
+        (-1, 1),
+        (_INT32_MAX, _INT32_MIN),
+        (_INT32_MIN, _INT32_MAX),
+        (_LONG_BOUND, 1),
+        (1, -_LONG_BOUND),
+    )
+    _LONG_ARITH_PAIR_EDGES = (
+        (None, None),
+        (None, 0),
+        (0, None),
+        (0, 0),
+        (1, -1),
+        (-1, 1),
+        (_LONG_ARITH_BOUND, 1),
+        (1, -_LONG_ARITH_BOUND),
+    )
+    # Sign-combo edges (plus NULL combinations) for the boolean tests.
+    # The bodies (``x > 0 and y > 0`` / ``x > 0 or y > 0``) raise in
+    # pure Python on a None input (``TypeError``), and the transpiler's
+    # NULL-guarded Compare also raises -- so the ``_run`` helper's "both
+    # raised" equivalence covers the NULL cases here.
+    _BOOLEAN_PAIR_EDGES = (
+        (None, None),
+        (None, 0),
+        (0, None),
+        (0, 0),
+        (1, -1),
+        (-1, 1),
+        (1, 1),
+        (-1, -1),
+        (_LONG_BOUND, 1),
+        (1, -_LONG_BOUND),
+    )
+
+    def _seed_examples(values, key="value"):
+        """Stack one ``@example`` decorator per seed value."""
+
+        def wrapper(method):
+            for v in reversed(values):
+                method = example(**{key: v})(method)
+            return method
+
+        return wrapper
+
+    def _seed_pair_examples(pairs, keys=("x", "y")):
+        def wrapper(method):
+            for v0, v1 in reversed(pairs):
+                method = example(**{keys[0]: v0, keys[1]: v1})(method)
+            return method
+
+        return wrapper
+
+
+# ---- The UDF templates we exercise --------------------------------------
+#
+# We keep these as module-level callables so that ``inspect.getsource`` works
+# (the transpiler reads source via inspection). They are deliberately written
+# the way a user would: idiomatic Python, including ``if x is not None``
+# guards and bare ``if x:`` truthiness checks.
+
+
+def plus_four(x):
+    if x is not None:
+        return x + 4
+
+
+def plus_four_unsafe(x):
+    return x + 4
+
+
+def plus_four_with_else(x):
+    if x is not None:
+        return x + 4
+    else:
+        return 0
+
+
+def is_none_branch(x):
+    if x is None:
+        return -1
+    else:
+        return x
+
+
+def truthy_bool_branch(x):
+    # The transpiler currently mishandles ``if x:`` for nullable bool inputs:
+    # Python treats ``None`` as falsy and takes the else branch, but a naive
+    # SQL lowering can produce NULL. This test is the canonical regression.
+    if x:
+        return 1
+    else:
+        return 2
+
+
+def add_then_mod(x):
+    if x is not None:
+        return (x + 7) % 5
+
+
+def minus_two(x):
+    # Exercises ast.Sub.
+    if x is not None:
+        return x - 2
+
+
+def times_three(x):
+    # Exercises ast.Mult.
+    if x is not None:
+        return x * 3
+
+
+def negate_truthy(x):
+    # Exercises ast.UnaryOp(Not). Same NULL-as-falsy semantics as
+    # ``truthy_bool_branch`` above, just inverted.
+    if not x:
+        return 0
+    else:
+        return 1
+
+
+def both_positive(x, y):
+    # Exercises ast.BoolOp(And) over Compare operands -- both operands
+    # are statically boolean, so the transpiler should lower to `&`.
+    # Kept as a single-statement body since the transpiler doesn't yet
+    # support multi-statement function bodies; NULL inputs flow through
+    # `>` to NULL on the Spark side and to a raise on the Python side,
+    # so the strategy below skips None.
+    return x > 0 and y > 0
+
+
+def either_positive(x, y):
+    # Exercises ast.BoolOp(Or) over Compare operands.
+    return x > 0 or y > 0
+
+
+def add_two(x, y):
+    # Multi-arg UDF -- exercises the parameter-index plumbing for
+    # functions with more than one positional argument.
+    if x is not None and y is not None:
+        return x + y
+    else:
+        return 0
+
+
+def eq_zero(x):
+    # Exercises ast.Compare(Eq) via _lower_eq with a non-None left and
+    # a literal 0 on the right. The None guard keeps the comparison
+    # itself away from NULL operands so the test stays single-path.
+    if x is not None:
+        return x == 0
+
+
+def neq_zero(x):
+    # Exercises ast.Compare(NotEq) through _lower_eq.
+    if x is not None:
+        return x != 0
+
+
+def eq_pair(x, y):
+    # Two-arg ``x == y`` exercising the full _lower_eq four-branch when
+    # chain that reproduces Python's None-equality semantics:
+    #   None == None -> True;  None == 0 -> False;  0 == None -> False.
+    # Note: no None guard, so every NULL combination runs through the
+    # transpiler's lowering.
+    return x == y
+
+
+def neq_pair(x, y):
+    # Sister of ``eq_pair`` for ast.Compare(NotEq).
+    return x != y
+
+
+# A lambda captured at module scope so ``inspect.getsource`` can read
+# its definition. Exercises the ``ast.Lambda`` branch in
+# ``_get_function_from_ast``.
+lambda_plus_four = lambda x: x + 4 if x is not None else 0  # noqa: E731
+
+
+# ----------------------------------------------------------------------------
+
+
+@unittest.skipUnless(_have_hypothesis and _hypothesis_enabled and _regular_spark, _skip_reason)
+class UDFTranspileHypothesisTests(ReusedSQLTestCase):
+    """Compare transpiled vs. interpreted Python UDF output on Hypothesis-generated inputs."""
+
+    # Markers we treat as "transpilation didn't actually happen" -- if any
+    # warning matches one of these, the differential comparison would
+    # collapse to interpreted-vs-interpreted and pass meaninglessly, so we
+    # fail loudly.
+    _BAD_TRANSPILE_WARNING_MARKERS = (
+        "Unable to transpile",
+        "Errors encountered during transpilation",
+        "Exception transpiling",
+        "ANSI mode",
+    )
+
+    def _run(self, func, return_type, df, *udf_arg_columns, kwargs=None):
+        """Run ``func`` as a UDF with transpilation on and off, return both rows.
+
+        ``udf_arg_columns`` and ``kwargs`` mirror what a caller would
+        write at the dataframe API: positional column names go in
+        ``udf_arg_columns`` and named-argument bindings go in ``kwargs``
+        (e.g. ``kwargs={"y": "b", "x": "a"}`` to bind UDF parameter ``y``
+        to column ``b`` and ``x`` to column ``a``). Use either, or both.
+
+        Asserts the transpiled code path was actually exercised:
+        ``transpiled`` must be non-empty after construction, and no
+        transpilation-related warning may fire. Without these checks both
+        runs could silently fall back to interpreted Python and the
+        differential assertion would succeed for the wrong reason.
+        """
+        func_name = getattr(func, "__name__", repr(func))
+        kwargs = kwargs or {}
+
+        transpile_on_conf = {
+            "spark.sql.experimental.optimizer.transpilePyUDFs": True,
+            # Transpilation requires ANSI; pin it on so the test result
+            # doesn't depend on the surrounding session default.
+            "spark.sql.ansi.enabled": True,
+        }
+        transpiled_error: Optional[Exception] = None
+        with warnings.catch_warnings(record=True) as caught:
+            warnings.simplefilter("always")
+            with self.sql_conf(transpile_on_conf):
+                transpiled_udf = UserDefinedFunction(func, return_type)
+                self.assertTrue(
+                    transpiled_udf.transpiled,
+                    f"transpilation produced no Catalyst expression for "
+                    f"{func_name!r} -- the differential comparison would be "
+                    "meaningless without it",
+                )
+                try:
+                    transpiled_value = df.select(
+                        transpiled_udf(*udf_arg_columns, **kwargs)
+                    ).collect()[0][0]
+                except Exception as e:
+                    transpiled_value = _SENTINEL_RAISED
+                    transpiled_error = e
+            bad = [
+                w
+                for w in caught
+                if any(marker in str(w.message) for marker in self._BAD_TRANSPILE_WARNING_MARKERS)
+            ]
+            self.assertFalse(
+                bad,
+                f"unexpected transpile warnings for {func_name!r}: {[str(w.message) for w in bad]}",
+            )
+
+        interpreted_error: Optional[Exception] = None
+        # Pin ANSI on for the interpreted path too so both sides see the same
+        # overflow semantics. Without this, the interpreted path would run with
+        # the ambient session default (likely False), causing LongType overflow
+        # to silently wrap in Python UDF results while ANSI raises on the
+        # transpiled path.
+        with self.sql_conf(
+            {
+                "spark.sql.experimental.optimizer.transpilePyUDFs": False,
+                "spark.sql.ansi.enabled": True,
+            }
+        ):
+            interpreted_udf = UserDefinedFunction(func, return_type)
+            try:
+                interpreted_value = df.select(
+                    interpreted_udf(*udf_arg_columns, **kwargs)
+                ).collect()[0][0]
+            except Exception as e:
+                interpreted_value = _SENTINEL_RAISED
+                interpreted_error = e
+
+        # If the transpiled path raises an exception we also need the interpreted path to raise one,
+        # however if the Python code (that in the interpreted path) raises an exception, the transpiled
+        # path may return a valid value.
+        if transpiled_error is not None:
+            self.assertIsNotNone(
+                interpreted_error,
+                f"{func_name!r}: transpiled raised {transpiled_error!r} but interpreted did not",
+            )
+        elif interpreted_error is not None:
+            interpreted_value = transpiled_value
+
+        return transpiled_value, interpreted_value
+
+    def _single_arg_df(self, value, dtype):
+        schema = StructType([StructField("a", dtype, nullable=True)])
+        return self.spark.createDataFrame([Row(a=value)], schema=schema)
+
+    def _two_long_arg_df(self, x, y):
+        schema = StructType(
+            [
+                StructField("a", LongType(), nullable=True),
+                StructField("b", LongType(), nullable=True),
+            ]
+        )
+        return self.spark.createDataFrame([Row(a=x, b=y)], schema=schema)
+
+    if _have_hypothesis:
+
+        @_hyp_settings
+        @given(value=_long_arith_strategy)
+        @_seed_examples(_LONG_ARITH_EDGES)
+        def test_plus_four_matches_python(self, value):
+            df = self._single_arg_df(value, LongType())
+            transpiled, interpreted = self._run(plus_four, LongType(), df, "a")
+            self.assertEqual(transpiled, interpreted, f"plus_four mismatch on {value!r}")
+
+        @_hyp_settings
+        @given(value=_long_arith_strategy)
+        @_seed_examples(_LONG_ARITH_EDGES)
+        def test_plus_four_unsafe_matches_python(self, value):
+            df = self._single_arg_df(value, LongType())
+            transpiled, interpreted = self._run(plus_four_unsafe, LongType(), df, "a")
+            self.assertEqual(transpiled, interpreted, f"plus_four mismatch on {value!r}")
+
+        @_hyp_settings
+        @given(value=_long_arith_strategy)
+        @_seed_examples(_LONG_ARITH_EDGES)
+        def test_plus_four_with_else_matches_python(self, value):
+            df = self._single_arg_df(value, LongType())
+            transpiled, interpreted = self._run(plus_four_with_else, LongType(), df, "a")
+            self.assertEqual(transpiled, interpreted, f"plus_four_with_else mismatch on {value!r}")
+
+        @_hyp_settings
+        @given(value=_long_strategy)
+        @_seed_examples(_LONG_EDGES)
+        def test_is_none_branch_matches_python(self, value):
+            df = self._single_arg_df(value, LongType())
+            transpiled, interpreted = self._run(is_none_branch, LongType(), df, "a")
+            self.assertEqual(transpiled, interpreted, f"is_none_branch mismatch on {value!r}")
+
+        @_hyp_settings
+        @given(value=_bool_strategy)
+        @_seed_examples(_BOOL_EDGES)
+        def test_truthy_bool_branch_falls_back(self, value):
+            # `if x:` on a bare parameter name is a bare truthiness test whose
+            # type is unknown at transpile time. The transpiler must refuse to
+            # lower it (Spark's coalesce(x, false) is unsound for non-boolean
+            # columns) and fall back to interpreted Python instead.
+            df = self._single_arg_df(value, BooleanType())
+            with self.sql_conf(
+                {
+                    "spark.sql.experimental.optimizer.transpilePyUDFs": True,
+                    "spark.sql.ansi.enabled": True,
+                }
+            ):
+                pudf = UserDefinedFunction(truthy_bool_branch, LongType())
+                self.assertEqual(
+                    [],
+                    pudf.transpiled,
+                    "truthy_bool_branch: bare truthiness test must NOT transpile",
+                )
+                interpreted = df.select(pudf("a")).collect()[0][0]
+            expected = 1 if value else 2
+            self.assertEqual(interpreted, expected, f"truthy_bool_branch mismatch on {value!r}")
+
+        @_hyp_settings
+        @given(value=_long_arith_strategy)
+        # add_then_mod is the case that surfaced the Python-vs-SQL mod
+        # sign mismatch; the seed values cover the four sign combinations
+        # of `(x + 7) % 5` so we always re-prove the pmod fix on every
+        # run regardless of the random seed.
+        @_seed_examples((*_LONG_ARITH_EDGES, -2, -8, 8, 100, -100))
+        def test_add_then_mod_matches_python(self, value):
+            df = self._single_arg_df(value, LongType())
+            transpiled, interpreted = self._run(add_then_mod, LongType(), df, "a")
+            self.assertEqual(transpiled, interpreted, f"add_then_mod mismatch on {value!r}")
+
+        @_hyp_settings
+        @given(value=_long_arith_strategy)
+        @_seed_examples(_LONG_ARITH_EDGES)
+        def test_minus_two_matches_python(self, value):
+            df = self._single_arg_df(value, LongType())
+            transpiled, interpreted = self._run(minus_two, LongType(), df, "a")
+            self.assertEqual(transpiled, interpreted, f"minus_two mismatch on {value!r}")
+
+        @_hyp_settings
+        @given(value=_long_arith_strategy)
+        @_seed_examples(_LONG_ARITH_EDGES)
+        def test_times_three_matches_python(self, value):
+            df = self._single_arg_df(value, LongType())
+            transpiled, interpreted = self._run(times_three, LongType(), df, "a")
+            self.assertEqual(transpiled, interpreted, f"times_three mismatch on {value!r}")
+
+        @_hyp_settings
+        @given(value=_bool_strategy)
+        @_seed_examples(_BOOL_EDGES)
+        def test_negate_truthy_falls_back(self, value):
+            # `if not x:` where x is a bare parameter name is unknown-type at
+            # transpile time. The transpiler must refuse (Spark's `~` is
+            # bitwise, not Python truthiness) and fall back to interpreted Python.
+            df = self._single_arg_df(value, BooleanType())
+            with self.sql_conf(
+                {
+                    "spark.sql.experimental.optimizer.transpilePyUDFs": True,
+                    "spark.sql.ansi.enabled": True,
+                }
+            ):
+                pudf = UserDefinedFunction(negate_truthy, LongType())
+                self.assertEqual(
+                    [],
+                    pudf.transpiled,
+                    "negate_truthy: bare `not x` must NOT transpile",
+                )
+                interpreted = df.select(pudf("a")).collect()[0][0]
+            expected = 0 if not value else 1
+            self.assertEqual(interpreted, expected, f"negate_truthy mismatch on {value!r}")
+
+        @_hyp_settings
+        @given(x=_long_arith_strategy, y=_long_arith_strategy)
+        @_seed_pair_examples(_LONG_ARITH_PAIR_EDGES)
+        def test_add_two_matches_python(self, x, y):
+            df = self._two_long_arg_df(x, y)
+            transpiled, interpreted = self._run(add_two, LongType(), df, "a", "b")
+            self.assertEqual(transpiled, interpreted, f"add_two mismatch on (x={x!r}, y={y!r})")
+
+        @_hyp_settings
+        @given(x=_long_arith_strategy, y=_long_arith_strategy)
+        @_seed_pair_examples(_LONG_ARITH_PAIR_EDGES)
+        def test_add_two_named_args_matches_python(self, x, y):
+            # Same UDF as above but called with kwargs (and intentionally
+            # in reversed order) to exercise the named-argument codepath
+            # documented in the udf() reference. The Python side resolves
+            # the kwargs to the function's positional params; the
+            # transpiled side has to align ``_udf_param_0`` /
+            # ``_udf_param_1`` with the same resolved positions, so any
+            # mistake here produces a swapped-argument bug.
+            df = self._two_long_arg_df(x, y)
+            transpiled, interpreted = self._run(
+                add_two,
+                LongType(),
+                df,
+                kwargs={"y": "b", "x": "a"},
+            )
+            self.assertEqual(
+                transpiled,
+                interpreted,
+                f"add_two named-args mismatch on (x={x!r}, y={y!r})",
+            )
+
+        @_hyp_settings
+        @given(x=_long_strategy, y=_long_strategy)
+        @_seed_pair_examples(_BOOLEAN_PAIR_EDGES)
+        def test_both_positive_matches_python(self, x, y):
+            df = self._two_long_arg_df(x, y)
+            transpiled, interpreted = self._run(both_positive, BooleanType(), df, "a", "b")
+            self.assertEqual(
+                transpiled, interpreted, f"both_positive mismatch on (x={x!r}, y={y!r})"
+            )
+
+        @_hyp_settings
+        @given(x=_long_strategy, y=_long_strategy)
+        @_seed_pair_examples(_BOOLEAN_PAIR_EDGES)
+        def test_either_positive_matches_python(self, x, y):
+            df = self._two_long_arg_df(x, y)
+            transpiled, interpreted = self._run(either_positive, BooleanType(), df, "a", "b")
+            self.assertEqual(
+                transpiled, interpreted, f"either_positive mismatch on (x={x!r}, y={y!r})"
+            )
+
+        @_hyp_settings
+        @given(x=_long_strategy, y=_long_strategy)
+        @_seed_pair_examples(_LONG_PAIR_EDGES)
+        def test_eq_pair_matches_python(self, x, y):
+            # Python's ``==`` has different NULL semantics from SQL ``=``:
+            # ``None == None`` is True, ``None == n`` is False. The
+            # transpiler's _lower_eq reproduces those semantics, so the
+            # transpiled and interpreted runs must agree on every NULL
+            # combination as well as the non-NULL cases.
+            df = self._two_long_arg_df(x, y)
+            transpiled, interpreted = self._run(eq_pair, BooleanType(), df, "a", "b")
+            self.assertEqual(transpiled, interpreted, f"eq_pair mismatch on (x={x!r}, y={y!r})")
+
+        @_hyp_settings
+        @given(x=_long_strategy, y=_long_strategy)
+        @_seed_pair_examples(_LONG_PAIR_EDGES)
+        def test_neq_pair_matches_python(self, x, y):
+            # Sister of ``test_eq_pair_matches_python`` for the NotEq arm.
+            df = self._two_long_arg_df(x, y)
+            transpiled, interpreted = self._run(neq_pair, BooleanType(), df, "a", "b")
+            self.assertEqual(transpiled, interpreted, f"neq_pair mismatch on (x={x!r}, y={y!r})")
+
+        @_hyp_settings
+        @given(value=_long_strategy)
+        @_seed_examples(_LONG_EDGES)
+        def test_eq_zero_matches_python(self, value):
+            # Single-arg ``x == 0`` with a None guard, exercising _lower_eq's
+            # non-None-on-both-sides arm.
+            df = self._single_arg_df(value, LongType())
+            transpiled, interpreted = self._run(eq_zero, BooleanType(), df, "a")
+            self.assertEqual(transpiled, interpreted, f"eq_zero mismatch on {value!r}")
+
+        @_hyp_settings
+        @given(value=_long_strategy)
+        @_seed_examples(_LONG_EDGES)
+        def test_neq_zero_matches_python(self, value):
+            # Sister of ``test_eq_zero_matches_python`` for the NotEq arm.
+            df = self._single_arg_df(value, LongType())
+            transpiled, interpreted = self._run(neq_zero, BooleanType(), df, "a")
+            self.assertEqual(transpiled, interpreted, f"neq_zero mismatch on {value!r}")
+
+        @_hyp_settings
+        @given(value=_long_arith_strategy)
+        @_seed_examples(_LONG_ARITH_EDGES)
+        def test_lambda_plus_four_matches_python(self, value):
+            df = self._single_arg_df(value, LongType())
+            transpiled, interpreted = self._run(lambda_plus_four, LongType(), df, "a")
+            self.assertEqual(transpiled, interpreted, f"lambda_plus_four mismatch on {value!r}")
+
+
+class UDFTranspileHypothesisGatingTests(unittest.TestCase):
+    """Smoke tests that always run, regardless of the env gate.
+
+    These don't talk to Spark; they just verify the gating / skipping logic
+    so a misconfigured environment doesn't silently skip everything forever.
+    """
+
+    def test_env_var_name_is_documented(self):
+        # If we ever rename the env var, the docstring needs to follow.
+        self.assertIn(_HYPOTHESIS_ENV, __doc__)
+
+    def test_skip_reason_mentions_env_var(self):
+        self.assertIn(_HYPOTHESIS_ENV, _skip_reason)
+
+    def test_env_gate_is_value_based(self):
+        # CI always sets RUN_HYPOTHESIS (to "true" or "false") via the
+        # transpile precondition in build_and_test.yml. If this gate ever
+        # regresses to presence-based, the very slow suite silently runs on
+        # every PySpark job. Pin the contract from both directions.
+        for opted_in in ("1", "true", "TRUE", "yes", " True "):
+            self.assertTrue(_env_opts_in(opted_in), f"{opted_in!r} must opt in")
+        for opted_out in (None, "", "0", "false", "FALSE", "no", "off"):
+            self.assertFalse(_env_opts_in(opted_out), f"{opted_out!r} must NOT opt in")
+
+
+if __name__ == "__main__":
+    from pyspark.testing import main
+
+    main()
diff --git a/python/pyspark/sql/tests/test_udf_transpile_parity.py b/python/pyspark/sql/tests/test_udf_transpile_parity.py
new file mode 100644
index 000000000000..a8276df0b162
--- /dev/null
+++ b/python/pyspark/sql/tests/test_udf_transpile_parity.py
@@ -0,0 +1,110 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""
+Parity tests that re-run the existing UDF test suites with Python-to-Catalyst
+transpilation enabled.
+
+Transpilation is only attempted when both
+``spark.sql.experimental.optimizer.transpilePyUDFs`` and
+``spark.sql.ansi.enabled`` are true, and it is designed to fall back to
+interpreted Python rather than risk semantic drift. These classes re-run the
+shared UDF mixins under that configuration so we can confirm that turning on the
+experimental feature does not change UDF results compared with the default
+(transpilation off) runs covered by the original concrete classes
+(``UDFTests``, ``UDFCombinationsTests``, ``UnifiedUDFTests``).
+
+Transpilation is currently only supported in regular (non-Connect) Spark, so
+these classes are guarded with ``is_remote_only()`` and are intentionally not
+inherited into the Spark Connect parity tests. The companion suites that test
+the transpiler directly live in ``test_udf_transpile_unit.py`` and
+``test_udf_transpile_hypothesis.py``.
+
+Note on configuration: enabling transpilation requires ANSI mode, so an "on"
+run is unavoidably also an ANSI run. All inherited tests currently pass as-is
+under this configuration, so no per-test overrides are defined here. If a future
+change makes an inherited test diverge purely due to ANSI semantics or because
+transpilation bypasses a Python-side effect (rather than a genuine result
+change), override it here with a documented ``unittest.skip`` rather than
+editing the inherited test body.
+"""
+
+import unittest
+
+from pyspark.sql.tests.test_udf import BaseUDFTestsMixin
+from pyspark.sql.tests.test_udf_combinations import UDFCombinationsTestsMixin
+from pyspark.sql.tests.test_unified_udf import UnifiedUDFTestsMixin
+from pyspark.testing.sqlutils import ReusedSQLTestCase
+from pyspark.testing.utils import (
+    have_pandas,
+    have_pyarrow,
+    pandas_requirement_message,
+    pyarrow_requirement_message,
+)
+from pyspark.util import is_remote_only
+
+
+# Transpilation is gated on both of these being enabled, both at UDF
+# construction time (python/pyspark/sql/udf.py) and again in the Catalyst
+# optimizer (the ConvertToCatalyst rule).
+# spark.conf.set requires strings, so we use "true" rather than Python True here.
+_TRANSPILE_CONF = {
+    "spark.sql.experimental.optimizer.transpilePyUDFs": "true",
+    "spark.sql.ansi.enabled": "true",
+}
+
+_NON_CONNECT_ONLY = "UDF transpilation is only supported in regular (non-Connect) Spark."
+
+
+def _enable_transpilation(cls):
+    for key, value in _TRANSPILE_CONF.items():
+        cls.spark.conf.set(key, value)
+
+
+@unittest.skipIf(is_remote_only(), _NON_CONNECT_ONLY)
+class TranspiledUDFParityTests(BaseUDFTestsMixin, ReusedSQLTestCase):
+    @classmethod
+    def setUpClass(cls):
+        ReusedSQLTestCase.setUpClass()
+        cls.spark.conf.set("spark.sql.execution.pythonUDF.arrow.enabled", "false")
+        _enable_transpilation(cls)
+
+
+@unittest.skipIf(is_remote_only(), _NON_CONNECT_ONLY)
+class TranspiledUDFCombinationsParityTests(UDFCombinationsTestsMixin, ReusedSQLTestCase):
+    @classmethod
+    def setUpClass(cls):
+        ReusedSQLTestCase.setUpClass()
+        cls.spark.conf.set("spark.sql.execution.pythonUDF.arrow.enabled", "false")
+        _enable_transpilation(cls)
+
+
+@unittest.skipIf(is_remote_only(), _NON_CONNECT_ONLY)
+@unittest.skipIf(
+    not have_pandas or not have_pyarrow,
+    pandas_requirement_message or pyarrow_requirement_message,
+)
+class TranspiledUnifiedUDFParityTests(UnifiedUDFTestsMixin, ReusedSQLTestCase):
+    @classmethod
+    def setUpClass(cls):
+        ReusedSQLTestCase.setUpClass()
+        _enable_transpilation(cls)
+
+
+if __name__ == "__main__":
+    from pyspark.testing import main
+
+    main()
diff --git a/python/pyspark/sql/tests/test_udf_transpile_unit.py b/python/pyspark/sql/tests/test_udf_transpile_unit.py
new file mode 100644
index 000000000000..de753b0e7bc0
--- /dev/null
+++ b/python/pyspark/sql/tests/test_udf_transpile_unit.py
@@ -0,0 +1,1736 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""
+Unit tests for UDF transpilation.
+
+These were previously interleaved with the broader UDF mixin in
+``test_udf.py``. They are split out because UDF transpilation is currently
+only supported in regular (non-Connect) Spark, so they should not be
+inherited into the Spark Connect parity test class. The companion
+property-based suite lives in ``test_udf_transpile_hypothesis.py``.
+"""
+
+import unittest
+
+from pyspark.sql import Row
+from pyspark.sql.types import (
+    BinaryType,
+    BooleanType,
+    DoubleType,
+    LongType,
+    StringType,
+)
+from pyspark.sql.udf import UserDefinedFunction
+from pyspark.testing.sqlutils import ReusedSQLTestCase
+from pyspark.util import is_remote_only
+
+
+# Both flags must be on for the transpiler to attempt a rewrite (at UDF
+# construction time and again in the optimizer); ANSI is required because
+# transpilation targets ANSI semantics.
+_TRANSPILE_ON = {
+    "spark.sql.experimental.optimizer.transpilePyUDFs": True,
+    "spark.sql.ansi.enabled": True,
+}
+
+
+@unittest.skipIf(
+    is_remote_only(),
+    "UDF transpilation is only supported in regular (non-Connect) Spark.",
+)
+class UDFTranspileUnitTests(ReusedSQLTestCase):
+    def test_udf_transpile_basic(self):
+        # Test callable object
+        class PlusFour:
+            def __call__(self, col):
+                return col + 4
+
+        with self.sql_conf(
+            {
+                "spark.sql.experimental.optimizer.transpilePyUDFs": True,
+                "spark.sql.ansi.enabled": True,
+            }
+        ):
+            # Make sure we can transpile the object
+            call = PlusFour()
+            pudf = UserDefinedFunction(call, LongType())
+            self.assertTrue(pudf.transpiled)
+            # Now make sure we can run the transpiled UDF*
+            input_df = self.spark.createDataFrame([Row(a=1)])
+            transformed_df = input_df.select(pudf("a"))
+            [row] = transformed_df.collect()
+            self.assertEqual(row[0], 5)
+
+        with self.sql_conf({"spark.sql.experimental.optimizer.transpilePyUDFs": False}):
+            call = PlusFour()
+            pudf = UserDefinedFunction(call, LongType())
+            self.assertEqual([], pudf.transpiled)
+            # Now make sure we can run the UDF
+            input_df = self.spark.createDataFrame([Row(a=1)])
+            transformed_df = input_df.select(pudf("a"))
+            [row] = transformed_df.collect()
+            self.assertEqual(row[0], 5)
+
+    def test_udf_transpile_with_nones(self):
+        # Test callable object
+        class PlusFour:
+            def __call__(self, col):
+                if col is not None:
+                    return col + 4
+
+        with self.sql_conf(
+            {
+                "spark.sql.experimental.optimizer.transpilePyUDFs": True,
+                "spark.sql.ansi.enabled": True,
+            }
+        ):
+            # Make sure we can transpile the object
+            call = PlusFour()
+            pudf = UserDefinedFunction(call, LongType())
+            self.assertTrue(pudf.transpiled)
+            # Now make sure we can run the transpiled UDF*
+            input_df = self.spark.createDataFrame([Row(a=1)])
+            transformed_df = input_df.select(pudf("a").alias("result"))
+            [row] = transformed_df.collect()
+            self.assertEqual(row[0], 5)
+            physical_plan = transformed_df._jdf.queryExecution().executedPlan().toString()
+            self.assertNotIn("UDF", physical_plan)
+
+        with self.sql_conf({"spark.sql.experimental.optimizer.transpilePyUDFs": False}):
+            call = PlusFour()
+            pudf = UserDefinedFunction(call, LongType())
+            self.assertEqual([], pudf.transpiled)
+            # Now make sure we can run the UDF
+            input_df = self.spark.createDataFrame([Row(a=1)])
+            transformed_df = input_df.select(pudf("a").alias("result"))
+            [row] = transformed_df.collect()
+            self.assertEqual(row[0], 5)
+            physical_plan = transformed_df._jdf.queryExecution().executedPlan().toString()
+            self.assertIn("UDF", physical_plan)
+
+    def test_udf_not_transpilable(self):
+        class UnsupportedEx:
+            def __call__(self, col):
+                if col is not None:
+                    return col in "4"
+
+        with self.sql_conf({"spark.sql.experimental.optimizer.transpilePyUDFs": True}):
+            call = UnsupportedEx()
+            pudf = UserDefinedFunction(call, BooleanType())
+            self.assertEqual([], pudf.transpiled)
+
+    def test_udf_transpile_requires_ansi(self):
+        # Transpilation targets ANSI semantics. With ANSI off the transpiler
+        # must skip rewriting (and warn the user) so we don't silently
+        # diverge from the Python interpretation; with ANSI on it should
+        # produce a Catalyst expression.
+        import warnings
+
+        def plus_four(x):
+            if x is not None:
+                return x + 4
+
+        with self.sql_conf(
+            {
+                "spark.sql.experimental.optimizer.transpilePyUDFs": True,
+                "spark.sql.ansi.enabled": False,
+            }
+        ):
+            with warnings.catch_warnings(record=True) as caught:
+                warnings.simplefilter("always")
+                pudf = UserDefinedFunction(plus_four, LongType())
+            self.assertEqual([], pudf.transpiled)
+            ansi_warnings = [w for w in caught if "ANSI mode" in str(w.message)]
+            self.assertTrue(
+                ansi_warnings,
+                "expected an 'ANSI mode' warning when transpilation is "
+                "requested but ANSI is disabled",
+            )
+
+        with self.sql_conf(
+            {
+                "spark.sql.experimental.optimizer.transpilePyUDFs": True,
+                "spark.sql.ansi.enabled": True,
+            }
+        ):
+            pudf = UserDefinedFunction(plus_four, LongType())
+            self.assertTrue(
+                pudf.transpiled,
+                "expected transpilation to produce a Catalyst expression "
+                "when both transpilePyUDFs and ANSI mode are enabled",
+            )
+
+    def test_udf_transpile_falls_back_for_unsupported_patterns(self):
+        # The transpiler intentionally only handles a small subset of
+        # Python AST today. Everything outside that subset must
+        # gracefully fall back to interpreted Python (with an empty
+        # `transpiled` list and a UserWarning) rather than break the
+        # UDF -- the "don't break people's Spark code" promise. This test
+        # walks the most common unsupported shapes, registers each as a
+        # UDF with transpilation on, and asserts (a) construction does
+        # not raise, (b) `transpiled == []`, (c) the UDF still produces
+        # the correct interpreted result.
+
+        def divide_by_two(x):  # `/` -- ast.Div, not handled.
+            if x is not None:
+                return x / 2
+
+        def floor_divide_by_two(x):  # `//` -- ast.FloorDiv, not handled.
+            if x is not None:
+                return x // 2
+
+        def bit_and_one(x):  # `&` -- ast.BitAnd, not handled.
+            if x is not None:
+                return x & 1
+
+        def bit_or_one(x):  # `|` -- ast.BitOr, not handled.
+            if x is not None:
+                return x | 1
+
+        def left_shift(x):  # `<<` -- ast.LShift, not handled.
+            if x is not None:
+                return x << 1
+
+        def multi_statement(x):  # > 1 top-level statement, not handled.
+            y = 1
+            return x + y if x is not None else 0
+
+        def func_closure_capture(x):
+            offset = 7
+            if x is not None:
+                return x + offset
+
+        cases = [
+            ("divide_by_two", divide_by_two, DoubleType(), Row(a=4.0), 2.0),
+            ("floor_divide_by_two", floor_divide_by_two, LongType(), Row(a=5), 2),
+            ("bit_and_one", bit_and_one, LongType(), Row(a=5), 1),
+            ("bit_or_one", bit_or_one, LongType(), Row(a=4), 5),
+            ("left_shift", left_shift, LongType(), Row(a=3), 6),
+            ("multi_statement", multi_statement, LongType(), Row(a=5), 6),
+            ("func_closure_capture", func_closure_capture, LongType(), Row(a=10), 17),
+        ]
+
+        with self.sql_conf(
+            {
+                "spark.sql.experimental.optimizer.transpilePyUDFs": True,
+                "spark.sql.ansi.enabled": True,
+            }
+        ):
+            for label, func, return_type, row, expected in cases:
+                with self.subTest(case=label):
+                    import warnings as _warnings
+
+                    with _warnings.catch_warnings(record=True) as caught_warnings:
+                        _warnings.simplefilter("always")
+                        pudf = UserDefinedFunction(func, return_type)
+                    self.assertEqual(
+                        [],
+                        pudf.transpiled,
+                        f"{label}: transpiler should not produce a Catalyst "
+                        "expression for this AST shape",
+                    )
+                    fallback = [
+                        w
+                        for w in caught_warnings
+                        if "Unable to transpile" in str(w.message)
+                        or "Errors encountered" in str(w.message)
+                        or "Exception transpiling" in str(w.message)
+                    ]
+                    self.assertTrue(
+                        fallback,
+                        f"{label}: expected a fallback warning when the "
+                        "transpiler can't lower the function",
+                    )
+                    df = self.spark.createDataFrame([row])
+                    [result] = df.select(pudf("a")).collect()
+                    self.assertEqual(
+                        result[0],
+                        expected,
+                        f"{label}: interpreted UDF result diverged from expected",
+                    )
+
+    def test_udf_transpile_boolean_and_or_lowered(self):
+        # When `and`/`or` operands are syntactically boolean (Compare
+        # results in this case), the transpiler should lower to bitwise
+        # `&`/`|` and produce results matching the interpreted UDF.
+        # Each UDF is a single top-level statement (the transpiler
+        # doesn't support multi-statement bodies yet).
+        from pyspark.sql.types import StructField, StructType
+
+        def both_positive(x, y):
+            return x > 0 and y > 0
+
+        def either_positive(x, y):
+            return x > 0 or y > 0
+
+        schema = StructType(
+            [
+                StructField("a", LongType(), nullable=True),
+                StructField("b", LongType(), nullable=True),
+            ]
+        )
+
+        with self.sql_conf(
+            {
+                "spark.sql.experimental.optimizer.transpilePyUDFs": True,
+                "spark.sql.ansi.enabled": True,
+            }
+        ):
+            # NULL inputs propagate through `>` to NULL, which then
+            # passes through `&` / `|` per SQL three-valued logic. We
+            # only assert on non-NULL inputs here since Python's
+            # interpreted `x > 0 and y > 0` would raise on None; the
+            # NULL handling itself is covered by the hypothesis suite.
+            for func, x, y, expected in [
+                (both_positive, 1, 2, True),
+                (both_positive, 1, -1, False),
+                (both_positive, -1, -1, False),
+                (either_positive, -1, 2, True),
+                (either_positive, -1, -1, False),
+                (either_positive, 1, 1, True),
+            ]:
+                with self.subTest(func=func.__name__, x=x, y=y):
+                    pudf = UserDefinedFunction(func, BooleanType())
+                    self.assertTrue(
+                        pudf.transpiled,
+                        f"{func.__name__}: bool-typed and/or should transpile",
+                    )
+                    df = self.spark.createDataFrame([Row(a=x, b=y)], schema=schema)
+                    [row] = df.select(pudf("a", "b")).collect()
+                    self.assertEqual(row[0], expected)
+
+    def test_udf_transpile_less_than_zero(self):
+        # Restored from the unsupported-patterns matrix: now that the
+        # transpiler handles ast.Lt, `x < 0` should lower to a Catalyst
+        # expression and match interpreted Python. The ``is not None``
+        # guard short-circuits None inputs through the else branch, so
+        # the comparison itself never sees a NULL in this UDF.
+        from pyspark.sql.types import StructField, StructType
+
+        def less_than_zero(x):
+            if x is not None:
+                return x < 0
+
+        schema = StructType([StructField("a", LongType(), nullable=True)])
+        with self.sql_conf(
+            {
+                "spark.sql.experimental.optimizer.transpilePyUDFs": True,
+                "spark.sql.ansi.enabled": True,
+            }
+        ):
+            pudf = UserDefinedFunction(less_than_zero, BooleanType())
+            self.assertTrue(pudf.transpiled, "less_than_zero should now transpile")
+            for value, expected in [(-1, True), (0, False), (5, False), (None, None)]:
+                with self.subTest(value=value):
+                    df = self.spark.createDataFrame([Row(a=value)], schema=schema)
+                    [row] = df.select(pudf("a")).collect()
+                    self.assertEqual(row[0], expected)
+
+    def test_udf_transpile_compare_with_none_raises(self):
+        # When a comparison's operand is NULL in Spark, Python would have
+        # raised TypeError ('>' not supported between NoneType and int).
+        # The transpiler wraps Compare ops with a raise_error guard so
+        # the rewritten plan fails loudly instead of silently producing
+        # NULL three-valued-logic results.
+        from pyspark.sql.types import StructField, StructType
+
+        def gt_zero(x):
+            return x > 0
+
+        schema = StructType([StructField("a", LongType(), nullable=True)])
+        with self.sql_conf(
+            {
+                "spark.sql.experimental.optimizer.transpilePyUDFs": True,
+                "spark.sql.ansi.enabled": True,
+            }
+        ):
+            pudf = UserDefinedFunction(gt_zero, BooleanType())
+            self.assertTrue(pudf.transpiled, "gt_zero should transpile")
+            df = self.spark.createDataFrame([Row(a=None)], schema=schema)
+            with self.assertRaises(Exception) as ctx:
+                df.select(pudf("a")).collect()
+            self.assertIn("cannot compare NULL", str(ctx.exception))
+
+    def test_udf_transpile_eq_none_semantics(self):
+        # Python ``==``/``!=`` differ from Spark's three-valued NULL equality:
+        # in Python ``None == None`` is ``True`` and ``None == 0`` is ``False``,
+        # whereas SQL ``NULL = NULL`` and ``NULL = 0`` both yield ``NULL``. The
+        # transpiler's ``_lower_eq`` reproduces Python's semantics; this test
+        # exercises every arm of that logic.
+        from pyspark.sql.types import StructField, StructType
+
+        def x_eq_zero(x):
+            if x is not None:
+                return x == 0
+            else:
+                return None
+
+        def x_neq_zero(x):
+            if x is not None:
+                return x != 0
+            else:
+                return None
+
+        def x_eq_y(x, y):
+            return x == y
+
+        def x_neq_y(x, y):
+            return x != y
+
+        long_schema = StructType([StructField("a", LongType(), nullable=True)])
+        two_col_schema = StructType(
+            [
+                StructField("a", LongType(), nullable=True),
+                StructField("b", LongType(), nullable=True),
+            ]
+        )
+        with self.sql_conf(
+            {
+                "spark.sql.experimental.optimizer.transpilePyUDFs": True,
+                "spark.sql.ansi.enabled": True,
+            }
+        ):
+            # Single-arg ``x == 0`` / ``x != 0`` with a None guard.
+            pudf_eq = UserDefinedFunction(x_eq_zero, BooleanType())
+            pudf_neq = UserDefinedFunction(x_neq_zero, BooleanType())
+            self.assertTrue(pudf_eq.transpiled, "x == 0 should transpile")
+            self.assertTrue(pudf_neq.transpiled, "x != 0 should transpile")
+            for value, eq_expected, neq_expected in [
+                (0, True, False),
+                (1, False, True),
+                (-3, False, True),
+                (None, None, None),
+            ]:
+                with self.subTest(value=value):
+                    df = self.spark.createDataFrame([Row(a=value)], schema=long_schema)
+                    [row_eq] = df.select(pudf_eq("a")).collect()
+                    [row_neq] = df.select(pudf_neq("a")).collect()
+                    self.assertEqual(row_eq[0], eq_expected)
+                    self.assertEqual(row_neq[0], neq_expected)
+
+            # Two-arg ``x == y`` / ``x != y`` exercising every NULL combination.
+            pudf_eq_xy = UserDefinedFunction(x_eq_y, BooleanType())
+            pudf_neq_xy = UserDefinedFunction(x_neq_y, BooleanType())
+            self.assertTrue(pudf_eq_xy.transpiled, "x == y should transpile")
+            self.assertTrue(pudf_neq_xy.transpiled, "x != y should transpile")
+            # Python semantics:
+            #   None == None -> True;     None != None -> False
+            #   None == 0    -> False;    None != 0    -> True
+            #   0    == None -> False;    0    != None -> True
+            #   1    == 1    -> True;     1    != 1    -> False
+            #   1    == 2    -> False;    1    != 2    -> True
+            for x, y, eq_expected, neq_expected in [
+                (None, None, True, False),
+                (None, 0, False, True),
+                (0, None, False, True),
+                (1, 1, True, False),
+                (1, 2, False, True),
+            ]:
+                with self.subTest(x=x, y=y):
+                    df = self.spark.createDataFrame([Row(a=x, b=y)], schema=two_col_schema)
+                    [row_eq] = df.select(pudf_eq_xy("a", "b")).collect()
+                    [row_neq] = df.select(pudf_neq_xy("a", "b")).collect()
+                    self.assertEqual(row_eq[0], eq_expected, f"({x} == {y})")
+                    self.assertEqual(row_neq[0], neq_expected, f"({x} != {y})")
+
+    def test_udf_transpile_lte_gte(self):
+        # ``<=`` and ``>=`` go through the same ``_lower_value_compare`` path
+        # as ``<`` / ``>`` (and so share the NULL-raises-TypeError guard), but
+        # the entry points are not exercised elsewhere. Cover both with a None
+        # guard so the comparison only sees non-NULL operands here.
+        from pyspark.sql.types import StructField, StructType
+
+        def lte_zero(x):
+            if x is not None:
+                return x <= 0
+
+        def gte_zero(x):
+            if x is not None:
+                return x >= 0
+
+        schema = StructType([StructField("a", LongType(), nullable=True)])
+        with self.sql_conf(
+            {
+                "spark.sql.experimental.optimizer.transpilePyUDFs": True,
+                "spark.sql.ansi.enabled": True,
+            }
+        ):
+            pudf_lte = UserDefinedFunction(lte_zero, BooleanType())
+            pudf_gte = UserDefinedFunction(gte_zero, BooleanType())
+            self.assertTrue(pudf_lte.transpiled, "x <= 0 should transpile")
+            self.assertTrue(pudf_gte.transpiled, "x >= 0 should transpile")
+            for value, lte_expected, gte_expected in [
+                (-1, True, False),
+                (0, True, True),
+                (1, False, True),
+                (None, None, None),
+            ]:
+                with self.subTest(value=value):
+                    df = self.spark.createDataFrame([Row(a=value)], schema=schema)
+                    [row_lte] = df.select(pudf_lte("a")).collect()
+                    [row_gte] = df.select(pudf_gte("a")).collect()
+                    self.assertEqual(row_lte[0], lte_expected)
+                    self.assertEqual(row_gte[0], gte_expected)
+
+    def test_udf_transpile_chained_comparison_falls_back(self):
+        # ``a < b < c`` is a chained comparison: Python evaluates it as
+        # ``(a < b) and (b < c)``. The transpiler refuses chained Compare
+        # nodes (``len(ops) != 1``) and must fall back to interpreted Python.
+        import warnings as _warnings
+        from pyspark.sql.types import StructField, StructType
+
+        def chained(x):
+            return 0 < x < 10
+
+        schema = StructType([StructField("a", LongType(), nullable=False)])
+        with self.sql_conf(
+            {
+                "spark.sql.experimental.optimizer.transpilePyUDFs": True,
+                "spark.sql.ansi.enabled": True,
+            }
+        ):
+            with _warnings.catch_warnings(record=True) as caught:
+                _warnings.simplefilter("always")
+                pudf = UserDefinedFunction(chained, BooleanType())
+            self.assertEqual([], pudf.transpiled, "chained comparison must NOT transpile")
+            fallback = [
+                w
+                for w in caught
+                if "Unable to transpile" in str(w.message) or "Errors encountered" in str(w.message)
+            ]
+            self.assertTrue(fallback, "expected a fallback warning")
+            for value, expected in [(5, True), (0, False), (10, False), (-3, False)]:
+                with self.subTest(value=value):
+                    df = self.spark.createDataFrame([Row(a=value)], schema=schema)
+                    [row] = df.select(pudf("a")).collect()
+                    self.assertEqual(row[0], expected)
+
+    def test_udf_transpile_multi_row(self):
+        # Every other transpile test uses a 1-row DataFrame; this one runs
+        # the same arithmetic transpile on a multi-row input to catch any
+        # column-reference / batch-boundary bug that single-row tests can't.
+        from pyspark.sql.types import StructField, StructType
+
+        def plus_four(x):
+            if x is not None:
+                return x + 4
+
+        schema = StructType([StructField("a", LongType(), nullable=True)])
+        with self.sql_conf(
+            {
+                "spark.sql.experimental.optimizer.transpilePyUDFs": True,
+                "spark.sql.ansi.enabled": True,
+            }
+        ):
+            pudf = UserDefinedFunction(plus_four, LongType())
+            self.assertTrue(pudf.transpiled)
+            inputs = [Row(a=v) for v in [-3, -1, 0, 1, 7, None, 100]]
+            df = self.spark.createDataFrame(inputs, schema=schema)
+            transformed_df = df.select(pudf("a").alias("result"))
+            rows = transformed_df.collect()
+            actual = [row[0] for row in rows]
+            expected = [None if v is None else v + 4 for v in [-3, -1, 0, 1, 7, None, 100]]
+            self.assertEqual(actual, expected)
+            # Plan should also have the UDF stripped under the rewrite.
+            physical_plan = transformed_df._jdf.queryExecution().executedPlan().toString()
+            self.assertNotIn("UDF", physical_plan)
+
+    def test_udf_transpile_falls_back_for_non_boolean_short_circuit(self):
+        # Python's `x or 0` returns x if truthy else 0; Spark's `|` is
+        # bitwise, so we'd silently produce wrong results. The transpiler
+        # must refuse, fall back to interpreted Python, and still produce
+        # the correct result.
+        import warnings as _warnings
+        from pyspark.sql.types import StructField, StructType
+
+        def or_zero(x):
+            return x or 0
+
+        def and_one(x):
+            return x and 1
+
+        def not_int(x):
+            return not 0 + x  # operand is BinOp, statically non-boolean
+
+        long_schema = StructType([StructField("a", LongType(), nullable=True)])
+
+        cases = [
+            ("or_zero", or_zero, LongType(), long_schema, Row(a=5), 5),
+            ("or_zero_none", or_zero, LongType(), long_schema, Row(a=None), 0),
+            ("and_one", and_one, LongType(), long_schema, Row(a=5), 1),
+            ("and_one_zero", and_one, LongType(), long_schema, Row(a=0), 0),
+            ("not_int", not_int, BooleanType(), long_schema, Row(a=0), True),
+            ("not_int_nonzero", not_int, BooleanType(), long_schema, Row(a=3), False),
+        ]
+        with self.sql_conf(
+            {
+                "spark.sql.experimental.optimizer.transpilePyUDFs": True,
+                "spark.sql.ansi.enabled": True,
+            }
+        ):
+            for label, func, return_type, schema, row, expected in cases:
+                with self.subTest(case=label):
+                    with _warnings.catch_warnings(record=True) as caught:
+                        _warnings.simplefilter("always")
+                        pudf = UserDefinedFunction(func, return_type)
+                    self.assertEqual(
+                        [],
+                        pudf.transpiled,
+                        f"{label}: non-boolean and/or/not must NOT be lowered",
+                    )
+                    fallback = [
+                        w
+                        for w in caught
+                        if "Unable to transpile" in str(w.message)
+                        or "Errors encountered" in str(w.message)
+                    ]
+                    self.assertTrue(fallback, f"{label}: expected a fallback warning")
+                    df = self.spark.createDataFrame([row], schema=schema)
+                    [result] = df.select(pudf("a")).collect()
+                    self.assertEqual(result[0], expected, f"{label}: interpreted mismatch")
+
+    def test_udf_transpile_falls_back_for_bare_truthiness_test(self):
+        # A bare `if x:` applied to a non-boolean column cannot be soundly
+        # lowered: Python truthiness is type-dependent (0, "", [], None are
+        # falsy) and the transpiler has no input type information at this
+        # point. Emitting coalesce(x, false) either fails Spark analysis for
+        # non-boolean columns or silently produces wrong answers.  The
+        # transpiler must refuse and fall back to interpreted Python.
+        import warnings as _warnings
+        from pyspark.sql.types import StructField, StructType
+
+        def truthy_int(x):
+            if x:
+                return x
+            return -1
+
+        def truthy_string(x):
+            return x if x else "default"
+
+        long_schema = StructType([StructField("a", LongType(), nullable=True)])
+        str_schema = StructType([StructField("a", StringType(), nullable=True)])
+
+        cases = [
+            ("truthy_int_zero", truthy_int, LongType(), long_schema, Row(a=0), -1),
+            ("truthy_int_nonzero", truthy_int, LongType(), long_schema, Row(a=3), 3),
+            ("truthy_string_empty", truthy_string, StringType(), str_schema, Row(a=""), "default"),
+            ("truthy_string_val", truthy_string, StringType(), str_schema, Row(a="hi"), "hi"),
+        ]
+
+        with self.sql_conf(
+            {
+                "spark.sql.experimental.optimizer.transpilePyUDFs": True,
+                "spark.sql.ansi.enabled": True,
+            }
+        ):
+            for label, func, return_type, schema, row, expected in cases:
+                with self.subTest(case=label):
+                    with _warnings.catch_warnings(record=True) as caught:
+                        _warnings.simplefilter("always")
+                        pudf = UserDefinedFunction(func, return_type)
+                    self.assertEqual(
+                        [],
+                        pudf.transpiled,
+                        f"{label}: bare truthiness test must NOT be lowered to Catalyst",
+                    )
+                    fallback = [
+                        w
+                        for w in caught
+                        if "Unable to transpile" in str(w.message)
+                        or "Errors encountered" in str(w.message)
+                    ]
+                    self.assertTrue(fallback, f"{label}: expected a fallback warning")
+                    df = self.spark.createDataFrame([row], schema=schema)
+                    [result] = df.select(pudf("a")).collect()
+                    self.assertEqual(result[0], expected, f"{label}: interpreted mismatch")
+
+    def test_udf_transpile_falls_back_for_mismatched_branch_types(self):
+        # An if/ternary whose two branches produce different Spark categories
+        # (e.g. numeric vs string) would lower to a CASE WHEN whose branch
+        # values share no common type under ANSI. That node is carried as a
+        # child of the TranspiledPythonUDF and is type-checked by CheckAnalysis
+        # before ConvertToCatalyst can drop it, so without a guard the whole
+        # query would fail analysis instead of falling back. The transpiler must
+        # refuse and run the UDF as interpreted Python.
+        import warnings as _warnings
+        from pyspark.sql.types import StructField, StructType
+
+        def mixed_ternary(x):
+            return 1 if x > 0 else "neg"
+
+        def mixed_if(x):
+            # Single top-level `if`/`else` so the If-statement lowering path
+            # (not the "more than one statement" fallback) exercises the guard.
+            if x > 0:
+                return "pos"
+            else:
+                return x
+
+        # Positive control: matching-category branches must still transpile, so
+        # the guard does not over-refuse.
+        def homogeneous(x):
+            return x if x > 0 else 0
+
+        long_schema = StructType([StructField("a", LongType(), nullable=True)])
+
+        with self.sql_conf(
+            {
+                "spark.sql.experimental.optimizer.transpilePyUDFs": True,
+                "spark.sql.ansi.enabled": True,
+            }
+        ):
+            # Inputs are chosen to take the string-returning branch so the
+            # interpreted result is unambiguous.
+            mismatch_cases = [
+                ("mixed_ternary", mixed_ternary, Row(a=-3), "neg"),
+                ("mixed_if", mixed_if, Row(a=10), "pos"),
+            ]
+            for label, func, row, expected in mismatch_cases:
+                with self.subTest(case=label):
+                    with _warnings.catch_warnings(record=True) as caught:
+                        _warnings.simplefilter("always")
+                        pudf = UserDefinedFunction(func, StringType())
+                    self.assertEqual(
+                        [],
+                        pudf.transpiled,
+                        f"{label}: mismatched branch types must NOT be lowered to Catalyst",
+                    )
+                    fallback = [w for w in caught if "Unable to transpile" in str(w.message)]
+                    self.assertTrue(fallback, f"{label}: expected a fallback warning")
+                    df = self.spark.createDataFrame([row], schema=long_schema)
+                    # Must run without an analysis failure and match interpreted Python.
+                    [result] = df.select(pudf("a")).collect()
+                    self.assertEqual(result[0], expected, f"{label}: interpreted mismatch")
+
+            with self.subTest(case="homogeneous"):
+                pudf = UserDefinedFunction(homogeneous, LongType())
+                self.assertNotEqual(
+                    [],
+                    pudf.transpiled,
+                    "matching-category branches must still transpile",
+                )
+                df = self.spark.createDataFrame([Row(a=5), Row(a=-3)], schema=long_schema)
+                results = [r[0] for r in df.select(pudf("a")).collect()]
+                self.assertEqual(results, [5, 0], "homogeneous branch result mismatch")
+
+    def test_udf_transpile_falls_back_for_cross_category_eq(self):
+        # `x == True` on a numeric column would lower to `x = true`, which
+        # fails ANSI analysis (BIGINT vs BOOLEAN) while the option is still a
+        # child of the TranspiledPythonUDF -- breaking a working UDF. The
+        # category gate must refuse so it runs as interpreted Python.
+        import warnings as _warnings
+        from pyspark.sql.types import StructField, StructType
+
+        def eq_true(x):
+            return x == True  # noqa: E712
+
+        long_schema = StructType([StructField("a", LongType(), nullable=True)])
+        with self.sql_conf(_TRANSPILE_ON):
+            with _warnings.catch_warnings(record=True):
+                _warnings.simplefilter("always")
+                pudf = UserDefinedFunction(eq_true, BooleanType())
+            self.assertEqual([], pudf.transpiled, "cross-category == must not transpile")
+            df = self.spark.createDataFrame([Row(a=5), Row(a=1)], schema=long_schema)
+            results = [r[0] for r in df.select(pudf("a")).collect()]
+            self.assertEqual(results, [5 == True, 1 == True])
+
+    def test_udf_transpile_falls_back_for_nested_ternary_eq(self):
+        # A ternary operand used inside `==` must contribute its branches'
+        # category, not the old "numeric" catch-all: `("5" if c else "6") == 5`
+        # previously passed the equality guard as numeric-vs-numeric and
+        # Spark's string-number coercion silently returned True where Python's
+        # cross-type == is False. (Reported by Codex review on PR #34.)
+        import warnings as _warnings
+        from pyspark.sql.types import StructField, StructType
+
+        def nested_ternary_eq(x):
+            return ("5" if x > 0 else "6") == 5
+
+        def none_branch_ternary_eq(x):
+            return ("5" if x > 0 else None) == 5
+
+        long_schema = StructType([StructField("a", LongType(), nullable=True)])
+        df = self.spark.createDataFrame([Row(a=5), Row(a=-5)], schema=long_schema)
+        with self.sql_conf(_TRANSPILE_ON):
+            for func in [nested_ternary_eq, none_branch_ternary_eq]:
+                with self.subTest(func=func.__name__):
+                    with _warnings.catch_warnings(record=True):
+                        _warnings.simplefilter("always")
+                        pudf = UserDefinedFunction(func, BooleanType())
+                    self.assertEqual(
+                        [], pudf.transpiled, "string-ternary == int must not transpile"
+                    )
+                    results = [r[0] for r in df.select(pudf("a")).collect()]
+                    self.assertEqual(results, [False, False], "must match Python's ==")
+
+    def test_udf_transpile_str_int_compare_matches_python(self):
+        # Comparing a value against a string literal (``x == "5"`` / ``x < "5"``)
+        # under the untyped-parameter path produces two candidate options -- a
+        # numeric variant and a string variant. The numeric variant mixes
+        # categories (numeric column vs string literal): Python compares such
+        # values as unequal / raises TypeError, while a lowered ``x = '5'`` would
+        # coerce under ANSI and silently diverge -- so ``_lower_eq`` /
+        # ``_lower_value_compare`` refuse it. Only the string variant survives.
+        # When the UDF is applied to a numeric column, ResolveTranspiledPython-
+        # UDFOptions drops the string option (no category match) and the UDF
+        # falls back to interpreted Python. We verify the observable guarantee on
+        # both sides: ``==`` returns Python's result (no coerced ``True``), and
+        # ``<`` raises on the Python side directly and surfaces the same error
+        # through Spark rather than returning a coerced answer.
+        from pyspark.errors import PythonException
+        from pyspark.sql.types import StructField, StructType
+
+        def eq_str(x):
+            return x == "5"
+
+        def lt_str(x):
+            return x < "5"
+
+        long_schema = StructType([StructField("a", LongType(), nullable=True)])
+        with self.sql_conf(_TRANSPILE_ON):
+            df = self.spark.createDataFrame([Row(a=5), Row(a=1)], schema=long_schema)
+
+            # ``==`` : numeric column -> string option dropped -> interpreted
+            # Python, so the result matches ``5 == "5"`` (False), not a coerced
+            # ``True`` from ``bigint = '5'``.
+            pudf_eq = UserDefinedFunction(eq_str, BooleanType())
+            results = [r[0] for r in df.select(pudf_eq("a")).collect()]
+            self.assertEqual(results, [5 == "5", 1 == "5"], "must match Python's ==")
+
+            # ``<`` : Python raises TypeError for ``int < str``; the transpiled
+            # string option is dropped for a numeric column, so Spark runs the
+            # interpreted UDF and surfaces the same error rather than coercing.
+            # Asserting PythonException (not a bare Exception) pins this to the
+            # interpreted-fallback path: a Catalyst AnalysisException here would
+            # instead mean the string option was wrongly kept and broke the
+            # query rather than falling back.
+            pudf_lt = UserDefinedFunction(lt_str, BooleanType())
+            with self.assertRaises(TypeError):
+                lt_str(5)
+            with self.assertRaises(PythonException) as ctx:
+                df.select(pudf_lt("a")).collect()
+            self.assertIn("not supported between", str(ctx.exception))
+
+    def test_udf_transpile_falls_back_for_bool_arithmetic(self):
+        # `(x > 0) + 1` is valid Python (True + 1 == 2), but the lowered
+        # Add(boolean, int) fails ANSI analysis. The category of a
+        # boolean-producing operand is now "bool" (not the numeric catch-all),
+        # so this refuses and runs as interpreted Python.
+        import warnings as _warnings
+        from pyspark.sql.types import StructField, StructType
+
+        def bool_plus_one(x):
+            return (x > 0) + 1
+
+        long_schema = StructType([StructField("a", LongType(), nullable=True)])
+        with self.sql_conf(_TRANSPILE_ON):
+            with _warnings.catch_warnings(record=True):
+                _warnings.simplefilter("always")
+                pudf = UserDefinedFunction(bool_plus_one, LongType())
+            self.assertEqual([], pudf.transpiled, "bool arithmetic must not transpile")
+            df = self.spark.createDataFrame([Row(a=5), Row(a=-5)], schema=long_schema)
+            results = [r[0] for r in df.select(pudf("a")).collect()]
+            self.assertEqual(results, [2, 1])
+
+    def test_udf_transpile_falls_back_for_return_wrapped_bool_branch(self):
+        # If-statement branches arrive as ast.Return nodes; the branch-category
+        # guard must see through the wrapper. A boolean-returning branch vs a
+        # numeric one previously slipped past the guard and failed analysis
+        # (CASE WHEN [BOOLEAN, INT]) instead of falling back.
+        import warnings as _warnings
+        from pyspark.sql.types import StructField, StructType
+
+        def mixed(x):
+            if x > 0:
+                return x > 5
+            else:
+                return 1
+
+        long_schema = StructType([StructField("a", LongType(), nullable=True)])
+        with self.sql_conf(_TRANSPILE_ON):
+            with _warnings.catch_warnings(record=True):
+                _warnings.simplefilter("always")
+                pudf = UserDefinedFunction(mixed, LongType())
+            self.assertEqual([], pudf.transpiled, "bool-vs-int branches must not transpile")
+            df = self.spark.createDataFrame([Row(a=-3)], schema=long_schema)
+            [result] = df.select(pudf("a")).collect()
+            self.assertEqual(result[0], 1)
+
+    def test_udf_transpile_falls_back_for_plain_self_param(self):
+        # A plain function whose first parameter is literally named `self`
+        # must not be confused with a bound __call__ method: stripping it
+        # previously emitted `_udf_param_-1` and threw an AnalysisException
+        # at call construction.
+        import warnings as _warnings
+
+        def weird(self, other):
+            return self + other
+
+        with self.sql_conf(_TRANSPILE_ON):
+            with _warnings.catch_warnings(record=True):
+                _warnings.simplefilter("always")
+                pudf = UserDefinedFunction(weird, LongType())
+            self.assertEqual([], pudf.transpiled, "plain 'self' param must not transpile")
+            df = self.spark.createDataFrame([Row(a=2, b=3)])
+            [result] = df.select(pudf("a", "b")).collect()
+            self.assertEqual(result[0], 5)
+
+    def test_udf_transpile_falls_back_for_wraps_decorated_function(self):
+        # inspect.getsource follows __wrapped__, so a functools.wraps-decorated
+        # UDF previously transpiled the WRAPPED function's source while the
+        # interpreted path ran the wrapper -- a silent wrong result.
+        import functools
+        import warnings as _warnings
+        from pyspark.sql.types import StructField, StructType
+
+        def base(x):
+            return x + 1
+
+        @functools.wraps(base)
+        def wrapper(x):
+            return base(x) * 10
+
+        long_schema = StructType([StructField("a", LongType(), nullable=True)])
+        with self.sql_conf(_TRANSPILE_ON):
+            with _warnings.catch_warnings(record=True):
+                _warnings.simplefilter("always")
+                pudf = UserDefinedFunction(wrapper, LongType())
+            self.assertEqual([], pudf.transpiled, "wraps-decorated UDF must not transpile")
+            df = self.spark.createDataFrame([Row(a=5)], schema=long_schema)
+            [result] = df.select(pudf("a")).collect()
+            self.assertEqual(result[0], 60, "must run the wrapper, not the wrapped source")
+
+    def test_udf_transpile_falls_back_for_none_in_boolop(self):
+        # Python's `None and x` short-circuits to None; Spark's three-valued
+        # `null AND false` is false. A literal None operand must force a
+        # fallback rather than silently diverge.
+        import warnings as _warnings
+        from pyspark.sql.types import StructField, StructType
+
+        def none_and(x):
+            return None and (x > 0)
+
+        long_schema = StructType([StructField("a", LongType(), nullable=True)])
+        with self.sql_conf(_TRANSPILE_ON):
+            with _warnings.catch_warnings(record=True):
+                _warnings.simplefilter("always")
+                pudf = UserDefinedFunction(none_and, BooleanType())
+            self.assertEqual([], pudf.transpiled, "literal None in and/or must not transpile")
+            df = self.spark.createDataFrame([Row(a=-5)], schema=long_schema)
+            [result] = df.select(pudf("a")).collect()
+            self.assertIsNone(result[0])
+
+    def test_udf_transpile_falls_back_for_uncastable_return_type(self):
+        # The lowered expression is cast to the declared return type; a return
+        # type no atomic lowering can be cast to (arrays, maps, datetimes, ...)
+        # would make that Cast fail CheckAnalysis and break the whole query
+        # (the options are children of TranspiledPythonUDF), so such UDFs must
+        # fall back at construction instead. Interpreted execution still works
+        # (the pickled-UDF converter nulls the type-mismatched results).
+        import warnings as _warnings
+        from pyspark.sql.types import ArrayType, TimestampType
+
+        plus_one = lambda x: x + 1  # noqa: E731
+        with self.sql_conf(_TRANSPILE_ON):
+            for rt in (ArrayType(LongType()), TimestampType()):
+                with _warnings.catch_warnings(record=True):
+                    _warnings.simplefilter("always")
+                    pudf = UserDefinedFunction(plus_one, rt)
+                self.assertEqual([], pudf.transpiled, f"return type {rt} must not transpile")
+            # Interpreted execution keeps working; an int result for an array
+            # return type is nulled by the pickled-UDF converter. (Timestamp
+            # is not exercised here: its converter accepts ints as micros.)
+            with _warnings.catch_warnings(record=True):
+                _warnings.simplefilter("always")
+                array_udf = UserDefinedFunction(plus_one, ArrayType(LongType()))
+            df = self.spark.createDataFrame([Row(a=1)])
+            [result] = df.select(array_udf("a")).collect()
+            self.assertIsNone(result[0], "interpreted fallback nulls the mismatch")
+
+    def test_udf_transpile_falls_back_for_cross_category_return_cast(self):
+        # Per-variant guard: the body category must MATCH the declared return
+        # type's category. Un-castable combos (binary body -> numeric return,
+        # boolean body -> binary return) would fail analysis outright, and
+        # analysis-valid cross-category casts (string -> long, numeric ->
+        # boolean, anything -> decimal) diverge from the interpreted path,
+        # which nulls type-mismatched results instead of casting -- e.g.
+        # `def f(s: str): return s` declared LongType() would return 123 for
+        # '123' (or raise CAST_INVALID_INPUT) where interpreted returns NULL.
+        import warnings as _warnings
+        from pyspark.sql.types import DecimalType
+
+        def bytes_to_long(x: bytes):
+            return x
+
+        def bool_to_binary(x):
+            return (x > 0) if x is not None else None
+
+        def str_ident(s: str):
+            return s
+
+        def plus_one(x):
+            return x + 1
+
+        with self.sql_conf(_TRANSPILE_ON):
+            for func, rt, label in (
+                (bytes_to_long, LongType(), "binary body -> numeric return"),
+                (bool_to_binary, BinaryType(), "boolean body -> binary return"),
+                (bytes_to_long, StringType(), "binary body -> string return"),
+                (str_ident, LongType(), "string body -> numeric return"),
+                (plus_one, BooleanType(), "numeric body -> boolean return"),
+                (plus_one, DecimalType(10, 2), "numeric body -> decimal return"),
+            ):
+                with _warnings.catch_warnings(record=True):
+                    _warnings.simplefilter("always")
+                    pudf = UserDefinedFunction(func, rt)
+                self.assertEqual([], pudf.transpiled, f"{label} must not transpile")
+            # Interpreted execution of the Codex-flagged example: NULL, not a
+            # cast. (The transpiled cast would have returned 123.)
+            with _warnings.catch_warnings(record=True):
+                _warnings.simplefilter("always")
+                str_long = UserDefinedFunction(str_ident, LongType())
+            df = self.spark.createDataFrame([("123",)], "s string")
+            self.assertIsNone(df.select(str_long("s")).first()[0])
+
+    def test_udf_transpile_falls_back_for_non_numeric_unary(self):
+        # Unary +/- only lower for numeric operands: Python raises TypeError
+        # on `+s`/`-s` for strings while Spark's ANSI string promotion would
+        # silently coerce (`-'5'` -> -5.0), and `-x` on a boolean would fail
+        # analysis outright rather than fall back.
+        import warnings as _warnings
+
+        def neg_str(s: str):
+            return -s
+
+        def pos_str(s: str):
+            return +s
+
+        def neg_bool(x: bool):
+            return -x
+
+        with self.sql_conf(_TRANSPILE_ON):
+            for func in (neg_str, pos_str, neg_bool):
+                with _warnings.catch_warnings(record=True):
+                    _warnings.simplefilter("always")
+                    pudf = UserDefinedFunction(func, LongType())
+                self.assertEqual([], pudf.transpiled, f"{func.__name__} must not transpile")
+        # Numeric unary still lowers and matches Python.
+        neg = lambda x: -x  # noqa: E731
+        self.assertEqual(self._vals(neg, LongType(), "a long", [(5,), (-3,)]), [-5, 3])
+
+    def test_udf_transpile_falls_back_for_self_reference(self):
+        # A __call__ body that references bare `self` has no column
+        # equivalent; the offset scheme previously emitted `_udf_param_-1`,
+        # which the JVM builder rejected with an AnalysisException at call
+        # construction instead of falling back to interpreted Python.
+        import warnings as _warnings
+
+        class PickSelf:
+            def __call__(self, x):
+                return x if x is not None else self
+
+        with self.sql_conf(_TRANSPILE_ON):
+            with _warnings.catch_warnings(record=True):
+                _warnings.simplefilter("always")
+                pudf = UserDefinedFunction(PickSelf(), LongType())
+            self.assertEqual([], pudf.transpiled, "`self` reference must not transpile")
+            # Interpreted execution still works (previously the call itself
+            # raised). Only non-null rows are exercised: a row that RETURNS
+            # `self` would fail JVM-side unpickling of the instance, which is
+            # interpreted-UDF behavior unrelated to this guard.
+            df = self.spark.createDataFrame([(2,), (7,)], "a long")
+            results = [r[0] for r in df.select(pudf("a")).collect()]
+            self.assertEqual(results, [2, 7])
+
+    def test_udf_transpile_preserves_auto_column_name(self):
+        # The auto-generated column name must stay `f(a)` whether or not the
+        # rewrite engages; the TranspiledPythonUDF wrapper (and its option
+        # children) must not leak into user-visible schema names.
+        from pyspark.sql.types import StructField, StructType
+
+        def plus_four(x):
+            return x + 4
+
+        long_schema = StructType([StructField("a", LongType(), nullable=True)])
+        df = self.spark.createDataFrame([Row(a=1)], schema=long_schema)
+        with self.sql_conf(_TRANSPILE_ON):
+            pudf = UserDefinedFunction(plus_four, LongType())
+            self.assertTrue(pudf.transpiled)
+            self.assertEqual(df.select(pudf("a")).columns, ["plus_four(a)"])
+
+    def test_udf_transpile_arity_mismatch_falls_back(self):
+        # Calling with the wrong number of arguments is a user error that must
+        # surface as the standard Python-side TypeError, not be silently
+        # absorbed by a transpiled constant (zero-param case) nor raise a
+        # misleading "internal error" AnalysisException (too-few-args case).
+        import warnings as _warnings
+        from pyspark.errors import PythonException
+        from pyspark.sql.types import StructField, StructType
+
+        def zero():
+            return 42
+
+        def two(x, y):
+            return x + y
+
+        long_schema = StructType([StructField("a", LongType(), nullable=True)])
+        df = self.spark.createDataFrame([Row(a=5)], schema=long_schema)
+        with self.sql_conf(_TRANSPILE_ON):
+            with _warnings.catch_warnings(record=True):
+                _warnings.simplefilter("always")
+                pudf_zero = UserDefinedFunction(zero, LongType())
+                pudf_two = UserDefinedFunction(two, LongType())
+            with self.assertRaises(PythonException):
+                df.select(pudf_zero("a")).collect()
+            with self.assertRaises(PythonException):
+                df.select(pudf_two("a")).collect()
+
+    def test_udf_transpile_decimal_input_falls_back(self):
+        # Python receives decimal.Decimal objects, which raise TypeError when
+        # mixed with float literals; the transpiled numeric lowering would
+        # silently succeed. Decimal columns must fall back to interpreted
+        # Python (pruned by input category at analysis time).
+        from pyspark.errors import PythonException
+
+        def add_half(x):
+            return x + 1.5
+
+        with self.sql_conf(_TRANSPILE_ON):
+            # DoubleType: the return type must category-match the numeric body
+            # for the option to be emitted (a string return type would itself
+            # force a fallback before the decimal-input pruning under test).
+            pudf = UserDefinedFunction(add_half, DoubleType())
+            self.assertTrue(pudf.transpiled, "numeric option should still be produced")
+            df = self.spark.sql("SELECT CAST(1.0 AS DECIMAL(10,2)) AS d")
+            with self.assertRaises(PythonException):
+                df.select(pudf("d")).collect()
+
+    def test_udf_transpile_collated_string_falls_back(self):
+        # Under a non-binary collation Spark's `=` follows collation rules
+        # ('abc' = 'ABC' is true under UTF8_LCASE) while Python compares
+        # codepoints. Collated columns must fall back to interpreted Python.
+        def eq_abc(s):
+            return s == "ABC"
+
+        with self.sql_conf(_TRANSPILE_ON):
+            pudf = UserDefinedFunction(eq_abc, BooleanType())
+            self.assertTrue(pudf.transpiled, "string option should still be produced")
+            df = self.spark.sql("SELECT 'abc' COLLATE UTF8_LCASE AS s")
+            [result] = df.select(pudf("s")).collect()
+            self.assertIs(result[0], False, "must match Python, not collation semantics")
+
+    def test_udf_transpile_is_none_semantics(self):
+        # `x is None` and `None is x` (and their `is not` variants) should
+        # transpile to isNull/isNotNull. Any other identity check (`x is 0`,
+        # `x is y`, `x is True`) must NOT transpile -- Python's `is` is an
+        # object-identity test with no SQL equivalent outside of None.
+        import warnings as _warnings
+        from pyspark.sql.types import StructField, StructType
+
+        long_schema = StructType([StructField("a", LongType(), nullable=True)])
+
+        def x_is_none(x):
+            return x is None
+
+        def x_is_not_none(x):
+            if x is not None:
+                return x + 1
+
+        def none_is_x(x):
+            return None is x
+
+        def none_is_not_x(x):
+            if None is not x:
+                return x + 1
+
+        def x_is_zero(x):
+            return x is 0  # noqa: F632  identity vs equality
+
+        def x_is_true(x):
+            return x is True
+
+        def x_is_y(x, y):
+            return x is y
+
+        with self.sql_conf(
+            {
+                "spark.sql.experimental.optimizer.transpilePyUDFs": True,
+                "spark.sql.ansi.enabled": True,
+            }
+        ):
+            # `x is None` and `None is x` should transpile and produce
+            # identical results.
+            for func, label in [(x_is_none, "x_is_none"), (none_is_x, "none_is_x")]:
+                with self.subTest(case=label):
+                    pudf = UserDefinedFunction(func, BooleanType())
+                    self.assertTrue(
+                        pudf.transpiled,
+                        f"{label}: expected transpilation to succeed",
+                    )
+                    df = self.spark.createDataFrame([Row(a=None)], schema=long_schema)
+                    [row] = df.select(pudf("a")).collect()
+                    self.assertTrue(row[0], f"{label}: None is None should be True")
+                    df = self.spark.createDataFrame([Row(a=1)], schema=long_schema)
+                    [row] = df.select(pudf("a")).collect()
+                    self.assertFalse(row[0], f"{label}: 1 is None should be False")
+
+            # `x is not None` and `None is not x` should transpile.
+            for func, label in [
+                (x_is_not_none, "x_is_not_none"),
+                (none_is_not_x, "none_is_not_x"),
+            ]:
+                with self.subTest(case=label):
+                    pudf = UserDefinedFunction(func, LongType())
+                    self.assertTrue(
+                        pudf.transpiled,
+                        f"{label}: expected transpilation to succeed",
+                    )
+                    df = self.spark.createDataFrame([Row(a=2)], schema=long_schema)
+                    [row] = df.select(pudf("a")).collect()
+                    self.assertEqual(row[0], 3, f"{label}: non-None input should return x+1")
+                    df = self.spark.createDataFrame([Row(a=None)], schema=long_schema)
+                    [row] = df.select(pudf("a")).collect()
+                    self.assertIsNone(row[0], f"{label}: None input should return None")
+
+            # Non-None identity checks must NOT transpile and must still
+            # return correct results via interpreted Python.
+            bool_schema = StructType([StructField("a", BooleanType(), nullable=True)])
+            two_col_schema = StructType(
+                [
+                    StructField("a", LongType(), nullable=True),
+                    StructField("b", LongType(), nullable=True),
+                ]
+            )
+            non_none_cases = [
+                # CPython interns small ints so `0 is 0` happens to be True in CPython,
+                # but that is an implementation detail. The transpiler must still refuse
+                # to lower these to isNull/isNotNull. We just verify: (a) no transpile,
+                # (b) the interpreted result matches what Python actually produces.
+                ("x_is_zero", x_is_zero, BooleanType(), long_schema, Row(a=0), True),
+                # `True is True` is True because bool singletons are interned.
+                ("x_is_true", x_is_true, BooleanType(), bool_schema, Row(a=True), True),
+                ("x_is_y", x_is_y, BooleanType(), two_col_schema, Row(a=1, b=1), True),
+            ]
+            for label, func, return_type, schema, row, expected in non_none_cases:
+                with self.subTest(case=label):
+                    with _warnings.catch_warnings(record=True) as caught:
+                        _warnings.simplefilter("always")
+                        pudf = UserDefinedFunction(func, return_type)
+                    self.assertEqual(
+                        [],
+                        pudf.transpiled,
+                        f"{label}: non-None identity check must NOT transpile",
+                    )
+                    fallback = [
+                        w
+                        for w in caught
+                        if "Unable to transpile" in str(w.message)
+                        or "Errors encountered" in str(w.message)
+                    ]
+                    self.assertTrue(fallback, f"{label}: expected a fallback warning")
+                    df = self.spark.createDataFrame([row], schema=schema)
+                    args = ["a", "b"] if "b" in schema.fieldNames() else ["a"]
+                    [result] = df.select(pudf(*args)).collect()
+                    self.assertEqual(result[0], expected, f"{label}: interpreted result mismatch")
+
+    def test_udf_transpile_not_bare_param_falls_back(self):
+        # `not x` where x is a bare UDF parameter (unknown type at
+        # transpile time) must NOT be lowered: Spark's `~` is bitwise, not
+        # Python truthiness, so `not 0` would produce True via Python but
+        # Spark's `~0L` is -1 (truthy). The transpiler must refuse and fall
+        # back to interpreted Python.
+        import warnings as _warnings
+        from pyspark.sql.types import StructField, StructType
+
+        def not_x(x):
+            return not x
+
+        long_schema = StructType([StructField("a", LongType(), nullable=True)])
+
+        with self.sql_conf(
+            {
+                "spark.sql.experimental.optimizer.transpilePyUDFs": True,
+                "spark.sql.ansi.enabled": True,
+            }
+        ):
+            with _warnings.catch_warnings(record=True) as caught:
+                _warnings.simplefilter("always")
+                pudf = UserDefinedFunction(not_x, BooleanType())
+            self.assertEqual([], pudf.transpiled, "not x on bare param must NOT transpile")
+            fallback = [
+                w
+                for w in caught
+                if "Unable to transpile" in str(w.message) or "Errors encountered" in str(w.message)
+            ]
+            self.assertTrue(fallback, "expected a fallback warning for `not x`")
+            # Verify interpreted result is still correct.
+            for value, expected in [(0, True), (1, False), (None, True)]:
+                with self.subTest(value=value):
+                    df = self.spark.createDataFrame([Row(a=value)], schema=long_schema)
+                    [row] = df.select(pudf("a")).collect()
+                    self.assertEqual(row[0], expected)
+
+    def test_udf_transpile_and_or_bare_param_falls_back(self):
+        # `x and y` / `x or y` where x/y are bare UDF parameters (unknown
+        # type) must NOT be lowered: Python returns one of the operands
+        # (truthiness semantics) while Spark's `&`/`|` are bitwise. The
+        # transpiler must refuse and fall back.
+        import warnings as _warnings
+        from pyspark.sql.types import StructField, StructType
+
+        def x_and_y(x, y):
+            return x and y
+
+        def x_or_y(x, y):
+            return x or y
+
+        schema = StructType(
+            [
+                StructField("a", LongType(), nullable=True),
+                StructField("b", LongType(), nullable=True),
+            ]
+        )
+
+        with self.sql_conf(
+            {
+                "spark.sql.experimental.optimizer.transpilePyUDFs": True,
+                "spark.sql.ansi.enabled": True,
+            }
+        ):
+            for func, label, row, expected in [
+                (x_and_y, "x_and_y_falsy", Row(a=0, b=5), 0),
+                (x_and_y, "x_and_y_truthy", Row(a=3, b=5), 5),
+                (x_or_y, "x_or_y_falsy_left", Row(a=0, b=5), 5),
+                (x_or_y, "x_or_y_truthy_left", Row(a=3, b=0), 3),
+            ]:
+                with self.subTest(case=label):
+                    with _warnings.catch_warnings(record=True) as caught:
+                        _warnings.simplefilter("always")
+                        pudf = UserDefinedFunction(func, LongType())
+                    self.assertEqual(
+                        [],
+                        pudf.transpiled,
+                        f"{label}: and/or on bare params must NOT transpile",
+                    )
+                    fallback = [
+                        w
+                        for w in caught
+                        if "Unable to transpile" in str(w.message)
+                        or "Errors encountered" in str(w.message)
+                    ]
+                    self.assertTrue(fallback, f"{label}: expected a fallback warning")
+                    df = self.spark.createDataFrame([row], schema=schema)
+                    [result] = df.select(pudf("a", "b")).collect()
+                    self.assertEqual(result[0], expected, f"{label}: interpreted result mismatch")
+
+    def test_cannot_convert_column_into_bool_includes_column_repr(self):
+        # The error fired by ``Column.__bool__`` should name the offending
+        # column so users can see which expression triggered the fallback.
+        from pyspark.errors import PySparkValueError
+
+        df = self.spark.createDataFrame([Row(a=1, b=2)])
+        col_a = df["a"]
+        with self.assertRaises(PySparkValueError) as ctx:
+            bool(col_a)
+        message = str(ctx.exception)
+        self.assertIn("Cannot convert column into bool", message)
+        # Column's stringification is JVM-side and may render the column
+        # as ``a`` (unresolved) or with a backtick variant, so we just
+        # require the column name appears somewhere in the message.
+        self.assertIn("a", message)
+
+    # ------------------------------------------------------------------
+    # Edge cases (SPARK-55206 follow-up). Helpers build a UDF with
+    # transpilation on; `_vals` runs it and returns outputs (asserting it
+    # transpiled), `_raises` asserts it raises. Arg columns come from the
+    # schema. Operator cases are table-driven. Plan-elision checks count
+    # `EvalPython` nodes because an ordering compare's `raise_error` message
+    # contains "UDF" (so the "UDF" substring is unreliable).
+    # ------------------------------------------------------------------
+
+    def _vals(self, func, return_type, schema, rows):
+        with self.sql_conf(_TRANSPILE_ON):
+            u = UserDefinedFunction(func, return_type)
+            self.assertTrue(u.transpiled, str(func))
+            df = self.spark.createDataFrame(rows, schema)
+            return [r[0] for r in df.select(u(*df.columns)).collect()]
+
+    def _raises(self, func, schema, rows, needle="numeric"):
+        with self.sql_conf(_TRANSPILE_ON):
+            u = UserDefinedFunction(func, LongType())
+            self.assertTrue(u.transpiled, str(func))
+            df = self.spark.createDataFrame(rows, schema)
+            with self.assertRaises(Exception) as ctx:
+                df.select(u(*df.columns)).collect()
+            self.assertIn(needle, str(ctx.exception).lower(), str(func))
+
+    @staticmethod
+    def _eval_python_count(df):
+        return df._jdf.queryExecution().executedPlan().toString().count("EvalPython")
+
+    def test_udf_transpile_lowers_operators(self):
+        # Operators lower to Catalyst and match Python: modulo sign-parity,
+        # non-commutative -/* (parameter order), unary nesting, constant
+        # body, not(compare), nested boolean, string ==/<, reversed-operand and
+        # column-to-column comparisons, if/elif/else, and assigned lambdas.
+        L, B = LongType(), BooleanType()
+        modulo = lambda x, y: x % y  # noqa: E731
+        subtract = lambda a, b: a - b  # noqa: E731
+        multiply = lambda a, b: a * b  # noqa: E731
+        double_neg = lambda x: --x  # noqa: E731
+        unary_pm = lambda x: +(-x)  # noqa: E731
+        constant = lambda x: 42  # noqa: E731
+        not_pos = lambda x: (not (x > 0)) if x is not None else None  # noqa: E731
+        nested = lambda x, y, z: ((x > 0) and (y > 0)) or (z == 0)  # noqa: E731
+        str_eq = lambda x: (x == "foo") if x is not None else None  # noqa: E731
+        str_lt = lambda x: (x < "m") if x is not None else None  # noqa: E731
+        rev_lt = lambda x: (0 < x) if x is not None else None  # noqa: E731
+        rev_eq = lambda x: 5 == x  # noqa: E731
+        none_eq = lambda x: None == x  # noqa: E711,E731
+        col_lt = lambda a, b: (a < b) if a is not None and b is not None else None  # noqa: E731
+        assigned = lambda v: v + 1  # noqa: E731
+
+        def if_elif_else(x):
+            if x is None:
+                return -1
+            elif x == 0:
+                return 0
+            else:
+                return 1
+
+        # (func, return_type, schema, rows, expected); arg columns come from the schema.
+        cases = [
+            (modulo, L, "a long, b long", [(7, 3), (7, -3), (-7, 3), (-7, -3)], [1, -2, 2, -1]),
+            (subtract, L, "a long, b long", [(5, 3), (3, 5)], [2, -2]),
+            (multiply, L, "a long, b long", [(4, 3), (-2, 5)], [12, -10]),
+            (double_neg, L, "a long", [(5,), (-3,)], [5, -3]),
+            (unary_pm, L, "a long", [(5,), (-3,)], [-5, 3]),
+            (constant, L, "a long", [(1,), (999,)], [42, 42]),
+            (not_pos, B, "a long", [(1,), (0,), (-1,), (None,)], [False, True, True, None]),
+            (str_eq, B, "a string", [("foo",), ("bar",), (None,)], [True, False, None]),
+            (str_lt, B, "a string", [("a",), ("z",), (None,)], [True, False, None]),
+            (rev_lt, B, "a long", [(1,), (0,), (-1,)], [True, False, False]),
+            (rev_eq, B, "a long", [(5,), (3,), (None,)], [True, False, False]),
+            (none_eq, B, "a long", [(None,), (5,)], [True, False]),
+            (col_lt, B, "a long, b long", [(1, 2), (2, 1), (1, 1)], [True, False, False]),
+            (if_elif_else, L, "a long", [(None,), (0,), (5,), (-3,)], [-1, 0, 1, 1]),
+            (assigned, L, "a long", [(1,), (10,)], [2, 11]),
+            (
+                nested,
+                B,
+                "a long, b long, c long",
+                [(1, 1, 5), (-1, 1, 0), (-1, 1, 5)],
+                [True, True, False],
+            ),
+        ]
+        for i, (func, rt, schema, rows, expected) in enumerate(cases):
+            with self.subTest(case=i):
+                self.assertEqual(self._vals(func, rt, schema, rows), expected, f"case {i}: {rows}")
+
+    def test_udf_transpile_callable_object_self_offset(self):
+        # A callable instance carries `self`; the extractor offsets it so a/b
+        # map to _udf_param_0/_udf_param_1 (non-commutative body proves order).
+        class SubAB:
+            def __call__(self, a, b):
+                return a - b
+
+        self.assertEqual(
+            self._vals(SubAB(), LongType(), "a long, b long", [(5, 3), (3, 5)]), [2, -2]
+        )
+
+    def test_udf_transpile_plan_elision(self):
+        # Transpiled UDFs are elided in filter (not just select); a mixed
+        # non-convertible -> convertible -> non-convertible chain inlines only
+        # the middle UDF, leaving exactly two Python eval nodes.
+        offset = 3
+        gt5 = lambda x: (x > 5) if x is not None else None  # noqa: E731
+        add_offset = lambda x: x + offset  # noqa: E731  closure -> fallback
+        plus_one = lambda x: x + 1  # noqa: E731  convertible
+        div_two = lambda x: x / 2  # noqa: E731  `/` -> fallback
+        with self.sql_conf(_TRANSPILE_ON):
+            f = UserDefinedFunction(gt5, BooleanType())
+            self.assertTrue(f.transpiled)
+            fdf = self.spark.createDataFrame([(3,), (7,), (1,), (None,)], "a long").filter(f("a"))
+            self.assertEqual([r[0] for r in fdf.collect()], [7])
+            self.assertEqual(0, self._eval_python_count(fdf))
+
+            u1 = UserDefinedFunction(add_offset, LongType())
+            u2 = UserDefinedFunction(plus_one, LongType())
+            u3 = UserDefinedFunction(div_two, DoubleType())
+            self.assertEqual(([], True, []), (u1.transpiled, bool(u2.transpiled), u3.transpiled))
+            chained = (
+                self.spark.createDataFrame([(10,)], "a long")
+                .select(u1("a").alias("x"))
+                .select(u2("x").alias("y"))
+                .select(u3("y").alias("z"))
+            )
+            self.assertEqual(chained.first()[0], 7.0)  # ((10 + 3) + 1) / 2
+            self.assertEqual(2, self._eval_python_count(chained))
+
+    def test_udf_transpile_config_toggle_no_stale_nodes(self):
+        # Built with the flags on, executed with them off -> clean fallback to
+        # interpreted Python (the optimizer drops the transpiled node), no error.
+        plus_one = lambda x: x + 1  # noqa: E731
+        with self.sql_conf(_TRANSPILE_ON):
+            u = UserDefinedFunction(plus_one, LongType())
+            self.assertTrue(u.transpiled)
+        with self.sql_conf(
+            {
+                "spark.sql.experimental.optimizer.transpilePyUDFs": False,
+                "spark.sql.ansi.enabled": False,
+            }
+        ):
+            df = self.spark.createDataFrame([(1,), (5,)], "a long")
+            self.assertEqual([r[0] for r in df.select(u("a")).collect()], [2, 6])
+
+    def test_udf_transpile_casts_to_return_type(self):
+        # The lowered expression is cast to the declared return type.
+        plus_one = lambda x: x + 1  # noqa: E731
+        with self.sql_conf(_TRANSPILE_ON):
+            d = UserDefinedFunction(plus_one, DoubleType())
+            col = self.spark.createDataFrame([(1,)], "a long").select(d("a").alias("r"))
+            self.assertEqual(col.schema["r"].dataType, DoubleType())
+            self.assertEqual(col.first()[0], 2.0)
+        self.assertEqual(self._vals(plus_one, LongType(), "a long", [(1,)]), [2])
+
+    def test_udf_transpile_falls_back(self):
+        # Shapes that must NOT transpile (and still compute via Python):
+        # inline/wrapped/partial lambdas, default/variadic/keyword-only args, and
+        # `%` string formatting. (String `+`/`*` now lower to concat/repeat -- see
+        # test_udf_transpile_string_operands -- but `%` as a format is not handled.)
+        import functools
+
+        def wrapper(fn):
+            return fn
+
+        def with_default(a, b=0):
+            return a + 10 * b
+
+        def with_varargs(a, *rest):
+            return a
+
+        def with_kwargs(a, **opts):
+            return a
+
+        base = lambda v, w: v + w  # noqa: E731
+        percent_fmt = lambda x: "n=%d" % x  # noqa: E731
+        with self.sql_conf(_TRANSPILE_ON):
+            # inline / wrapped / partial lambdas -> source can't be extracted
+            self.assertEqual([], UserDefinedFunction(lambda v: v + 1, LongType()).transpiled)
+            self.assertEqual(
+                [], UserDefinedFunction(wrapper(lambda v: v + 1), LongType()).transpiled
+            )
+            self.assertEqual(
+                [], UserDefinedFunction(functools.partial(base, 1), LongType()).transpiled
+            )
+            # default / variadic / keyword-only args, and `%` string formatting
+            for func, rt in [
+                (with_default, LongType()),
+                (with_varargs, LongType()),
+                (with_kwargs, LongType()),
+                (percent_fmt, StringType()),
+            ]:
+                with self.subTest(func=func):
+                    self.assertEqual([], UserDefinedFunction(func, rt).transpiled)
+            # Fell back -> interpreted Python still computes correctly.
+            wd = UserDefinedFunction(with_default, LongType())
+            num = self.spark.createDataFrame([(5,)], "a long")
+            self.assertEqual(
+                [num.select(wd("a")).first()[0], num.select(wd("a", "a")).first()[0]], [5, 55]
+            )
+
+    def test_udf_transpile_known_value_divergences(self):
+        # Transpile but DIVERGE from Python (documented in transpile.py; pinned so
+        # a future fix is noticed): unguarded arithmetic on NULL yields NULL
+        # (Python raises TypeError), and NaN > 0 is True (Python False; Spark
+        # orders NaN highest). Mixed str/numeric arithmetic is handled or falls
+        # back -- see test_udf_transpile_string_operands{,_fall_back}.
+        unguarded = lambda x: x + 1  # noqa: E731
+        nan_gt = lambda x: (x > 0) if x is not None else None  # noqa: E731
+        eq_strlit = lambda x: (x == "5") if x is not None else None  # noqa: E731
+        self.assertEqual(self._vals(unguarded, LongType(), "a long", [(None,), (5,)]), [None, 6])
+        self.assertEqual(
+            self._vals(nan_gt, BooleanType(), "a double", [(float("nan"),), (1.0,)]), [True, True]
+        )
+        # `x == "5"` used to be pinned as a coercion divergence (int == "5" ->
+        # True). The eq category gate now drops the numeric variant, so on a
+        # long column the string option is pruned and the UDF falls back to
+        # interpreted Python -- matching Python's cross-type == (always False).
+        self.assertEqual(
+            self._vals(eq_strlit, BooleanType(), "a long", [(5,), (3,)]), [False, False]
+        )
+
+    def test_udf_transpile_overflow_and_modulo_zero_raise(self):
+        # Transpiled arithmetic that raises at runtime: `*` overflow raises under
+        # ANSI where Python promotes to a big int (a real divergence, SPARK-55210),
+        # while `% 0` raises in both Spark and Python (compatible -- pinned here so
+        # it isn't mistaken for a divergence).
+        overflow = lambda x: x * x  # noqa: E731
+        modulo_zero = lambda x: x % 0  # noqa: E731
+        self._raises(overflow, "a long", [(4000000000,)], "overflow")
+        self._raises(modulo_zero, "a long", [(5,)], "zero")
+
+    def test_udf_transpile_string_operands(self):
+        # Textual `+`/`*` lower to Catalyst string ops and match Python: `str +
+        # str` -> concat, and `str * int` / `int * str` -> repeat (including a
+        # string column times a numeric literal). The transpiler emits a string-
+        # typed variant whose declared categories the JVM matches against the bound
+        # column types (see UserDefinedPythonFunction.builder).
+        S = StringType()
+        add = lambda a, b: a + b  # noqa: E731
+        mul = lambda a, b: a * b  # noqa: E731
+        mul3 = lambda a: a * 3  # noqa: E731
+        concat_right = lambda a: a + "!"  # noqa: E731
+        concat_left = lambda a: "pre-" + a  # noqa: E731
+        repeat_lit = lambda x: "ab" * x  # noqa: E731
+        # (func, return_type, schema, rows, expected); arg columns come from schema.
+        cases = [
+            (add, S, "a string, b string", [("x", "y"), ("a", "b")], ["xy", "ab"]),
+            (mul, S, "a string, b long", [("ab", 3)], ["ababab"]),
+            (mul, S, "a long, b string", [(3, "ab")], ["ababab"]),
+            (mul3, S, "a string", [("2",), ("ab",)], ["222", "ababab"]),
+            (concat_right, S, "a string", [("hi",)], ["hi!"]),
+            (concat_left, S, "a string", [("x",)], ["pre-x"]),
+            (repeat_lit, S, "a long", [(3,)], ["ababab"]),
+        ]
+        for i, (func, rt, schema, rows, expected) in enumerate(cases):
+            with self.subTest(case=i):
+                self.assertEqual(self._vals(func, rt, schema, rows), expected, f"case {i}")
+
+    def test_udf_transpile_string_operands_fall_back(self):
+        # Operand/type combos with no valid string lowering for the bound column
+        # types fall back to the Python UDF, which raises the same way CPython does:
+        # `str + int` (and reversed), `str - int`, `str * str`, `str % int`, and a
+        # string column plus a numeric literal. The transpiler still emits numeric
+        # (and/or concat/repeat) variants, but none match the column types, so the
+        # JVM drops them and runs Python -- matching its TypeError.
+        add = lambda a, b: a + b  # noqa: E731
+        sub = lambda a, b: a - b  # noqa: E731
+        mul = lambda a, b: a * b  # noqa: E731
+        mod = lambda a, b: a % b  # noqa: E731
+        add5 = lambda a: a + 5  # noqa: E731
+        # needle="" -> assert only that it raises (the message is CPython's).
+        for func, schema, rows in [
+            (add, "a string, b long", [("10", 5)]),  # str + int
+            (add, "a long, b string", [(5, "10")]),  # int + str
+            (sub, "a string, b long", [("10", 5)]),  # str - int
+            (mul, "a string, b string", [("a", "b")]),  # str * str
+            (mod, "a string, b long", [("10", 3)]),  # str % int
+            (add5, "a string", [("10",)]),  # str column + numeric literal
+        ]:
+            with self.subTest(func=func, schema=schema):
+                self._raises(func, schema, rows, needle="")
+
+    def test_udf_transpile_power_falls_back(self):
+        # `**` is intentionally not lowered (Spark's pow is DOUBLE and loses
+        # precision for large ints), so a UDF using it falls back to interpreted
+        # Python. TODO(SPARK-55210): revisit once an exact integer-power lowering
+        # exists.
+        square = lambda x: x**2  # noqa: E731
+        with self.sql_conf(_TRANSPILE_ON):
+            self.assertFalse(UserDefinedFunction(square, LongType()).transpiled)
+
+    def test_udf_transpile_non_numeric_constant_falls_back(self):
+        # bool/None constants have no faithful numeric/string lowering, so
+        # arithmetic against them must fall back rather than emit an option that
+        # crashes analysis (`x * True`) or silently returns NULL (`x + None`).
+        mul_bool = lambda x: x * True  # noqa: E731
+        add_none = lambda x: x + None  # noqa: E731
+        with self.sql_conf(_TRANSPILE_ON):
+            self.assertFalse(UserDefinedFunction(mul_bool, LongType()).transpiled)
+            self.assertFalse(UserDefinedFunction(add_none, LongType()).transpiled)
+
+    def test_udf_transpile_mixed_type_comparison_falls_back(self):
+        # Python forbids ordering across types (`a < b` for int/str -> TypeError);
+        # Spark would coerce and return a wrong boolean. A comparison whose
+        # operand categories differ is dropped (so int-vs-str `<` falls back),
+        # while a same-category comparison still transpiles.
+        def lt_mixed(a: int, b: str):
+            return (a < b) if a is not None and b is not None else None
+
+        def lt_same(a: int, b: int):
+            return (a < b) if a is not None and b is not None else None
+
+        with self.sql_conf(_TRANSPILE_ON):
+            self.assertFalse(UserDefinedFunction(lt_mixed, BooleanType()).transpiled)
+            self.assertTrue(UserDefinedFunction(lt_same, BooleanType()).transpiled)
+
+    def test_udf_transpile_skips_nondeterministic(self):
+        # A nondeterministic UDF must not be transpiled: the optimizer could
+        # fold/reorder/duplicate the plain expression, dropping the barrier.
+        # Holds whether marked at construction or via asNondeterministic().
+        plus_one = lambda x: x + 1  # noqa: E731
+        with self.sql_conf(_TRANSPILE_ON):
+            self.assertTrue(UserDefinedFunction(plus_one, LongType()).transpiled)
+            self.assertFalse(
+                UserDefinedFunction(plus_one, LongType()).asNondeterministic().transpiled
+            )
+            self.assertFalse(
+                UserDefinedFunction(plus_one, LongType(), deterministic=False).transpiled
+            )
+
+    def test_udf_transpile_bool_and_binary_params(self):
+        # bool/bytes annotations map to the "bool"/"binary" categories and match
+        # Boolean/Binary columns. Identity and same-category comparison transpile
+        # (and match Python); boolean arithmetic has no lowering and falls back.
+        def bool_ident(x: bool):
+            return x
+
+        def bool_lt(a: bool, b: bool):
+            return (a < b) if a is not None and b is not None else None
+
+        def bool_add(x: bool):
+            return x + 1  # no boolean arithmetic lowering -> fall back
+
+        def bytes_ident(x: bytes):
+            return x
+
+        self.assertEqual(
+            self._vals(bool_ident, BooleanType(), "a boolean", [(True,), (False,), (None,)]),
+            [True, False, None],
+        )
+        self.assertEqual(
+            self._vals(
+                bool_lt,
+                BooleanType(),
+                "a boolean, b boolean",
+                [(False, True), (True, False), (True, True)],
+            ),
+            [True, False, False],
+        )
+        with self.sql_conf(_TRANSPILE_ON):
+            self.assertFalse(UserDefinedFunction(bool_add, LongType()).transpiled)
+            self.assertTrue(UserDefinedFunction(bytes_ident, BinaryType()).transpiled)
+
+    def test_param_category_combos_caps_preserve_typed_pins(self):
+        # With more than three untyped params the cap collapses the untyped ones
+        # to numeric/string but keeps each typed param pinned (here a: str).
+        import ast as _ast
+
+        from pyspark.sql.transpile import _param_category_combos
+
+        fn = _ast.parse("def f(a: str, b, c, d, e): return a").body[0]
+        combos = _param_category_combos(fn, ["a", "b", "c", "d", "e"])
+        self.assertEqual(len(combos), 2)
+        for combo in combos:
+            self.assertEqual(combo[0], "string")
+
+
+if __name__ == "__main__":
+    from pyspark.testing import main
+
+    main()
diff --git a/python/pyspark/sql/transpile.py b/python/pyspark/sql/transpile.py
new file mode 100644
index 000000000000..ef4ca9a90894
--- /dev/null
+++ b/python/pyspark/sql/transpile.py
@@ -0,0 +1,1098 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""
+Experimental tools for transpiling UDFS.
+
+Transpilation is only attempted when both
+``spark.sql.experimental.optimizer.transpilePyUDFs=true`` and
+``spark.sql.ansi.enabled=true``. The generated Catalyst expressions
+target ANSI-mode SQL semantics (overflow raises, divide-by-zero raises,
+etc.); running them under non-ANSI mode would silently diverge from the
+Python interpretation in ways we don't currently track. If you flip
+transpilation on with ANSI off the UDF will fall back to interpreted
+Python execution and a warning is logged at UDF construction time.
+
+Python's ``+`` and ``*`` are overloaded for text (concat / repeat), so an
+untyped parameter is transpiled into one option per input-type category
+(numeric and string) and the JVM picks the one matching the bound column
+types -- falling back to interpreted Python when none fit. Annotating the
+UDF's parameters (e.g. ``def f(a: int, b: str)``) pins each category and
+keeps the option matrix small; prefer doing so. To bound plan growth,
+functions with more than three untyped parameters only emit the
+all-numeric and all-string variants.
+"""
+
+import ast
+from typing import Any, Callable, List, Optional, Tuple, TYPE_CHECKING
+import inspect
+import itertools
+import textwrap
+from pyspark.errors import UnsupportedOperationException
+from pyspark.sql.column import Column
+from pyspark.sql.types import (
+    BinaryType,
+    BooleanType,
+    DataType,
+    DecimalType,
+    NumericType,
+    StringType,
+)
+from pyspark.sql.functions import (
+    abs as _abs,
+    coalesce,
+    col,
+    concat,
+    lit,
+    pmod,
+    raise_error,
+    repeat,
+    when,
+)
+
+
+if TYPE_CHECKING:
+    from pyspark.sql import SparkSession
+    from pyspark.sql._typing import DataTypeOrString
+
+
+class AbstractTranspiler(object):
+    """Base class for transpilers. All experimental."""
+
+    varieties: dict[str, type["AbstractTranspiler"]] = {}
+    # Specify the "friendly" name a user can add to spark.sql.experimental.optimizer.pyTranspilers
+    # to enable this transpiler.
+    variety: str = ""
+
+    @classmethod
+    def register(cls) -> None:
+        AbstractTranspiler.varieties[cls.variety] = cls
+
+    def _transpile_from_ast(
+        self,
+        src: Optional[str],
+        ast_info: ast.AST,
+        function_ast: ast.FunctionDef,
+        params: List[str],
+        returnType: "DataTypeOrString",
+        param_categories: Optional[dict] = None,
+    ) -> Optional[Column]:
+        pass
+
+
+def _is_definitely_basic_type(node: ast.AST) -> bool:
+    """
+    Return True when ``node`` is statically guaranteed to produce a Python
+    basic/builtin type (int, float, str, bool, None, lists, etc.).
+    All ast.Name's are treated as basic types for now this will need to be updated
+    if/when we add free variables / closures to transpilation.
+    """
+    match node:
+        case ast.Constant():
+            return True
+        case ast.BinOp(left=left, right=right):
+            return _is_definitely_basic_type(left) and _is_definitely_basic_type(right)
+        case ast.UnaryOp(operand=operand):
+            return _is_definitely_basic_type(operand)
+        case ast.Name():
+            return True
+        case _:
+            return False
+
+
+def _is_definitely_boolean(node: ast.AST) -> bool:
+    """Return True when ``node`` is statically guaranteed to produce a Python
+    ``bool`` (or ``None``, which round-trips through ``coalesce``).
+
+    Used to gate ``if``/ternary lowering: we only allow the test expression
+    into Catalyst's ``when(coalesce(test, false), ...)`` form when it provably
+    produces a boolean. Everything else (bare Name, arithmetic, function calls,
+    subscript, ...) must force a fallback to interpreted Python instead of
+    silently diverging.
+    """
+    match node:
+        case ast.Constant(value=v):
+            return v is None or isinstance(v, bool)
+        case ast.Compare(left=left, comparators=comparators):
+            # All comparison operators of simple types bool
+            return all(_is_definitely_basic_type(v) for v in comparators + [left])
+        case ast.BoolOp(values=values):
+            return all(_is_definitely_boolean(v) for v in values)
+        case ast.UnaryOp(op=ast.Not()):
+            # `not x` always produces bool.
+            return True
+        case ast.IfExp(body=body, orelse=orelse):
+            # Ternary is boolean only if both branches are.
+            return _is_definitely_boolean(body) and _is_definitely_boolean(orelse)
+        case _:
+            return False
+
+
+class CatalystTranspiler(AbstractTranspiler):
+    """Transpiler that attempts to convert a Python UDF into native Spark SQL expressions."""
+
+    variety = "catalyst"
+
+    # TODO (SPARK-55218): handle implicit-None return bodies like
+    # ``def f(x): x + x`` -- no return statement means return None;
+    # we should lower to lit(None) and optionally warn since it's
+    # likely a mistake.
+    def _convert_branch(self, params: List[str], statements: List[ast.stmt], slot: str) -> Column:
+        """Lower a single-statement if-body / if-else block.
+
+        ``slot`` is just used to disambiguate the multi-statement error
+        message between the body and the else arm.
+        """
+        if len(statements) > 1:
+            raise UnsupportedOperationException(
+                f"if statements with more than one expression in the {slot} "
+                "are not currently supported by the transpiler"
+            )
+        if len(statements) == 0:
+            return lit(None)
+        return self._convert_chunk(params, statements[0])
+
+    def _safe_category(self, params: List[str], node: Optional[ast.AST]) -> Optional[str]:
+        """Best-effort input-type category for an if/else branch, or ``None`` when
+        it can't be pinned down statically.
+
+        Used only to compare the two branches of an if/ternary. A ``None`` result
+        means "treat as compatible" (don't force a fallback): the node is absent,
+        is a bare ``None`` literal (which unifies with any branch type via
+        ``coalesce``/``Cast``), or its category can't be determined.
+        """
+        if node is None:
+            return None
+        # If-statement branches arrive as ``Return`` statements; classify the
+        # returned value, not the statement wrapper (``_is_definitely_boolean``
+        # has no ``Return`` case, so without this a boolean-returning branch
+        # would fall through to ``_category``'s numeric catch-all).
+        if isinstance(node, ast.Return):
+            return self._safe_category(params, node.value)
+        # An if-statement's category is its branches' common category (the
+        # ``_category`` catch-all would mislabel every ``ast.If`` "numeric").
+        # Mismatched branches return None ("can't be pinned down"); the
+        # branch-compatibility check in ``_convert_if_like`` raises for them.
+        if isinstance(node, ast.If):
+            body_c = self._safe_category(params, node.body[0]) if node.body else None
+            else_c = self._safe_category(params, node.orelse[0]) if node.orelse else None
+            if body_c is not None and else_c is not None and body_c != else_c:
+                return None
+            return body_c if body_c is not None else else_c
+        if isinstance(node, ast.Constant) and node.value is None:
+            return None
+        # Comparisons / ``not`` / boolean ops produce a boolean column; classify
+        # them as "bool" (``_category``'s catch-all would mislabel them numeric).
+        if _is_definitely_boolean(node):
+            return "bool"
+        try:
+            return self._category(params, node)
+        except UnsupportedOperationException:
+            return None
+
+    def _convert_if_like(
+        self,
+        params: List[str],
+        test_col: Column,
+        body_col: Column,
+        else_col: Column,
+        test_node: ast.AST,
+        body_node: Optional[ast.AST],
+        else_node: Optional[ast.AST],
+    ) -> Column:
+        # We cannot soundly lower a generic Python truthiness test here.
+        # Python truthiness depends on the runtime input type and value:
+        # for example, 0, 0.0, "", empty collections, and None are all
+        # falsy, while most other values are truthy. The transpiler does
+        # not have enough input type information at this point to decide
+        # whether ``test_col`` is a boolean expression or a bare value
+        # whose truthiness would need Python-specific handling. Emitting
+        # ``when(coalesce(test_col, false), ...)`` is therefore unsound:
+        # it can either fail Spark analysis for non-boolean columns or
+        # silently diverge from Python semantics. Fail closed so the UDF
+        # falls back to interpreted Python execution instead.
+        if not _is_definitely_boolean(test_node):
+            raise UnsupportedOperationException(
+                f"bare truthiness tests ({ast.dump(test_node)}) in if-expressions are "
+                " not currently supported by the transpiler"
+            )
+        # When the two branches resolve to concrete but different categories
+        # (e.g. numeric vs string), the lowered ``when(...).otherwise(...)`` is a
+        # CASE WHEN whose branch values share no common type under ANSI. That node
+        # is carried as a child of the TranspiledPythonUDF and is type-checked by
+        # CheckAnalysis *before* ConvertToCatalyst can drop it, so it would fail
+        # the whole query rather than fall back. Refuse here so the UDF runs as
+        # interpreted Python instead. Branches whose category we can't pin down
+        # (e.g. a bare ``None``) are treated as compatible and don't force this.
+        body_cat = self._safe_category(params, body_node)
+        else_cat = self._safe_category(params, else_node)
+        if body_cat is not None and else_cat is not None and body_cat != else_cat:
+            raise UnsupportedOperationException(
+                f"if/else branches have incompatible categories ({body_cat} vs "
+                f"{else_cat}); the lowered CASE WHEN has no common type under ANSI, "
+                "so the transpiler falls back to interpreted Python"
+            )
+        safe_test = coalesce(test_col, lit(False))
+        return when(safe_test, body_col).otherwise(else_col)
+
+    def _lower_eq(
+        self,
+        params: List[str],
+        left_node: ast.AST,
+        right_node: ast.AST,
+        equal: bool,
+    ) -> Column:
+        """Lower ``==`` / ``!=`` with Python's None-equality semantics.
+
+        Unlike ordering operators, Python doesn't raise on ``None == x`` /
+        ``None != x``: ``None == None`` is True, ``None == 0`` is False,
+        and ``!=`` is the negation. Spark's ``==`` returns NULL on NULL
+        operands (three-valued logic), which would round-trip through
+        the UDF as ``None`` rather than the bool Python would have
+        produced. Hand-roll the four cases via ``when`` branches.
+
+        When the two operands resolve to concrete but DIFFERENT categories
+        (e.g. ``x == True`` on a numeric column, or ``x == "5"`` under the
+        numeric variant), the lowered ``=`` either fails analysis under ANSI
+        (bool vs bigint) -- which would break a working UDF since the option
+        is type-checked before ConvertToCatalyst can drop it -- or coerces
+        where Python's ``==`` is simply False. Refuse those so the UDF falls
+        back to interpreted Python. A ``None`` literal operand stays allowed
+        (the four-branch NULL handling above reproduces Python exactly).
+
+        One value-level difference remains (needs runtime values, so it is
+        documented, not guarded): Spark treats ``NaN = NaN`` as true, while
+        Python's ``nan == nan`` is False.
+        """
+        lc = self._safe_category(params, left_node)
+        rc = self._safe_category(params, right_node)
+        if lc is not None and rc is not None and lc != rc:
+            raise UnsupportedOperationException(
+                f"`==`/`!=` operands have incompatible categories ({lc} vs {rc}); "
+                "Python compares across types as unequal while Spark would coerce "
+                "or fail analysis, so the transpiler falls back to interpreted Python"
+            )
+        left_col = self._convert_chunk(params, left_node)
+        right_col = self._convert_chunk(params, right_node)
+        left_null = left_col.isNull()
+        right_null = right_col.isNull()
+        if equal:
+            both_null_val: Column = lit(True)
+            one_null_val: Column = lit(False)
+            value_cmp = left_col == right_col
+        else:
+            both_null_val = lit(False)
+            one_null_val = lit(True)
+            value_cmp = left_col != right_col
+        return (
+            when(left_null & right_null, both_null_val)
+            .when(left_null | right_null, one_null_val)
+            .otherwise(value_cmp)
+        )
+
+    def _lower_value_compare(
+        self,
+        params: List[str],
+        left_node: ast.AST,
+        right_node: ast.AST,
+        op: Callable[[Column, Column], Column],
+        op_repr: str,
+    ) -> Column:
+        """Lower a value comparison (``<``, ``<=``, ``>``, ``>=``).
+
+        Python raises ``TypeError`` when an operand of these operators is
+        ``None`` (e.g. ``None > 0``), whereas Spark's three-valued logic
+        returns ``NULL``. To stay faithful to the source UDF we guard the
+        comparison: if either operand is ``NULL`` we raise via
+        ``raise_error``, otherwise we evaluate ``left op right`` as usual.
+        Callers that have already proven the operand non-null (``if x is
+        not None: x > 0``) take the otherwise branch, so they never trip
+        the raise.
+
+        Python also forbids ordering across types (``1 < "a"`` -> TypeError),
+        whereas Spark would coerce the operands and return a (wrong) boolean.
+        We therefore only lower when both operands share a category; a
+        mismatch raises so this variant is dropped and the UDF falls back to
+        interpreted Python rather than silently diverging.
+
+        One value-level difference from Python remains (it needs runtime
+        value info, so it is documented, not guarded): Spark orders ``NaN``
+        as greater than every value, whereas Python's ``NaN`` comparisons
+        are all ``False``.
+        """
+        lc = self._category(params, left_node)
+        rc = self._category(params, right_node)
+        if lc != rc:
+            raise UnsupportedOperationException(
+                f"`{op_repr}` compares operands of different categories "
+                f"({lc} vs {rc}); Python would raise TypeError, so the "
+                "transpiler falls back to interpreted Python"
+            )
+        left_col = self._convert_chunk(params, left_node)
+        right_col = self._convert_chunk(params, right_node)
+        null_guard = left_col.isNull() | right_col.isNull()
+        err = lit(
+            "Python UDF transpiler: cannot compare NULL with operator "
+            f"`{op_repr}`; Python would raise TypeError here. Add an "
+            "`is not None` guard or filter NULLs upstream."
+        )
+        return when(null_guard, raise_error(err)).otherwise(op(left_col, right_col))
+
+    def _category(self, params: List[str], node: ast.AST) -> str:
+        """Infer ``"numeric"`` or ``"string"`` for ``node`` under the current
+        ``self._param_categories`` assumption (set per input-type variant).
+
+        Drives operator selection (``+`` -> add vs concat, ``*`` -> multiply vs
+        repeat) and raises ``UnsupportedOperationException`` when an operator's
+        operands are type-incompatible, so the caller drops that variant and the
+        JVM picks another option / falls back to the Python UDF.
+        """
+        match node:
+            case ast.Constant(value=v):
+                # bool subclasses int, so classify it first: int/float -> numeric,
+                # str -> string, bool -> bool, bytes -> binary. None/complex/
+                # Ellipsis have no usable Spark column type, so raise to drop this
+                # variant and fall back rather than emit an option that fails
+                # CheckAnalysis or silently diverges (e.g. `x + None` -> NULL where
+                # Python raises TypeError).
+                if isinstance(v, bool):
+                    return "bool"
+                if isinstance(v, bytes):
+                    return "binary"
+                if isinstance(v, (int, float)):
+                    return "numeric"
+                if isinstance(v, str):
+                    return "string"
+                raise UnsupportedOperationException(
+                    f"constant {v!r} ({type(v).__name__}) has no usable column "
+                    "category; falling back to interpreted Python"
+                )
+            case ast.Name(id=name) if name in params:
+                index = params.index(name)
+                if params and params[0] == "self":
+                    index -= 1
+                return self._param_categories.get(index, "numeric")
+            case ast.BinOp(left=left, op=op, right=right):
+                lc = self._category(params, left)
+                rc = self._category(params, right)
+                if isinstance(op, ast.Add) and lc == rc:
+                    return lc  # str + str -> str, num + num -> num
+                if isinstance(op, ast.Mult):
+                    if {lc, rc} == {"numeric", "numeric"}:
+                        return "numeric"
+                    if {lc, rc} == {"numeric", "string"}:
+                        return "string"  # str * int / int * str -> repeat
+                if isinstance(op, (ast.Sub, ast.Mod)) and lc == rc == "numeric":
+                    return "numeric"
+                raise UnsupportedOperationException(
+                    f"operands of `{type(op).__name__}` are not type-compatible "
+                    "for this input-type variant"
+                )
+            case ast.Return(value=value) if value is not None:
+                return self._category(params, value)
+            case ast.IfExp(body=if_body, orelse=if_orelse):
+                # A ternary's category is its branches' common category. Without
+                # this arm the catch-all labeled every IfExp "numeric", so e.g.
+                # `("5" if c else "6") == 5` passed the equality guard as
+                # numeric-vs-numeric and Spark's string-number coercion silently
+                # diverged from Python's cross-type `==` (always False). A
+                # None-literal branch adopts the other branch's category (NULL
+                # unifies with any type in the lowered CASE WHEN); mismatched or
+                # all-None branches raise so the variant is dropped.
+                def branch_category(b: ast.AST) -> Optional[str]:
+                    if isinstance(b, ast.Constant) and b.value is None:
+                        return None
+                    return self._category(params, b)
+
+                body_cat = branch_category(if_body)
+                else_cat = branch_category(if_orelse)
+                if body_cat is not None and else_cat is not None and body_cat != else_cat:
+                    raise UnsupportedOperationException(
+                        f"ternary branches have mismatched categories ({body_cat} "
+                        f"vs {else_cat}) and cannot drive operator selection"
+                    )
+                result_cat = body_cat if body_cat is not None else else_cat
+                if result_cat is None:
+                    raise UnsupportedOperationException(
+                        "ternary with all-None branches has no usable column category"
+                    )
+                return result_cat
+            case _ if _is_definitely_boolean(node):
+                # Comparisons, `not`, and boolean ops produce a boolean column.
+                # Labeling them "numeric" (the old catch-all) let booleans into
+                # arithmetic/equality lowerings where ANSI analysis fails (e.g.
+                # `(x > 0) + 1`, valid Python) instead of falling back.
+                return "bool"
+            case _:
+                # Remaining nodes (unsupported calls, subscripts, ...) don't
+                # drive concat/repeat selection and are rejected later by
+                # `_convert_chunk`; treat as numeric for category purposes.
+                return "numeric"
+
+    def _convert_chunk(self, params: List[str], body: ast.AST | None) -> Column:
+        match body:
+            case None:
+                # Special case literal None, the implicit return None
+                return lit(None)
+            case ast.UnaryOp(op=ast.Not(), operand=operand):
+                # Python's `not None` is `True` (None is falsy), but Spark's
+                # `~NULL` is `NULL`. Coalesce against `lit(True)` so a NULL
+                # operand mirrors Python's "None is falsy" rule. We only
+                # accept operands that are statically known to be boolean;
+                # for non-boolean operands (e.g. `not 0`, `not x` where x is
+                # a bare parameter name) Spark's `~` is bitwise, not Python
+                # truthiness, so we bail and let the caller fall back to
+                # interpreted Python rather than silently diverge.
+                if not _is_definitely_boolean(operand):
+                    raise UnsupportedOperationException(
+                        "`not` operand type is not statically known to be "
+                        "boolean; Spark's `~` is bitwise, not Python "
+                        "truthiness, so the transpiler refuses to lower this "
+                        "and the UDF falls back to interpreted Python"
+                    )
+                return coalesce(self._convert_chunk(params, operand).__invert__(), lit(True))
+            case ast.UnaryOp(op=(ast.USub() | ast.UAdd()) as op, operand=operand):
+                # `-x` / `+x` -- like the binary arithmetic operators, only
+                # lower for numeric operands. Python raises TypeError for
+                # unary +/- on strings, but Spark's ANSI string promotion
+                # would silently coerce the string to double (`-'5'` ->
+                # -5.0), and a boolean operand emits UnaryMinus(bool), which
+                # fails CheckAnalysis outright -- breaking the query instead
+                # of falling back, since the option is type-checked as a
+                # child of TranspiledPythonUDF before ConvertToCatalyst can
+                # drop it. Fail closed for every non-numeric category.
+                if self._category(params, operand) != "numeric":
+                    raise UnsupportedOperationException(
+                        "unary `+`/`-` is only supported for numeric operands "
+                        "(Python raises TypeError on strings, and Spark would "
+                        "coerce or fail analysis); the transpiler falls back "
+                        "to interpreted Python"
+                    )
+                if isinstance(op, ast.USub):
+                    # Handles both literal negative ints (USub on a Constant)
+                    # and runtime negation of a column.
+                    return self._convert_chunk(params, operand).__neg__()
+                # `+x` -- identity, kept for symmetry with USub.
+                return self._convert_chunk(params, operand)
+            case ast.BoolOp(op=op, values=values):
+                # Python `and` / `or` short-circuit and return one of the
+                # operands rather than a strict boolean. For the booleans
+                # produced by Compare / UnaryOp(Not) / nested BoolOps this
+                # maps cleanly onto Spark Column `&` / `|`. For
+                # non-boolean operands (including bare parameter names whose
+                # runtime type is unknown) the right semantics would require
+                # Python's truthiness rules (0 / "" / None / [] all
+                # falsy), which we can't faithfully reproduce without the
+                # input column types -- Spark's `&` / `|` would silently
+                # do bitwise instead. Require all operands to be statically
+                # known boolean so the caller falls back to interpreted
+                # Python rather than producing a plan whose results diverge.
+                if not all(_is_definitely_boolean(v) for v in values):
+                    raise UnsupportedOperationException(
+                        "`and` / `or` operand type is not statically known "
+                        "to be boolean; Spark's `&` / `|` are bitwise, not "
+                        "Python truthiness, so the transpiler refuses to "
+                        "lower this and the UDF falls back to interpreted "
+                        "Python"
+                    )
+                # A literal None operand short-circuits differently: Python's
+                # `None and (x > 0)` returns None regardless of x, but Spark's
+                # three-valued `null AND false` is false (and `null OR true` is
+                # true), so the lowered form diverges. `_is_definitely_boolean`
+                # accepts None for `not`/if-test contexts where coalesce handles
+                # it; here it must force a fallback instead.
+                if any(isinstance(v, ast.Constant) and v.value is None for v in values):
+                    raise UnsupportedOperationException(
+                        "literal None operand in `and` / `or` cannot be lowered: "
+                        "Spark's three-valued logic diverges from Python's "
+                        "short-circuit-return-operand semantics, so the UDF "
+                        "falls back to interpreted Python"
+                    )
+                cols = [self._convert_chunk(params, v) for v in values]
+                if isinstance(op, ast.And):
+                    result = cols[0]
+                    for c in cols[1:]:
+                        result = result & c
+                    return result
+                if isinstance(op, ast.Or):
+                    result = cols[0]
+                    for c in cols[1:]:
+                        result = result | c
+                    return result
+                raise UnsupportedOperationException(f"BoolOp operator {op} is not supported")
+            case ast.IfExp(test=test, body=body_expr, orelse=orelse_expr):
+                # Ternary `body if test else orelse` -- shares the
+                # NULL-as-falsy lowering with the if-statement case.
+                return self._convert_if_like(
+                    params,
+                    self._convert_chunk(params, test),
+                    self._convert_chunk(params, body_expr),
+                    self._convert_chunk(params, orelse_expr),
+                    test,
+                    body_expr,
+                    orelse_expr,
+                )
+            case ast.If(test, success, orelse):
+                return self._convert_if_like(
+                    params,
+                    self._convert_chunk(params, test),
+                    self._convert_branch(params, success, "body"),
+                    self._convert_branch(params, orelse, "else body"),
+                    test,
+                    success[0] if success else None,
+                    orelse[0] if orelse else None,
+                )
+            case ast.Compare(left, ops, comps):
+                if len(ops) != 1 or len(comps) != 1:
+                    raise UnsupportedOperationException(
+                        "chained comparisons (e.g. `a < b < c`) are not supported by the transpiler"
+                    )
+                comp = comps[0]
+                match ops[0]:
+                    case ast.Is() | ast.IsNot():
+                        # Only lower `x is None` / `None is x` (and their
+                        # `is not` variants) to isNull/isNotNull. For any
+                        # other comparator (e.g. `x is 0`, `x is y`) Python
+                        # performs an object-identity check that has no SQL
+                        # equivalent, so we must fall back to interpreted
+                        # Python rather than silently emitting a null check.
+                        is_none_left = isinstance(left, ast.Constant) and left.value is None
+                        is_none_right = isinstance(comp, ast.Constant) and comp.value is None
+                        if not (is_none_left or is_none_right):
+                            raise UnsupportedOperationException(
+                                "`is`/`is not` is only supported when one "
+                                "operand is the literal None; other identity "
+                                "checks (e.g. `x is 0`, `x is y`) cannot be "
+                                "lowered to SQL and the UDF falls back to "
+                                "interpreted Python"
+                            )
+                        subject_node = comp if is_none_left else left
+                        subject_col = self._convert_chunk(params, subject_node)
+                        if isinstance(ops[0], ast.Is):
+                            return subject_col.isNull()
+                        else:
+                            return subject_col.isNotNull()
+                    case ast.Eq():
+                        return self._lower_eq(params, left, comp, equal=True)
+                    case ast.NotEq():
+                        return self._lower_eq(params, left, comp, equal=False)
+                    case ast.Lt():
+                        return self._lower_value_compare(
+                            params, left, comp, lambda l, r: l < r, "<"
+                        )
+                    case ast.LtE():
+                        return self._lower_value_compare(
+                            params, left, comp, lambda l, r: l <= r, "<="
+                        )
+                    case ast.Gt():
+                        return self._lower_value_compare(
+                            params, left, comp, lambda l, r: l > r, ">"
+                        )
+                    case ast.GtE():
+                        return self._lower_value_compare(
+                            params, left, comp, lambda l, r: l >= r, ">="
+                        )
+                    case _:
+                        raise UnsupportedOperationException(
+                            f"comparison operator {type(ops[0]).__name__} "
+                            "is not supported by the transpiler"
+                        )
+            case ast.BinOp(left=left, op=op, right=right):
+                # Operator selection is driven by the operand *categories* under
+                # the current input-type variant (see ``_category``): Python's
+                # `+` / `*` are overloaded for text. `+` -> add (num,num) or
+                # concat (str,str); `*` -> multiply (num,num) or repeat (str,int
+                # / int,str); `-` / `%` are numeric-only. Combos that don't fit
+                # (str+int, str-str, ...) raise so this variant is dropped and
+                # the JVM picks another option or falls back to the Python UDF.
+                #
+                # `**` is intentionally NOT lowered: Spark's `pow` is DOUBLE and
+                # loses precision for large integers, so it would silently return
+                # wrong results. TODO (SPARK-55210): add an exact integer-power
+                # lowering and re-enable it.
+                #
+                # Value-level divergences remain documented (need runtime value
+                # info, not type): overflow raises ARITHMETIC_OVERFLOW under ANSI
+                # where Python promotes to a big int; arithmetic is not
+                # NULL-guarded (`x + 1` on NULL -> NULL vs Python TypeError).
+                # TODO (SPARK-55210): map overflow / divide-by-zero precisely.
+                lc = self._category(params, left)
+                rc = self._category(params, right)
+                left_col = self._convert_chunk(params, left)
+                right_col = self._convert_chunk(params, right)
+                match op:
+                    case ast.Add():
+                        if lc == rc == "string":
+                            return concat(left_col, right_col)
+                        if lc == rc == "numeric":
+                            return left_col.__add__(right_col)
+                    case ast.Sub():
+                        if lc == rc == "numeric":
+                            return left_col.__sub__(right_col)
+                    case ast.Mult():
+                        if lc == "numeric" and rc == "numeric":
+                            return left_col.__mul__(right_col)
+                        if lc == "string" and rc == "numeric":
+                            return repeat(left_col, right_col.cast("int"))
+                        if lc == "numeric" and rc == "string":
+                            return repeat(right_col, left_col.cast("int"))
+                    case ast.Mod():
+                        if lc == rc == "numeric":
+                            # Python's `%` takes the sign of the divisor; Spark's
+                            # takes the dividend's. `sign(b) * pmod(sign(b) * a,
+                            # abs(b))` reproduces Python for every non-zero divisor
+                            # except at the LongType overflow boundaries -- `a =
+                            # Long.MinValue` with `b < 0` (the `sign(b) * a` negate
+                            # overflows) and `b = Long.MinValue` (the `abs(b)`
+                            # overflows) -- where this raises ARITHMETIC_OVERFLOW
+                            # under ANSI while Python returns a value. That matches
+                            # the documented overflow caveat for `+`/`-`/`*` above.
+                            # Use a CASE-based integer sign rather than sign() to
+                            # avoid promoting operands to DoubleType, which loses
+                            # precision near LongType boundaries.
+                            sb = (
+                                when(right_col > 0, lit(1))
+                                .when(right_col < 0, lit(-1))
+                                .otherwise(lit(0))
+                            )
+                            return sb * pmod(sb * left_col, _abs(right_col))
+                    case _:
+                        raise UnsupportedOperationException(
+                            f"binary operator {type(op).__name__} is not "
+                            "supported by the transpiler"
+                        )
+                raise UnsupportedOperationException(
+                    f"`{type(op).__name__}` operands are not type-compatible for "
+                    "this input-type variant"
+                )
+            case ast.Return(value=value):
+                return self._convert_chunk(params, value)
+            case ast.Constant(value=value):
+                # Avoid circular import issue.
+                return lit(value)
+            case ast.Name(id=name, ctx=ast.Load()):
+                # Insert columns referencing the param indexes for children
+                if name in params:
+                    param_index = params.index(name)
+                    # Special hack for self on callables
+                    if params[0] == "self":
+                        # A body that references the receiver itself (e.g.
+                        # ``return self``) has no column equivalent: ``self``
+                        # is not an argument at the call site, and offsetting
+                        # would emit ``_udf_param_-1``, which the JVM builder
+                        # rejects with an AnalysisException at call
+                        # construction instead of falling back. Refuse so the
+                        # UDF stays interpreted.
+                        if name == "self":
+                            raise UnsupportedOperationException(
+                                "references to `self` in a callable's body "
+                                "are not supported by the transpiler; falling "
+                                "back to interpreted Python"
+                            )
+                        param_index -= 1
+                    return col(f"_udf_param_{param_index}")
+                else:
+                    # TODO (SPARK-55207): Handle assignments, class vars, and closures
+                    # via scope evaluation.
+                    raise UnsupportedOperationException(
+                        f"name {name!r} is not in the UDF's parameter list "
+                        "and free variables / closures are not supported"
+                    )
+            case _:
+                raise UnsupportedOperationException(
+                    f"AST node {type(body).__name__} is not supported by the "
+                    f"transpiler ({ast.dump(body)[:120]})"
+                )
+
+    def _transpile_from_ast(
+        self,
+        src: Optional[str],
+        ast_info: ast.AST,
+        function_ast: ast.FunctionDef,
+        params: List[str],
+        returnType: "DataTypeOrString",
+        param_categories: Optional[dict] = None,
+    ) -> Optional[Column]:
+        # Short circuit on nothing to transpile.
+        if src == "" or ast_info is None:
+            return None
+        # Per-variant input-type assumption ({public_param_index -> category}),
+        # read by ``_category`` to choose str vs numeric operators.
+        self._param_categories = param_categories or {}
+        function_body = function_ast.body
+        if len(function_body) != 1:
+            raise UnsupportedOperationException(
+                "functions with more than one top-level statement are not "
+                "supported by the transpiler"
+            )
+        # Refuse variants whose body category does not MATCH the declared
+        # return type's category. Two distinct failure modes hide here:
+        #
+        # * A cast that can never resolve (binary -> numeric, bool -> binary):
+        #   the options are type-checked by CheckAnalysis as children of
+        #   TranspiledPythonUDF before ConvertToCatalyst could drop them, so
+        #   the whole query fails instead of falling back.
+        # * A cast that IS analysis-valid but that the interpreted
+        #   SQL_BATCHED_UDF path never performs: EvaluatePython.makeFromJava
+        #   accepts only the expected JVM types for the declared return type
+        #   and nulls everything else. E.g. `def f(s: str): return s` declared
+        #   LongType() returns NULL interpreted, but a lowered
+        #   cast(string as bigint) would return 123 for '123' (or raise
+        #   CAST_INVALID_INPUT for 'abc') -- a silent divergence.
+        #
+        # So require the strict match: numeric -> non-decimal NumericType
+        # (DecimalType is excluded like it is for inputs: the interpreted
+        # converter accepts only decimal.Decimal results there and nulls the
+        # ints/floats these lowerings produce), string -> StringType, bool ->
+        # BooleanType, binary -> BinaryType. An unknown category (e.g. a bare
+        # None body) lowers to NULL, which every return type accepts as NULL
+        # on both paths. Within-numeric conversions (e.g. a bigint body cast
+        # to a double return type) are intentionally still allowed and
+        # documented as the transpiled-cast behavior pinned by
+        # test_udf_transpile_casts_to_return_type.
+        if isinstance(returnType, DataType):
+            body_cat = self._safe_category(params, function_body[0])
+            cast_ok = (
+                body_cat is None
+                or (
+                    body_cat == "numeric"
+                    and isinstance(returnType, NumericType)
+                    and not isinstance(returnType, DecimalType)
+                )
+                or (body_cat == "string" and isinstance(returnType, StringType))
+                or (body_cat == "bool" and isinstance(returnType, BooleanType))
+                or (body_cat == "binary" and isinstance(returnType, BinaryType))
+            )
+            if not cast_ok:
+                raise UnsupportedOperationException(
+                    f"a {body_cat}-typed lowering does not match the declared "
+                    f"return type {returnType.simpleString()}; the interpreted "
+                    "path would return NULL where the lowered cast would "
+                    "convert (or fail), so the transpiler falls back to "
+                    "interpreted Python"
+                )
+        converted = self._convert_chunk(params, function_body[0])
+        # Cast to the declared return type so the rewritten plan reports a
+        # known data type to the optimizer's plan validator (otherwise it
+        # sees an UnresolvedFunction tree and reports VOID, which fails
+        # the schema-stability check on this rule).
+        return converted.cast(returnType)
+
+
+CatalystTranspiler.register()
+
+
+def _get_transpilers(session: "SparkSession") -> List[AbstractTranspiler]:
+    """Get the transpilers we should try."""
+    configured_transpilers = session.conf.get("spark.sql.experimental.optimizer.pyTranspilers")
+    if not configured_transpilers:
+        return []
+    transpiler_names = configured_transpilers.split(",")
+    return [
+        AbstractTranspiler.varieties[name]()
+        for name in transpiler_names
+        if name in AbstractTranspiler.varieties
+    ]
+
+
+def _annotation_category(annotation: Optional[ast.AST]) -> Optional[str]:
+    """Map a parameter's type annotation to a category
+    (``"numeric"``/``"string"``/``"bool"``/``"binary"``), or ``None`` when it's
+    absent or unrecognised (the caller then tries both numeric and string)."""
+    name: Optional[str] = None
+    if isinstance(annotation, ast.Name):
+        name = annotation.id
+    elif isinstance(annotation, ast.Constant) and isinstance(annotation.value, str):
+        name = annotation.value  # stringized annotation, e.g. def f(a: "int")
+    # str -> "string", int/float -> "numeric", bool -> "bool", bytes -> "binary"
+    # (matching the constant handling in ``_category``). complex and anything
+    # unrecognised return None so the caller tries both numeric and string.
+    if name == "str":
+        return "string"
+    if name in ("int", "float"):
+        return "numeric"
+    if name == "bool":
+        return "bool"
+    if name == "bytes":
+        return "binary"
+    return None
+
+
+def _param_category_combos(function_ast: ast.FunctionDef, public_params: List[str]) -> List[dict]:
+    """Per-variant maps ``{public_param_index -> category}`` where category is
+    one of ``"numeric"``/``"string"``/``"bool"``/``"binary"``.
+
+    A typed param (``def f(a: str, b: int)``) is pinned to its category; an
+    untyped param is tried as both numeric and string. To cap plan growth, when
+    more than three params are untyped we collapse the untyped ones to the
+    all-numeric and all-string variants (encourage typing inputs to keep the
+    matrix small) while keeping every typed param pinned.
+    """
+    n = len(public_params)
+    public_args = function_ast.args.args[len(function_ast.args.args) - n :]
+    candidates: List[List[str]] = []
+    untyped = 0
+    for arg in public_args:
+        cat = _annotation_category(arg.annotation)
+        if cat is None:
+            candidates.append(["numeric", "string"])
+            untyped += 1
+        else:
+            candidates.append([cat])
+    if untyped > 3:
+        # Cap the 2**untyped blow-up, but keep each typed param pinned to its
+        # category (a single-element ``candidates`` entry); only the untyped
+        # params collapse to the all-numeric / all-string pair.
+        return [
+            {i: c[0] if len(c) == 1 else fill for i, c in enumerate(candidates)}
+            for fill in ("numeric", "string")
+        ]
+    return [{i: choice[i] for i in range(n)} for choice in itertools.product(*candidates)] or [{}]
+
+
+def _get_src_ast_from_func(func: Callable) -> Tuple[Optional[str], Optional[ast.AST]]:
+    """Try and get the AST from a given callable"""
+    # Note: consider maybe dill? (see the JYTHON PR)
+    # inspect getsource does not work for functions defined in vanilla
+    # repl, but does for those in files or in ipython.
+    # It also fails when we give it an instance of a callable class.
+    try:
+        src = inspect.getsource(func)
+        src = textwrap.dedent(src).strip()
+        ast_info = ast.parse(src)
+    except Exception:
+        try:
+            # getattr keeps mypy happy: `__call__` on a bare Callable is
+            # not attribute-accessible in the type system.
+            src = inspect.getsource(getattr(func, "__call__"))
+            src = textwrap.dedent(src).strip()
+            ast_info = ast.parse(src)
+        except Exception:
+            # No usable source (REPL/stdin definition, builtin, ...) --
+            # return cleanly so the caller reports "cannot transpile"
+            # instead of surfacing an UnboundLocalError as the reason.
+            return None, None
+    return src, ast_info
+
+
+def _get_parameter_list(node: ast.FunctionDef) -> list[str]:
+    """Return the positional argument names in order."""
+    return [arg.arg for arg in node.args.args]
+
+
+def _get_function_from_ast(body: ast.AST) -> ast.FunctionDef | None:
+    """
+    Extract a :class:`ast.FunctionDef` node from an AST produced by
+    ``ast.parse(inspect.getsource(udf_func))``.
+
+    Handles the following source patterns (in order):
+
+    * ``f = lambda x: x + 1`` -- lambda bound directly to a name
+    * ``lambda x: x + 1`` -- bare expression (getsource on a raw lambda)
+    * ``def f(x): ... return x + 1``
+    * a class with a ``__call__`` method
+
+    Returns ``None`` when no single unambiguous function can be identified --
+    notably, a lambda wrapped in a call such as
+    ``f = some_wrapper(lambda x: x + 1)`` parses as ``Assign(value=Call(...))``,
+    which is not unwrapped here and so falls back to interpreted Python. Local
+    class variables are likewise unsupported.
+    """
+    if not hasattr(body, "body") or not body.body:
+        return None
+
+    stmt = body.body[0]
+
+    # Grab the value side of a top level assign (e.g. x = lambda ...)
+    if isinstance(stmt, ast.Assign):
+        stmt = stmt.value
+
+    # Bare ``lambda x: ...`` (when ``inspect.getsource`` returns a raw
+    # lambda expression at module top level) parses as ``Expr(Lambda)``.
+    if isinstance(stmt, ast.Expr) and isinstance(stmt.value, ast.Lambda):
+        stmt = stmt.value
+
+    if isinstance(stmt, ast.Lambda):
+        # Synthesize a one-statement FunctionDef wrapping the lambda body so
+        # the rest of the transpiler can treat lambdas and ``def`` uniformly.
+        # ``ast.FunctionDef``'s overloads in mypy's typeshed require
+        # keyword-only ``type_params`` on 3.12+, which doesn't exist at
+        # runtime on every Python we support (the field was added in
+        # 3.12 -- before that, passing it raises). Drop to ``Any`` so we
+        # avoid the overload resolution entirely; constructing the node
+        # via keyword args is well-defined at runtime even when the typed
+        # overloads disagree.
+        fn_ctor: Any = ast.FunctionDef
+        return fn_ctor(
+            name="<lambda>",
+            args=stmt.args,
+            body=[ast.Return(value=stmt.body)],
+            decorator_list=[],
+        )
+
+    if isinstance(stmt, ast.FunctionDef):
+        return stmt
+    return None
+
+
+def _transpile_func(
+    session: "SparkSession",
+    func: Callable[..., Any],
+    returnType: "DataTypeOrString",
+) -> Tuple[List[Column], List[str], List[str], List[List[str]]]:
+    """
+    An experimental internal function that attempts to transpile a callable function.
+
+    Returns
+    -------
+    list of transpiled options (one per backend x input-type variant)
+    list of errors as strings
+    list of positional parameter names (excluding ``self`` for callable
+    instances) -- needed so the caller can resolve named-argument
+    invocations to positional order at call time, since the ``_udf_param_N``
+    substitution in :class:`UserDefinedPythonFunction` is positional.
+    list of per-option input-type categories (``"numeric"`` / ``"string"`` per
+    public param) -- the JVM picks the option whose categories match the bound
+    column types, or falls back to the Python UDF when none match.
+    """
+    try:
+        # The transpiler lowers to atomic (numeric/string/boolean/binary)
+        # expressions and casts the result to the declared return type. For a
+        # return type no lowering can even category-match (arrays, maps,
+        # structs, datetimes, ...), that Cast either never resolves -- and
+        # because the options ride along as children of TranspiledPythonUDF,
+        # an unresolvable Cast fails the WHOLE query at CheckAnalysis instead
+        # of falling back -- or diverges from the interpreted converter, which
+        # nulls type-mismatched results. Restrict transpilation to return
+        # types some lowering can match (the strict per-variant body-category
+        # check lives in ``_transpile_from_ast``); everything else falls back
+        # to interpreted Python.
+        if isinstance(returnType, str):
+            from pyspark.sql.types import _parse_datatype_string
+
+            returnType = _parse_datatype_string(returnType)
+        if not isinstance(returnType, (NumericType, StringType, BooleanType, BinaryType)):
+            return (
+                [],
+                [
+                    f"return type {returnType.simpleString()} is not supported by "
+                    "the transpiler (no lowered expression can be cast to it "
+                    "under ANSI rules); falling back to interpreted Python"
+                ],
+                [],
+                [],
+            )
+        # A functools.wraps-style decorator makes ``inspect.getsource`` return
+        # the WRAPPED function's source (getsource follows ``__wrapped__``),
+        # while the UDF actually executes the wrapper. Transpiling would
+        # silently reproduce the wrong behavior, so refuse and fall back.
+        if (
+            getattr(func, "__wrapped__", None) is not None
+            or getattr(getattr(func, "__call__", None), "__wrapped__", None) is not None
+        ):
+            return (
+                [],
+                [
+                    "decorated callables (functools.wraps) are not supported: "
+                    "the visible source is the wrapped function's, not the "
+                    "wrapper's, so transpilation would change behavior"
+                ],
+                [],
+                [],
+            )
+        src, ast = _get_src_ast_from_func(func)
+        if ast is None:
+            return ([], ["Error getting ast for function, cannot transpile"], [], [])
+        # Get the lambda body and parameters
+        function_ast = _get_function_from_ast(ast)
+        if function_ast is None:
+            return ([], ["Error extracting function body from ast, cannot transpile"], [], [])
+        # Default, variadic (``*args`` / ``**kwargs``), keyword-only, and
+        # positional-only parameters can't be represented by the positional
+        # ``_udf_param_N`` placeholder scheme: a call site may omit a
+        # defaulted argument, leaving the placeholder referencing a position
+        # the call never bound, and ``_get_parameter_list`` only reads
+        # ``args``. Fall back to interpreted Python rather than emit an
+        # invalid plan.
+        fn_args = function_ast.args
+        if (
+            fn_args.defaults
+            or any(d is not None for d in fn_args.kw_defaults)
+            or fn_args.kwonlyargs
+            or fn_args.vararg is not None
+            or fn_args.kwarg is not None
+            or getattr(fn_args, "posonlyargs", [])
+        ):
+            return (
+                [],
+                [
+                    "functions with default, variadic, keyword-only, or "
+                    "positional-only arguments are not supported by the transpiler"
+                ],
+                [],
+                [],
+            )
+        params = _get_parameter_list(function_ast)
+        # The transpiler strips a leading ``self`` on the assumption that the
+        # source came from a bound ``__call__`` / method whose receiver is not
+        # supplied at the call site. A PLAIN function whose first parameter
+        # happens to be named ``self`` breaks that assumption: every arg IS
+        # supplied at the call site, and stripping would misnumber the
+        # ``_udf_param_N`` placeholders (emitting ``_udf_param_-1``). Refuse
+        # and fall back rather than guess.
+        if params and params[0] == "self" and inspect.isfunction(func):
+            return (
+                [],
+                [
+                    "plain function with first parameter named 'self' is "
+                    "ambiguous to the transpiler's self-stripping; falling "
+                    "back to interpreted Python"
+                ],
+                [],
+                [],
+            )
+        # Strip ``self`` for the caller-facing param list -- callers will
+        # match user-supplied kwargs against this, and the user doesn't
+        # name ``self`` at the call site.
+        public_params = params[1:] if params and params[0] == "self" else list(params)
+        transpiled: list[Column] = []
+        input_categories: list[list[str]] = []
+        errors = []
+        # One transpiled option per (backend x input-type variant). Untyped
+        # params are tried as both numeric and string so the JVM can pick the
+        # option matching the actual column types (or fall back if none match).
+        combos = _param_category_combos(function_ast, public_params)
+        # Maybe multiple transpilers (think CUDA, etc.).
+        transpilers = _get_transpilers(session)
+        for transpiler in transpilers:
+            for combo in combos:
+                try:
+                    transpiled_column = transpiler._transpile_from_ast(
+                        src, ast, function_ast, params, returnType, combo
+                    )
+                    if transpiled_column is not None:
+                        transpiled.append(transpiled_column)
+                        input_categories.append(
+                            [combo.get(i, "numeric") for i in range(len(public_params))]
+                        )
+                except Exception as e:
+                    errors.append(str(e))
+        return (transpiled, errors, public_params, input_categories)
+    except Exception as e:
+        # Don't re-raise: an inability to transpile must never break a
+        # working UDF. The caller treats an empty ``transpiled`` list as a
+        # silent fall-back to interpreted Python.
+        return ([], [str(e)], [], [])
diff --git a/python/pyspark/sql/udf.py b/python/pyspark/sql/udf.py
index e586c560deeb..dbfc1b465086 100644
--- a/python/pyspark/sql/udf.py
+++ b/python/pyspark/sql/udf.py
@@ -206,6 +206,98 @@ class UserDefinedFunction:
         )
         self.evalType = evalType
         self.deterministic = deterministic
+        # Extract Python UDF details if transpilation is enabled.
+        self.transpiled: list = []
+        self._transpiled_param_names: list[str] = []
+        # Per-option input-type categories ("numeric"/"string" per public param),
+        # parallel to ``self.transpiled``; the JVM picks the option matching the
+        # actual column types or falls back to interpreted Python.
+        self._transpiled_input_categories: list = []
+        # When we have a transpiled rewrite, ``__call__`` resolves any
+        # user-supplied kwargs against this positional parameter list so
+        # the JVM-side ``_udf_param_N`` substitution sees the inputs in
+        # the right order. Empty list when transpilation didn't happen.
+        from pyspark.sql import SparkSession
+
+        session = SparkSession._instantiatedSession
+
+        # A nondeterministic UDF must not be transpiled: replacing it with a plain
+        # Catalyst expression would let the optimizer fold/reorder/duplicate it,
+        # discarding the nondeterminism barrier. (asNondeterministic() also clears
+        # any options set here, for the udf(f).asNondeterministic() ordering.)
+        # Conf values are compared case-insensitively: `SET conf=True` stores
+        # the literal "True", which would otherwise silently disable
+        # transpilation (or mis-trigger the ANSI warning below).
+        #
+        # Each conf read is a JVM roundtrip, so keep the default construction
+        # path cheap: the experimental gate is only read for deterministic
+        # batched UDFs (the only shape we transpile), and the ANSI conf is only
+        # read once the gate is known to be on. When ``default`` is given it is
+        # passed through to ``RuntimeConfig.get`` so construction never depends
+        # on the JVM having the (experimental) conf registered -- e.g. a newer
+        # Python client against an older driver. No default is passed for
+        # ``spark.sql.ansi.enabled``: its registered default is dynamic
+        # (environment-driven) and must be respected when the key is unset.
+        def _conf_is_true(key: str, default: Optional[str] = None) -> bool:
+            if session is None:
+                return False
+            if default is None:
+                value = session.conf.get(key)
+            else:
+                value = session.conf.get(key, default)
+            return value is not None and value.lower() == "true"
+
+        try:
+            transpile_enabled = (
+                deterministic
+                and evalType == PythonEvalType.SQL_BATCHED_UDF
+                and _conf_is_true("spark.sql.experimental.optimizer.transpilePyUDFs", "false")
+            )
+            # Transpilation only attempts to reproduce ANSI-mode Spark SQL
+            # semantics (no silent integer overflow, divide-by-zero raises,
+            # etc.). Running it against non-ANSI Spark would balloon the test
+            # matrix we'd have to maintain to verify Python-vs-SQL equivalence,
+            # so we gate on ANSI here and warn the user instead of trying to
+            # transpile in a mode we don't claim to support yet.
+            if transpile_enabled and not _conf_is_true("spark.sql.ansi.enabled"):
+                warnings.warn(
+                    "Python UDF transpilation "
+                    "(spark.sql.experimental.optimizer.transpilePyUDFs) is only "
+                    "supported when ANSI mode is enabled "
+                    "(spark.sql.ansi.enabled=true). Skipping transpilation for "
+                    f"{func} -- enable ANSI mode or set transpilePyUDFs=false to "
+                    "silence this warning.",
+                    RuntimeWarning,
+                )
+                transpile_enabled = False
+            if transpile_enabled and session:
+                # Import only if needed, also avoid circular import loops.
+                from pyspark.sql.transpile import _transpile_func
+
+                # ``self.returnType`` parses (and caches) the declared return
+                # type; the transpiler needs the parsed form to decide whether
+                # the final Cast to it can resolve at all. The parse is reused
+                # later by ``_create_judf``, so this adds no extra JVM work.
+                (
+                    self.transpiled,
+                    errors,
+                    self._transpiled_param_names,
+                    self._transpiled_input_categories,
+                ) = _transpile_func(session, func, self.returnType)
+                if not self.transpiled:
+                    detail = f": {errors}" if errors else ""
+                    warnings.warn(f"Unable to transpile UDF {func}{detail}")
+        except Exception as e:
+            # An inability to transpile must never break a working UDF -- fall
+            # back to interpreted Python execution and surface the failure as a
+            # warning so users can opt to investigate without losing their
+            # query. The conf reads above are included: a session whose JVM
+            # cannot answer them should degrade to "no transpilation", not
+            # break UDF definition.
+            warnings.warn(f"Exception transpiling UDF {func}: {e}")
+            self.transpiled = []
+            self._transpiled_param_names = []
+            self._transpiled_input_categories = []
 
     @staticmethod
     def _check_return_type(returnType: DataType, evalType: int) -> None:
@@ -411,8 +503,11 @@ class UserDefinedFunction:
             self._judf_placeholder = self._create_judf(self.func)
         return self._judf_placeholder
 
-    def _create_judf(self, func: Callable[..., Any]) -> "JavaObject":
+    def _create_judf(
+        self, func: Callable[..., Any], include_transpiled: bool = True
+    ) -> "JavaObject":
         from pyspark.sql import SparkSession
+        from pyspark.sql.classic.column import _to_java_column_opt
 
         spark = SparkSession._getActiveSessionOrCreate()
         sc = spark.sparkContext
@@ -420,8 +515,16 @@ class UserDefinedFunction:
         wrapped_func = _wrap_function(sc, func, self.returnType)
         jdt = spark._jsparkSession.parseDataType(self.returnType.json())
         assert sc._jvm is not None
+        transpiled = self.transpiled if include_transpiled else []
+        input_categories = self._transpiled_input_categories if include_transpiled else []
         judf = getattr(sc._jvm, "org.apache.spark.sql.execution.python.UserDefinedPythonFunction")(
-            self._name, wrapped_func, jdt, self.evalType, self.deterministic
+            self._name,
+            wrapped_func,
+            jdt,
+            self.evalType,
+            self.deterministic,
+            map(_to_java_column_opt, transpiled),
+            input_categories,
         )
         return judf
 
@@ -430,6 +533,32 @@ class UserDefinedFunction:
 
         sc = get_active_spark_context()
 
+        # Transpilation rewrites the UDF into a Catalyst expression that
+        # references its inputs positionally via ``_udf_param_N`` (see
+        # ``UserDefinedPythonFunction.builder.resolveUDFParams``). If the
+        # caller used kwargs, the JVM-side substitution would otherwise
+        # splice ``NamedArgumentExpression`` wrappers into the rewritten
+        # tree (and into nested function calls like ``isnotnull``, which
+        # rejects named arguments). Resolve kwargs to positional here
+        # using the parameter list captured at transpilation time so the
+        # rewritten expression sees plain column refs in declared order.
+        if kwargs and self.transpiled and self._transpiled_param_names:
+            params = self._transpiled_param_names
+            ordered: list = list(args)
+            remaining_kwargs = dict(kwargs)
+            for pname in params[len(args) :]:
+                if pname in remaining_kwargs:
+                    ordered.append(remaining_kwargs.pop(pname))
+                else:
+                    # Caller didn't supply this param positionally or by
+                    # name -- bail out of the rewrite and let the regular
+                    # JVM-side path raise a user-facing error.
+                    break
+            else:
+                if not remaining_kwargs:
+                    args = tuple(ordered)
+                    kwargs = {}
+
         assert sc._jvm is not None
         jcols = [_to_java_column(arg) for arg in args] + [
             sc._jvm.PythonSQLUtils.namedArgumentExpression(key, _to_java_column(value))
@@ -477,7 +606,11 @@ class UserDefinedFunction:
                     return profiler.profile(f, *args, **kwargs)
 
                 func.__signature__ = inspect.signature(f)  # type: ignore[attr-defined]
-                judf = self._create_judf(func)
+                # Profiling requires the Python function to actually execute,
+                # and the transpiled path never runs it (it also produces a
+                # TranspiledPythonUDF, which has no resultId for the profiler
+                # to key on). Build this call's judf without transpiled options.
+                judf = self._create_judf(func, include_transpiled=False)
                 jUDFExpr = judf.builderWithColumns(_to_seq(sc, jcols))
                 jPythonUDF = judf.fromUDFExpr(jUDFExpr)
                 id = jUDFExpr.resultId().id()
@@ -499,7 +632,9 @@ class UserDefinedFunction:
                     )
 
                 func.__signature__ = inspect.signature(f)  # type: ignore[attr-defined]
-                judf = self._create_judf(func)
+                # See the profiler branch above: no transpiled options while
+                # profiling, since only the interpreted path runs the function.
+                judf = self._create_judf(func, include_transpiled=False)
                 jUDFExpr = judf.builderWithColumns(_to_seq(sc, jcols))
                 jPythonUDF = judf.fromUDFExpr(jUDFExpr)
                 id = jUDFExpr.resultId().id()
@@ -557,6 +692,14 @@ class UserDefinedFunction:
         # with 'deterministic' updated. See SPARK-23233.
         self._judf_placeholder = None
         self.deterministic = False
+        # A transpiled rewrite replaces the (now nondeterministic) Python UDF
+        # with a plain Catalyst expression, which the optimizer is free to
+        # fold, reorder, or duplicate -- discarding the nondeterminism barrier
+        # the caller just asked for. Drop any transpiled options so a
+        # nondeterministic UDF always runs as interpreted Python.
+        self.transpiled = []
+        self._transpiled_param_names = []
+        self._transpiled_input_categories = []
         return self
 
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 8c02c92634ba..220e997eaa18 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -638,7 +638,8 @@ class Analyzer(
       Seq(
         ResolveWithCTE,
         ExtractDistributedSequenceID,
-        ResolveAsOfJoin) ++
+        ResolveAsOfJoin,
+        ResolveTranspiledPythonUDFOptions) ++
       Seq(ResolveUpdateEventTimeWatermarkColumn) ++
       extendedResolutionRules ++
       Seq(NameStreamingSources) : _*),
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveTranspiledPythonUDFOptions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveTranspiledPythonUDFOptions.scala
new file mode 100644
index 000000000000..188b3e5ae22d
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveTranspiledPythonUDFOptions.scala
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.analysis
+
+import org.apache.spark.sql.catalyst.expressions.TranspiledPythonUDF
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.catalyst.trees.TreePattern.TRANSPILED_PYTHON_UDF
+import org.apache.spark.sql.types.{BinaryType, BooleanType, DataType, DecimalType, NumericType, StringType}
+
+/**
+ * Prunes the per-input-type options carried by a [[TranspiledPythonUDF]] down to those whose
+ * declared categories match the resolved argument types.
+ *
+ * A Python operator such as `a + b` is overloaded for text, so the transpiler emits one option
+ * per input-type variant -- a numeric `Add` and a string `concat`, say -- each tagged with the
+ * input-type categories it expects. Those options are children of the node, so leaving a
+ * type-incompatible one in place (a numeric `Add` over string columns) would make `CheckAnalysis`
+ * reject the whole plan. We can only choose once the argument types are known, which is after
+ * reference resolution -- hence a rule here rather than in the builder, which runs at
+ * call-construction time before the columns are bound -- and we must run before `CheckAnalysis`.
+ *
+ * Matching is strict by category (a numeric option only for numeric columns, a string option only
+ * for string columns). We deliberately do not lean on implicit type coercion, which would, e.g.,
+ * make a numeric `Add` "valid" over a string column and silently diverge from Python's
+ * `TypeError`. When no option matches, the list is emptied and `ConvertToCatalyst` falls back to
+ * the original Python UDF.
+ */
+object ResolveTranspiledPythonUDFOptions extends Rule[LogicalPlan] {
+  def apply(plan: LogicalPlan): LogicalPlan = {
+    if (!plan.containsPattern(TRANSPILED_PYTHON_UDF)) {
+      plan
+    } else {
+      plan.resolveOperatorsWithPruning(_.containsPattern(TRANSPILED_PYTHON_UDF)) {
+        case op if op.containsPattern(TRANSPILED_PYTHON_UDF) =>
+          // Bottom-up so a nested TranspiledPythonUDF (a transpiled UDF feeding another) is pruned
+          // -- and thus resolved -- before its parent's input types are inspected.
+          op.transformExpressionsUpWithPruning(_.containsPattern(TRANSPILED_PYTHON_UDF)) {
+            case t: TranspiledPythonUDF
+                if t.optionInputCategories.nonEmpty && t.pythonUDFExpr.childrenResolved =>
+              val argTypes = t.pythonUDFExpr.children.map(_.dataType)
+              val kept = t.transpiledOptions.zip(t.optionInputCategories).collect {
+                case (option, categories) if optionMatchesTypes(categories, argTypes) => option
+              }
+              t.copy(transpiledOptions = kept, optionInputCategories = Nil)
+          }
+      }
+    }
+  }
+
+  // True when each declared category matches the corresponding argument type:
+  // "numeric" -> NumericType, "string" -> StringType, "bool" -> BooleanType,
+  // "binary" -> BinaryType. "string" matches only StringType (not BinaryType): a
+  // bytes/BinaryType column is tagged "binary" instead, so the string lowerings
+  // (e.g. `repeat`) never see it. Empty categories means "no restriction", so the
+  // option is kept.
+  //
+  // Two deliberate exclusions keep the transpiled semantics faithful to Python:
+  // - DecimalType is NOT "numeric": Python receives decimal.Decimal objects,
+  //   which raise TypeError when mixed with float literals and carry different
+  //   precision semantics than Spark's decimal arithmetic, so decimal columns
+  //   fall back to interpreted Python.
+  // - "string" requires the default UTF8_BINARY collation: under a non-binary
+  //   collation (e.g. UTF8_LCASE) Spark's `=`/`<`/`concat` follow collation
+  //   rules while Python compares codepoints, so `'abc' == 'ABC'` would return
+  //   true where Python returns False.
+  private def optionMatchesTypes(categories: Seq[String], argTypes: Seq[DataType]): Boolean = {
+    if (categories.isEmpty) {
+      true
+    } else if (categories.length != argTypes.length) {
+      false
+    } else {
+      categories.zip(argTypes).forall {
+        case ("numeric", dt) => dt.isInstanceOf[NumericType] && !dt.isInstanceOf[DecimalType]
+        case ("string", st: StringType) => st.isUTF8BinaryCollation
+        case ("bool", dt) => dt.isInstanceOf[BooleanType]
+        case ("binary", dt) => dt.isInstanceOf[BinaryType]
+        case _ => false
+      }
+    }
+  }
+}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala
index 834f3b0debd0..4959df90a53e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala
@@ -519,6 +519,9 @@ trait NonSQLExpression extends Expression {
       case a: Attribute => new PrettyAttribute(a)
       case a: Alias => PrettyAttribute(a.sql, a.dataType)
       case p: PythonFuncExpression => PrettyPythonUDF(p.name, p.dataType, p.children)
+      // Render a transpiled UDF like the UDF it wraps (options must not leak
+      // into user-visible strings).
+      case t: TranspiledPythonUDF => PrettyPythonUDF(t.name, t.dataType, t.pythonUDFExpr.children)
     }.toString
   }
 }
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/PythonUDF.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/PythonUDF.scala
index 99619d9de96c..48da53ccf7c2 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/PythonUDF.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/PythonUDF.scala
@@ -23,7 +23,8 @@ import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.analysis.UnresolvedException
 import org.apache.spark.sql.catalyst.expressions.aggregate.AggregateFunction
 import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode}
-import org.apache.spark.sql.catalyst.trees.TreePattern.{PYTHON_UDF, TreePattern}
+import org.apache.spark.sql.catalyst.trees.TreePattern.{PYTHON_UDF, TRANSPILED_PYTHON_UDF,
+  TreePattern}
 import org.apache.spark.sql.catalyst.util.toPrettySQL
 import org.apache.spark.sql.errors.{QueryCompilationErrors, QueryExecutionErrors}
 import org.apache.spark.sql.types._
@@ -89,6 +90,41 @@ trait PythonFuncExpression extends NonSQLExpression with UserDefinedExpression {
   override def nullable: Boolean = true
 }
 
+
+case class TranspiledPythonUDF(
+  name: String,
+  pythonUDFExpr: Expression,
+  transpiledOptions: List[Expression],
+  // Per-option input-type categories ("numeric"/"string" per public param),
+  // parallel to `transpiledOptions`. ResolveTranspiledPythonUDFOptions prunes the
+  // options to those whose categories match the resolved input types (before
+  // CheckAnalysis can reject a type-incompatible option) and clears this field;
+  // ConvertToCatalyst then picks the first survivor or falls back to the Python
+  // UDF. Empty means "no restriction" (kept as-is).
+  optionInputCategories: List[List[String]] = Nil) extends Expression with Unevaluable {
+  require(
+    optionInputCategories.isEmpty || optionInputCategories.length == transpiledOptions.length,
+    s"optionInputCategories (${optionInputCategories.length}) must be parallel to " +
+    s"transpiledOptions (${transpiledOptions.length}) or empty"
+  )
+  override def children: Seq[Expression] = pythonUDFExpr +: transpiledOptions
+  override def dataType: DataType = pythonUDFExpr.dataType
+  override def nullable: Boolean = pythonUDFExpr.nullable
+  override protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]):
+      TranspiledPythonUDF =
+    copy(pythonUDFExpr = newChildren.head, transpiledOptions = newChildren.tail.toList)
+  final override val nodePatterns: Seq[TreePattern] = Seq(TRANSPILED_PYTHON_UDF)
+
+  // True when every direct input to pythonUDFExpr is a plain PythonUDF (not a
+  // TranspiledPythonUDF). Used to decide whether to preserve the UDF batch pipeline
+  // rather than inserting a Catalyst node in the middle of a Python UDF chain.
+  def hasOnlyPythonUDFInputs: Boolean =
+    pythonUDFExpr.children.nonEmpty &&
+    pythonUDFExpr.children.forall {
+      _.isInstanceOf[PythonUDF]
+    }
+}
+
 /**
  * A serialized version of a Python lambda function. This is a special expression, which needs a
  * dedicated physical operator to execute it, and thus can't be pushed down to data sources.
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
index fec31c21aa53..76fed0428be8 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
@@ -175,6 +175,13 @@ abstract class Optimizer(catalogManager: CatalogManager)
         PushDownPredicates))
 
     val batches: Seq[Batch] = flattenBatches(Seq(
+    // UDF substitution rules should be executed before any other optimization rules
+    // so that the substituted Catalyst alternatives go through the same finalization
+    // (FinishAnalysis, RewriteWithExpression, etc.) and downstream optimization as
+    // any other expression. Anything ConvertToCatalyst leaves behind -- including
+    // RuntimeReplaceable nodes inside transpiled options -- is still rewritten by
+    // the FinishAnalysis batch that runs immediately after.
+    Batch("Convert python UDFs to Catalyst", Once, ConvertToCatalyst),
     Batch("Finish Analysis", FixedPoint(1), FinishAnalysis),
     // We must run this batch after `ReplaceExpressions`, as `RuntimeReplaceable` expression
     // may produce `With` expressions that need to be rewritten.
@@ -305,6 +312,10 @@ abstract class Optimizer(catalogManager: CatalogManager)
    */
   def nonExcludableRules: Seq[String] =
     Seq(
+      // ConvertToCatalyst is the only rule that strips the Unevaluable
+      // TranspiledPythonUDF node; excluding it would leak that node into
+      // execution, so it must never be excludable.
+      ConvertToCatalyst.ruleName,
       FinishAnalysis.ruleName,
       RewriteDistinctAggregates.ruleName,
       ReplaceDeduplicateWithAggregate.ruleName,
@@ -1007,6 +1018,80 @@ object LimitPushDown extends Rule[LogicalPlan] {
   }
 }
 
+/**
+ * Attempt to convert UDFS to Catalyst expressions.
+ */
+object ConvertToCatalyst extends Rule[LogicalPlan] {
+  def apply(plan: LogicalPlan): LogicalPlan = {
+    // Short circuit if there are no Transpiled Python UDFs in the plan.
+    if (!plan.containsPattern(TRANSPILED_PYTHON_UDF)) {
+      return plan
+    }
+    // Traverse subquery plans too: this batch runs Once, and later rules (e.g.
+    // PullupCorrelatedPredicates) can move expressions from a subquery into the
+    // outer plan, so an Unevaluable TranspiledPythonUDF left inside a subquery
+    // here could otherwise escape and reach execution un-stripped.
+    plan.transformDownWithSubqueriesAndPruning(
+      _.containsPattern(TRANSPILED_PYTHON_UDF), ruleId) {
+      case p => p.transformExpressionsWithPruning(_.containsPattern(TRANSPILED_PYTHON_UDF)) {
+        case s: TranspiledPythonUDF => applyExpr(s, parentIsUdf = false)
+      }
+    }
+  }
+
+  def applyExpr(expression: Expression, parentIsUdf: Boolean = false): Expression = {
+    expression match {
+      case s: TranspiledPythonUDF =>
+        // We _shouldn't_ have these nodes if ANSI is not enabled or transpilation is disabled
+        // but if someone changed it while running we'll want to strip the nodes out.
+        if (!conf.getConf(SQLConf.ANSI_ENABLED)) {
+          logWarning(log"Skipping Python UDF transpilation: " +
+            log"${MDC(LogKeys.CONFIG, SQLConf.ANSI_ENABLED.key)} is disabled. The transpiler " +
+            log"targets ANSI semantics and refuses to rewrite plans under non-ANSI mode. " +
+            log"Enable ANSI or disable transpilation to silence this warning.")
+          s.pythonUDFExpr.mapChildren(applyExpr(_, parentIsUdf = true))
+        } else if (!conf.getConf(SQLConf.ATTEMPT_TRANSPILATION_OF_PYTHON_UDFS)) {
+          logWarning(log"Skipping Python UDF transpilation: " +
+            log"${MDC(LogKeys.CONFIG, SQLConf.ATTEMPT_TRANSPILATION_OF_PYTHON_UDFS.key)} " +
+            log"is disabled but we still got TranspiledPythonUDFs in our plan.")
+          s.pythonUDFExpr.mapChildren(applyExpr(_, parentIsUdf = true))
+        } else if (!parentIsUdf || !s.hasOnlyPythonUDFInputs) {
+          // Walk the full list of transpiled options and pick the first one,
+          // falling back to the original Python UDF if none are available.
+          // Options whose declared input-type categories don't match the bound
+          // column types are already pruned during analysis by
+          // ResolveTranspiledPythonUDFOptions, so any option that reaches here is
+          // safe to use. If you're plugging in your own transpilation, please add
+          // a separate ConvertToX so you can choose your desired transpiled nodes.
+          // NOTE: the substituted option is used as-is, with no cast back to the
+          // UDF's declared return type. The built-in transpiler guarantees each
+          // option's dataType already matches; a custom transpiler MUST do the
+          // same (or insert its own Cast), or it will silently change the output
+          // schema.
+          val firstEvaluable = s.transpiledOptions.headOption
+          firstEvaluable match {
+            case None =>
+              s.pythonUDFExpr.mapChildren(applyExpr(_, parentIsUdf = true))
+            case Some(catalystExpr) =>
+              // Recursively apply to the children first because we may use them as inputs in parent
+              catalystExpr.mapChildren(applyExpr(_, parentIsUdf = false))
+          }
+        } else {
+          // We should avoid converting a UDF node where that could break pipelining.
+          // For example: (UDF -> UDF -> UDF) is often cheaper than UDF -> Catalyst -> UDF.
+          s.pythonUDFExpr.mapChildren(applyExpr(_, parentIsUdf = true))
+        }
+      case _ =>
+        // Not a TranspiledPythonUDF: recurse down, telling the children whether
+        // this node is itself a scalar Python UDF so a transpiled child can
+        // preserve the UDF batch pipeline (e.g. an outer UDF that could not be
+        // transpiled wrapping one that could).
+        expression.mapChildren(
+          applyExpr(_, parentIsUdf = isScalarPythonUDF(expression)))
+    }
+  }
+}
+
 /**
  * Pushes Project operator to both sides of a Union operator.
  * Operations that are safe to pushdown are listed as follows.
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/RuleIdCollection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/RuleIdCollection.scala
index a71b4bc6380f..d2c1ab9a8bc2 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/RuleIdCollection.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/rules/RuleIdCollection.scala
@@ -133,6 +133,7 @@ object RuleIdCollection {
       "org.apache.spark.sql.catalyst.optimizer.CombineUnions" ::
       "org.apache.spark.sql.catalyst.optimizer.ConstantFolding" ::
       "org.apache.spark.sql.catalyst.optimizer.ConstantPropagation" ::
+      "org.apache.spark.sql.catalyst.optimizer.ConvertToCatalyst" ::
       "org.apache.spark.sql.catalyst.optimizer.ConvertToLocalRelation" ::
       "org.apache.spark.sql.catalyst.optimizer.CostBasedJoinReorder" ::
       "org.apache.spark.sql.catalyst.optimizer.DecimalAggregates" ::
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreePatterns.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreePatterns.scala
index dfb815414dd3..990808bf30a5 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreePatterns.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreePatterns.scala
@@ -106,6 +106,7 @@ object TreePattern extends Enumeration  {
   val TEMP_RESOLVED_COLUMN: Value = Value
   val TIME_WINDOW: Value = Value
   val TIME_ZONE_AWARE_EXPRESSION: Value = Value
+  val TRANSPILED_PYTHON_UDF: Value = Value
   val TRUE_OR_FALSE_LITERAL: Value = Value
   val USER_DEFINED_AGGREGATION: Value = Value
   val VARIANT_GET: Value = Value
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/package.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/package.scala
index a79e53ab85ef..2a412c538b0c 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/package.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/package.scala
@@ -130,6 +130,11 @@ package object util extends Logging {
     case c: Cast if !c.containsTag(Cast.USER_SPECIFIED_CAST) =>
       PrettyAttribute(usePrettyExpression(c.child, shouldTrimTempResolvedColumn).sql, c.dataType)
     case p: PythonFuncExpression => PrettyPythonUDF(p.name, p.dataType, p.children)
+    // Present a transpiled UDF exactly like the UDF it wraps, so auto-generated
+    // column names stay `f(a)` whether or not transpilation engages (the node
+    // carries the rewrite options as extra children, which must not leak into
+    // user-visible names).
+    case t: TranspiledPythonUDF => PrettyPythonUDF(t.name, t.dataType, t.pythonUDFExpr.children)
   }
 
   def quoteIdentifier(name: String): String = {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala
index bf11971e99fe..65b5f2b9e3a2 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala
@@ -67,6 +67,22 @@ private[sql] object QueryCompilationErrors extends QueryErrorsBase with Compilat
     )
   }
 
+  def invalidUDFParameterPlaceholder(placeholder: String): Throwable = {
+    new AnalysisException(
+      errorClass = "INVALID_UDF_PARAMETER_PLACEHOLDER",
+      messageParameters = Map("placeholder" -> placeholder)
+    )
+  }
+
+  def invalidUDFParameterPlaceholderIndex(index: Int, numParams: Int): Throwable = {
+    new AnalysisException(
+      errorClass = "INVALID_UDF_PARAMETER_PLACEHOLDER_INDEX",
+      messageParameters = Map(
+        "index" -> index.toString,
+        "numParams" -> numParams.toString)
+    )
+  }
+
   def positionalAndNamedArgumentDoubleReference(
       routineName: String, parameterName: String): Throwable = {
     val errorClass =
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index beb8d5ee1458..1a0896df0417 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -610,6 +610,34 @@ object SQLConf {
     .booleanConf
     .createWithDefault(true)
 
+  val ATTEMPT_TRANSPILATION_OF_PYTHON_UDFS =
+    buildConf("spark.sql.experimental.optimizer.transpilePyUDFs")
+    .withBindingPolicy(ConfigBindingPolicy.SESSION)
+    .doc("When true, attempt to transpile Python UDFs to Catalyst expressions. " +
+        "Transpilation also requires ANSI mode (spark.sql.ansi.enabled=true) -- " +
+        "the rewritten expressions target ANSI semantics, so with ANSI off the " +
+        "transpiler falls back to interpreted Python and a warning is logged at " +
+        "UDF construction. Transpiled UDFS attempt to match the Python functionality but " +
+        "may not be 100% equivalent. Some known differences include: overflows from input types " +
+        "(you can precast to decimal to avoid), type coercion on comparison, and implicit " +
+        "returns. This initial version only works with non-Connect Spark; Spark Connect " +
+        "support is to follow. This is an *experimental* feature."
+    )
+    .version("4.3.0")
+    .booleanConf
+    .createWithDefault(false)
+
+
+  val PYTHON_UDF_TRANSPILERS =
+    buildConf("spark.sql.experimental.optimizer.pyTranspilers")
+    .withBindingPolicy(ConfigBindingPolicy.SESSION)
+    .doc("Comma-separated list of Python transpilers to attempt, in order. " +
+      "The first transpiler that successfully produces a Catalyst expression " +
+      "is used. Default: catalyst.")
+    .version("4.3.0")
+    .stringConf
+    .createWithDefault("catalyst")
+
   val OPTIMIZER_EXCLUDED_RULES = buildConf("spark.sql.optimizer.excludedRules")
     .doc("Configures a list of rules to be disabled in the optimizer, in which the rules are " +
       "specified by their rule names and separated by comma. It is not guaranteed that all the " +
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveTranspiledPythonUDFOptionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveTranspiledPythonUDFOptionsSuite.scala
new file mode 100644
index 000000000000..f9295eb17b00
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveTranspiledPythonUDFOptionsSuite.scala
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.analysis
+
+import org.apache.spark.api.python.PythonEvalType
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.expressions.{Add, Alias, Concat, Expression, Literal, PythonUDF, TranspiledPythonUDF}
+import org.apache.spark.sql.catalyst.plans.PlanTest
+import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, Project}
+import org.apache.spark.sql.types.LongType
+
+/**
+ * Unit tests for [[ResolveTranspiledPythonUDFOptions]], which prunes a
+ * TranspiledPythonUDF's per-input-type options to those whose declared categories match the
+ * resolved argument types. func=null in the leaf PythonUDF is intentional: these structural
+ * tests don't execute Python.
+ */
+class ResolveTranspiledPythonUDFOptionsSuite extends PlanTest {
+
+  private def pyUDF(children: Seq[Expression]): PythonUDF =
+    PythonUDF("udf", null, LongType, children,
+      PythonEvalType.SQL_BATCHED_UDF, udfDeterministic = true)
+
+  // Runs the rule on a Project that wraps the node, and returns the (possibly pruned) node.
+  private def prune(node: TranspiledPythonUDF, rel: LocalRelation): TranspiledPythonUDF = {
+    val rewritten = ResolveTranspiledPythonUDFOptions(Project(Seq(Alias(node, "r")()), rel))
+    rewritten.expressions.flatMap(_.collect { case t: TranspiledPythonUDF => t }).head
+  }
+
+  test("keeps the numeric option for numeric columns and drops the string one") {
+    val a = $"a".long
+    val b = $"b".long
+    val numericOpt = Add(a, b)
+    val stringOpt = Concat(Seq(a, b))
+    val node = TranspiledPythonUDF("udf", pyUDF(Seq(a, b)), List(numericOpt, stringOpt),
+      List(List("numeric", "numeric"), List("string", "string")))
+    val pruned = prune(node, LocalRelation(a, b))
+    assert(pruned.transpiledOptions == List(numericOpt))
+    assert(pruned.optionInputCategories.isEmpty)
+  }
+
+  test("keeps the string option for string columns and drops the numeric one") {
+    val a = $"a".string
+    val b = $"b".string
+    val numericOpt = Add(a, b)
+    val stringOpt = Concat(Seq(a, b))
+    val node = TranspiledPythonUDF("udf", pyUDF(Seq(a, b)), List(numericOpt, stringOpt),
+      List(List("numeric", "numeric"), List("string", "string")))
+    val pruned = prune(node, LocalRelation(a, b))
+    assert(pruned.transpiledOptions == List(stringOpt))
+    assert(pruned.optionInputCategories.isEmpty)
+  }
+
+  test("empties the options when no category set matches (falls back to Python UDF)") {
+    val a = $"a".string
+    val b = $"b".long
+    val node = TranspiledPythonUDF("udf", pyUDF(Seq(a, b)),
+      List(Add(a, b), Concat(Seq(a, b))),
+      List(List("numeric", "numeric"), List("string", "string")))
+    val pruned = prune(node, LocalRelation(a, b))
+    assert(pruned.transpiledOptions.isEmpty)
+    assert(pruned.optionInputCategories.isEmpty)
+  }
+
+  test("matches binary columns against neither category (string is StringType only)") {
+    val a = $"a".binary
+    val node = TranspiledPythonUDF("udf", pyUDF(Seq(a)),
+      List(Concat(Seq(a, a))), List(List("string")))
+    val pruned = prune(node, LocalRelation(a))
+    assert(pruned.transpiledOptions.isEmpty)
+  }
+
+  test("leaves options untouched when categories are empty (no restriction)") {
+    val a = $"a".long
+    val onlyOpt = Add(a, Literal(1L))
+    val node = TranspiledPythonUDF("udf", pyUDF(Seq(a)), List(onlyOpt), Nil)
+    val pruned = prune(node, LocalRelation(a))
+    assert(pruned.transpiledOptions == List(onlyOpt))
+  }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ConvertToCatalystSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ConvertToCatalystSuite.scala
new file mode 100644
index 000000000000..23baf25a5897
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/ConvertToCatalystSuite.scala
@@ -0,0 +1,237 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.optimizer
+
+import org.apache.spark.api.python.PythonEvalType
+import org.apache.spark.sql.catalyst.dsl.expressions._
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateExpression, Count}
+import org.apache.spark.sql.catalyst.plans.PlanTest
+import org.apache.spark.sql.catalyst.plans.logical.{Filter, LocalRelation, Project}
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.types.{BooleanType, IntegerType, LongType}
+
+/**
+ * Unit tests for the ConvertToCatalyst optimizer rule, which rewrites
+ * TranspiledPythonUDF nodes to their Catalyst equivalents.
+ *
+ * These tests exercise the rule directly via applyExpr rather than running the
+ * full optimizer pipeline, which means no JVM/Python bridge is required.
+ */
+class ConvertToCatalystSuite extends PlanTest {
+
+  private val attrA = $"a".long
+
+  // A leaf PythonUDF that takes one column argument. func=null is intentional:
+  // structural tests don't need an executable PythonFunction.
+  private def makePyUDF(input: Expression = attrA): PythonUDF =
+    PythonUDF("udf", null, LongType, Seq(input),
+      PythonEvalType.SQL_BATCHED_UDF, udfDeterministic = true)
+
+  // A leaf PythonUDAF (grouped-agg pandas eval type, return type Long for parity
+  // with Count's output). func=null is intentional, as with makePyUDF.
+  private def makePyUDAF(input: Expression = attrA): PythonUDAF =
+    PythonUDAF("agg", null, LongType, Seq(input),
+      udfDeterministic = true,
+      evalType = PythonEvalType.SQL_GROUPED_AGG_PANDAS_UDF)
+
+  // A TranspiledPythonUDF wrapping pyUDF with a single Catalyst option.
+  private def makeTPUDF(pyUDF: PythonUDF, catalystOpt: Expression): TranspiledPythonUDF =
+    TranspiledPythonUDF("udf", pyUDF, List(catalystOpt))
+
+  private val catalystExpr: Expression = Add(attrA, Literal(4L))
+
+  // ---- helpers ----
+
+  // Both ANSI and ATTEMPT_TRANSPILATION must be true for the transpile path to fire.
+  private def transpileOn[T](block: => T): T =
+    withSQLConf(
+      SQLConf.ANSI_ENABLED.key -> "true",
+      SQLConf.ATTEMPT_TRANSPILATION_OF_PYTHON_UDFS.key -> "true") { block }
+
+  private def ansiOff[T](block: => T): T =
+    withSQLConf(
+      SQLConf.ANSI_ENABLED.key -> "false",
+      SQLConf.ATTEMPT_TRANSPILATION_OF_PYTHON_UDFS.key -> "true") { block }
+
+  private def transpileOff[T](block: => T): T =
+    withSQLConf(
+      SQLConf.ANSI_ENABLED.key -> "true",
+      SQLConf.ATTEMPT_TRANSPILATION_OF_PYTHON_UDFS.key -> "false") { block }
+
+  // ---- tests ----
+
+  test("transpiles when not nested (parentIsUdf = false)") {
+    transpileOn {
+      val tpudf = makeTPUDF(makePyUDF(), catalystExpr)
+      val result = ConvertToCatalyst.applyExpr(tpudf, parentIsUdf = false)
+      assert(!result.isInstanceOf[TranspiledPythonUDF])
+      assert(!result.isInstanceOf[PythonUDF])
+    }
+  }
+
+  test("prevents transpilation when parentIsUdf=true and inputs are plain PythonUDFs") {
+    // PythonUDF -> TranspiledPythonUDF -> PythonUDF: the middle node should NOT be
+    // transpiled when called from an outer UDF context, to preserve the batch pipeline.
+    transpileOn {
+      val innerPyUDF = makePyUDF(attrA)
+      val outerPyUDF = makePyUDF(innerPyUDF)
+      val outerTPUDF = makeTPUDF(outerPyUDF, Add(innerPyUDF, Literal(4L)))
+      val result = ConvertToCatalyst.applyExpr(outerTPUDF, parentIsUdf = true)
+      assert(result.isInstanceOf[PythonUDF])
+      assert(!result.isInstanceOf[TranspiledPythonUDF])
+    }
+  }
+
+  test("does not prevent transpilation when input to pythonUDFExpr is a TranspiledPythonUDF") {
+    // When the input to a TPUDF is itself a TranspiledPythonUDF (has a Catalyst alternative),
+    // hasOnlyPythonUDFInputs returns false so the outer TPUDF still transpiles.
+    transpileOn {
+      val innerPyUDF = makePyUDF(attrA)
+      val innerTPUDF = makeTPUDF(innerPyUDF, catalystExpr)
+      val outerPyUDF = makePyUDF(innerTPUDF)
+      val outerTPUDF = makeTPUDF(outerPyUDF, Add(innerTPUDF, Literal(4L)))
+      val result = ConvertToCatalyst.applyExpr(outerTPUDF, parentIsUdf = true)
+      assert(!result.isInstanceOf[TranspiledPythonUDF])
+      assert(!result.isInstanceOf[PythonUDF])
+    }
+  }
+
+  test("hasOnlyPythonUDFInputs unit test") {
+    val innerPyUDF = makePyUDF(attrA)
+    val innerTPUDF = makeTPUDF(innerPyUDF, catalystExpr)
+
+    // pythonUDFExpr's child is a plain PythonUDF -> true
+    assert(makeTPUDF(makePyUDF(innerPyUDF), catalystExpr).hasOnlyPythonUDFInputs)
+    // pythonUDFExpr's child is a TranspiledPythonUDF -> false
+    assert(!makeTPUDF(makePyUDF(innerTPUDF), catalystExpr).hasOnlyPythonUDFInputs)
+    // pythonUDFExpr's child is a plain column (leaf) -> false
+    assert(!makeTPUDF(makePyUDF(attrA), catalystExpr).hasOnlyPythonUDFInputs)
+    // zero-arg pythonUDFExpr -> false (nonEmpty guard)
+    val zeroPyUDF = PythonUDF("udf", null, LongType, Seq.empty,
+      PythonEvalType.SQL_BATCHED_UDF, udfDeterministic = true)
+    assert(!TranspiledPythonUDF("udf", zeroPyUDF, List(Literal(42L))).hasOnlyPythonUDFInputs)
+  }
+
+  test("falls back to PythonUDF when ANSI is disabled") {
+    ansiOff {
+      val tpudf = makeTPUDF(makePyUDF(), catalystExpr)
+      val result = ConvertToCatalyst.applyExpr(tpudf, parentIsUdf = false)
+      assert(result.isInstanceOf[PythonUDF])
+      assert(!result.isInstanceOf[TranspiledPythonUDF])
+    }
+  }
+
+  test("falls back to PythonUDF when transpilation is disabled") {
+    transpileOff {
+      val tpudf = makeTPUDF(makePyUDF(), catalystExpr)
+      val result = ConvertToCatalyst.applyExpr(tpudf, parentIsUdf = false)
+      assert(result.isInstanceOf[PythonUDF])
+      assert(!result.isInstanceOf[TranspiledPythonUDF])
+    }
+  }
+
+  test("falls back to PythonUDF when transpiledOptions is empty") {
+    transpileOn {
+      val pyUDF = makePyUDF()
+      val tpudf = TranspiledPythonUDF("udf", pyUDF, List())
+      val result = ConvertToCatalyst.applyExpr(tpudf, parentIsUdf = false)
+      assert(result.isInstanceOf[PythonUDF])
+      assert(!result.isInstanceOf[TranspiledPythonUDF])
+    }
+  }
+
+  test("apply(plan) reaches TranspiledPythonUDF nodes below the root") {
+    // Regression test for the traversal bug where ``plan.mapExpressions`` only
+    // walks expressions on the root plan node. With that bug, a TPUDF inside a
+    // Filter (or any non-root node) would survive the optimizer rule as an
+    // ``Unevaluable`` expression and crash at execution. The fix uses
+    // ``transformAllExpressionsWithPruning`` which descends through child
+    // plans; this test pins that contract.
+    transpileOn {
+      val attrB = $"b".long
+      val relation = LocalRelation(attrA, attrB)
+      // The TPUDF lives in the Filter's condition (boolean), not at the root.
+      val booleanTPUDF = TranspiledPythonUDF(
+        "udf",
+        PythonUDF("udf", null, BooleanType, Seq(attrA),
+          PythonEvalType.SQL_BATCHED_UDF, udfDeterministic = true),
+        List(GreaterThan(attrA, Literal(0L))))
+      val plan = Project(Seq(attrB), Filter(booleanTPUDF, relation))
+      val rewritten = ConvertToCatalyst.apply(plan)
+      // No TranspiledPythonUDF should remain anywhere in the rewritten plan.
+      val leftover = rewritten.collect {
+        case p if p.expressions.exists(_.find(_.isInstanceOf[TranspiledPythonUDF]).isDefined) =>
+          p
+      }
+      assert(leftover.isEmpty,
+        s"TranspiledPythonUDF survived ConvertToCatalyst.apply: $rewritten")
+      // The Filter's condition must be the resolved Catalyst expression, not a fallback PythonUDF.
+      val filterCond = rewritten.asInstanceOf[Project].child.asInstanceOf[Filter].condition
+      assert(filterCond == GreaterThan(attrA, Literal(0L)),
+        s"Filter condition was not rewritten to GreaterThan: $filterCond")
+    }
+  }
+
+  test("uses pre-coerced transpiledOptions as-is (analysis is responsible for coercion)") {
+    // The Analyzer coerces transpiledOptions before the optimizer runs, because
+    // TranspiledPythonUDF.children exposes them to the resolver's generic coercion pass.
+    // ConvertToCatalyst must not re-run coercion; it simply selects the first non-null option.
+    // This test simulates what analysis would produce for `def f(x: Long): return x + 4`
+    // where the integer literal has already been cast to LongType.
+    transpileOn {
+      val preCoerced = Add(attrA, Cast(Literal(4, IntegerType), LongType))
+      val tpudf = makeTPUDF(makePyUDF(), preCoerced)
+      val result = ConvertToCatalyst.applyExpr(tpudf, parentIsUdf = false)
+      assert(result == preCoerced,
+        s"Expected pre-coerced expression unchanged, got: $result")
+    }
+  }
+
+  // ---- UDAF cases (post-fromUDFExpr shape) ----
+  //
+  // After UserDefinedPythonFunction.fromUDFExpr lifts a PythonUDAF inside a
+  // TranspiledPythonUDF, the wrapper holds an AggregateExpression instead of a
+  // bare PythonUDAF. These tests pin the optimizer rule's behavior on that shape.
+
+  test("transpiles TranspiledPythonUDF wrapping AggregateExpression(PythonUDAF)") {
+    transpileOn {
+      val pyAgg = makePyUDAF().toAggregateExpression()
+      val catalystAgg = Count(Seq(attrA)).toAggregateExpression()
+      val tpudf = TranspiledPythonUDF("agg", pyAgg, List(catalystAgg))
+      val result = ConvertToCatalyst.applyExpr(tpudf, parentIsUdf = false)
+      assert(result == catalystAgg,
+        s"Expected catalyst aggregate alternative, got: $result")
+    }
+  }
+
+  test("falls back to AggregateExpression(PythonUDAF) when ANSI is off (UDAF)") {
+    ansiOff {
+      val pyAgg = makePyUDAF().toAggregateExpression()
+      val catalystAgg = Count(Seq(attrA)).toAggregateExpression()
+      val tpudf = TranspiledPythonUDF("agg", pyAgg, List(catalystAgg))
+      val result = ConvertToCatalyst.applyExpr(tpudf, parentIsUdf = false)
+      result match {
+        case ae: AggregateExpression =>
+          assert(ae.aggregateFunction.isInstanceOf[PythonUDAF],
+            s"Expected aggregateFunction to be PythonUDAF, got: ${ae.aggregateFunction}")
+        case other => fail(s"Expected AggregateExpression(PythonUDAF, ...), got: $other")
+      }
+    }
+  }
+}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/UserDefinedPythonFunction.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/UserDefinedPythonFunction.scala
index 41bffaca65cf..cfd1fb4ea222 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/python/UserDefinedPythonFunction.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/python/UserDefinedPythonFunction.scala
@@ -18,18 +18,22 @@
 package org.apache.spark.sql.execution.python
 
 import java.io.{DataInputStream, DataOutputStream}
+import java.util.{List => JList}
 
 import scala.collection.mutable.ArrayBuffer
+import scala.jdk.CollectionConverters._
 
 import net.razorvine.pickle.Pickler
 
 import org.apache.spark.api.python.{PythonEvalType, PythonFunction, PythonWorkerUtils, SpecialLengths}
 import org.apache.spark.sql.{Column, TableArg}
-import org.apache.spark.sql.catalyst.expressions.{Alias, Ascending, Descending, Expression, FunctionTableSubqueryArgumentExpression, NamedArgumentExpression, NullsFirst, NullsLast, PythonUDAF, PythonUDF, PythonUDTF, PythonUDTFAnalyzeResult, PythonUDTFSelectedExpression, SortOrder, UnresolvedPolymorphicPythonUDTF, UnresolvedTableArgPlanId}
+import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute
+import org.apache.spark.sql.catalyst.expressions.{Alias, Ascending, Descending, Expression, FunctionTableSubqueryArgumentExpression, NamedArgumentExpression, NullsFirst, NullsLast, PythonUDAF, PythonUDF, PythonUDTF, PythonUDTFAnalyzeResult, PythonUDTFSelectedExpression, SortOrder, TranspiledPythonUDF, UnresolvedPolymorphicPythonUDTF, UnresolvedTableArgPlanId}
 import org.apache.spark.sql.catalyst.parser.ParserInterface
 import org.apache.spark.sql.catalyst.plans.logical.{Generate, LogicalPlan, NamedParametersSupport, OneRowRelation}
 import org.apache.spark.sql.classic.{DataFrame, Dataset, SparkSession}
 import org.apache.spark.sql.classic.ClassicConversions._
+import org.apache.spark.sql.classic.ColumnConversions
 import org.apache.spark.sql.classic.ExpressionUtils.expression
 import org.apache.spark.sql.errors.QueryCompilationErrors
 import org.apache.spark.sql.internal.{SQLConf, TableValuedFunctionArgument}
@@ -43,7 +47,16 @@ case class UserDefinedPythonFunction(
     func: PythonFunction,
     dataType: DataType,
     pythonEvalType: Int,
-    udfDeterministic: Boolean) {
+    udfDeterministic: Boolean,
+    // TODO: Add support for transpilation with Spark Connect and remove the default value.
+    transpiled: JList[Column] = Nil.asJava,
+    // Per-option input-type categories ("numeric"/"string" per public param),
+    // parallel to `transpiled` (same length). The analyzer rule
+    // ResolveTranspiledPythonUDFOptions later keeps only the options whose
+    // categories match the bound argument types; when none match, the call
+    // falls back to the plain Python UDF. `builder` requires the two lists to
+    // be parallel and skips transpilation otherwise.
+    transpiledInputTypes: JList[JList[String]] = Nil.asJava) {
 
   def builder(e: Seq[Expression]): Expression = {
     if (pythonEvalType == PythonEvalType.SQL_BATCHED_UDF
@@ -63,8 +76,13 @@ case class UserDefinedPythonFunction(
     } else if (e.exists(_.isInstanceOf[NamedArgumentExpression])) {
       throw QueryCompilationErrors.namedArgumentsNotSupported(name)
     }
+    val transpiledExprs: List[Expression] = transpiled.asScala.map(
+      column => ColumnConversions.expression(column)).toList
+    val optionInputTypes: List[List[String]] =
+      transpiledInputTypes.asScala.map(_.asScala.toList).toList
+
 
-    if (pythonEvalType == PythonEvalType.SQL_GROUPED_AGG_PANDAS_UDF
+    val udfExpr = if (pythonEvalType == PythonEvalType.SQL_GROUPED_AGG_PANDAS_UDF
       || pythonEvalType == PythonEvalType.SQL_GROUPED_AGG_PANDAS_ITER_UDF
       || pythonEvalType == PythonEvalType.SQL_GROUPED_AGG_ARROW_UDF
       || pythonEvalType == PythonEvalType.SQL_GROUPED_AGG_ARROW_ITER_UDF) {
@@ -72,6 +90,71 @@ case class UserDefinedPythonFunction(
     } else {
       PythonUDF(name, func, dataType, e, pythonEvalType, udfDeterministic)
     }
+    // The ``_udf_param_N`` substitution below is positional, so a UDF
+    // call site that supplied named arguments (e.g. SQL ``name => val``
+    // or pyspark ``udf(b=col)``) would splice ``NamedArgumentExpression``
+    // wrappers into the rewritten Catalyst tree and confuse downstream
+    // function resolution (``isnotnull`` etc. reject named parameters).
+    // The Python ``__call__`` shim resolves kwargs to positional before
+    // they reach this builder; SQL named arguments don't go through that
+    // shim, so we conservatively skip transpilation here when any child
+    // is a ``NamedArgumentExpression`` and let the regular Python UDF
+    // path execute.
+    val transpiledExprsForUse =
+      if (e.exists(_.isInstanceOf[NamedArgumentExpression])) Nil else transpiledExprs
+    val optionInputTypesForUse =
+      if (e.exists(_.isInstanceOf[NamedArgumentExpression])) Nil else optionInputTypes
+    // If we have possible transpiled expressions insert the node carrying every
+    // option plus its declared input-type categories. We can't pick here: this
+    // builder runs at call-construction time, before the argument columns are
+    // bound, so their types aren't known yet. ResolveTranspiledPythonUDFOptions
+    // prunes the options to those matching the resolved input types (once known,
+    // and before CheckAnalysis), and ConvertToCatalyst picks the survivor.
+    // Only build the node when every option carries its parallel input-type
+    // categories. ResolveTranspiledPythonUDFOptions prunes type-incompatible
+    // options using those categories, but only when they are present (its guard
+    // is `optionInputCategories.nonEmpty`); an empty or mismatched categories
+    // list would leave a type-invalid option to fail CheckAnalysis instead of
+    // falling back. If the two lists don't line up, skip transpilation.
+    // A call-site arity mismatch (user passed more or fewer args than the UDF's
+    // parameters) must fall back to the plain Python UDF so the standard runtime
+    // TypeError surfaces. Each option's category list has exactly one entry per
+    // public parameter, so a length mismatch against the bound children detects
+    // both directions: too few args would otherwise trip the placeholder bounds
+    // check below as a misleading "internal error", and too many args on a
+    // zero/fewer-param UDF would otherwise silently succeed where Python raises.
+    if (transpiledExprsForUse.nonEmpty &&
+        optionInputTypesForUse.length == transpiledExprsForUse.length &&
+        optionInputTypesForUse.forall(_.length == e.length)) {
+      val udfChildren = udfExpr.children.toArray
+      // Resolve the `_udf_param_N` placeholders the transpiler emits into the bound
+      // UDF arguments. Apply this ONLY to the transpiled options -- never to
+      // `udfExpr` itself, whose children are the user's argument expressions. A user
+      // column literally named `_udf_param_N` passed as an argument must not be
+      // rewritten, so we leave `udfExpr` untouched.
+      def resolveUDFParams(expression: Expression, children: Array[Expression]): Expression = {
+        expression match {
+          case UnresolvedAttribute(nameParts)
+              if nameParts.length == 1 && nameParts.head.startsWith("_udf_param_") =>
+            val suffix = nameParts.head.stripPrefix("_udf_param_")
+            val index = suffix.toIntOption.getOrElse {
+              throw QueryCompilationErrors.invalidUDFParameterPlaceholder(nameParts.head)
+            }
+            if (index >= 0 && index < children.length) {
+              children(index)
+            } else {
+              throw QueryCompilationErrors.invalidUDFParameterPlaceholderIndex(
+                index, children.length)
+            }
+          case _ =>
+            expression.mapChildren(resolveUDFParams(_, children))
+        }
+      }
+      val resolvedOptions = transpiledExprsForUse.map(resolveUDFParams(_, udfChildren))
+      TranspiledPythonUDF(name, udfExpr, resolvedOptions, optionInputTypesForUse)
+    } else {
+      udfExpr
+    }
   }
 
   def builderWithColumns(e: Seq[Column]): Expression = builder(e.map(expression))
@@ -86,6 +169,8 @@ case class UserDefinedPythonFunction(
    */
   def fromUDFExpr(expr: Expression): Column = {
     Column(expr match {
+      case TranspiledPythonUDF(name, udaf: PythonUDAF, transpiled, inputCategories) =>
+        TranspiledPythonUDF(name, udaf.toAggregateExpression(), transpiled, inputCategories)
       case udaf: PythonUDAF => udaf.toAggregateExpression()
       case _ => expr
     })


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org