You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ho...@apache.org on 2026/09/28 06:19:04 UTC

(spark) branch branch-4.3 updated: [SPARK-55214][PYTHON][SQL] Transpile string len() to Catalyst length

This is an automated email from the ASF dual-hosted git repository.

asf-gitbox-commits pushed a commit to branch branch-4.3
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/branch-4.3 by this push:
     new 310f540a8cfc [SPARK-55214][PYTHON][SQL] Transpile string len() to Catalyst length
310f540a8cfc is described below

commit 310f540a8cfcf5fc890f5583d331afc1c11e365f
Author: Monica G <mo...@gmail.com>
AuthorDate: Sun Sep 27 23:18:15 2026 -0700

    [SPARK-55214][PYTHON][SQL] Transpile string len() to Catalyst length
    
    ### What changes were proposed in this pull request?
    
    This is a scoped slice of [SPARK-55214](https://issues.apache.org/jira/browse/SPARK-55214) (parent [SPARK-54783](https://issues.apache.org/jira/browse/SPARK-54783) / [SPIP](https://docs.google.com/document/d/1cHc6tiR4yO3hppTzrK1F1w9RwyEPMvaeEuL2ub2LURg/edit?usp=sharing)): lower Python `len(s)` on a string UDF argument to Catalyst `length`.
    
    String `+` / `*` already lower to `concat` / `repeat`. This PR only adds `len`. Other `len` arguments (numbers, lists, ...) still fall back to interpreted Python. NULL stays NULL, matching the existing unguarded-NULL caveat for `x + 1`.
    
    Remaining string ops (`upper` / `lower` / `strip` / slicing / ...) are out of scope here.
    
    ### Why are the changes needed?
    
    Python UDF `len(s)` is a common, cheap string operation that still paid the Python worker tax. Mapping it to native `length` is the smallest missing string op that can land before 4.3 RC.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Only with the experimental flags already required for transpilation (`spark.sql.experimental.optimizer.transpilePyUDFs=true` and `spark.sql.ansi.enabled=true`):
    
    ```python
    strlen = udf(lambda x: len(x), "long")
    df.select(strlen(col("s")))
    ```
    
    Previously this stayed an interpreted Python UDF. Now it can run as Catalyst `length`. Default (flags off) is unchanged.
    
    ### How was this patch tested?
    
    Added `test_udf_transpile_string_len` in `python/pyspark/sql/tests/test_udf_transpile_unit.py` (empty / ASCII / concat / NULL, plus `len` on a long column falling back). Ran:
    
    ```
    python/run-tests --testnames pyspark.sql.tests.test_udf_transpile_unit
    ```
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Yes, co-authored using Cursor Grok 4.6
    
    Closes #59081 from mounikavas/SPARK-55214.
    
    Lead-authored-by: Monica G <mo...@gmail.com>
    Co-authored-by: Holden Karau <ho...@pigscanfly.ca>
    Signed-off-by: Holden Karau <ho...@snowflake.com>
    (cherry picked from commit 47301f96b5e6a72585bcd161b5c4b84113fcc1ec)
    Signed-off-by: Holden Karau <ho...@snowflake.com>
---
 .../pyspark/sql/tests/test_udf_transpile_unit.py   | 25 ++++++++++++++++
 python/pyspark/sql/transpile.py                    | 33 ++++++++++++++++++++++
 2 files changed, 58 insertions(+)

diff --git a/python/pyspark/sql/tests/test_udf_transpile_unit.py b/python/pyspark/sql/tests/test_udf_transpile_unit.py
index bccf1dc683e0..ea32a5648db7 100644
--- a/python/pyspark/sql/tests/test_udf_transpile_unit.py
+++ b/python/pyspark/sql/tests/test_udf_transpile_unit.py
@@ -2454,6 +2454,31 @@ class UDFTranspileUnitTests(ReusedSQLTestCase):
             with self.subTest(case=i):
                 self.assertEqual(self._vals(func, rt, schema, rows), expected, f"case {i}")
 
+    def test_udf_transpile_string_len(self):
+        # SPARK-55214: Empty, ASCII, and concat inputs match Python.
+        # Unguarded ``len(NULL)`` raises like CPython (not Spark length's NULL).
+        # A proven-non-null branch still returns None for NULL rows. ``len`` on
+        # a numeric column has no matching string option, so the UDF falls back
+        # and Python raises TypeError.
+        L = LongType()
+        strlen = lambda x: len(x)  # noqa: E731
+        len_concat = lambda a, b: len(a + b)  # noqa: E731
+        strlen_guarded = lambda x: len(x) if x is not None else None  # noqa: E731
+        self.assertEqual(
+            self._vals(strlen, L, "a string", [("",), ("ab",), ("a",)]),
+            [0, 2, 1],
+        )
+        self.assertEqual(
+            self._vals(len_concat, L, "a string, b string", [("x", "yz"), ("", "ab")]),
+            [3, 2],
+        )
+        self.assertEqual(
+            self._vals(strlen_guarded, L, "a string", [("ab",), (None,)]),
+            [2, None],
+        )
+        self._raises(strlen, "a string", [(None,)], needle="len()")
+        self._raises(strlen, "a long", [(5,)], needle="")
+
     def test_udf_transpile_string_operands_fall_back(self):
         # Operand/type combos with no valid string lowering for the bound column
         # types fall back to the Python UDF, which raises the same way CPython does:
diff --git a/python/pyspark/sql/transpile.py b/python/pyspark/sql/transpile.py
index dd2c99789788..61b88903c89f 100644
--- a/python/pyspark/sql/transpile.py
+++ b/python/pyspark/sql/transpile.py
@@ -35,6 +35,10 @@ keeps the option matrix small; prefer doing so. To bound plan growth,
 functions with more than three untyped parameters only emit the
 all-numeric and all-string variants.
 
+``len(s)`` lowers to Catalyst ``length`` for a string operand (SPARK-55214).
+Other ``len`` arguments (numbers, lists, ...) stay interpreted Python.
+``len(None)`` raises (via ``raise_error`` on NULL), matching CPython.
+
 A lambda is lowered only when its source names it directly and alone: bind it
 to a name (``f = lambda x: x + 1``, annotated if you like) and give it a line
 of its own. Passed straight to ``udf(...)``, wrapped in another call, returned
@@ -504,6 +508,15 @@ class CatalystTranspiler(AbstractTranspiler):
                     f"operands of `{type(op).__name__}` are not type-compatible "
                     "for this input-type variant"
                 )
+            case ast.Call(func=ast.Name(id="len"), args=[arg], keywords=[]):
+                # ``len(s)`` is an int. Only strings are lowered (SPARK-55214);
+                # other argument categories raise so this variant is dropped.
+                if self._category(params, arg) != "string":
+                    raise UnsupportedOperationException(
+                        "`len` is only lowered for string operands; other "
+                        "types fall back to interpreted Python"
+                    )
+                return "numeric"
             case ast.Return(value=value) if value is not None:
                 return self._category(params, value)
             case ast.IfExp(body=if_body, orelse=if_orelse):
@@ -801,6 +814,26 @@ class CatalystTranspiler(AbstractTranspiler):
                         f"name {name!r} is not in the UDF's parameter list "
                         "and free variables / closures are not supported"
                     )
+            case ast.Call(func=ast.Name(id="len"), args=[arg], keywords=[]):
+                # SPARK-55214: Python ``len`` on a str is the number of Unicode
+                # code points; Spark ``length`` on a string column is character
+                # length -- they match for well-formed UTF-8. ``len(None)``
+                # raises TypeError in Python, while Spark ``length(NULL)`` is
+                # NULL, so guard like value comparisons: raise on NULL, else
+                # ``length``. A caller that already proved non-null (``if x is
+                # not None: return len(x)``) takes the otherwise branch.
+                if self._category(params, arg) != "string":
+                    raise UnsupportedOperationException(
+                        "`len` is only lowered for string operands; other "
+                        "types fall back to interpreted Python"
+                    )
+                arg_col = self._convert_chunk(params, arg)
+                err = lit(
+                    "Python UDF transpiler: cannot call len() on NULL; "
+                    "Python would raise TypeError here. Add an "
+                    "`is not None` guard or filter NULLs upstream."
+                )
+                return when(arg_col.isNull(), raise_error(err)).otherwise(length(arg_col))
             case _:
                 raise UnsupportedOperationException(
                     f"AST node {type(body).__name__} is not supported by the "


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org