(iceberg-python) branch main updated: fix: correct NOT STARTS WITH projection for truncated partitions (#3528)

[ke...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

kevinjqliu pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-python.git


The following commit(s) were added to refs/heads/main by this push:
     new fe7c74922 fix: correct NOT STARTS WITH projection for truncated partitions (#3528)
fe7c74922 is described below

commit fe7c749229be6a439a13371348ab741bba6fc940
Author: GayathriSrividya <ga...@gmail.com>
AuthorDate: Mon Jun 22 01:10:46 2026 +0530

    fix: correct NOT STARTS WITH projection for truncated partitions (#3528)
    
    closes #3493
    
    ## Summary
    
    Fixes incorrect projection of `NOT STARTS WITH` predicates for truncated
    string/binary partition fields. The current implementation unsafely
    truncates the filter literal without checking its length relative to the
    truncate width.
    
    ## Root Cause
    
    The `TruncateTransform.project` method calls `_truncate_array` which
    blindly truncates the literal for both `STARTS WITH` and `NOT STARTS
    WITH` predicates:
    
    ```python
    elif isinstance(pred, BoundNotStartsWith):
        return NotStartsWith(Reference(name), _transform_literal(transform, boundary))
    ```
    
    For `NOT STARTS WITH "hello"` with `truncate[2]`, this produces:
    - Current (unsafe): `NOT STARTS WITH "he"`
    - Problem: The truncated partition contains all values starting with
    "he" (from "hello", "heat", "hear", etc.), so we cannot safely exclude
    all non-"hello" rows
    
    ## Solution
    
    Add special handling for `BoundNotStartsWith` in the `project` method
    following the Java/Go reference behavior:
    
    - **prefix_length < truncate_width**: Keep original `NOT STARTS WITH`
    literal (safe)
    - **prefix_length == truncate_width**: Project to `!=` instead (safe
    equality check)
    - **prefix_length > truncate_width**: Return `None` (no inclusive
    projection possible)
    
    ### pyiceberg/transforms.py
    - Add explicit `NOT STARTS WITH` handling before calling
    `_truncate_array`
    - Check literal length vs truncate width and apply correct projection
    rules
    
    ### tests/test_transforms.py
    - Update `test_projection_truncate_string_not_starts_with` to expect
    `None` (prefix_length > width is unsafe)
    - Add `test_projection_truncate_string_not_starts_with_shorter_literal`
    (prefix_length == width → `!=`)
    - Add `test_projection_truncate_string_not_starts_with_original_literal`
    (prefix_length < width → original)
    
    ## Validation
    - `make lint` ✓ (all pre-commit hooks pass)
    - `pytest tests/test_transforms.py` → 280 passed ✓
    - All 13 string truncate projection tests pass
    
    ---------
    
    Co-authored-by: Gayathri Srividya Rajavarapu <ga...@Gayathris-MacBook-Air.local>
    Co-authored-by: Kevin Liu <ke...@users.noreply.github.com>
---
 pyiceberg/transforms.py  | 11 ++++++++++-
 tests/test_transforms.py | 20 +++++++++++++++++---
 2 files changed, 27 insertions(+), 4 deletions(-)

diff --git a/pyiceberg/transforms.py b/pyiceberg/transforms.py
index 739e18a6e..1b603b6bb 100644
--- a/pyiceberg/transforms.py
+++ b/pyiceberg/transforms.py
@@ -813,7 +813,16 @@ class TruncateTransform(Transform[S, S]):
                 return _truncate_number(name, pred, self.transform(field_type))
         elif isinstance(field_type, (BinaryType, StringType)):
             if isinstance(pred, BoundLiteralPredicate):
-                return _truncate_array(name, pred, self.transform(field_type))
+                if isinstance(pred, BoundNotStartsWith):
+                    literal_width = len(pred.literal.value)
+                    if literal_width < self.width:
+                        return pred.as_unbound(name, pred.literal.value)
+                    elif literal_width == self.width:
+                        return NotEqualTo(name, pred.literal.value)
+                    else:
+                        return None
+                else:
+                    return _truncate_array(name, pred, self.transform(field_type))
 
     def strict_project(self, name: str, pred: BoundPredicate) -> UnboundPredicate | None:
         field_type = pred.term.ref().field.field_type
diff --git a/tests/test_transforms.py b/tests/test_transforms.py
index 96500907c..381e93c68 100644
--- a/tests/test_transforms.py
+++ b/tests/test_transforms.py
@@ -1025,10 +1025,24 @@ def test_projection_truncate_string_starts_with(bound_reference_str: BoundRefere
     ) == StartsWith(term="name", literal=literal("he"))
 
 
-def test_projection_truncate_string_not_starts_with(bound_reference_str: BoundReference) -> None:
+def test_projection_truncate_string_not_starts_with_longer_literal(bound_reference_str: BoundReference) -> None:
+    # Not a valid projection: return None because pruning on "he" could drop qualifying rows like "help".
+    assert TruncateTransform(2).project("name", BoundNotStartsWith(term=bound_reference_str, literal=literal("hello"))) is None
+
+
+def test_projection_truncate_string_not_starts_with_shorter_literal(bound_reference_str: BoundReference) -> None:
+def test_projection_truncate_string_not_starts_with_equal_width_literal(bound_reference_str: BoundReference) -> None:
+    # Valid projection: improve NOT STARTS WITH "he" to partition != "he".
+    assert TruncateTransform(2).project(
+        "name", BoundNotStartsWith(term=bound_reference_str, literal=literal("he"))
+    ) == NotEqualTo(term="name", literal=literal("he"))
+
+
+def test_projection_truncate_string_not_starts_with_shorter_literal(bound_reference_str: BoundReference) -> None:
+    # Valid projection: pass the NOT STARTS WITH literal "h" through unchanged.
     assert TruncateTransform(2).project(
-        "name", BoundNotStartsWith(term=bound_reference_str, literal=literal("hello"))
-    ) == NotStartsWith(term="name", literal=literal("he"))
+        "name", BoundNotStartsWith(term=bound_reference_str, literal=literal("h"))
+    ) == NotStartsWith(term="name", literal=literal("h"))
 
 
 def _test_projection(lhs: UnboundPredicate | None, rhs: UnboundPredicate | None) -> None: