You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by mb...@apache.org on 2026/07/14 19:35:35 UTC
(datafusion) branch branch-54 updated: [branch-54] fix: regex simplification of anchored patterns produces wrong results (backport #22727) (#23578)
This is an automated email from the ASF dual-hosted git repository.
mbutrovich pushed a commit to branch branch-54
in repository https://gitbox.apache.org/repos/asf/datafusion.git
The following commit(s) were added to refs/heads/branch-54 by this push:
new 9d41b02983 [branch-54] fix: regex simplification of anchored patterns produces wrong results (backport #22727) (#23578)
9d41b02983 is described below
commit 9d41b02983058d692373c7ce18f6bba3b3afd5cc
Author: Matt Butrovich <mb...@users.noreply.github.com>
AuthorDate: Tue Jul 14 15:35:30 2026 -0400
[branch-54] fix: regex simplification of anchored patterns produces wrong results (backport #22727) (#23578)
## Which issue does this PR close?
- Backport of #22727 to `branch-54` (for 54.1.0, tracked in #22547).
- Fixes wrong results #22726 (regression introduced in 54.0.0 by #20581,
which extended the rewrite to these types).
## Rationale for this change
Anchored-regex simplification produces wrong results or errors on
`Utf8View` and `LargeUtf8` columns in 54.0.0. No API changes,
correctness-only fix, so it fits the backport criteria.
## What changes are included in this PR?
Clean cherry-pick of #22727. No adaptation required.
## Are these changes tested?
Yes. Carries the original regression coverage (`predicates.slt`), all
tests pass.
## Are there any user-facing changes?
Anchored regex predicates on `Utf8View`/`LargeUtf8` now return correct
results. No API changes.
Co-authored-by: linfeng <33...@users.noreply.github.com>
---
.../optimizer/src/simplify_expressions/regex.rs | 32 +++--
datafusion/sqllogictest/test_files/predicates.slt | 159 +++++++++++++++++++++
2 files changed, 181 insertions(+), 10 deletions(-)
diff --git a/datafusion/optimizer/src/simplify_expressions/regex.rs b/datafusion/optimizer/src/simplify_expressions/regex.rs
index b341c328e9..df4c344b2e 100644
--- a/datafusion/optimizer/src/simplify_expressions/regex.rs
+++ b/datafusion/optimizer/src/simplify_expressions/regex.rs
@@ -283,20 +283,23 @@ fn partial_anchored_literal_to_like(v: &[Hir]) -> Option<String> {
/// Extracts a string literal expression assuming that [`is_anchored_literal`]
/// returned true.
-fn anchored_literal_to_expr(v: &[Hir]) -> Option<Expr> {
+fn anchored_literal_to_expr(v: &[Hir], string_scalar: &StringScalar) -> Option<Expr> {
match v.len() {
- 2 => Some(lit("")),
+ 2 => Some(string_scalar.to_expr("")),
3 => {
let HirKind::Literal(l) = v[1].kind() else {
return None;
};
- like_str_from_literal(l).map(lit)
+ like_str_from_literal(l).map(|s| string_scalar.to_expr(s))
}
_ => None,
}
}
-fn anchored_alternation_to_exprs(v: &[Hir]) -> Option<Vec<Expr>> {
+fn anchored_alternation_to_exprs(
+ v: &[Hir],
+ string_scalar: &StringScalar,
+) -> Option<Vec<Expr>> {
if 3 != v.len() {
return None;
}
@@ -308,7 +311,8 @@ fn anchored_alternation_to_exprs(v: &[Hir]) -> Option<Vec<Expr>> {
for hir in alters {
let mut is_safe = false;
if let HirKind::Literal(l) = hir.kind()
- && let Some(safe_literal) = str_from_literal(l).map(lit)
+ && let Some(safe_literal) =
+ str_from_literal(l).map(|s| string_scalar.to_expr(s))
{
literals.push(safe_literal);
is_safe = true;
@@ -321,7 +325,9 @@ fn anchored_alternation_to_exprs(v: &[Hir]) -> Option<Vec<Expr>> {
return Some(literals);
} else if let HirKind::Literal(l) = sub.kind() {
- if let Some(safe_literal) = str_from_literal(l).map(lit) {
+ if let Some(safe_literal) =
+ str_from_literal(l).map(|s| string_scalar.to_expr(s))
+ {
return Some(vec![safe_literal]);
}
return None;
@@ -351,12 +357,18 @@ fn lower_simple(
));
}
HirKind::Concat(inner) if is_anchored_literal(inner) => {
- return anchored_literal_to_expr(inner).map(|right| {
- mode.expr_matches_literal(Box::new(left.clone()), Box::new(right))
+ return anchored_literal_to_expr(inner, string_scalar).map(|right| {
+ if mode.i {
+ // Case-insensitive: use ILIKE for exact match (no wildcards)
+ mode.expr(Box::new(left.clone()), Box::new(right))
+ } else {
+ // Case-sensitive: use Eq / NotEq
+ mode.expr_matches_literal(Box::new(left.clone()), Box::new(right))
+ }
});
}
- HirKind::Concat(inner) if is_anchored_capture(inner) => {
- return anchored_alternation_to_exprs(inner)
+ HirKind::Concat(inner) if !mode.i && is_anchored_capture(inner) => {
+ return anchored_alternation_to_exprs(inner, string_scalar)
.map(|right| left.clone().in_list(right, mode.not));
}
HirKind::Concat(inner) => {
diff --git a/datafusion/sqllogictest/test_files/predicates.slt b/datafusion/sqllogictest/test_files/predicates.slt
index d45c3e0b45..280d21cd71 100644
--- a/datafusion/sqllogictest/test_files/predicates.slt
+++ b/datafusion/sqllogictest/test_files/predicates.slt
@@ -204,12 +204,171 @@ SELECT * FROM test WHERE column1 ~ 'z'
----
Bazzz
+query T
+SELECT * FROM test WHERE column1 ~ '^Bazzz$'
+----
+Bazzz
+
+query T
+SELECT * FROM test WHERE column1 ~ '^(foo|Bazzz)$'
+----
+foo
+Bazzz
+
+statement ok
+CREATE TABLE test_regex_utf8view(s VARCHAR) AS VALUES ('foo'), ('Bazzz');
+
+statement ok
+set datafusion.explain.logical_plan_only = true
+
+# `~` anchored literal -> `= Utf8View(..)`
+query TT
+EXPLAIN SELECT * FROM test_regex_utf8view WHERE s ~ '^Bazzz$'
+----
+logical_plan
+01)Filter: test_regex_utf8view.s = Utf8View("Bazzz")
+02)--TableScan: test_regex_utf8view projection=[s]
+
+# `~*` anchored literal -> `ILIKE Utf8View(..)`
+query TT
+EXPLAIN SELECT * FROM test_regex_utf8view WHERE s ~* '^bazzz$'
+----
+logical_plan
+01)Filter: test_regex_utf8view.s ILIKE Utf8View("bazzz")
+02)--TableScan: test_regex_utf8view projection=[s]
+
+# `~` anchored alternation -> OR of `= Utf8View(..)` comparisons.
+query TT
+EXPLAIN SELECT * FROM test_regex_utf8view WHERE s ~ '^(foo|Bazzz)$'
+----
+logical_plan
+01)Filter: test_regex_utf8view.s = Utf8View("foo") OR test_regex_utf8view.s = Utf8View("Bazzz")
+02)--TableScan: test_regex_utf8view projection=[s]
+
+# `~*` anchored alternation -> NOT simplified: it falls back to a regex match,
+# because `IN`/`=` cannot express case-insensitive matching.
+query TT
+EXPLAIN SELECT * FROM test_regex_utf8view WHERE s ~* '^(foo|bazzz)$'
+----
+logical_plan
+01)Filter: test_regex_utf8view.s ~* Utf8View("^(foo|bazzz)$")
+02)--TableScan: test_regex_utf8view projection=[s]
+
+# `!~` -> `!= Utf8View(..)`
+query TT
+EXPLAIN SELECT * FROM test_regex_utf8view WHERE s !~ '^Bazzz$'
+----
+logical_plan
+01)Filter: test_regex_utf8view.s != Utf8View("Bazzz")
+02)--TableScan: test_regex_utf8view projection=[s]
+
+# `!~*` -> `NOT ILIKE Utf8View(..)`
+query TT
+EXPLAIN SELECT * FROM test_regex_utf8view WHERE s !~* '^bazzz$'
+----
+logical_plan
+01)Filter: test_regex_utf8view.s NOT ILIKE Utf8View("bazzz")
+02)--TableScan: test_regex_utf8view projection=[s]
+
+# `!~` anchored alternation -> AND of `!= Utf8View(..)` comparisons.
+query TT
+EXPLAIN SELECT * FROM test_regex_utf8view WHERE s !~ '^(foo|Bazzz)$'
+----
+logical_plan
+01)Filter: test_regex_utf8view.s != Utf8View("foo") AND test_regex_utf8view.s != Utf8View("Bazzz")
+02)--TableScan: test_regex_utf8view projection=[s]
+
+# `!~*` anchored alternation -> NOT simplified: it falls back to a regex match,
+# same reason as the `~*` alternation above.
+query TT
+EXPLAIN SELECT * FROM test_regex_utf8view WHERE s !~* '^(foo|bazzz)$'
+----
+logical_plan
+01)Filter: test_regex_utf8view.s !~* Utf8View("^(foo|bazzz)$")
+02)--TableScan: test_regex_utf8view projection=[s]
+
+statement ok
+set datafusion.explain.logical_plan_only = false
+
+# Result assertions
+query T
+SELECT * FROM test_regex_utf8view WHERE s ~ '^Bazzz$'
+----
+Bazzz
+
+query T
+SELECT * FROM test_regex_utf8view WHERE s ~ '^(foo|Bazzz)$'
+----
+foo
+Bazzz
+
+# Case-insensitive anchored match over Utf8View: must be simplified to ILIKE
+# (not a case-sensitive Eq) and must keep operand types as Utf8View.
+query T
+SELECT * FROM test_regex_utf8view WHERE s ~* '^bazzz$'
+----
+Bazzz
+
+# Case-insensitive anchored alternation over Utf8View
+query T rowsort
+SELECT * FROM test_regex_utf8view WHERE s ~* '^(foo|bazzz)$'
+----
+Bazzz
+foo
+
+query T rowsort
+SELECT * FROM test_regex_utf8view WHERE s !~ '^Bazzz$'
+----
+foo
+
+query T rowsort
+SELECT * FROM test_regex_utf8view WHERE s !~* '^bazzz$'
+----
+foo
+
+# Both rows match the alternation, so the negated forms return nothing.
+query T rowsort
+SELECT * FROM test_regex_utf8view WHERE s !~ '^(foo|Bazzz)$'
+----
+
+query T rowsort
+SELECT * FROM test_regex_utf8view WHERE s !~* '^(foo|bazzz)$'
+----
+
+statement ok
+DROP TABLE test_regex_utf8view;
+
query T
SELECT * FROM test WHERE column1 ~* 'z'
----
Bazzz
ZZZZZ
+query T
+SELECT * FROM test WHERE column1 ~* '^barrr$'
+----
+Barrr
+
+query T
+SELECT * FROM test WHERE column1 ~* '^(barrr|bazzz)$'
+----
+Barrr
+Bazzz
+
+query T rowsort
+SELECT * FROM test WHERE column1 !~ '^Bazzz$'
+----
+Barrr
+ZZZZZ
+foo
+
+query T rowsort
+SELECT * FROM test WHERE column1 !~* '^barrr$'
+----
+Bazzz
+ZZZZZ
+foo
+
query T
SELECT * FROM test WHERE column1 !~ 'z'
----
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org