You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/09/10 13:55:57 UTC

(datafusion) branch gh-readonly-queue/main/pr-25046-46ec37d87695c1e96fdcbde1a7b9aa1e919d1719 created (now 935748de22)

This is an automated email from the ASF dual-hosted git repository.

github-merge-queue[bot] pushed a change to branch gh-readonly-queue/main/pr-25046-46ec37d87695c1e96fdcbde1a7b9aa1e919d1719
in repository https://gitbox.apache.org/repos/asf/datafusion.git


      at 935748de22 fix: accept empty flags in regexp_like and regexp_match (#25046)

This branch includes the following new commits:

     new 935748de22 fix: accept empty flags in regexp_like and regexp_match (#25046)

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.



---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org


(datafusion) 01/01: fix: accept empty flags in regexp_like and regexp_match (#25046)

Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-25046-46ec37d87695c1e96fdcbde1a7b9aa1e919d1719
in repository https://gitbox.apache.org/repos/asf/datafusion.git

commit 935748de2291b9ffeb80d6f2ef92b81c9f03bfe4
Author: Peter Lee <pe...@gmail.com>
AuthorDate: Thu Sep 10 13:55:34 2026 +0000

    fix: accept empty flags in regexp_like and regexp_match (#25046)
    
    ## Which issue does this PR close?
    
    Closes #25021.
    
    ## Rationale for this change
    
    An empty flags string should behave like omitted flags. `regexp_match`
    currently rejects it, and `regexp_like` can accept or reject it
    depending on whether the optimizer simplifies the call.
    
    ## What changes are included in this PR?
    
    Treat empty flags as no flags before calling Arrow's regex kernels.
    Apply the same handling to scalar evaluation and the regex compilation
    size check during planning.
    
    ## What is the testing strategy for this PR?
    
    Regression tests cover every combination of scalar and array arguments
    for `Utf8`, `LargeUtf8`, and `Utf8View`, including empty patterns,
    captures, and batches containing empty, null, and nonempty flags. The
    planning test also verifies that empty flags preserve the compilation
    size limit. SQLLogicTest cases in `regexp_match.slt` and
    `regexp_like.slt` cover literal and column flags, case sensitivity,
    empty patterns, and nulls; both files pass.
    
    All 35 regex tests pass on the current base. The extended workspace
    suite passed before rebasing: 11,259 Rust tests plus SQLLogicTests.
    Formatting, workspace Clippy with all targets and features, and the
    repository lint suite pass.
    
    Existing regex benchmarks were run against `main` at `6ab4ce660` using
    the `ci` profile. Timings varied substantially under concurrent machine
    load, so they do not support a performance conclusion.
    
    ## Are there any user-facing changes?
    
    Yes. `regexp_match('foobarbaz', 'b..', '')` returns `[bar]`, and
    `regexp_like('foobarbaz', 'b..', flags)` returns `true` when the flags
    column contains an empty string.
---
 datafusion/functions/src/regex/mod.rs              | 113 +++++++++++++++++++++
 datafusion/functions/src/regex/regexplike.rs       |  15 +--
 datafusion/functions/src/regex/regexpmatch.rs      |   8 +-
 .../sqllogictest/test_files/regexp/regexp_like.slt |  33 ++++++
 .../test_files/regexp/regexp_match.slt             |  33 ++++++
 5 files changed, 194 insertions(+), 8 deletions(-)

diff --git a/datafusion/functions/src/regex/mod.rs b/datafusion/functions/src/regex/mod.rs
index 614fcc5af0..3877251c66 100644
--- a/datafusion/functions/src/regex/mod.rs
+++ b/datafusion/functions/src/regex/mod.rs
@@ -17,7 +17,10 @@
 
 //! "regex" DataFusion functions
 
+use arrow::array::ArrayRef;
+use arrow::compute::kernels::{cmp::eq, nullif::nullif};
 use arrow::error::ArrowError;
+use datafusion_common::{Result, ScalarValue};
 use regex::Regex;
 use std::collections::HashMap;
 use std::collections::hash_map::Entry;
@@ -28,6 +31,14 @@ pub mod regexplike;
 pub mod regexpmatch;
 pub mod regexpreplace;
 
+/// Arrow's regex kernels treat null flags as no flags, but reject empty flags.
+/// Normalize empty strings without copying the string buffers.
+fn normalize_empty_flags(flags: &ArrayRef) -> Result<ArrayRef> {
+    let empty =
+        ScalarValue::try_from_string(String::new(), flags.data_type())?.to_scalar()?;
+    Ok(nullif(flags, &eq(flags, &empty)?)?)
+}
+
 // create UDFs
 make_udf_function!(regexpcount::RegexpCountFunc, regexp_count);
 make_udf_function!(regexpinstr::RegexpInstrFunc, regexp_instr);
@@ -185,6 +196,108 @@ pub fn compile_regex(regex: &str, flags: Option<&str>) -> Result<Regex, ArrowErr
 mod tests {
     use super::start_to_byte_offset;
 
+    #[test]
+    fn empty_flags_match_omitted_flags() {
+        use arrow::array::StringArray;
+        use arrow::compute::cast;
+        use arrow::datatypes::{DataType, Field};
+        use datafusion_common::config::ConfigOptions;
+        use datafusion_expr::{ColumnarValue, ScalarFunctionArgs};
+
+        use super::*;
+
+        for udf in [regexp_like(), regexp_match()] {
+            for data_type in [DataType::Utf8, DataType::LargeUtf8, DataType::Utf8View] {
+                // Exercise every scalar/array combination, bypassing simplification.
+                for shape in 0..8 {
+                    for pattern in ["b..", "B..", "", "(b)(..)"] {
+                        let args = ["foobarbaz", pattern, ""]
+                            .into_iter()
+                            .enumerate()
+                            .map(|(i, value)| {
+                                let scalar = ScalarValue::try_from_string(
+                                    value.to_string(),
+                                    &data_type,
+                                )
+                                .unwrap();
+                                if shape & (1 << i) == 0 {
+                                    ColumnarValue::Scalar(scalar)
+                                } else {
+                                    ColumnarValue::Array(
+                                        scalar.to_array_of_size(3).unwrap(),
+                                    )
+                                }
+                            })
+                            .collect::<Vec<_>>();
+                        let invoke = |args: Vec<ColumnarValue>| {
+                            let types = args
+                                .iter()
+                                .map(ColumnarValue::data_type)
+                                .collect::<Vec<_>>();
+                            let return_type = udf.return_type(&types).unwrap();
+                            udf.invoke_with_args(ScalarFunctionArgs {
+                                arg_fields: types
+                                    .into_iter()
+                                    .map(|t| Arc::new(Field::new("arg", t, true)))
+                                    .collect(),
+                                args,
+                                number_rows: 3,
+                                return_field: Arc::new(Field::new(
+                                    "result",
+                                    return_type,
+                                    true,
+                                )),
+                                config_options: Arc::new(ConfigOptions::default()),
+                            })
+                            .unwrap()
+                            .to_array(3)
+                            .unwrap()
+                        };
+                        assert_eq!(
+                            &invoke(args.clone()),
+                            &invoke(args[..2].to_vec()),
+                            "{} {data_type:?} shape={shape} pattern={pattern:?}",
+                            udf.name()
+                        );
+                    }
+                }
+
+                // Empty and null flags both mean no flags; preserve nonempty flags
+                // and null values/patterns in the same batch.
+                let arrays = [
+                    vec![
+                        Some("abc"),
+                        Some("ABC"),
+                        Some("ABC"),
+                        None,
+                        Some("abc"),
+                        Some("abc"),
+                    ],
+                    vec![Some("a"), Some("a"), Some("a"), Some("a"), None, Some("a")],
+                    vec![Some(""), Some("i"), Some(""), Some(""), Some(""), None],
+                ]
+                .map(|values| cast(&StringArray::from(values), &data_type).unwrap());
+                let expected_flags = cast(
+                    &StringArray::from(vec![None, Some("i"), None, None, None, None]),
+                    &data_type,
+                )
+                .unwrap();
+                let kernel = if udf.name() == "regexp_like" {
+                    regexplike::regexp_like
+                } else {
+                    regexpmatch::regexp_match
+                };
+                let expected = kernel(&[
+                    Arc::clone(&arrays[0]),
+                    Arc::clone(&arrays[1]),
+                    expected_flags,
+                ])
+                .unwrap();
+                assert_eq!(&kernel(&arrays).unwrap(), &expected);
+            }
+        }
+    }
+
     #[test]
     fn start_to_byte_offset_ascii() {
         assert_eq!(start_to_byte_offset("abc", 1), Some(0));
diff --git a/datafusion/functions/src/regex/regexplike.rs b/datafusion/functions/src/regex/regexplike.rs
index 18114931cf..08cd7b0651 100644
--- a/datafusion/functions/src/regex/regexplike.rs
+++ b/datafusion/functions/src/regex/regexplike.rs
@@ -180,7 +180,7 @@ impl ScalarUDFImpl for RegexpLikeFunc {
             return true;
         }
 
-        let pattern = match flags {
+        let pattern = match flags.filter(|flags| !flags.is_empty()) {
             Some(flags) => Cow::Owned(format!("(?{flags}){pattern}")),
             None => Cow::Borrowed(pattern),
         };
@@ -308,10 +308,11 @@ pub fn regexp_like(args: &[ArrayRef]) -> Result<ArrayRef> {
     match args.len() {
         2 => handle_regexp_like(&args[0], &args[1], None),
         3 => {
-            let flags = match args[2].data_type() {
-                Utf8 => args[2].as_string::<i32>(),
+            let flags = super::normalize_empty_flags(&args[2])?;
+            let flags = match flags.data_type() {
+                Utf8 => flags.as_string::<i32>(),
                 LargeUtf8 => {
-                    let large_string_array = args[2].as_string::<i64>();
+                    let large_string_array = flags.as_string::<i64>();
                     let string_vec: Vec<Option<&str>> = (0..large_string_array.len())
                         .map(|i| {
                             if large_string_array.is_null(i) {
@@ -325,7 +326,7 @@ pub fn regexp_like(args: &[ArrayRef]) -> Result<ArrayRef> {
                     &GenericStringArray::<i32>::from(string_vec)
                 }
                 _ => {
-                    let string_view_array = args[2].as_string_view();
+                    let string_view_array = flags.as_string_view();
                     let string_vec: Vec<Option<String>> = (0..string_view_array.len())
                         .map(|i| {
                             if string_view_array.is_null(i) {
@@ -374,6 +375,7 @@ fn regexp_like_array_scalar(
     let Some(pattern) = pattern else {
         return Ok(Arc::new(BooleanArray::new_null(values.len())));
     };
+    let flags = flags.filter(|flags| !flags.is_empty());
     let array = match values.data_type() {
         Utf8 => {
             let array = values.as_string::<i32>();
@@ -412,7 +414,7 @@ fn regexp_like_scalar(
 
     let value = value.unwrap();
     let pattern = pattern.unwrap();
-    let pattern = match flags {
+    let pattern = match flags.filter(|flags| !flags.is_empty()) {
         Some(flagz) => format!("(?{flagz}){pattern}"),
         None => pattern.to_string(),
     };
@@ -679,6 +681,7 @@ mod tests {
 
         assert!(should_evaluate(args("^a+$", None)));
         assert!(!should_evaluate(args("a{5}{5}{5}{5}{5}{5}", None)));
+        assert!(!should_evaluate(args("a{5}{5}{5}{5}{5}{5}", Some(""))));
         assert!(!should_evaluate(args("a{5}{5}{5}{5}{5}{5}", Some("m"))));
 
         let null_value = vec![
diff --git a/datafusion/functions/src/regex/regexpmatch.rs b/datafusion/functions/src/regex/regexpmatch.rs
index ce7c437a54..7ca8ba82d5 100644
--- a/datafusion/functions/src/regex/regexpmatch.rs
+++ b/datafusion/functions/src/regex/regexpmatch.rs
@@ -192,7 +192,10 @@ fn regexp_match_scalar_pattern(args: &[ColumnarValue]) -> Result<Option<ArrayRef
     }
 
     let pattern = pattern.to_scalar()?;
-    let flags = flags.map(ScalarValue::to_scalar).transpose()?;
+    let flags = flags
+        .filter(|flags| flags.try_as_str() != Some(Some("")))
+        .map(ScalarValue::to_scalar)
+        .transpose()?;
 
     regexp::regexp_match(
         values,
@@ -237,7 +240,8 @@ pub fn regexp_match(args: &[ArrayRef]) -> Result<ArrayRef> {
                 }
             }
 
-            regexp::regexp_match(&args[0], &args[1], Some(&args[2]))
+            let flags = super::normalize_empty_flags(&args[2])?;
+            regexp::regexp_match(&args[0], &args[1], Some(&flags))
                 .map_err(|e| arrow_datafusion_err!(e))
         }
         other => exec_err!(
diff --git a/datafusion/sqllogictest/test_files/regexp/regexp_like.slt b/datafusion/sqllogictest/test_files/regexp/regexp_like.slt
index 30fa913896..14531fd466 100644
--- a/datafusion/sqllogictest/test_files/regexp/regexp_like.slt
+++ b/datafusion/sqllogictest/test_files/regexp/regexp_like.slt
@@ -368,3 +368,36 @@ select * from regexp_test where regexp_like('f', regexp_replace((('v\r') like ('
 
 statement ok
 drop table if exists dict_table;
+
+# Empty flags behave like omitted flags for literals and table columns.
+query BBB
+SELECT regexp_like('foobarbaz', 'b..', ''),
+       regexp_like('fooBAR', 'b..', ''),
+       regexp_like('foobarbaz', '', '');
+----
+true false true
+
+statement ok
+CREATE TABLE regexp_empty_flags (str VARCHAR, pattern VARCHAR, flags VARCHAR) AS VALUES
+    ('foobarbaz', 'b..', ''),
+    ('fooBAR', 'b..', ''),
+    ('fooBAR', 'b..', 'i'),
+    ('foobarbaz', '', ''),
+    (NULL, 'b..', ''),
+    ('foobarbaz', NULL, ''),
+    ('foobarbaz', 'b..', NULL);
+
+query BB
+SELECT regexp_like(str, pattern, flags), regexp_like(str, 'b..', '')
+FROM regexp_empty_flags;
+----
+true true
+false false
+true false
+true true
+NULL NULL
+NULL true
+true true
+
+statement ok
+DROP TABLE regexp_empty_flags;
diff --git a/datafusion/sqllogictest/test_files/regexp/regexp_match.slt b/datafusion/sqllogictest/test_files/regexp/regexp_match.slt
index e79af4774a..1bc97b5be6 100644
--- a/datafusion/sqllogictest/test_files/regexp/regexp_match.slt
+++ b/datafusion/sqllogictest/test_files/regexp/regexp_match.slt
@@ -199,3 +199,36 @@ query B
 select null !~* 'abc';
 ----
 NULL
+
+# Empty flags behave like omitted flags for literals and table columns.
+query ???
+SELECT regexp_match('foobarbaz', 'b..', ''),
+       regexp_match('fooBAR', 'b..', ''),
+       regexp_match('foobarbaz', '', '');
+----
+[bar] NULL []
+
+statement ok
+CREATE TABLE regexp_empty_flags (str VARCHAR, pattern VARCHAR, flags VARCHAR) AS VALUES
+    ('foobarbaz', 'b..', ''),
+    ('fooBAR', 'b..', ''),
+    ('fooBAR', 'b..', 'i'),
+    ('foobarbaz', '', ''),
+    (NULL, 'b..', ''),
+    ('foobarbaz', NULL, ''),
+    ('foobarbaz', 'b..', NULL);
+
+query ??
+SELECT regexp_match(str, pattern, flags), regexp_match(str, 'b..', '')
+FROM regexp_empty_flags;
+----
+[bar] [bar]
+NULL NULL
+[BAR] NULL
+[] [bar]
+NULL NULL
+NULL [bar]
+[bar] [bar]
+
+statement ok
+DROP TABLE regexp_empty_flags;


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org