You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/09/14 14:04:05 UTC

(datafusion-comet) 01/01: test: cover string expression routing and native opt-in (#5915)

This is an automated email from the ASF dual-hosted git repository.

github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-5915-481aefea9c60592612650de73bd2e1c7aa173979
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git

commit 1fa558ef7a3e19f35c43e63f1127a3f167225f05
Author: KUAN-HAO HUANG <10...@users.noreply.github.com>
AuthorDate: Mon Sep 14 14:03:40 2026 +0000

    test: cover string expression routing and native opt-in (#5915)
---
 .../string/routing_case_conversion_disabled.sql    | 31 +++++++++++++++
 .../string/routing_case_conversion_enabled.sql     | 31 +++++++++++++++
 .../string/routing_case_conversion_opt_in.sql      | 31 +++++++++++++++
 .../string/routing_string_opt_in_disabled.sql      | 36 +++++++++++++++++
 .../string/routing_string_opt_in_enabled.sql       | 36 +++++++++++++++++
 .../string/routing_string_opt_in_opt_in.sql        | 36 +++++++++++++++++
 .../string/routing_strings_disabled.sql            | 46 ++++++++++++++++++++++
 .../expressions/string/routing_strings_enabled.sql | 46 ++++++++++++++++++++++
 .../expressions/string/routing_strings_opt_in.sql  | 46 ++++++++++++++++++++++
 .../string/routing_unbase64_disabled.sql           | 33 ++++++++++++++++
 .../string/routing_unbase64_enabled.sql            | 33 ++++++++++++++++
 11 files changed, 405 insertions(+)

diff --git a/spark/src/test/resources/sql-tests/expressions/string/routing_case_conversion_disabled.sql b/spark/src/test/resources/sql-tests/expressions/string/routing_case_conversion_disabled.sql
new file mode 100644
index 0000000000..3c09468db5
--- /dev/null
+++ b/spark/src/test/resources/sql-tests/expressions/string/routing_case_conversion_disabled.sql
@@ -0,0 +1,31 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- Config: spark.comet.exec.scalaUDF.codegen.enabled=false
+-- Config: spark.comet.caseConversion.enabled=false
+
+statement
+CREATE TABLE routing_case_conversion(s STRING) USING parquet
+
+statement
+INSERT INTO routing_case_conversion VALUES ('Hello world'), (''), (NULL)
+
+query expect_fallback(upper: spark.comet.exec.scalaUDF.codegen.enabled=false)
+SELECT upper(s) FROM routing_case_conversion
+
+query expect_fallback(lower: spark.comet.exec.scalaUDF.codegen.enabled=false)
+SELECT lower(s) FROM routing_case_conversion
diff --git a/spark/src/test/resources/sql-tests/expressions/string/routing_case_conversion_enabled.sql b/spark/src/test/resources/sql-tests/expressions/string/routing_case_conversion_enabled.sql
new file mode 100644
index 0000000000..e8eef4cfb1
--- /dev/null
+++ b/spark/src/test/resources/sql-tests/expressions/string/routing_case_conversion_enabled.sql
@@ -0,0 +1,31 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- Config: spark.comet.exec.scalaUDF.codegen.enabled=true
+-- Config: spark.comet.caseConversion.enabled=false
+
+statement
+CREATE TABLE routing_case_conversion(s STRING) USING parquet
+
+statement
+INSERT INTO routing_case_conversion VALUES ('Hello world'), (''), (NULL)
+
+query expect_dispatch(upper)
+SELECT upper(s) FROM routing_case_conversion
+
+query expect_dispatch(lower)
+SELECT lower(s) FROM routing_case_conversion
diff --git a/spark/src/test/resources/sql-tests/expressions/string/routing_case_conversion_opt_in.sql b/spark/src/test/resources/sql-tests/expressions/string/routing_case_conversion_opt_in.sql
new file mode 100644
index 0000000000..c9cbaf065a
--- /dev/null
+++ b/spark/src/test/resources/sql-tests/expressions/string/routing_case_conversion_opt_in.sql
@@ -0,0 +1,31 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- Config: spark.comet.exec.scalaUDF.codegen.enabled=false
+-- Config: spark.comet.caseConversion.enabled=true
+
+statement
+CREATE TABLE routing_case_conversion(s STRING) USING parquet
+
+statement
+INSERT INTO routing_case_conversion VALUES ('Hello world'), (''), (NULL)
+
+query expect_native(upper)
+SELECT upper(s) FROM routing_case_conversion
+
+query expect_native(lower)
+SELECT lower(s) FROM routing_case_conversion
diff --git a/spark/src/test/resources/sql-tests/expressions/string/routing_string_opt_in_disabled.sql b/spark/src/test/resources/sql-tests/expressions/string/routing_string_opt_in_disabled.sql
new file mode 100644
index 0000000000..96357a6b96
--- /dev/null
+++ b/spark/src/test/resources/sql-tests/expressions/string/routing_string_opt_in_disabled.sql
@@ -0,0 +1,36 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- Config: spark.comet.exec.scalaUDF.codegen.enabled=false
+-- Config: spark.comet.expression.InitCap.allowIncompatible=false
+-- Config: spark.comet.expression.StringReplace.allowIncompatible=false
+-- Config: spark.comet.expression.GetJsonObject.allowIncompatible=false
+
+statement
+CREATE TABLE routing_string_opt_in(s STRING, j STRING) USING parquet
+
+statement
+INSERT INTO routing_string_opt_in VALUES ('hello world', '{"a":1}'), ('', '{}'), (NULL, NULL)
+
+query expect_fallback(initcap: spark.comet.exec.scalaUDF.codegen.enabled=false)
+SELECT initcap(s) FROM routing_string_opt_in
+
+query expect_fallback(replace: spark.comet.exec.scalaUDF.codegen.enabled=false)
+SELECT replace(s, 'l', 'L') FROM routing_string_opt_in
+
+query expect_fallback(get_json_object: spark.comet.exec.scalaUDF.codegen.enabled=false)
+SELECT get_json_object(j, '$.a') FROM routing_string_opt_in
diff --git a/spark/src/test/resources/sql-tests/expressions/string/routing_string_opt_in_enabled.sql b/spark/src/test/resources/sql-tests/expressions/string/routing_string_opt_in_enabled.sql
new file mode 100644
index 0000000000..ca2f091c05
--- /dev/null
+++ b/spark/src/test/resources/sql-tests/expressions/string/routing_string_opt_in_enabled.sql
@@ -0,0 +1,36 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- Config: spark.comet.exec.scalaUDF.codegen.enabled=true
+-- Config: spark.comet.expression.InitCap.allowIncompatible=false
+-- Config: spark.comet.expression.StringReplace.allowIncompatible=false
+-- Config: spark.comet.expression.GetJsonObject.allowIncompatible=false
+
+statement
+CREATE TABLE routing_string_opt_in(s STRING, j STRING) USING parquet
+
+statement
+INSERT INTO routing_string_opt_in VALUES ('hello world', '{"a":1}'), ('', '{}'), (NULL, NULL)
+
+query expect_dispatch(initcap)
+SELECT initcap(s) FROM routing_string_opt_in
+
+query expect_dispatch(replace)
+SELECT replace(s, 'l', 'L') FROM routing_string_opt_in
+
+query expect_dispatch(get_json_object)
+SELECT get_json_object(j, '$.a') FROM routing_string_opt_in
diff --git a/spark/src/test/resources/sql-tests/expressions/string/routing_string_opt_in_opt_in.sql b/spark/src/test/resources/sql-tests/expressions/string/routing_string_opt_in_opt_in.sql
new file mode 100644
index 0000000000..f87062cd8e
--- /dev/null
+++ b/spark/src/test/resources/sql-tests/expressions/string/routing_string_opt_in_opt_in.sql
@@ -0,0 +1,36 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- Config: spark.comet.exec.scalaUDF.codegen.enabled=false
+-- Config: spark.comet.expression.InitCap.allowIncompatible=true
+-- Config: spark.comet.expression.StringReplace.allowIncompatible=true
+-- Config: spark.comet.expression.GetJsonObject.allowIncompatible=true
+
+statement
+CREATE TABLE routing_string_opt_in(s STRING, j STRING) USING parquet
+
+statement
+INSERT INTO routing_string_opt_in VALUES ('hello world', '{"a":1}'), ('', '{}'), (NULL, NULL)
+
+query expect_native(initcap)
+SELECT initcap(s) FROM routing_string_opt_in
+
+query expect_native(replace)
+SELECT replace(s, 'l', 'L') FROM routing_string_opt_in
+
+query expect_native(get_json_object)
+SELECT get_json_object(j, '$.a') FROM routing_string_opt_in
diff --git a/spark/src/test/resources/sql-tests/expressions/string/routing_strings_disabled.sql b/spark/src/test/resources/sql-tests/expressions/string/routing_strings_disabled.sql
new file mode 100644
index 0000000000..234831ece6
--- /dev/null
+++ b/spark/src/test/resources/sql-tests/expressions/string/routing_strings_disabled.sql
@@ -0,0 +1,46 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- Config: spark.comet.exec.scalaUDF.codegen.enabled=false
+-- Config: spark.comet.expression.StringTranslate.allowIncompatible=false
+
+statement
+CREATE TABLE routing_strings(s STRING, p STRING, b BINARY, a ARRAY<INT>) USING parquet
+
+statement
+INSERT INTO routing_strings VALUES ('h_llo', 'h_llo', unhex('4142'), array(1, 2)), ('hello', 'h_llo', unhex(''), array()), (NULL, NULL, NULL, NULL)
+
+query expect_native(like)
+SELECT s LIKE p FROM routing_strings
+
+query expect_native(concat)
+SELECT concat(s, '!') FROM routing_strings
+
+query expect_native(concat_ws)
+SELECT concat_ws('-', s, 'x') FROM routing_strings
+
+query expect_fallback(like: spark.comet.exec.scalaUDF.codegen.enabled=false)
+SELECT s LIKE 'h$_llo' ESCAPE '$' FROM routing_strings
+
+query expect_fallback(concat: spark.comet.exec.scalaUDF.codegen.enabled=false)
+SELECT concat(b, b), concat(a, a) FROM routing_strings
+
+query expect_fallback(concat_ws: spark.comet.exec.scalaUDF.codegen.enabled=false)
+SELECT concat_ws('-', 'a', 'b')
+
+query expect_fallback(translate: spark.comet.exec.scalaUDF.codegen.enabled=false)
+SELECT translate(s, 'hl', 'HL') FROM routing_strings
diff --git a/spark/src/test/resources/sql-tests/expressions/string/routing_strings_enabled.sql b/spark/src/test/resources/sql-tests/expressions/string/routing_strings_enabled.sql
new file mode 100644
index 0000000000..da1db9cd03
--- /dev/null
+++ b/spark/src/test/resources/sql-tests/expressions/string/routing_strings_enabled.sql
@@ -0,0 +1,46 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- Config: spark.comet.exec.scalaUDF.codegen.enabled=true
+-- Config: spark.comet.expression.StringTranslate.allowIncompatible=false
+
+statement
+CREATE TABLE routing_strings(s STRING, p STRING, b BINARY, a ARRAY<INT>) USING parquet
+
+statement
+INSERT INTO routing_strings VALUES ('h_llo', 'h_llo', unhex('4142'), array(1, 2)), ('hello', 'h_llo', unhex(''), array()), (NULL, NULL, NULL, NULL)
+
+query expect_native(like)
+SELECT s LIKE p FROM routing_strings
+
+query expect_native(concat)
+SELECT concat(s, '!') FROM routing_strings
+
+query expect_native(concat_ws)
+SELECT concat_ws('-', s, 'x') FROM routing_strings
+
+query expect_dispatch(like)
+SELECT s LIKE 'h$_llo' ESCAPE '$' FROM routing_strings
+
+query expect_dispatch(concat)
+SELECT concat(b, b), concat(a, a) FROM routing_strings
+
+query expect_dispatch(concat_ws)
+SELECT concat_ws('-', 'a', 'b')
+
+query expect_dispatch(translate)
+SELECT translate(s, 'hl', 'HL') FROM routing_strings
diff --git a/spark/src/test/resources/sql-tests/expressions/string/routing_strings_opt_in.sql b/spark/src/test/resources/sql-tests/expressions/string/routing_strings_opt_in.sql
new file mode 100644
index 0000000000..b64e34aa05
--- /dev/null
+++ b/spark/src/test/resources/sql-tests/expressions/string/routing_strings_opt_in.sql
@@ -0,0 +1,46 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- Config: spark.comet.exec.scalaUDF.codegen.enabled=false
+-- Config: spark.comet.expression.StringTranslate.allowIncompatible=true
+
+statement
+CREATE TABLE routing_strings(s STRING, p STRING, b BINARY, a ARRAY<INT>) USING parquet
+
+statement
+INSERT INTO routing_strings VALUES ('h_llo', 'h_llo', unhex('4142'), array(1, 2)), ('hello', 'h_llo', unhex(''), array()), (NULL, NULL, NULL, NULL)
+
+query expect_native(like)
+SELECT s LIKE p FROM routing_strings
+
+query expect_native(concat)
+SELECT concat(s, '!') FROM routing_strings
+
+query expect_native(concat_ws)
+SELECT concat_ws('-', s, 'x') FROM routing_strings
+
+query expect_fallback(like: spark.comet.exec.scalaUDF.codegen.enabled=false)
+SELECT s LIKE 'h$_llo' ESCAPE '$' FROM routing_strings
+
+query expect_fallback(concat: spark.comet.exec.scalaUDF.codegen.enabled=false)
+SELECT concat(b, b), concat(a, a) FROM routing_strings
+
+query expect_fallback(concat_ws: spark.comet.exec.scalaUDF.codegen.enabled=false)
+SELECT concat_ws('-', 'a', 'b')
+
+query expect_native(translate)
+SELECT translate(s, 'hl', 'HL') FROM routing_strings
diff --git a/spark/src/test/resources/sql-tests/expressions/string/routing_unbase64_disabled.sql b/spark/src/test/resources/sql-tests/expressions/string/routing_unbase64_disabled.sql
new file mode 100644
index 0000000000..fe39fcea1f
--- /dev/null
+++ b/spark/src/test/resources/sql-tests/expressions/string/routing_unbase64_disabled.sql
@@ -0,0 +1,33 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- Config: spark.comet.exec.scalaUDF.codegen.enabled=false
+
+statement
+CREATE TABLE routing_unbase64(s STRING) USING parquet
+
+statement
+INSERT INTO routing_unbase64 VALUES ('aGVsbG8='), (''), (NULL)
+
+query expect_native(unbase64)
+SELECT unbase64(s) FROM routing_unbase64
+
+query expect_fallback(unbase64: spark.comet.exec.scalaUDF.codegen.enabled=false)
+SELECT unbase64(concat(s, '')) FROM routing_unbase64
+
+query expect_fallback(unbase64: spark.comet.exec.scalaUDF.codegen.enabled=false)
+SELECT to_binary(s, 'base64') FROM routing_unbase64
diff --git a/spark/src/test/resources/sql-tests/expressions/string/routing_unbase64_enabled.sql b/spark/src/test/resources/sql-tests/expressions/string/routing_unbase64_enabled.sql
new file mode 100644
index 0000000000..9e238f1dc9
--- /dev/null
+++ b/spark/src/test/resources/sql-tests/expressions/string/routing_unbase64_enabled.sql
@@ -0,0 +1,33 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- Config: spark.comet.exec.scalaUDF.codegen.enabled=true
+
+statement
+CREATE TABLE routing_unbase64(s STRING) USING parquet
+
+statement
+INSERT INTO routing_unbase64 VALUES ('aGVsbG8='), (''), (NULL)
+
+query expect_native(unbase64)
+SELECT unbase64(s) FROM routing_unbase64
+
+query expect_dispatch(unbase64)
+SELECT unbase64(concat(s, '')) FROM routing_unbase64
+
+query expect_dispatch(unbase64)
+SELECT to_binary(s, 'base64') FROM routing_unbase64


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org